From d3e830c7369940c5d1de005c4be26a552257089b Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Tue, 6 Jul 2021 13:08:30 -0400 Subject: [PATCH] refactor sync tools to use common vars and stuff maybe --- sync/common | 51 ++++++++++++++++++++++++++++ sync/prep-staging-8.sh | 68 +++++++++++-------------------------- sync/sync-to-prod.sh | 16 ++++++--- sync/sync-to-staging-sig.sh | 18 +++++++--- sync/sync-to-staging.sh | 18 +++++++--- 5 files changed, 108 insertions(+), 63 deletions(-) create mode 100644 sync/common diff --git a/sync/common b/sync/common new file mode 100644 index 0000000..cd322f8 --- /dev/null +++ b/sync/common @@ -0,0 +1,51 @@ +# To be sourced by scripts to use + +# Revision must always start with a major number +REVISION=8.4 + +MAJOR="${REVISION:0:1}" +MINOR="${REVISION:2:1}" + +# comment or blank if needed +APPEND_TO_DIR="-RC2" + +STAGING_ROOT="/mnt/repos-staging" +PRODUCTION_ROOT="/mnt/repos-production" + +# relative to ${ENV}_ROOT +CATEGORY_STUB="mirror/pub/rocky" +RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}" + +# Define arches we support +ARCHES=(x86_64 aarch64) + +# Set all repos that have no comps/groups associated with them. This is even in +# cases where repos will not be available by normal means. It's just for +# consistency. +NONMODS_REPOS=( + extras + Devel + nfv + storage/gluster9 + plus +) + +# These repos have comps/groups, except for debuginfo and sources +MODS_REPOS=( + BaseOS + AppStream + HighAvailability + ResilientStorage + PowerTools +) + +ALL_REPOS=( + "${NONMODS_REPOS[@]}" + "${MODS_REPOS[@]}" +) + +# These repos have modules +MODS=( + AppStream + PowerTools +) diff --git a/sync/prep-staging-8.sh b/sync/prep-staging-8.sh index 1f61dd9..1179f57 100644 --- a/sync/prep-staging-8.sh +++ b/sync/prep-staging-8.sh @@ -10,49 +10,19 @@ # and provide everything, even if it's not installable). # # Compose dir example: /mnt/repos-staging/mirror/pub/rocky/8.4-RC2 -# Revision must always start with a major number -REVISION=8.4 -#MAJOR="${REVISION:0:1}" -# comment or blank if needed -APPEND_TO_DIR="-RC2" -COMPOSE_DIR="/mnt/repos-staging/mirror/pub/rocky/${REVISION}${APPEND_TO_DIR}" -ARCHES=(x86_64 aarch64) -# Set all repos that have no comps/groups associated with them. This is even in -# cases where repos will not be available by normal means. It's just for -# consistency. -NONMODS_REPOS=( - extras - Devel - nfv - storage/gluster9 - plus -) - -# These repos have comps/groups, except for debuginfo and sources -MODS_REPOS=( - BaseOS - AppStream - HighAvailability - ResilientStorage - PowerTools -) - -# These repos have modules -MODS=( - AppStream - PowerTools -) +# Source common variables +source $(dirname "$0")/common echo "** Updating source repos" -for y in "${NONMODS_REPOS[@]}" "${MODS_REPOS[@]}"; do - test -d "${COMPOSE_DIR}/${y}/${x}/${z}" +for y in "${ALL_REPOS[@]}"; do + test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" ret_val=$? if [ "$ret_val" -eq 0 ]; then - createrepo --update "${COMPOSE_DIR}/${y}/source/tree" \ + createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree" \ "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" else - echo "${COMPOSE_DIR}/${y}/source/tree does not exist" + echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree does not exist" fi done @@ -64,41 +34,41 @@ for x in "${ARCHES[@]}"; do for y in "${NONMODS_REPOS[@]}"; do # os and debug/tree directories for z in os debug/tree; do - test -d "${COMPOSE_DIR}/${y}/${x}/${z}" + test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" ret_val=$? if [ "$ret_val" -eq 0 ]; then - createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \ + createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \ "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" else - echo "${COMPOSE_DIR}/${y}/${x}/${z} does not exist" + echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z} does not exist" fi done done # repos with comps/groups involved, but only debug for y in "${MODS_REPOS[@]}"; do - test -d "${COMPOSE_DIR}/${y}/${x}/debug/tree" + test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree" ret_val=$? if [ "$ret_val" -eq 0 ]; then - createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \ + createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \ "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" else - echo "${COMPOSE_DIR}/${y}/${x}/debug/tree does not exist" + echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree does not exist" fi done echo "** Update all repos with comps/groups" for y in "${MODS_REPOS[@]}"; do echo "${y}: ${x}" - test -d "${COMPOSE_DIR}/${y}/${x}/os" + test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os" ret_val=$? if [ "$ret_val" -eq 0 ]; then - createrepo --update "${COMPOSE_DIR}/${y}/${x}/os" \ + createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os" \ --groupfile="/mnt/compose/8/latest-Rocky-8/work/${x}/comps/comps-${y}.${x}.xml" \ --xz --revision=${REVISION} \ "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \ --workers=8 --checksum=sha256 else - echo "${COMPOSE_DIR}/${y}/${x}/os does not exist" + echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist" fi done @@ -106,14 +76,14 @@ for x in "${ARCHES[@]}"; do for y in "${MODS[@]}"; do echo "Modules: ${y} ${x}" cp "/mnt/compose/8_metadata/${x}/${y}-modules.yaml" /tmp/modules.yaml - test -d "${COMPOSE_DIR}/${y}/${x}/os" + test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os" ret_val=$? if [ "$ret_val" -eq 0 ]; then modifyrepo --mdtype=modules /tmp/modules.yaml \ - "${COMPOSE_DIR}/${y}/${x}/os/repodata" \ + "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os/repodata" \ --compress --compress-type=gz else - echo "${COMPOSE_DIR}/${y}/${x}/os does not exist" + echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist" fi rm /tmp/modules.yaml @@ -121,7 +91,7 @@ for x in "${ARCHES[@]}"; do done echo "** Fix variants" - TREEINFO_VAR="${COMPOSE_DIR}/BaseOS/${x}/os/.treeinfo" + TREEINFO_VAR="${STAGING_ROOT}/${RELEASE_DIR}/BaseOS/${x}/os/.treeinfo" test -f "${TREEINFO_VAR}" treeinfo_retval=$? test -x /usr/bin/python3 diff --git a/sync/sync-to-prod.sh b/sync/sync-to-prod.sh index 7b5ad1e..f8279ba 100644 --- a/sync/sync-to-prod.sh +++ b/sync/sync-to-prod.sh @@ -1,11 +1,19 @@ #!/bin/bash # Syncs everything from staging to production -REVISION=${1} -cd "/mnt/repos-staging/mirror/pub/rocky/${REVISION}" + +# Source common variables +source $(dirname "$0")/common + +REV=${1} + +cd "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" ret_val=$? + if [ $ret_val -eq "0" ]; then - mkdir -p "/mnt/repos-production/mirror/pub/rocky/${REVISION:0:3}" - sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-production/mirror/pub/rocky/${REVISION:0:3} + TARGET="${PRODUCTION_ROOT}/${CATEGORY_STUB}/${REV:0:3}" + mkdir -p "${TARGET}" + sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ + {} ${TARGET} else echo "Failed to change directory" fi diff --git a/sync/sync-to-staging-sig.sh b/sync/sync-to-staging-sig.sh index aa7e5d8..06561f1 100644 --- a/sync/sync-to-staging-sig.sh +++ b/sync/sync-to-staging-sig.sh @@ -1,17 +1,25 @@ #!/bin/bash + +# Source common variables +source $(dirname "$0")/common + # Major Version (eg, 8) -MAJOR=${1} +MAJ=${1} # Short name (eg, NFV, extras, Rocky, gluster9) SHORT=${2} # The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX -REVISION=${3} +REV=${3} # Note, this should be lowercase. eg, storage. SIG=${4} -cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR} + +cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ} ret_val=$? + if [ $ret_val -eq "0" ]; then - mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG} - sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG} + local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${SIG} + mkdir -p ${TARGET} + sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ + {} ${TARGET} else echo "Failed to change directory" fi diff --git a/sync/sync-to-staging.sh b/sync/sync-to-staging.sh index 45b1466..7581d3d 100644 --- a/sync/sync-to-staging.sh +++ b/sync/sync-to-staging.sh @@ -1,15 +1,23 @@ #!/bin/bash + +# Source common variables +source $(dirname "$0")/common + # Major Version (eg, 8) -MAJOR=${1} +MAJ=${1} # Short name (eg, NFV, extras, Rocky, gluster9) SHORT=${2} # The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX -REVISION=${3} -cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR} +REV=${3} + +cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ} ret_val=$? + if [ $ret_val -eq "0" ]; then - mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION} - sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION} + local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV} + mkdir -p ${TARGET} + sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ + {} ${TARGET} else echo "Failed to change directory" fi