refactor sync tools to use common vars and stuff maybe

sync/common

@@ -0,0 +1,51 @@
+# To be sourced by scripts to use
+
+# Revision must always start with a major number
+REVISION=8.4
+
+MAJOR="${REVISION:0:1}"
+MINOR="${REVISION:2:1}"
+
+# comment or blank if needed
+APPEND_TO_DIR="-RC2"
+
+STAGING_ROOT="/mnt/repos-staging"
+PRODUCTION_ROOT="/mnt/repos-production"
+
+# relative to ${ENV}_ROOT
+CATEGORY_STUB="mirror/pub/rocky"
+RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}"
+
+# Define arches we support
+ARCHES=(x86_64 aarch64)
+
+# Set all repos that have no comps/groups associated with them. This is even in
+# cases where repos will not be available by normal means. It's just for
+# consistency.
+NONMODS_REPOS=(
+  extras
+  Devel
+  nfv
+  storage/gluster9
+  plus
+)
+
+# These repos have comps/groups, except for debuginfo and sources
+MODS_REPOS=(
+  BaseOS
+  AppStream
+  HighAvailability
+  ResilientStorage
+  PowerTools
+)
+
+ALL_REPOS=(
+  "${NONMODS_REPOS[@]}"
+  "${MODS_REPOS[@]}"
+)
+
+# These repos have modules
+MODS=(
+  AppStream
+  PowerTools
+)

sync/prep-staging-8.sh

@@ -10,49 +10,19 @@
 # and provide everything, even if it's not installable).
 #
 # Compose dir example: /mnt/repos-staging/mirror/pub/rocky/8.4-RC2
-# Revision must always start with a major number
-REVISION=8.4
-#MAJOR="${REVISION:0:1}"
-# comment or blank if needed
-APPEND_TO_DIR="-RC2"
-COMPOSE_DIR="/mnt/repos-staging/mirror/pub/rocky/${REVISION}${APPEND_TO_DIR}"
-ARCHES=(x86_64 aarch64)
 
-# Set all repos that have no comps/groups associated with them. This is even in
+# Source common variables
-# cases where repos will not be available by normal means. It's just for
+source $(dirname "$0")/common
-# consistency.
-NONMODS_REPOS=(
-  extras
-  Devel
-  nfv
-  storage/gluster9
-  plus
-)
-
-# These repos have comps/groups, except for debuginfo and sources
-MODS_REPOS=(
-  BaseOS
-  AppStream
-  HighAvailability
-  ResilientStorage
-  PowerTools
-)
-
-# These repos have modules
-MODS=(
-  AppStream
-  PowerTools
-)
 
 echo "** Updating source repos"
-for y in "${NONMODS_REPOS[@]}" "${MODS_REPOS[@]}"; do
+for y in "${ALL_REPOS[@]}"; do
-  test -d "${COMPOSE_DIR}/${y}/${x}/${z}"
+  test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}"
   ret_val=$?
   if [ "$ret_val" -eq 0 ]; then
-    createrepo --update "${COMPOSE_DIR}/${y}/source/tree" \
+    createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree" \
       "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
   else
-    echo "${COMPOSE_DIR}/${y}/source/tree does not exist"
+    echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree does not exist"
   fi
 done
 
@@ -64,41 +34,41 @@ for x in "${ARCHES[@]}"; do
   for y in "${NONMODS_REPOS[@]}"; do
     # os and debug/tree directories
     for z in os debug/tree; do
-      test -d "${COMPOSE_DIR}/${y}/${x}/${z}"
+      test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}"
       ret_val=$?
       if [ "$ret_val" -eq 0 ]; then
-        createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \
+        createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \
           "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
       else
-        echo "${COMPOSE_DIR}/${y}/${x}/${z} does not exist"
+        echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z} does not exist"
       fi
     done
   done
   # repos with comps/groups involved, but only debug
   for y in "${MODS_REPOS[@]}"; do
-    test -d "${COMPOSE_DIR}/${y}/${x}/debug/tree"
+    test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree"
     ret_val=$?
     if [ "$ret_val" -eq 0 ]; then
-      createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \
+      createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \
         "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
     else
-      echo "${COMPOSE_DIR}/${y}/${x}/debug/tree does not exist"
+      echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree does not exist"
     fi
   done
 
   echo "** Update all repos with comps/groups"
   for y in "${MODS_REPOS[@]}"; do
     echo "${y}: ${x}"
-    test -d "${COMPOSE_DIR}/${y}/${x}/os"
+    test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os"
     ret_val=$?
     if [ "$ret_val" -eq 0 ]; then
-      createrepo --update "${COMPOSE_DIR}/${y}/${x}/os" \
+      createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os" \
         --groupfile="/mnt/compose/8/latest-Rocky-8/work/${x}/comps/comps-${y}.${x}.xml" \
         --xz --revision=${REVISION} \
         "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \
         --workers=8 --checksum=sha256
     else
-      echo "${COMPOSE_DIR}/${y}/${x}/os does not exist"
+      echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist"
     fi
   done
 
@@ -106,14 +76,14 @@ for x in "${ARCHES[@]}"; do
   for y in "${MODS[@]}"; do
     echo "Modules: ${y} ${x}"
     cp "/mnt/compose/8_metadata/${x}/${y}-modules.yaml" /tmp/modules.yaml
-    test -d "${COMPOSE_DIR}/${y}/${x}/os"
+    test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os"
     ret_val=$?
     if [ "$ret_val" -eq 0 ]; then
       modifyrepo --mdtype=modules /tmp/modules.yaml \
-        "${COMPOSE_DIR}/${y}/${x}/os/repodata" \
+        "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os/repodata" \
         --compress --compress-type=gz
     else
-      echo "${COMPOSE_DIR}/${y}/${x}/os does not exist"
+      echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist"
     fi
 
     rm /tmp/modules.yaml
@@ -121,7 +91,7 @@ for x in "${ARCHES[@]}"; do
   done
 
   echo "** Fix variants"
-  TREEINFO_VAR="${COMPOSE_DIR}/BaseOS/${x}/os/.treeinfo"
+  TREEINFO_VAR="${STAGING_ROOT}/${RELEASE_DIR}/BaseOS/${x}/os/.treeinfo"
   test -f "${TREEINFO_VAR}"
   treeinfo_retval=$?
   test -x /usr/bin/python3

sync/sync-to-prod.sh

@@ -1,11 +1,19 @@
 #!/bin/bash
 # Syncs everything from staging to production
-REVISION=${1}
+
-cd "/mnt/repos-staging/mirror/pub/rocky/${REVISION}"
+# Source common variables
+source $(dirname "$0")/common
+
+REV=${1}
+
+cd "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
 ret_val=$?
+
 if [ $ret_val -eq "0" ]; then
-  mkdir -p "/mnt/repos-production/mirror/pub/rocky/${REVISION:0:3}"
+  TARGET="${PRODUCTION_ROOT}/${CATEGORY_STUB}/${REV:0:3}"
-  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-production/mirror/pub/rocky/${REVISION:0:3}
+  mkdir -p "${TARGET}"
+  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
+      {} ${TARGET}
 else
   echo "Failed to change directory"
 fi

sync/sync-to-staging-sig.sh

@@ -1,17 +1,25 @@
 #!/bin/bash
+
+# Source common variables
+source $(dirname "$0")/common
+
 # Major Version (eg, 8)
-MAJOR=${1}
+MAJ=${1}
 # Short name (eg, NFV, extras, Rocky, gluster9)
 SHORT=${2}
 # The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX
-REVISION=${3}
+REV=${3}
 # Note, this should be lowercase. eg, storage.
 SIG=${4}
-cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR}
+
+cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}
 ret_val=$?
+
 if [ $ret_val -eq "0" ]; then
-  mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG}
+  local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${SIG}
-  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG}
+  mkdir -p ${TARGET}
+  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
+      {} ${TARGET}
 else
   echo "Failed to change directory"
 fi

sync/sync-to-staging.sh

@@ -1,15 +1,23 @@
 #!/bin/bash
+
+# Source common variables
+source $(dirname "$0")/common
+
 # Major Version (eg, 8)
-MAJOR=${1}
+MAJ=${1}
 # Short name (eg, NFV, extras, Rocky, gluster9)
 SHORT=${2}
 # The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX
-REVISION=${3}
+REV=${3}
-cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR}
+
+cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}
 ret_val=$?
+
 if [ $ret_val -eq "0" ]; then
-  mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION}
+  local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV}
-  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION}
+  mkdir -p ${TARGET}
+  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
+      {} ${TARGET}
 else
   echo "Failed to change directory"
 fi