refactor sync tools to use common vars and stuff maybe

2021-07-06 13:08:30 -04:00 · 2021-07-06 13:08:30 -04:00 · d3e830c736
commit d3e830c736
parent 091e3d1d08
5 changed files with 108 additions and 63 deletions
--- a/sync/common
+++ b/sync/common
@ -0,0 +1,51 @@
+# To be sourced by scripts to use
+
+# Revision must always start with a major number
+REVISION=8.4
+
+MAJOR="${REVISION:0:1}"
+MINOR="${REVISION:2:1}"
+
+# comment or blank if needed
+APPEND_TO_DIR="-RC2"
+
+STAGING_ROOT="/mnt/repos-staging"
+PRODUCTION_ROOT="/mnt/repos-production"
+
+# relative to ${ENV}_ROOT
+CATEGORY_STUB="mirror/pub/rocky"
+RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}"
+
+# Define arches we support
+ARCHES=(x86_64 aarch64)
+
+# Set all repos that have no comps/groups associated with them. This is even in
+# cases where repos will not be available by normal means. It's just for
+# consistency.
+NONMODS_REPOS=(
+  extras
+  Devel
+  nfv
+  storage/gluster9
+  plus
+)
+
+# These repos have comps/groups, except for debuginfo and sources
+MODS_REPOS=(
+  BaseOS
+  AppStream
+  HighAvailability
+  ResilientStorage
+  PowerTools
+)
+
+ALL_REPOS=(
+  "${NONMODS_REPOS[@]}"
+  "${MODS_REPOS[@]}"
+)
+
+# These repos have modules
+MODS=(
+  AppStream
+  PowerTools
+)
--- a/sync/prep-staging-8.sh
+++ b/sync/prep-staging-8.sh
@ -10,49 +10,19 @@
 # and provide everything, even if it's not installable).
 #
 # Compose dir example: /mnt/repos-staging/mirror/pub/rocky/8.4-RC2
-# Revision must always start with a major number
-REVISION=8.4
-#MAJOR="${REVISION:0:1}"
-# comment or blank if needed
-APPEND_TO_DIR="-RC2"
-COMPOSE_DIR="/mnt/repos-staging/mirror/pub/rocky/${REVISION}${APPEND_TO_DIR}"
-ARCHES=(x86_64 aarch64)

-# Set all repos that have no comps/groups associated with them. This is even in
-# cases where repos will not be available by normal means. It's just for
-# consistency.
-NONMODS_REPOS=(
-  extras
-  Devel
-  nfv
-  storage/gluster9
-  plus
-)
-
-# These repos have comps/groups, except for debuginfo and sources
-MODS_REPOS=(
-  BaseOS
-  AppStream
-  HighAvailability
-  ResilientStorage
-  PowerTools
-)
-
-# These repos have modules
-MODS=(
-  AppStream
-  PowerTools
-)
+# Source common variables
+source $(dirname "$0")/common

 echo "** Updating source repos"
-for y in "${NONMODS_REPOS[@]}" "${MODS_REPOS[@]}"; do
-  test -d "${COMPOSE_DIR}/${y}/${x}/${z}"
+for y in "${ALL_REPOS[@]}"; do
+  test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}"
  ret_val=$?
  if [ "$ret_val" -eq 0 ]; then
-    createrepo --update "${COMPOSE_DIR}/${y}/source/tree" \
+    createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree" \
      "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
  else
-    echo "${COMPOSE_DIR}/${y}/source/tree does not exist"
+    echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree does not exist"
  fi
 done

@ -64,41 +34,41 @@ for x in "${ARCHES[@]}"; do
  for y in "${NONMODS_REPOS[@]}"; do
    # os and debug/tree directories
    for z in os debug/tree; do
-      test -d "${COMPOSE_DIR}/${y}/${x}/${z}"
+      test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}"
      ret_val=$?
      if [ "$ret_val" -eq 0 ]; then
-        createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \
+        createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \
          "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
      else
-        echo "${COMPOSE_DIR}/${y}/${x}/${z} does not exist"
+        echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z} does not exist"
      fi
    done
  done
  # repos with comps/groups involved, but only debug
  for y in "${MODS_REPOS[@]}"; do
-    test -d "${COMPOSE_DIR}/${y}/${x}/debug/tree"
+    test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree"
    ret_val=$?
    if [ "$ret_val" -eq 0 ]; then
-      createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \
+      createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \
        "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
    else
-      echo "${COMPOSE_DIR}/${y}/${x}/debug/tree does not exist"
+      echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree does not exist"
    fi
  done

  echo "** Update all repos with comps/groups"
  for y in "${MODS_REPOS[@]}"; do
    echo "${y}: ${x}"
-    test -d "${COMPOSE_DIR}/${y}/${x}/os"
+    test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os"
    ret_val=$?
    if [ "$ret_val" -eq 0 ]; then
-      createrepo --update "${COMPOSE_DIR}/${y}/${x}/os" \
+      createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os" \
        --groupfile="/mnt/compose/8/latest-Rocky-8/work/${x}/comps/comps-${y}.${x}.xml" \
        --xz --revision=${REVISION} \
        "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \
        --workers=8 --checksum=sha256
    else
-      echo "${COMPOSE_DIR}/${y}/${x}/os does not exist"
+      echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist"
    fi
  done

@ -106,14 +76,14 @@ for x in "${ARCHES[@]}"; do
  for y in "${MODS[@]}"; do
    echo "Modules: ${y} ${x}"
    cp "/mnt/compose/8_metadata/${x}/${y}-modules.yaml" /tmp/modules.yaml
-    test -d "${COMPOSE_DIR}/${y}/${x}/os"
+    test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os"
    ret_val=$?
    if [ "$ret_val" -eq 0 ]; then
      modifyrepo --mdtype=modules /tmp/modules.yaml \
-        "${COMPOSE_DIR}/${y}/${x}/os/repodata" \
+        "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os/repodata" \
        --compress --compress-type=gz
    else
-      echo "${COMPOSE_DIR}/${y}/${x}/os does not exist"
+      echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist"
    fi

    rm /tmp/modules.yaml
@ -121,7 +91,7 @@ for x in "${ARCHES[@]}"; do
  done

  echo "** Fix variants"
-  TREEINFO_VAR="${COMPOSE_DIR}/BaseOS/${x}/os/.treeinfo"
+  TREEINFO_VAR="${STAGING_ROOT}/${RELEASE_DIR}/BaseOS/${x}/os/.treeinfo"
  test -f "${TREEINFO_VAR}"
  treeinfo_retval=$?
  test -x /usr/bin/python3
--- a/sync/sync-to-prod.sh
+++ b/sync/sync-to-prod.sh
@ -1,11 +1,19 @@
 #!/bin/bash
 # Syncs everything from staging to production
-REVISION=${1}
-cd "/mnt/repos-staging/mirror/pub/rocky/${REVISION}"
+
+# Source common variables
+source $(dirname "$0")/common
+
+REV=${1}
+
+cd "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
 ret_val=$?
+
 if [ $ret_val -eq "0" ]; then
-  mkdir -p "/mnt/repos-production/mirror/pub/rocky/${REVISION:0:3}"
-  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-production/mirror/pub/rocky/${REVISION:0:3}
+  TARGET="${PRODUCTION_ROOT}/${CATEGORY_STUB}/${REV:0:3}"
+  mkdir -p "${TARGET}"
+  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
+      {} ${TARGET}
 else
  echo "Failed to change directory"
 fi
--- a/sync/sync-to-staging-sig.sh
+++ b/sync/sync-to-staging-sig.sh
@ -1,17 +1,25 @@
 #!/bin/bash
+
+# Source common variables
+source $(dirname "$0")/common
+
 # Major Version (eg, 8)
-MAJOR=${1}
+MAJ=${1}
 # Short name (eg, NFV, extras, Rocky, gluster9)
 SHORT=${2}
 # The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX
-REVISION=${3}
+REV=${3}
 # Note, this should be lowercase. eg, storage.
 SIG=${4}
-cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR}
+
+cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}
 ret_val=$?
+
 if [ $ret_val -eq "0" ]; then
-  mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG}
-  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG}
+  local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${SIG}
+  mkdir -p ${TARGET}
+  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
+      {} ${TARGET}
 else
  echo "Failed to change directory"
 fi
--- a/sync/sync-to-staging.sh
+++ b/sync/sync-to-staging.sh
@ -1,15 +1,23 @@
 #!/bin/bash
+
+# Source common variables
+source $(dirname "$0")/common
+
 # Major Version (eg, 8)
-MAJOR=${1}
+MAJ=${1}
 # Short name (eg, NFV, extras, Rocky, gluster9)
 SHORT=${2}
 # The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX
-REVISION=${3}
-cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR}
+REV=${3}
+
+cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}
 ret_val=$?
+
 if [ $ret_val -eq "0" ]; then
-  mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION}
-  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION}
+  local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV}
+  mkdir -p ${TARGET}
+  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
+      {} ${TARGET}
 else
  echo "Failed to change directory"
 fi