Merge branch 'devel' into 'main'

Refactor to utilize common variable/config locations that can be shared across scripts See merge request release-engineering/public/toolkit!1
2021-07-06 18:27:16 +00:00 · 2021-07-06 18:27:16 +00:00 · 327437eeff
parent 11df58a0b7 d3e830c736
commit 327437eeff
8 changed files with 173 additions and 63 deletions
--- a/mangle/README.md
+++ b/mangle/README.md
@ -0,0 +1 @@
+# Mirrormanager Mangling tools
--- a/mangle/common
+++ b/mangle/common
@ -0,0 +1,8 @@
+# vim: set syntax=bash
+
+# To be sourced by scripts as needed
+
+# The mirrorlist url
+MIRRORLIST_BASE="http://mirrors.rockylinux.org/mirrorlist"
+
+MIRROR_DISPLAY_COUNT=1
--- a/mangle/validate_repos
+++ b/mangle/validate_repos
@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+# Source mangle vars
+source $(dirname "$0")/common
+# Source sync / migrate vars for repository information
+source $(dirname "$0")/../sync/common
+
+# How many 
+ARG1=${1}
+NUM=$(( ${ARG1:-${MIRROR_DISPLAY_COUNT}} + 2))
+
+print_result () {
+    if [[ "${result}" =~ "invalid repo or arch" ]]; then
+        printf "# repo = %s arch = %s\n" "${_repo}" "${arch}"
+        printf "# FAIL: invalid repo or arch\n\n"
+    else
+        printf "%s\n# number of mirrors returned: %s\n\n" "$( echo "$result" | head -${NUM})" "$(( $(echo "$result" | wc -l) - 2 ))"
+    fi
+}
+
+cleanup_repo () {
+    # plus is actually 'rockyplus'. Others may as well(?)
+    if [[ "${repo}" =~ ^(plus)$ ]]; then
+        repo="rocky${repo}"
+    elif [[ "${repo}" =~ ^(nfv)$ ]]; then
+        repo="${repo^^}"
+    else
+        repo="${repo}"
+    fi
+    return 0
+}
+
+for repo in "${ALL_REPOS[@]}"; do 
+
+    # Business logic must be done, sometimes...
+    cleanup_repo "${repo}"
+
+    # Print a nice header
+    printf "================\n${repo}\n================\n"
+    for arch in "${ARCHES[@]}" "source"; do
+        if [[ "${arch}" == "source" ]]; then
+            _repo="${repo}-8-source"
+        else
+            _repo="${repo}-8"
+        fi
+
+        result=$(curl -s "${MIRRORLIST_BASE}?repo=${_repo}&arch=${arch}&time&country=global")
+        print_result
+
+        if [[ "${arch}" =~ ^(x86_|aarch)64$ ]]; then
+            result=$(curl -s "${MIRRORLIST_BASE}?repo=${_repo}-debug&arch=${arch}&time&country=global")
+            print_result
+        fi
+    done
+    echo
+done
--- a/sync/common
+++ b/sync/common
@ -0,0 +1,51 @@
+# To be sourced by scripts to use
+
+# Revision must always start with a major number
+REVISION=8.4
+
+MAJOR="${REVISION:0:1}"
+MINOR="${REVISION:2:1}"
+
+# comment or blank if needed
+APPEND_TO_DIR="-RC2"
+
+STAGING_ROOT="/mnt/repos-staging"
+PRODUCTION_ROOT="/mnt/repos-production"
+
+# relative to ${ENV}_ROOT
+CATEGORY_STUB="mirror/pub/rocky"
+RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}"
+
+# Define arches we support
+ARCHES=(x86_64 aarch64)
+
+# Set all repos that have no comps/groups associated with them. This is even in
+# cases where repos will not be available by normal means. It's just for
+# consistency.
+NONMODS_REPOS=(
+  extras
+  Devel
+  nfv
+  storage/gluster9
+  plus
+)
+
+# These repos have comps/groups, except for debuginfo and sources
+MODS_REPOS=(
+  BaseOS
+  AppStream
+  HighAvailability
+  ResilientStorage
+  PowerTools
+)
+
+ALL_REPOS=(
+  "${NONMODS_REPOS[@]}"
+  "${MODS_REPOS[@]}"
+)
+
+# These repos have modules
+MODS=(
+  AppStream
+  PowerTools
+)
--- a/sync/prep-staging-8.sh
+++ b/sync/prep-staging-8.sh
@ -10,49 +10,19 @@
 # and provide everything, even if it's not installable).
 #
 # Compose dir example: /mnt/repos-staging/mirror/pub/rocky/8.4-RC2
-# Revision must always start with a major number
-REVISION=8.4
-#MAJOR="${REVISION:0:1}"
-# comment or blank if needed
-APPEND_TO_DIR="-RC2"
-COMPOSE_DIR="/mnt/repos-staging/mirror/pub/rocky/${REVISION}${APPEND_TO_DIR}"
-ARCHES=(x86_64 aarch64)

-# Set all repos that have no comps/groups associated with them. This is even in
-# cases where repos will not be available by normal means. It's just for
-# consistency.
-NONMODS_REPOS=(
-  extras
-  Devel
-  nfv
-  storage/gluster9
-  plus
-)
-
-# These repos have comps/groups, except for debuginfo and sources
-MODS_REPOS=(
-  BaseOS
-  AppStream
-  HighAvailability
-  ResilientStorage
-  PowerTools
-)
-
-# These repos have modules
-MODS=(
-  AppStream
-  PowerTools
-)
+# Source common variables
+source $(dirname "$0")/common

 echo "** Updating source repos"
-for y in "${NONMODS_REPOS[@]}" "${MODS_REPOS[@]}"; do
-  test -d "${COMPOSE_DIR}/${y}/${x}/${z}"
+for y in "${ALL_REPOS[@]}"; do
+  test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}"
  ret_val=$?
  if [ "$ret_val" -eq 0 ]; then
-    createrepo --update "${COMPOSE_DIR}/${y}/source/tree" \
+    createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree" \
      "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
  else
-    echo "${COMPOSE_DIR}/${y}/source/tree does not exist"
+    echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree does not exist"
  fi
 done

@ -64,41 +34,41 @@ for x in "${ARCHES[@]}"; do
  for y in "${NONMODS_REPOS[@]}"; do
    # os and debug/tree directories
    for z in os debug/tree; do
-      test -d "${COMPOSE_DIR}/${y}/${x}/${z}"
+      test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}"
      ret_val=$?
      if [ "$ret_val" -eq 0 ]; then
-        createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \
+        createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \
          "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
      else
-        echo "${COMPOSE_DIR}/${y}/${x}/${z} does not exist"
+        echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z} does not exist"
      fi
    done
  done
  # repos with comps/groups involved, but only debug
  for y in "${MODS_REPOS[@]}"; do
-    test -d "${COMPOSE_DIR}/${y}/${x}/debug/tree"
+    test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree"
    ret_val=$?
    if [ "$ret_val" -eq 0 ]; then
-      createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \
+      createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \
        "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
    else
-      echo "${COMPOSE_DIR}/${y}/${x}/debug/tree does not exist"
+      echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree does not exist"
    fi
  done

  echo "** Update all repos with comps/groups"
  for y in "${MODS_REPOS[@]}"; do
    echo "${y}: ${x}"
-    test -d "${COMPOSE_DIR}/${y}/${x}/os"
+    test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os"
    ret_val=$?
    if [ "$ret_val" -eq 0 ]; then
-      createrepo --update "${COMPOSE_DIR}/${y}/${x}/os" \
+      createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os" \
        --groupfile="/mnt/compose/8/latest-Rocky-8/work/${x}/comps/comps-${y}.${x}.xml" \
        --xz --revision=${REVISION} \
        "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \
        --workers=8 --checksum=sha256
    else
-      echo "${COMPOSE_DIR}/${y}/${x}/os does not exist"
+      echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist"
    fi
  done

@ -106,14 +76,14 @@ for x in "${ARCHES[@]}"; do
  for y in "${MODS[@]}"; do
    echo "Modules: ${y} ${x}"
    cp "/mnt/compose/8_metadata/${x}/${y}-modules.yaml" /tmp/modules.yaml
-    test -d "${COMPOSE_DIR}/${y}/${x}/os"
+    test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os"
    ret_val=$?
    if [ "$ret_val" -eq 0 ]; then
      modifyrepo --mdtype=modules /tmp/modules.yaml \
-        "${COMPOSE_DIR}/${y}/${x}/os/repodata" \
+        "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os/repodata" \
        --compress --compress-type=gz
    else
-      echo "${COMPOSE_DIR}/${y}/${x}/os does not exist"
+      echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist"
    fi

    rm /tmp/modules.yaml
@ -121,7 +91,7 @@ for x in "${ARCHES[@]}"; do
  done

  echo "** Fix variants"
-  TREEINFO_VAR="${COMPOSE_DIR}/BaseOS/${x}/os/.treeinfo"
+  TREEINFO_VAR="${STAGING_ROOT}/${RELEASE_DIR}/BaseOS/${x}/os/.treeinfo"
  test -f "${TREEINFO_VAR}"
  treeinfo_retval=$?
  test -x /usr/bin/python3
--- a/sync/sync-to-prod.sh
+++ b/sync/sync-to-prod.sh
@ -1,11 +1,19 @@
 #!/bin/bash
 # Syncs everything from staging to production
-REVISION=${1}
-cd "/mnt/repos-staging/mirror/pub/rocky/${REVISION}"
+
+# Source common variables
+source $(dirname "$0")/common
+
+REV=${1}
+
+cd "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
 ret_val=$?
+
 if [ $ret_val -eq "0" ]; then
-  mkdir -p "/mnt/repos-production/mirror/pub/rocky/${REVISION:0:3}"
-  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-production/mirror/pub/rocky/${REVISION:0:3}
+  TARGET="${PRODUCTION_ROOT}/${CATEGORY_STUB}/${REV:0:3}"
+  mkdir -p "${TARGET}"
+  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
+      {} ${TARGET}
 else
  echo "Failed to change directory"
 fi
--- a/sync/sync-to-staging-sig.sh
+++ b/sync/sync-to-staging-sig.sh
@ -1,17 +1,25 @@
 #!/bin/bash
+
+# Source common variables
+source $(dirname "$0")/common
+
 # Major Version (eg, 8)
-MAJOR=${1}
+MAJ=${1}
 # Short name (eg, NFV, extras, Rocky, gluster9)
 SHORT=${2}
 # The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX
-REVISION=${3}
+REV=${3}
 # Note, this should be lowercase. eg, storage.
 SIG=${4}
-cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR}
+
+cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}
 ret_val=$?
+
 if [ $ret_val -eq "0" ]; then
-  mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG}
-  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG}
+  local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${SIG}
+  mkdir -p ${TARGET}
+  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
+      {} ${TARGET}
 else
  echo "Failed to change directory"
 fi
--- a/sync/sync-to-staging.sh
+++ b/sync/sync-to-staging.sh
@ -1,15 +1,23 @@
 #!/bin/bash
+
+# Source common variables
+source $(dirname "$0")/common
+
 # Major Version (eg, 8)
-MAJOR=${1}
+MAJ=${1}
 # Short name (eg, NFV, extras, Rocky, gluster9)
 SHORT=${2}
 # The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX
-REVISION=${3}
-cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR}
+REV=${3}
+
+cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}
 ret_val=$?
+
 if [ $ret_val -eq "0" ]; then
-  mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION}
-  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION}
+  local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV}
+  mkdir -p ${TARGET}
+  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
+      {} ${TARGET}
 else
  echo "Failed to change directory"
 fi