From 091e3d1d0887654d987106e3f6e9ab53e0409c0f Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Tue, 6 Jul 2021 13:06:40 -0400 Subject: [PATCH 1/2] Add validate_repo script and begin 'mangle' toolkit --- mangle/README.md | 1 + mangle/common | 8 +++++++ mangle/validate_repos | 56 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 mangle/README.md create mode 100644 mangle/common create mode 100755 mangle/validate_repos diff --git a/mangle/README.md b/mangle/README.md new file mode 100644 index 0000000..6711deb --- /dev/null +++ b/mangle/README.md @@ -0,0 +1 @@ +# Mirrormanager Mangling tools diff --git a/mangle/common b/mangle/common new file mode 100644 index 0000000..c6c28e8 --- /dev/null +++ b/mangle/common @@ -0,0 +1,8 @@ +# vim: set syntax=bash + +# To be sourced by scripts as needed + +# The mirrorlist url +MIRRORLIST_BASE="http://mirrors.rockylinux.org/mirrorlist" + +MIRROR_DISPLAY_COUNT=1 diff --git a/mangle/validate_repos b/mangle/validate_repos new file mode 100755 index 0000000..ecfd90e --- /dev/null +++ b/mangle/validate_repos @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Source mangle vars +source $(dirname "$0")/common +# Source sync / migrate vars for repository information +source $(dirname "$0")/../sync/common + +# How many +ARG1=${1} +NUM=$(( ${ARG1:-${MIRROR_DISPLAY_COUNT}} + 2)) + +print_result () { + if [[ "${result}" =~ "invalid repo or arch" ]]; then + printf "# repo = %s arch = %s\n" "${_repo}" "${arch}" + printf "# FAIL: invalid repo or arch\n\n" + else + printf "%s\n# number of mirrors returned: %s\n\n" "$( echo "$result" | head -${NUM})" "$(( $(echo "$result" | wc -l) - 2 ))" + fi +} + +cleanup_repo () { + # plus is actually 'rockyplus'. Others may as well(?) + if [[ "${repo}" =~ ^(plus)$ ]]; then + repo="rocky${repo}" + elif [[ "${repo}" =~ ^(nfv)$ ]]; then + repo="${repo^^}" + else + repo="${repo}" + fi + return 0 +} + +for repo in "${ALL_REPOS[@]}"; do + + # Business logic must be done, sometimes... + cleanup_repo "${repo}" + + # Print a nice header + printf "================\n${repo}\n================\n" + for arch in "${ARCHES[@]}" "source"; do + if [[ "${arch}" == "source" ]]; then + _repo="${repo}-8-source" + else + _repo="${repo}-8" + fi + + result=$(curl -s "${MIRRORLIST_BASE}?repo=${_repo}&arch=${arch}&time&country=global") + print_result + + if [[ "${arch}" =~ ^(x86_|aarch)64$ ]]; then + result=$(curl -s "${MIRRORLIST_BASE}?repo=${_repo}-debug&arch=${arch}&time&country=global") + print_result + fi + done + echo +done From d3e830c7369940c5d1de005c4be26a552257089b Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Tue, 6 Jul 2021 13:08:30 -0400 Subject: [PATCH 2/2] refactor sync tools to use common vars and stuff maybe --- sync/common | 51 ++++++++++++++++++++++++++++ sync/prep-staging-8.sh | 68 +++++++++++-------------------------- sync/sync-to-prod.sh | 16 ++++++--- sync/sync-to-staging-sig.sh | 18 +++++++--- sync/sync-to-staging.sh | 18 +++++++--- 5 files changed, 108 insertions(+), 63 deletions(-) create mode 100644 sync/common diff --git a/sync/common b/sync/common new file mode 100644 index 0000000..cd322f8 --- /dev/null +++ b/sync/common @@ -0,0 +1,51 @@ +# To be sourced by scripts to use + +# Revision must always start with a major number +REVISION=8.4 + +MAJOR="${REVISION:0:1}" +MINOR="${REVISION:2:1}" + +# comment or blank if needed +APPEND_TO_DIR="-RC2" + +STAGING_ROOT="/mnt/repos-staging" +PRODUCTION_ROOT="/mnt/repos-production" + +# relative to ${ENV}_ROOT +CATEGORY_STUB="mirror/pub/rocky" +RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}" + +# Define arches we support +ARCHES=(x86_64 aarch64) + +# Set all repos that have no comps/groups associated with them. This is even in +# cases where repos will not be available by normal means. It's just for +# consistency. +NONMODS_REPOS=( + extras + Devel + nfv + storage/gluster9 + plus +) + +# These repos have comps/groups, except for debuginfo and sources +MODS_REPOS=( + BaseOS + AppStream + HighAvailability + ResilientStorage + PowerTools +) + +ALL_REPOS=( + "${NONMODS_REPOS[@]}" + "${MODS_REPOS[@]}" +) + +# These repos have modules +MODS=( + AppStream + PowerTools +) diff --git a/sync/prep-staging-8.sh b/sync/prep-staging-8.sh index 1f61dd9..1179f57 100644 --- a/sync/prep-staging-8.sh +++ b/sync/prep-staging-8.sh @@ -10,49 +10,19 @@ # and provide everything, even if it's not installable). # # Compose dir example: /mnt/repos-staging/mirror/pub/rocky/8.4-RC2 -# Revision must always start with a major number -REVISION=8.4 -#MAJOR="${REVISION:0:1}" -# comment or blank if needed -APPEND_TO_DIR="-RC2" -COMPOSE_DIR="/mnt/repos-staging/mirror/pub/rocky/${REVISION}${APPEND_TO_DIR}" -ARCHES=(x86_64 aarch64) -# Set all repos that have no comps/groups associated with them. This is even in -# cases where repos will not be available by normal means. It's just for -# consistency. -NONMODS_REPOS=( - extras - Devel - nfv - storage/gluster9 - plus -) - -# These repos have comps/groups, except for debuginfo and sources -MODS_REPOS=( - BaseOS - AppStream - HighAvailability - ResilientStorage - PowerTools -) - -# These repos have modules -MODS=( - AppStream - PowerTools -) +# Source common variables +source $(dirname "$0")/common echo "** Updating source repos" -for y in "${NONMODS_REPOS[@]}" "${MODS_REPOS[@]}"; do - test -d "${COMPOSE_DIR}/${y}/${x}/${z}" +for y in "${ALL_REPOS[@]}"; do + test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" ret_val=$? if [ "$ret_val" -eq 0 ]; then - createrepo --update "${COMPOSE_DIR}/${y}/source/tree" \ + createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree" \ "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" else - echo "${COMPOSE_DIR}/${y}/source/tree does not exist" + echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree does not exist" fi done @@ -64,41 +34,41 @@ for x in "${ARCHES[@]}"; do for y in "${NONMODS_REPOS[@]}"; do # os and debug/tree directories for z in os debug/tree; do - test -d "${COMPOSE_DIR}/${y}/${x}/${z}" + test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" ret_val=$? if [ "$ret_val" -eq 0 ]; then - createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \ + createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \ "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" else - echo "${COMPOSE_DIR}/${y}/${x}/${z} does not exist" + echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z} does not exist" fi done done # repos with comps/groups involved, but only debug for y in "${MODS_REPOS[@]}"; do - test -d "${COMPOSE_DIR}/${y}/${x}/debug/tree" + test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree" ret_val=$? if [ "$ret_val" -eq 0 ]; then - createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \ + createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \ "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" else - echo "${COMPOSE_DIR}/${y}/${x}/debug/tree does not exist" + echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree does not exist" fi done echo "** Update all repos with comps/groups" for y in "${MODS_REPOS[@]}"; do echo "${y}: ${x}" - test -d "${COMPOSE_DIR}/${y}/${x}/os" + test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os" ret_val=$? if [ "$ret_val" -eq 0 ]; then - createrepo --update "${COMPOSE_DIR}/${y}/${x}/os" \ + createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os" \ --groupfile="/mnt/compose/8/latest-Rocky-8/work/${x}/comps/comps-${y}.${x}.xml" \ --xz --revision=${REVISION} \ "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \ --workers=8 --checksum=sha256 else - echo "${COMPOSE_DIR}/${y}/${x}/os does not exist" + echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist" fi done @@ -106,14 +76,14 @@ for x in "${ARCHES[@]}"; do for y in "${MODS[@]}"; do echo "Modules: ${y} ${x}" cp "/mnt/compose/8_metadata/${x}/${y}-modules.yaml" /tmp/modules.yaml - test -d "${COMPOSE_DIR}/${y}/${x}/os" + test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os" ret_val=$? if [ "$ret_val" -eq 0 ]; then modifyrepo --mdtype=modules /tmp/modules.yaml \ - "${COMPOSE_DIR}/${y}/${x}/os/repodata" \ + "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os/repodata" \ --compress --compress-type=gz else - echo "${COMPOSE_DIR}/${y}/${x}/os does not exist" + echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist" fi rm /tmp/modules.yaml @@ -121,7 +91,7 @@ for x in "${ARCHES[@]}"; do done echo "** Fix variants" - TREEINFO_VAR="${COMPOSE_DIR}/BaseOS/${x}/os/.treeinfo" + TREEINFO_VAR="${STAGING_ROOT}/${RELEASE_DIR}/BaseOS/${x}/os/.treeinfo" test -f "${TREEINFO_VAR}" treeinfo_retval=$? test -x /usr/bin/python3 diff --git a/sync/sync-to-prod.sh b/sync/sync-to-prod.sh index 7b5ad1e..f8279ba 100644 --- a/sync/sync-to-prod.sh +++ b/sync/sync-to-prod.sh @@ -1,11 +1,19 @@ #!/bin/bash # Syncs everything from staging to production -REVISION=${1} -cd "/mnt/repos-staging/mirror/pub/rocky/${REVISION}" + +# Source common variables +source $(dirname "$0")/common + +REV=${1} + +cd "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" ret_val=$? + if [ $ret_val -eq "0" ]; then - mkdir -p "/mnt/repos-production/mirror/pub/rocky/${REVISION:0:3}" - sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-production/mirror/pub/rocky/${REVISION:0:3} + TARGET="${PRODUCTION_ROOT}/${CATEGORY_STUB}/${REV:0:3}" + mkdir -p "${TARGET}" + sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ + {} ${TARGET} else echo "Failed to change directory" fi diff --git a/sync/sync-to-staging-sig.sh b/sync/sync-to-staging-sig.sh index aa7e5d8..06561f1 100644 --- a/sync/sync-to-staging-sig.sh +++ b/sync/sync-to-staging-sig.sh @@ -1,17 +1,25 @@ #!/bin/bash + +# Source common variables +source $(dirname "$0")/common + # Major Version (eg, 8) -MAJOR=${1} +MAJ=${1} # Short name (eg, NFV, extras, Rocky, gluster9) SHORT=${2} # The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX -REVISION=${3} +REV=${3} # Note, this should be lowercase. eg, storage. SIG=${4} -cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR} + +cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ} ret_val=$? + if [ $ret_val -eq "0" ]; then - mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG} - sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG} + local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${SIG} + mkdir -p ${TARGET} + sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ + {} ${TARGET} else echo "Failed to change directory" fi diff --git a/sync/sync-to-staging.sh b/sync/sync-to-staging.sh index 45b1466..7581d3d 100644 --- a/sync/sync-to-staging.sh +++ b/sync/sync-to-staging.sh @@ -1,15 +1,23 @@ #!/bin/bash + +# Source common variables +source $(dirname "$0")/common + # Major Version (eg, 8) -MAJOR=${1} +MAJ=${1} # Short name (eg, NFV, extras, Rocky, gluster9) SHORT=${2} # The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX -REVISION=${3} -cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR} +REV=${3} + +cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ} ret_val=$? + if [ $ret_val -eq "0" ]; then - mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION} - sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION} + local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV} + mkdir -p ${TARGET} + sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ + {} ${TARGET} else echo "Failed to change directory" fi