forked from sig_core/toolkit
Merge branch 'devel' into 'main'
Refactor to utilize common variable/config locations that can be shared across scripts See merge request release-engineering/public/toolkit!1
This commit is contained in:
commit
327437eeff
8 changed files with 173 additions and 63 deletions
1
mangle/README.md
Normal file
1
mangle/README.md
Normal file
|
|
@ -0,0 +1 @@
|
|||
# Mirrormanager Mangling tools
|
||||
8
mangle/common
Normal file
8
mangle/common
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
# vim: set syntax=bash
|
||||
|
||||
# To be sourced by scripts as needed
|
||||
|
||||
# The mirrorlist url
|
||||
MIRRORLIST_BASE="http://mirrors.rockylinux.org/mirrorlist"
|
||||
|
||||
MIRROR_DISPLAY_COUNT=1
|
||||
56
mangle/validate_repos
Executable file
56
mangle/validate_repos
Executable file
|
|
@ -0,0 +1,56 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Source mangle vars
|
||||
source $(dirname "$0")/common
|
||||
# Source sync / migrate vars for repository information
|
||||
source $(dirname "$0")/../sync/common
|
||||
|
||||
# How many
|
||||
ARG1=${1}
|
||||
NUM=$(( ${ARG1:-${MIRROR_DISPLAY_COUNT}} + 2))
|
||||
|
||||
print_result () {
|
||||
if [[ "${result}" =~ "invalid repo or arch" ]]; then
|
||||
printf "# repo = %s arch = %s\n" "${_repo}" "${arch}"
|
||||
printf "# FAIL: invalid repo or arch\n\n"
|
||||
else
|
||||
printf "%s\n# number of mirrors returned: %s\n\n" "$( echo "$result" | head -${NUM})" "$(( $(echo "$result" | wc -l) - 2 ))"
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup_repo () {
|
||||
# plus is actually 'rockyplus'. Others may as well(?)
|
||||
if [[ "${repo}" =~ ^(plus)$ ]]; then
|
||||
repo="rocky${repo}"
|
||||
elif [[ "${repo}" =~ ^(nfv)$ ]]; then
|
||||
repo="${repo^^}"
|
||||
else
|
||||
repo="${repo}"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
for repo in "${ALL_REPOS[@]}"; do
|
||||
|
||||
# Business logic must be done, sometimes...
|
||||
cleanup_repo "${repo}"
|
||||
|
||||
# Print a nice header
|
||||
printf "================\n${repo}\n================\n"
|
||||
for arch in "${ARCHES[@]}" "source"; do
|
||||
if [[ "${arch}" == "source" ]]; then
|
||||
_repo="${repo}-8-source"
|
||||
else
|
||||
_repo="${repo}-8"
|
||||
fi
|
||||
|
||||
result=$(curl -s "${MIRRORLIST_BASE}?repo=${_repo}&arch=${arch}&time&country=global")
|
||||
print_result
|
||||
|
||||
if [[ "${arch}" =~ ^(x86_|aarch)64$ ]]; then
|
||||
result=$(curl -s "${MIRRORLIST_BASE}?repo=${_repo}-debug&arch=${arch}&time&country=global")
|
||||
print_result
|
||||
fi
|
||||
done
|
||||
echo
|
||||
done
|
||||
51
sync/common
Normal file
51
sync/common
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
# To be sourced by scripts to use
|
||||
|
||||
# Revision must always start with a major number
|
||||
REVISION=8.4
|
||||
|
||||
MAJOR="${REVISION:0:1}"
|
||||
MINOR="${REVISION:2:1}"
|
||||
|
||||
# comment or blank if needed
|
||||
APPEND_TO_DIR="-RC2"
|
||||
|
||||
STAGING_ROOT="/mnt/repos-staging"
|
||||
PRODUCTION_ROOT="/mnt/repos-production"
|
||||
|
||||
# relative to ${ENV}_ROOT
|
||||
CATEGORY_STUB="mirror/pub/rocky"
|
||||
RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}"
|
||||
|
||||
# Define arches we support
|
||||
ARCHES=(x86_64 aarch64)
|
||||
|
||||
# Set all repos that have no comps/groups associated with them. This is even in
|
||||
# cases where repos will not be available by normal means. It's just for
|
||||
# consistency.
|
||||
NONMODS_REPOS=(
|
||||
extras
|
||||
Devel
|
||||
nfv
|
||||
storage/gluster9
|
||||
plus
|
||||
)
|
||||
|
||||
# These repos have comps/groups, except for debuginfo and sources
|
||||
MODS_REPOS=(
|
||||
BaseOS
|
||||
AppStream
|
||||
HighAvailability
|
||||
ResilientStorage
|
||||
PowerTools
|
||||
)
|
||||
|
||||
ALL_REPOS=(
|
||||
"${NONMODS_REPOS[@]}"
|
||||
"${MODS_REPOS[@]}"
|
||||
)
|
||||
|
||||
# These repos have modules
|
||||
MODS=(
|
||||
AppStream
|
||||
PowerTools
|
||||
)
|
||||
|
|
@ -10,49 +10,19 @@
|
|||
# and provide everything, even if it's not installable).
|
||||
#
|
||||
# Compose dir example: /mnt/repos-staging/mirror/pub/rocky/8.4-RC2
|
||||
# Revision must always start with a major number
|
||||
REVISION=8.4
|
||||
#MAJOR="${REVISION:0:1}"
|
||||
# comment or blank if needed
|
||||
APPEND_TO_DIR="-RC2"
|
||||
COMPOSE_DIR="/mnt/repos-staging/mirror/pub/rocky/${REVISION}${APPEND_TO_DIR}"
|
||||
ARCHES=(x86_64 aarch64)
|
||||
|
||||
# Set all repos that have no comps/groups associated with them. This is even in
|
||||
# cases where repos will not be available by normal means. It's just for
|
||||
# consistency.
|
||||
NONMODS_REPOS=(
|
||||
extras
|
||||
Devel
|
||||
nfv
|
||||
storage/gluster9
|
||||
plus
|
||||
)
|
||||
|
||||
# These repos have comps/groups, except for debuginfo and sources
|
||||
MODS_REPOS=(
|
||||
BaseOS
|
||||
AppStream
|
||||
HighAvailability
|
||||
ResilientStorage
|
||||
PowerTools
|
||||
)
|
||||
|
||||
# These repos have modules
|
||||
MODS=(
|
||||
AppStream
|
||||
PowerTools
|
||||
)
|
||||
# Source common variables
|
||||
source $(dirname "$0")/common
|
||||
|
||||
echo "** Updating source repos"
|
||||
for y in "${NONMODS_REPOS[@]}" "${MODS_REPOS[@]}"; do
|
||||
test -d "${COMPOSE_DIR}/${y}/${x}/${z}"
|
||||
for y in "${ALL_REPOS[@]}"; do
|
||||
test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}"
|
||||
ret_val=$?
|
||||
if [ "$ret_val" -eq 0 ]; then
|
||||
createrepo --update "${COMPOSE_DIR}/${y}/source/tree" \
|
||||
createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree" \
|
||||
"--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
|
||||
else
|
||||
echo "${COMPOSE_DIR}/${y}/source/tree does not exist"
|
||||
echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree does not exist"
|
||||
fi
|
||||
done
|
||||
|
||||
|
|
@ -64,41 +34,41 @@ for x in "${ARCHES[@]}"; do
|
|||
for y in "${NONMODS_REPOS[@]}"; do
|
||||
# os and debug/tree directories
|
||||
for z in os debug/tree; do
|
||||
test -d "${COMPOSE_DIR}/${y}/${x}/${z}"
|
||||
test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}"
|
||||
ret_val=$?
|
||||
if [ "$ret_val" -eq 0 ]; then
|
||||
createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \
|
||||
createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \
|
||||
"--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
|
||||
else
|
||||
echo "${COMPOSE_DIR}/${y}/${x}/${z} does not exist"
|
||||
echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z} does not exist"
|
||||
fi
|
||||
done
|
||||
done
|
||||
# repos with comps/groups involved, but only debug
|
||||
for y in "${MODS_REPOS[@]}"; do
|
||||
test -d "${COMPOSE_DIR}/${y}/${x}/debug/tree"
|
||||
test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree"
|
||||
ret_val=$?
|
||||
if [ "$ret_val" -eq 0 ]; then
|
||||
createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \
|
||||
createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \
|
||||
"--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
|
||||
else
|
||||
echo "${COMPOSE_DIR}/${y}/${x}/debug/tree does not exist"
|
||||
echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree does not exist"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "** Update all repos with comps/groups"
|
||||
for y in "${MODS_REPOS[@]}"; do
|
||||
echo "${y}: ${x}"
|
||||
test -d "${COMPOSE_DIR}/${y}/${x}/os"
|
||||
test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os"
|
||||
ret_val=$?
|
||||
if [ "$ret_val" -eq 0 ]; then
|
||||
createrepo --update "${COMPOSE_DIR}/${y}/${x}/os" \
|
||||
createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os" \
|
||||
--groupfile="/mnt/compose/8/latest-Rocky-8/work/${x}/comps/comps-${y}.${x}.xml" \
|
||||
--xz --revision=${REVISION} \
|
||||
"--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \
|
||||
--workers=8 --checksum=sha256
|
||||
else
|
||||
echo "${COMPOSE_DIR}/${y}/${x}/os does not exist"
|
||||
echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist"
|
||||
fi
|
||||
done
|
||||
|
||||
|
|
@ -106,14 +76,14 @@ for x in "${ARCHES[@]}"; do
|
|||
for y in "${MODS[@]}"; do
|
||||
echo "Modules: ${y} ${x}"
|
||||
cp "/mnt/compose/8_metadata/${x}/${y}-modules.yaml" /tmp/modules.yaml
|
||||
test -d "${COMPOSE_DIR}/${y}/${x}/os"
|
||||
test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os"
|
||||
ret_val=$?
|
||||
if [ "$ret_val" -eq 0 ]; then
|
||||
modifyrepo --mdtype=modules /tmp/modules.yaml \
|
||||
"${COMPOSE_DIR}/${y}/${x}/os/repodata" \
|
||||
"${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os/repodata" \
|
||||
--compress --compress-type=gz
|
||||
else
|
||||
echo "${COMPOSE_DIR}/${y}/${x}/os does not exist"
|
||||
echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist"
|
||||
fi
|
||||
|
||||
rm /tmp/modules.yaml
|
||||
|
|
@ -121,7 +91,7 @@ for x in "${ARCHES[@]}"; do
|
|||
done
|
||||
|
||||
echo "** Fix variants"
|
||||
TREEINFO_VAR="${COMPOSE_DIR}/BaseOS/${x}/os/.treeinfo"
|
||||
TREEINFO_VAR="${STAGING_ROOT}/${RELEASE_DIR}/BaseOS/${x}/os/.treeinfo"
|
||||
test -f "${TREEINFO_VAR}"
|
||||
treeinfo_retval=$?
|
||||
test -x /usr/bin/python3
|
||||
|
|
|
|||
|
|
@ -1,11 +1,19 @@
|
|||
#!/bin/bash
|
||||
# Syncs everything from staging to production
|
||||
REVISION=${1}
|
||||
cd "/mnt/repos-staging/mirror/pub/rocky/${REVISION}"
|
||||
|
||||
# Source common variables
|
||||
source $(dirname "$0")/common
|
||||
|
||||
REV=${1}
|
||||
|
||||
cd "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
|
||||
ret_val=$?
|
||||
|
||||
if [ $ret_val -eq "0" ]; then
|
||||
mkdir -p "/mnt/repos-production/mirror/pub/rocky/${REVISION:0:3}"
|
||||
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-production/mirror/pub/rocky/${REVISION:0:3}
|
||||
TARGET="${PRODUCTION_ROOT}/${CATEGORY_STUB}/${REV:0:3}"
|
||||
mkdir -p "${TARGET}"
|
||||
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
|
||||
{} ${TARGET}
|
||||
else
|
||||
echo "Failed to change directory"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -1,17 +1,25 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Source common variables
|
||||
source $(dirname "$0")/common
|
||||
|
||||
# Major Version (eg, 8)
|
||||
MAJOR=${1}
|
||||
MAJ=${1}
|
||||
# Short name (eg, NFV, extras, Rocky, gluster9)
|
||||
SHORT=${2}
|
||||
# The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX
|
||||
REVISION=${3}
|
||||
REV=${3}
|
||||
# Note, this should be lowercase. eg, storage.
|
||||
SIG=${4}
|
||||
cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR}
|
||||
|
||||
cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}
|
||||
ret_val=$?
|
||||
|
||||
if [ $ret_val -eq "0" ]; then
|
||||
mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG}
|
||||
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG}
|
||||
local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${SIG}
|
||||
mkdir -p ${TARGET}
|
||||
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
|
||||
{} ${TARGET}
|
||||
else
|
||||
echo "Failed to change directory"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -1,15 +1,23 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Source common variables
|
||||
source $(dirname "$0")/common
|
||||
|
||||
# Major Version (eg, 8)
|
||||
MAJOR=${1}
|
||||
MAJ=${1}
|
||||
# Short name (eg, NFV, extras, Rocky, gluster9)
|
||||
SHORT=${2}
|
||||
# The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX
|
||||
REVISION=${3}
|
||||
cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR}
|
||||
REV=${3}
|
||||
|
||||
cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}
|
||||
ret_val=$?
|
||||
|
||||
if [ $ret_val -eq "0" ]; then
|
||||
mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION}
|
||||
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION}
|
||||
local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV}
|
||||
mkdir -p ${TARGET}
|
||||
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
|
||||
{} ${TARGET}
|
||||
else
|
||||
echo "Failed to change directory"
|
||||
fi
|
||||
|
|
|
|||
Loading…
Reference in a new issue