refactor sync tools to use common vars and stuff maybe

This commit is contained in:
Neil Hanlon 2021-07-06 13:08:30 -04:00
parent 091e3d1d08
commit d3e830c736
Signed by untrusted user: neil
GPG Key ID: 705BC21EC3C70F34
5 changed files with 108 additions and 63 deletions

51
sync/common Normal file
View File

@ -0,0 +1,51 @@
# To be sourced by scripts to use
# Revision must always start with a major number
REVISION=8.4
MAJOR="${REVISION:0:1}"
MINOR="${REVISION:2:1}"
# comment or blank if needed
APPEND_TO_DIR="-RC2"
STAGING_ROOT="/mnt/repos-staging"
PRODUCTION_ROOT="/mnt/repos-production"
# relative to ${ENV}_ROOT
CATEGORY_STUB="mirror/pub/rocky"
RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}"
# Define arches we support
ARCHES=(x86_64 aarch64)
# Set all repos that have no comps/groups associated with them. This is even in
# cases where repos will not be available by normal means. It's just for
# consistency.
NONMODS_REPOS=(
extras
Devel
nfv
storage/gluster9
plus
)
# These repos have comps/groups, except for debuginfo and sources
MODS_REPOS=(
BaseOS
AppStream
HighAvailability
ResilientStorage
PowerTools
)
ALL_REPOS=(
"${NONMODS_REPOS[@]}"
"${MODS_REPOS[@]}"
)
# These repos have modules
MODS=(
AppStream
PowerTools
)

View File

@ -10,49 +10,19 @@
# and provide everything, even if it's not installable). # and provide everything, even if it's not installable).
# #
# Compose dir example: /mnt/repos-staging/mirror/pub/rocky/8.4-RC2 # Compose dir example: /mnt/repos-staging/mirror/pub/rocky/8.4-RC2
# Revision must always start with a major number
REVISION=8.4
#MAJOR="${REVISION:0:1}"
# comment or blank if needed
APPEND_TO_DIR="-RC2"
COMPOSE_DIR="/mnt/repos-staging/mirror/pub/rocky/${REVISION}${APPEND_TO_DIR}"
ARCHES=(x86_64 aarch64)
# Set all repos that have no comps/groups associated with them. This is even in # Source common variables
# cases where repos will not be available by normal means. It's just for source $(dirname "$0")/common
# consistency.
NONMODS_REPOS=(
extras
Devel
nfv
storage/gluster9
plus
)
# These repos have comps/groups, except for debuginfo and sources
MODS_REPOS=(
BaseOS
AppStream
HighAvailability
ResilientStorage
PowerTools
)
# These repos have modules
MODS=(
AppStream
PowerTools
)
echo "** Updating source repos" echo "** Updating source repos"
for y in "${NONMODS_REPOS[@]}" "${MODS_REPOS[@]}"; do for y in "${ALL_REPOS[@]}"; do
test -d "${COMPOSE_DIR}/${y}/${x}/${z}" test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}"
ret_val=$? ret_val=$?
if [ "$ret_val" -eq 0 ]; then if [ "$ret_val" -eq 0 ]; then
createrepo --update "${COMPOSE_DIR}/${y}/source/tree" \ createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree" \
"--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
else else
echo "${COMPOSE_DIR}/${y}/source/tree does not exist" echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree does not exist"
fi fi
done done
@ -64,41 +34,41 @@ for x in "${ARCHES[@]}"; do
for y in "${NONMODS_REPOS[@]}"; do for y in "${NONMODS_REPOS[@]}"; do
# os and debug/tree directories # os and debug/tree directories
for z in os debug/tree; do for z in os debug/tree; do
test -d "${COMPOSE_DIR}/${y}/${x}/${z}" test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}"
ret_val=$? ret_val=$?
if [ "$ret_val" -eq 0 ]; then if [ "$ret_val" -eq 0 ]; then
createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \ createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \
"--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
else else
echo "${COMPOSE_DIR}/${y}/${x}/${z} does not exist" echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z} does not exist"
fi fi
done done
done done
# repos with comps/groups involved, but only debug # repos with comps/groups involved, but only debug
for y in "${MODS_REPOS[@]}"; do for y in "${MODS_REPOS[@]}"; do
test -d "${COMPOSE_DIR}/${y}/${x}/debug/tree" test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree"
ret_val=$? ret_val=$?
if [ "$ret_val" -eq 0 ]; then if [ "$ret_val" -eq 0 ]; then
createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \ createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \
"--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
else else
echo "${COMPOSE_DIR}/${y}/${x}/debug/tree does not exist" echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree does not exist"
fi fi
done done
echo "** Update all repos with comps/groups" echo "** Update all repos with comps/groups"
for y in "${MODS_REPOS[@]}"; do for y in "${MODS_REPOS[@]}"; do
echo "${y}: ${x}" echo "${y}: ${x}"
test -d "${COMPOSE_DIR}/${y}/${x}/os" test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os"
ret_val=$? ret_val=$?
if [ "$ret_val" -eq 0 ]; then if [ "$ret_val" -eq 0 ]; then
createrepo --update "${COMPOSE_DIR}/${y}/${x}/os" \ createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os" \
--groupfile="/mnt/compose/8/latest-Rocky-8/work/${x}/comps/comps-${y}.${x}.xml" \ --groupfile="/mnt/compose/8/latest-Rocky-8/work/${x}/comps/comps-${y}.${x}.xml" \
--xz --revision=${REVISION} \ --xz --revision=${REVISION} \
"--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \ "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \
--workers=8 --checksum=sha256 --workers=8 --checksum=sha256
else else
echo "${COMPOSE_DIR}/${y}/${x}/os does not exist" echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist"
fi fi
done done
@ -106,14 +76,14 @@ for x in "${ARCHES[@]}"; do
for y in "${MODS[@]}"; do for y in "${MODS[@]}"; do
echo "Modules: ${y} ${x}" echo "Modules: ${y} ${x}"
cp "/mnt/compose/8_metadata/${x}/${y}-modules.yaml" /tmp/modules.yaml cp "/mnt/compose/8_metadata/${x}/${y}-modules.yaml" /tmp/modules.yaml
test -d "${COMPOSE_DIR}/${y}/${x}/os" test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os"
ret_val=$? ret_val=$?
if [ "$ret_val" -eq 0 ]; then if [ "$ret_val" -eq 0 ]; then
modifyrepo --mdtype=modules /tmp/modules.yaml \ modifyrepo --mdtype=modules /tmp/modules.yaml \
"${COMPOSE_DIR}/${y}/${x}/os/repodata" \ "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os/repodata" \
--compress --compress-type=gz --compress --compress-type=gz
else else
echo "${COMPOSE_DIR}/${y}/${x}/os does not exist" echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist"
fi fi
rm /tmp/modules.yaml rm /tmp/modules.yaml
@ -121,7 +91,7 @@ for x in "${ARCHES[@]}"; do
done done
echo "** Fix variants" echo "** Fix variants"
TREEINFO_VAR="${COMPOSE_DIR}/BaseOS/${x}/os/.treeinfo" TREEINFO_VAR="${STAGING_ROOT}/${RELEASE_DIR}/BaseOS/${x}/os/.treeinfo"
test -f "${TREEINFO_VAR}" test -f "${TREEINFO_VAR}"
treeinfo_retval=$? treeinfo_retval=$?
test -x /usr/bin/python3 test -x /usr/bin/python3

View File

@ -1,11 +1,19 @@
#!/bin/bash #!/bin/bash
# Syncs everything from staging to production # Syncs everything from staging to production
REVISION=${1}
cd "/mnt/repos-staging/mirror/pub/rocky/${REVISION}" # Source common variables
source $(dirname "$0")/common
REV=${1}
cd "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
ret_val=$? ret_val=$?
if [ $ret_val -eq "0" ]; then if [ $ret_val -eq "0" ]; then
mkdir -p "/mnt/repos-production/mirror/pub/rocky/${REVISION:0:3}" TARGET="${PRODUCTION_ROOT}/${CATEGORY_STUB}/${REV:0:3}"
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-production/mirror/pub/rocky/${REVISION:0:3} mkdir -p "${TARGET}"
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
{} ${TARGET}
else else
echo "Failed to change directory" echo "Failed to change directory"
fi fi

View File

@ -1,17 +1,25 @@
#!/bin/bash #!/bin/bash
# Source common variables
source $(dirname "$0")/common
# Major Version (eg, 8) # Major Version (eg, 8)
MAJOR=${1} MAJ=${1}
# Short name (eg, NFV, extras, Rocky, gluster9) # Short name (eg, NFV, extras, Rocky, gluster9)
SHORT=${2} SHORT=${2}
# The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX # The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX
REVISION=${3} REV=${3}
# Note, this should be lowercase. eg, storage. # Note, this should be lowercase. eg, storage.
SIG=${4} SIG=${4}
cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR}
cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}
ret_val=$? ret_val=$?
if [ $ret_val -eq "0" ]; then if [ $ret_val -eq "0" ]; then
mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG} local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${SIG}
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG} mkdir -p ${TARGET}
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
{} ${TARGET}
else else
echo "Failed to change directory" echo "Failed to change directory"
fi fi

View File

@ -1,15 +1,23 @@
#!/bin/bash #!/bin/bash
# Source common variables
source $(dirname "$0")/common
# Major Version (eg, 8) # Major Version (eg, 8)
MAJOR=${1} MAJ=${1}
# Short name (eg, NFV, extras, Rocky, gluster9) # Short name (eg, NFV, extras, Rocky, gluster9)
SHORT=${2} SHORT=${2}
# The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX # The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX
REVISION=${3} REV=${3}
cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR}
cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}
ret_val=$? ret_val=$?
if [ $ret_val -eq "0" ]; then if [ $ret_val -eq "0" ]; then
mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION} local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV}
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION} mkdir -p ${TARGET}
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
{} ${TARGET}
else else
echo "Failed to change directory" echo "Failed to change directory"
fi fi