refactor sync tools to use common vars and stuff maybe

This commit is contained in:
Neil Hanlon 2021-07-06 13:08:30 -04:00
parent 091e3d1d08
commit d3e830c736
Signed by: neil
GPG Key ID: 705BC21EC3C70F34
5 changed files with 108 additions and 63 deletions

51
sync/common Normal file
View File

@ -0,0 +1,51 @@
# To be sourced by scripts to use
# Revision must always start with a major number
REVISION=8.4
MAJOR="${REVISION:0:1}"
MINOR="${REVISION:2:1}"
# comment or blank if needed
APPEND_TO_DIR="-RC2"
STAGING_ROOT="/mnt/repos-staging"
PRODUCTION_ROOT="/mnt/repos-production"
# relative to ${ENV}_ROOT
CATEGORY_STUB="mirror/pub/rocky"
RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}"
# Define arches we support
ARCHES=(x86_64 aarch64)
# Set all repos that have no comps/groups associated with them. This is even in
# cases where repos will not be available by normal means. It's just for
# consistency.
NONMODS_REPOS=(
extras
Devel
nfv
storage/gluster9
plus
)
# These repos have comps/groups, except for debuginfo and sources
MODS_REPOS=(
BaseOS
AppStream
HighAvailability
ResilientStorage
PowerTools
)
ALL_REPOS=(
"${NONMODS_REPOS[@]}"
"${MODS_REPOS[@]}"
)
# These repos have modules
MODS=(
AppStream
PowerTools
)

View File

@ -10,49 +10,19 @@
# and provide everything, even if it's not installable).
#
# Compose dir example: /mnt/repos-staging/mirror/pub/rocky/8.4-RC2
# Revision must always start with a major number
REVISION=8.4
#MAJOR="${REVISION:0:1}"
# comment or blank if needed
APPEND_TO_DIR="-RC2"
COMPOSE_DIR="/mnt/repos-staging/mirror/pub/rocky/${REVISION}${APPEND_TO_DIR}"
ARCHES=(x86_64 aarch64)
# Set all repos that have no comps/groups associated with them. This is even in
# cases where repos will not be available by normal means. It's just for
# consistency.
NONMODS_REPOS=(
extras
Devel
nfv
storage/gluster9
plus
)
# These repos have comps/groups, except for debuginfo and sources
MODS_REPOS=(
BaseOS
AppStream
HighAvailability
ResilientStorage
PowerTools
)
# These repos have modules
MODS=(
AppStream
PowerTools
)
# Source common variables
source $(dirname "$0")/common
echo "** Updating source repos"
for y in "${NONMODS_REPOS[@]}" "${MODS_REPOS[@]}"; do
test -d "${COMPOSE_DIR}/${y}/${x}/${z}"
for y in "${ALL_REPOS[@]}"; do
test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}"
ret_val=$?
if [ "$ret_val" -eq 0 ]; then
createrepo --update "${COMPOSE_DIR}/${y}/source/tree" \
createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree" \
"--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
else
echo "${COMPOSE_DIR}/${y}/source/tree does not exist"
echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree does not exist"
fi
done
@ -64,41 +34,41 @@ for x in "${ARCHES[@]}"; do
for y in "${NONMODS_REPOS[@]}"; do
# os and debug/tree directories
for z in os debug/tree; do
test -d "${COMPOSE_DIR}/${y}/${x}/${z}"
test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}"
ret_val=$?
if [ "$ret_val" -eq 0 ]; then
createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \
createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \
"--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
else
echo "${COMPOSE_DIR}/${y}/${x}/${z} does not exist"
echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z} does not exist"
fi
done
done
# repos with comps/groups involved, but only debug
for y in "${MODS_REPOS[@]}"; do
test -d "${COMPOSE_DIR}/${y}/${x}/debug/tree"
test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree"
ret_val=$?
if [ "$ret_val" -eq 0 ]; then
createrepo --update "${COMPOSE_DIR}/${y}/${x}/${z}" \
createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \
"--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}"
else
echo "${COMPOSE_DIR}/${y}/${x}/debug/tree does not exist"
echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree does not exist"
fi
done
echo "** Update all repos with comps/groups"
for y in "${MODS_REPOS[@]}"; do
echo "${y}: ${x}"
test -d "${COMPOSE_DIR}/${y}/${x}/os"
test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os"
ret_val=$?
if [ "$ret_val" -eq 0 ]; then
createrepo --update "${COMPOSE_DIR}/${y}/${x}/os" \
createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os" \
--groupfile="/mnt/compose/8/latest-Rocky-8/work/${x}/comps/comps-${y}.${x}.xml" \
--xz --revision=${REVISION} \
"--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \
--workers=8 --checksum=sha256
else
echo "${COMPOSE_DIR}/${y}/${x}/os does not exist"
echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist"
fi
done
@ -106,14 +76,14 @@ for x in "${ARCHES[@]}"; do
for y in "${MODS[@]}"; do
echo "Modules: ${y} ${x}"
cp "/mnt/compose/8_metadata/${x}/${y}-modules.yaml" /tmp/modules.yaml
test -d "${COMPOSE_DIR}/${y}/${x}/os"
test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os"
ret_val=$?
if [ "$ret_val" -eq 0 ]; then
modifyrepo --mdtype=modules /tmp/modules.yaml \
"${COMPOSE_DIR}/${y}/${x}/os/repodata" \
"${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os/repodata" \
--compress --compress-type=gz
else
echo "${COMPOSE_DIR}/${y}/${x}/os does not exist"
echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os does not exist"
fi
rm /tmp/modules.yaml
@ -121,7 +91,7 @@ for x in "${ARCHES[@]}"; do
done
echo "** Fix variants"
TREEINFO_VAR="${COMPOSE_DIR}/BaseOS/${x}/os/.treeinfo"
TREEINFO_VAR="${STAGING_ROOT}/${RELEASE_DIR}/BaseOS/${x}/os/.treeinfo"
test -f "${TREEINFO_VAR}"
treeinfo_retval=$?
test -x /usr/bin/python3

View File

@ -1,11 +1,19 @@
#!/bin/bash
# Syncs everything from staging to production
REVISION=${1}
cd "/mnt/repos-staging/mirror/pub/rocky/${REVISION}"
# Source common variables
source $(dirname "$0")/common
REV=${1}
cd "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
ret_val=$?
if [ $ret_val -eq "0" ]; then
mkdir -p "/mnt/repos-production/mirror/pub/rocky/${REVISION:0:3}"
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-production/mirror/pub/rocky/${REVISION:0:3}
TARGET="${PRODUCTION_ROOT}/${CATEGORY_STUB}/${REV:0:3}"
mkdir -p "${TARGET}"
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
{} ${TARGET}
else
echo "Failed to change directory"
fi

View File

@ -1,17 +1,25 @@
#!/bin/bash
# Source common variables
source $(dirname "$0")/common
# Major Version (eg, 8)
MAJOR=${1}
MAJ=${1}
# Short name (eg, NFV, extras, Rocky, gluster9)
SHORT=${2}
# The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX
REVISION=${3}
REV=${3}
# Note, this should be lowercase. eg, storage.
SIG=${4}
cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR}
cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}
ret_val=$?
if [ $ret_val -eq "0" ]; then
mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG}
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION}/${SIG}
local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${SIG}
mkdir -p ${TARGET}
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
{} ${TARGET}
else
echo "Failed to change directory"
fi

View File

@ -1,15 +1,23 @@
#!/bin/bash
# Source common variables
source $(dirname "$0")/common
# Major Version (eg, 8)
MAJOR=${1}
MAJ=${1}
# Short name (eg, NFV, extras, Rocky, gluster9)
SHORT=${2}
# The directory where we're going to, usually MAJOR.MINOR, sometimes it's MAJOR.MINOR-RCX
REVISION=${3}
cd /mnt/compose/${MAJOR}/latest-${SHORT}-${MAJOR}
REV=${3}
cd /mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}
ret_val=$?
if [ $ret_val -eq "0" ]; then
mkdir -p /mnt/repos-staging/mirror/pub/rocky/${REVISION}
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} /mnt/repos-staging/mirror/pub/rocky/${REVISION}
local TARGET=${STAGING_ROOT}/${CATEGORY_STUB}/${REV}
mkdir -p ${TARGET}
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
{} ${TARGET}
else
echo "Failed to change directory"
fi