333 lines
10 KiB
Bash
333 lines
10 KiB
Bash
#!/bin/bash
|
|
# To be sourced by scripts to use
|
|
|
|
# Variables that can be overriden should be noted with optional context. It is
|
|
# expected that these values are here in this file (per variable or per set):
|
|
#
|
|
# * Allowed
|
|
# * Allowed with caveats
|
|
# * Not Allowed
|
|
# * Required
|
|
|
|
# Temporary probably. This makes it so if RLVER=... is called before the script
|
|
# it will set the version for the variables to call up. This was easier than
|
|
# creating duplicates of a bunch of stuff. Default version is 8.
|
|
if [ -z "$RLVER" ]; then
|
|
export RLVER=8
|
|
fi
|
|
|
|
if [ -z "$RLREL" ]; then
|
|
export RLREL=stable
|
|
fi
|
|
|
|
# Override: Not Allowed
|
|
STAGING_ROOT="/mnt/repos-staging"
|
|
PRODUCTION_ROOT="/mnt/repos-production"
|
|
COMPOSE_ROOT="/mnt/compose"
|
|
|
|
# Override: Not Allowed
|
|
# relative to ${ENV}_ROOT
|
|
CATEGORY_STUB="mirror/pub/rocky"
|
|
SIG_CATEGORY_STUB="mirror/pub/sig"
|
|
VAULT_STUB="mirror/vault/rocky"
|
|
SIG_VAULT_STUB="mirror/vault/sig"
|
|
|
|
# Override: Required
|
|
#RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}"
|
|
|
|
# Define arches we support
|
|
# Override: Not Allowed
|
|
ARCHES=(x86_64 aarch64)
|
|
|
|
# Source Major common
|
|
# Override: Not Allowed
|
|
test -f "$(dirname "${BASH_SOURCE[0]}")/common_${RLVER}" && source "$(dirname "${BASH_SOURCE[0]}")/common_${RLVER}"
|
|
if [ "$?" -ne 0 ]; then
|
|
echo "Could not source common_${RLVER}"
|
|
exit 1
|
|
fi
|
|
|
|
# Combined variables based on common data
|
|
# Override: Capable with caveats
|
|
REV="${REVISION}${APPEND_TO_DIR}"
|
|
|
|
# These repos have modules
|
|
# Override: Allowed
|
|
# This variable can (and probably should) be overriden by their common_X files
|
|
# that are sourced. The reason is because future EL's can potentially change up
|
|
# which repos are module based (whether adding/removing). This is something to
|
|
# keep in mind. For example, Fedora (sanely) has a "base" repo and then an
|
|
# updates repo for both their regular packages *and* their modularity repos.
|
|
# This obviously makes sense and I can only hope Red Hat decides to bring that
|
|
# back in some way.
|
|
#MODS=(
|
|
# AppStream
|
|
# PowerTools
|
|
#)
|
|
|
|
# Used to iterate over ISOs?
|
|
# Override: Allowed
|
|
VARIANTS=(boot minimal dvd1 dvd)
|
|
|
|
# Syncing functions
|
|
function parallel_rsync_no_delete_staging() {
|
|
local TARGET="${1}"
|
|
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} "${TARGET}"
|
|
}
|
|
|
|
function parallel_rsync_no_delete_prod() {
|
|
local TARGET="${1}"
|
|
sudo -l && find ./ -mindepth 1 -maxdepth 1 -type d -exec find {}/ -mindepth 1 -maxdepth 1 -type d \;|sed 's/^..//g' | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} "${TARGET}"
|
|
# shellcheck disable=SC2035
|
|
sudo -l && find ** -maxdepth 0 -type l | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} "${TARGET}"
|
|
}
|
|
|
|
function parallel_rsync_delete_staging() {
|
|
TARGET="${1}"
|
|
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable --delete {} "${TARGET}"
|
|
}
|
|
|
|
function parallel_rsync_delete_prod() {
|
|
local TARGET="${1}"
|
|
sudo -l && find ./ -mindepth 1 -maxdepth 1 -type d -exec find {}/ -mindepth 1 -maxdepth 1 -type d \;|sed 's/^..//g' | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable --delete {} "${TARGET}"
|
|
# shellcheck disable=SC2035
|
|
sudo -l && find ** -maxdepth 0 -type l | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} "${TARGET}"
|
|
}
|
|
|
|
# normal rsync
|
|
function rsync_no_delete_staging() {
|
|
local TARGET="${1}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable compose/ "${TARGET}"
|
|
}
|
|
|
|
function rsync_no_delete_staging_pungi() {
|
|
local TARGET="${1}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
|
|
--exclude ppc \
|
|
--exclude images \
|
|
--exclude boot \
|
|
--exclude .discinfo \
|
|
--exclude .treeinfo \
|
|
--exclude EFI \
|
|
--exclude isolinux \
|
|
--exclude media.repo \
|
|
--exclude EULA \
|
|
--exclude LICENSE \
|
|
--exclude COMMUNITY-CHARTER \
|
|
--exclude extra_files.json \
|
|
--exclude Minimal \
|
|
compose/ "${TARGET}"
|
|
}
|
|
|
|
function rsync_no_delete_staging_with_excludes() {
|
|
local TARGET="${1}"
|
|
local EXCLUDE="${2}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
|
|
--exclude "${EXCLUDE}" \
|
|
--exclude Minimal \
|
|
compose/ "${TARGET}"
|
|
}
|
|
|
|
function rsync_delete_staging_with_excludes() {
|
|
local TARGET="${1}"
|
|
local EXCLUDE="${2}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
|
|
--exclude "${EXCLUDE}" \
|
|
--exclude Minimal \
|
|
--delete \
|
|
compose/ "${TARGET}"
|
|
}
|
|
|
|
function rsync_delete_subdirectory() {
|
|
local SUBDIR="${1}"
|
|
local TARGET="${2}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
|
|
--delete \
|
|
"compose/${SUBDIR}/" "${TARGET}"
|
|
}
|
|
|
|
function rsync_delete_generic() {
|
|
local SOURCE="${1}"
|
|
local TARGET="${2}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
|
|
--delete \
|
|
"${SOURCE}/" "${TARGET}"
|
|
}
|
|
|
|
function rsync_no_delete_prod() {
|
|
local REV="${1}"
|
|
local TARGET="${2}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable "${REV}/" "${TARGET}"
|
|
}
|
|
|
|
function rsync_no_delete_prod_no_repodata() {
|
|
local REV="${1}"
|
|
local TARGET="${2}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable --exclude repodata "${REV}/" "${TARGET}"
|
|
}
|
|
|
|
function rsync_delete_staging() {
|
|
local TARGET="${1}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable --delete compose/ "${TARGET}"
|
|
}
|
|
|
|
function rsync_delete_prod() {
|
|
local REV="${1}"
|
|
local TARGET="${2}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable --delete "${REV}/" "${TARGET}"
|
|
}
|
|
|
|
function perform_hardlink() {
|
|
local TARGET="${1}"
|
|
hardlink -x '.*\.xml.*' -x ".*\.treeinfo" -x ".*\.discinfo" "${TARGET}"
|
|
}
|
|
|
|
# dry rsync
|
|
function rsync_no_delete_staging_dry() {
|
|
local TARGET="${1}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable --dry-run compose/ "${TARGET}"
|
|
}
|
|
|
|
function rsync_no_delete_staging_pungi_dry() {
|
|
local TARGET="${1}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
|
|
--exclude ppc \
|
|
--exclude images \
|
|
--exclude boot \
|
|
--exclude .discinfo \
|
|
--exclude .treeinfo \
|
|
--exclude EFI \
|
|
--exclude isolinux \
|
|
--exclude media.repo \
|
|
--exclude EULA \
|
|
--exclude LICENSE \
|
|
--exclude COMMUNITY-CHARTER \
|
|
--exclude extra_files.json \
|
|
--exclude Minimal \
|
|
--dry-run \
|
|
compose/ "${TARGET}"
|
|
}
|
|
|
|
function rsync_no_delete_staging_with_excludes_dry() {
|
|
local TARGET="${1}"
|
|
local EXCLUDE="${2}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
|
|
--exclude "${EXCLUDE}" \
|
|
--exclude Minimal \
|
|
--dry-run \
|
|
compose/ "${TARGET}"
|
|
}
|
|
|
|
function rsync_delete_staging_with_excludes_dry() {
|
|
local TARGET="${1}"
|
|
local EXCLUDE="${2}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
|
|
--exclude "${EXCLUDE}" \
|
|
--exclude Minimal \
|
|
--dry-run \
|
|
compose/ "${TARGET}"
|
|
}
|
|
|
|
function rsync_no_delete_prod_dry() {
|
|
local REV="${1}"
|
|
local TARGET="${2}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable --dry-run "${REV}/" "${TARGET}"
|
|
}
|
|
|
|
function rsync_delete_staging_dry() {
|
|
local TARGET="${1}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable --delete --dry-run compose/ "${TARGET}"
|
|
}
|
|
|
|
function rsync_delete_prod_dry() {
|
|
local REV="${1}"
|
|
local TARGET="${2}"
|
|
sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable --delete --dry-run "${REV}/" "${TARGET}"
|
|
}
|
|
|
|
|
|
function createrepo_only() {
|
|
local TARGET="${1}"
|
|
local TARGET_REV="${2}"
|
|
createrepo "${TARGET}" \
|
|
"--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \
|
|
--workers 12
|
|
}
|
|
|
|
function createrepo_update() {
|
|
local TARGET="${1}"
|
|
local TARGET_REV="${2}"
|
|
createrepo --update "${TARGET}" \
|
|
"--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \
|
|
--workers 12
|
|
}
|
|
|
|
function createrepo_comps() {
|
|
local TARGET="${1}"
|
|
local TARGET_REV="${2}"
|
|
local TARGET_COMPS="${3}"
|
|
createrepo --update "${TARGET}" \
|
|
--groupfile="${TARGET_COMPS}" \
|
|
--xz --revision="${TARGET_REV}" \
|
|
"--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \
|
|
--workers 12 --checksum=sha256
|
|
}
|
|
|
|
function modifyrepo_module() {
|
|
local TARGET="${1}"
|
|
local TARGET_YAML="${2}"
|
|
/bin/cp "${TARGET_YAML}" /tmp/modules.yaml
|
|
modifyrepo --mdtype=modules /tmp/modules.yaml \
|
|
"${TARGET}" --compress --compress-type=xz
|
|
|
|
/bin/rm /tmp/modules.yaml
|
|
}
|
|
|
|
function sign_data() {
|
|
local TARGET="${1}"
|
|
test -f /root/bin/sign-repo.sh && /root/bin/sign-repo.sh \
|
|
"${TARGET}"
|
|
}
|
|
|
|
function fix_metadata() {
|
|
local TARGET="${1}"
|
|
sed -i '/<open-size><\/open-size>/d' "${TARGET}"
|
|
}
|
|
|
|
function createrepo_updates() {
|
|
local TARGET="${1}"
|
|
local TARGET_REV="${2}"
|
|
createrepo "${TARGET}" \
|
|
"--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \
|
|
--workers 12 --update
|
|
}
|
|
|
|
export -f parallel_rsync_no_delete_staging
|
|
export -f parallel_rsync_no_delete_prod
|
|
export -f parallel_rsync_delete_staging
|
|
export -f parallel_rsync_delete_prod
|
|
# normal rsync
|
|
export -f rsync_no_delete_staging
|
|
export -f rsync_no_delete_staging_with_excludes
|
|
export -f rsync_no_delete_prod
|
|
export -f rsync_no_delete_prod_no_repodata
|
|
export -f rsync_delete_staging
|
|
export -f rsync_delete_staging_with_excludes
|
|
export -f rsync_delete_prod
|
|
export -f rsync_delete_subdirectory
|
|
export -f rsync_delete_generic
|
|
# dry rsync
|
|
export -f rsync_no_delete_staging_dry
|
|
export -f rsync_no_delete_staging_with_excludes_dry
|
|
export -f rsync_no_delete_prod_dry
|
|
export -f rsync_delete_staging_dry
|
|
export -f rsync_delete_staging_with_excludes_dry
|
|
export -f rsync_delete_prod_dry
|
|
|
|
export -f perform_hardlink
|
|
export -f createrepo_only
|
|
export -f createrepo_update
|
|
export -f createrepo_comps
|
|
export -f modifyrepo_module
|
|
export -f sign_data
|
|
export -f createrepo_updates
|