toolkit/sync/common

#!/bin/bash
# To be sourced by scripts to use

# Variables that can be overriden should be noted with optional context. It is
# expected that these values are here in this file (per variable or per set):
#
#   * Allowed
#   * Allowed with caveats
#   * Not Allowed
#   * Required

# Temporary probably. This makes it so if RLVER=... is called before the script
# it will set the version for the variables to call up. This was easier than
# creating duplicates of a bunch of stuff. Default version is 8.
if [ -z "$RLVER" ]; then
  export RLVER=8
fi

if [ -z "$RLREL" ]; then
  export RLREL=stable
fi

# Override: Not Allowed
STAGING_ROOT="/mnt/repos-staging"
PRODUCTION_ROOT="/mnt/repos-production"
COMPOSE_ROOT="/mnt/compose"

# Override: Not Allowed
# relative to ${ENV}_ROOT
CATEGORY_STUB="mirror/pub/rocky"
SIG_CATEGORY_STUB="mirror/pub/sig"
VAULT_STUB="mirror/vault/rocky"
SIG_VAULT_STUB="mirror/vault/sig"

# Override: Required
#RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}"

# Define arches we support
# Override: Not Allowed
ARCHES=(x86_64 aarch64)

# Source Major common
# Override: Not Allowed
test -f "$(dirname "${BASH_SOURCE[0]}")/common_${RLVER}" && source "$(dirname "${BASH_SOURCE[0]}")/common_${RLVER}"
if [ "$?" -ne 0 ]; then
  echo "Could not source common_${RLVER}"
  exit 1
fi

# Combined variables based on common data
# Override: Capable with caveats
REV="${REVISION}${APPEND_TO_DIR}"

# These repos have modules
# Override: Allowed
# This variable can (and probably should) be overriden by their common_X files
# that are sourced. The reason is because future EL's can potentially change up
# which repos are module based (whether adding/removing). This is something to
# keep in mind. For example, Fedora (sanely) has a "base" repo and then an
# updates repo for both their regular packages *and* their modularity repos.
# This obviously makes sense and I can only hope Red Hat decides to bring that
# back in some way.
#MODS=(
#  AppStream
#  PowerTools
#)

# Used to iterate over ISOs?
# Override: Allowed
VARIANTS=(boot minimal dvd1 dvd)

# Syncing functions
function parallel_rsync_no_delete_staging() {
  local TARGET="${1}"
  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} "${TARGET}"
}

function parallel_rsync_no_delete_prod() {
  local TARGET="${1}"
  sudo -l && find ./ -mindepth 1 -maxdepth 1 -type d -exec find {}/ -mindepth 1 -maxdepth 1 -type d \;|sed 's/^..//g' | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} "${TARGET}"
  # shellcheck disable=SC2035
  sudo -l && find ** -maxdepth 0 -type l | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} "${TARGET}"
}

function parallel_rsync_delete_staging() {
  TARGET="${1}"
  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable --delete {} "${TARGET}"
}

function parallel_rsync_delete_prod() {
  local TARGET="${1}"
  sudo -l && find ./ -mindepth 1 -maxdepth 1 -type d -exec find {}/ -mindepth 1 -maxdepth 1 -type d \;|sed 's/^..//g' | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable --delete {} "${TARGET}"
  # shellcheck disable=SC2035
  sudo -l && find ** -maxdepth 0 -type l | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable {} "${TARGET}"
}

# normal rsync
function rsync_no_delete_staging() {
  local TARGET="${1}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable compose/ "${TARGET}"
}

function rsync_no_delete_staging_pungi() {
  local TARGET="${1}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
    --exclude ppc \
    --exclude images \
    --exclude boot \
    --exclude .discinfo \
    --exclude .treeinfo \
    --exclude EFI \
    --exclude isolinux \
    --exclude media.repo \
    --exclude EULA \
    --exclude LICENSE \
    --exclude COMMUNITY-CHARTER \
    --exclude extra_files.json \
    --exclude Minimal \
    compose/ "${TARGET}"
}

function rsync_no_delete_staging_with_excludes() {
  local TARGET="${1}"
  local EXCLUDE="${2}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
    --exclude "${EXCLUDE}" \
    --exclude Minimal \
    compose/ "${TARGET}"
}

function rsync_delete_staging_with_excludes() {
  local TARGET="${1}"
  local EXCLUDE="${2}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
    --exclude "${EXCLUDE}" \
    --exclude Minimal \
    --delete \
    compose/ "${TARGET}"
}

function rsync_delete_subdirectory() {
  local SUBDIR="${1}"
  local TARGET="${2}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
   --delete \
   "compose/${SUBDIR}/" "${TARGET}"
}

function rsync_delete_generic() {
  local SOURCE="${1}"
  local TARGET="${2}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
    --delete \
    "${SOURCE}/" "${TARGET}"
}

function rsync_no_delete_prod() {
  local REV="${1}"
  local TARGET="${2}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable "${REV}/" "${TARGET}"
}

function rsync_no_delete_prod_no_repodata() {
  local REV="${1}"
  local TARGET="${2}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable --exclude repodata "${REV}/" "${TARGET}"
}

function rsync_delete_staging() {
  local TARGET="${1}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable --delete compose/ "${TARGET}"
}

function rsync_delete_prod() {
  local REV="${1}"
  local TARGET="${2}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable --delete "${REV}/" "${TARGET}"
}

function perform_hardlink() {
  local TARGET="${1}"
  hardlink -x '.*\.xml.*' -x ".*\.treeinfo" -x ".*\.discinfo" "${TARGET}"
}

# dry rsync
function rsync_no_delete_staging_dry() {
  local TARGET="${1}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable --dry-run compose/ "${TARGET}"
}

function rsync_no_delete_staging_pungi_dry() {
  local TARGET="${1}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
    --exclude ppc \
    --exclude images \
    --exclude boot \
    --exclude .discinfo \
    --exclude .treeinfo \
    --exclude EFI \
    --exclude isolinux \
    --exclude media.repo \
    --exclude EULA \
    --exclude LICENSE \
    --exclude COMMUNITY-CHARTER \
    --exclude extra_files.json \
    --exclude Minimal \
    --dry-run \
    compose/ "${TARGET}"
}

function rsync_no_delete_staging_with_excludes_dry() {
  local TARGET="${1}"
  local EXCLUDE="${2}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
    --exclude "${EXCLUDE}" \
    --exclude Minimal \
    --dry-run \
    compose/ "${TARGET}"
}

function rsync_delete_staging_with_excludes_dry() {
  local TARGET="${1}"
  local EXCLUDE="${2}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable \
    --exclude "${EXCLUDE}" \
    --exclude Minimal \
    --dry-run \
    compose/ "${TARGET}"
}

function rsync_no_delete_prod_dry() {
  local REV="${1}"
  local TARGET="${2}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable --dry-run "${REV}/" "${TARGET}"
}

function rsync_delete_staging_dry() {
  local TARGET="${1}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable --delete --dry-run compose/ "${TARGET}"
}

function rsync_delete_prod_dry() {
  local REV="${1}"
  local TARGET="${2}"
  sudo -l && rsync -vrlptDSHog --chown=10004:10005 --progress --human-readable --delete --dry-run "${REV}/" "${TARGET}"
}


function createrepo_only() {
  local TARGET="${1}"
  local TARGET_REV="${2}"
  createrepo "${TARGET}" \
    "--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \
    --workers 12
}

function createrepo_update() {
  local TARGET="${1}"
  local TARGET_REV="${2}"
  createrepo --update "${TARGET}" \
    "--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \
    --workers 12
}

function createrepo_comps() {
  local TARGET="${1}"
  local TARGET_REV="${2}"
  local TARGET_COMPS="${3}"
  createrepo --update "${TARGET}" \
    --groupfile="${TARGET_COMPS}" \
    --xz --revision="${TARGET_REV}" \
    "--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \
    --workers 12 --checksum=sha256
}

function modifyrepo_module() {
  local TARGET="${1}"
  local TARGET_YAML="${2}"
  /bin/cp "${TARGET_YAML}" /tmp/modules.yaml
  modifyrepo --mdtype=modules /tmp/modules.yaml \
    "${TARGET}" --compress --compress-type=xz

  /bin/rm /tmp/modules.yaml
}

function sign_data() {
  local TARGET="${1}"
  local VER="${2}"
  test -f "/root/bin/sign-repo-${VER}.sh" && "/root/bin/sign-repo-${VER}.sh" \
    "${TARGET}"
}

function fix_metadata() {
  local TARGET="${1}"
  sed -i '/<open-size><\/open-size>/d' "${TARGET}"
}

function createrepo_updates() {
  local TARGET="${1}"
  local TARGET_REV="${2}"
  createrepo "${TARGET}" \
    "--distro=cpe:/o:rocky:rocky:${TARGET_REV:0:1},Rocky Linux ${TARGET_REV:0:1}" \
    --workers 12 --update
}

export -f parallel_rsync_no_delete_staging
export -f parallel_rsync_no_delete_prod
export -f parallel_rsync_delete_staging
export -f parallel_rsync_delete_prod
# normal rsync
export -f rsync_no_delete_staging
export -f rsync_no_delete_staging_with_excludes
export -f rsync_no_delete_prod
export -f rsync_no_delete_prod_no_repodata
export -f rsync_delete_staging
export -f rsync_delete_staging_with_excludes
export -f rsync_delete_prod
export -f rsync_delete_subdirectory
export -f rsync_delete_generic
# dry rsync
export -f rsync_no_delete_staging_dry
export -f rsync_no_delete_staging_with_excludes_dry
export -f rsync_no_delete_prod_dry
export -f rsync_delete_staging_dry
export -f rsync_delete_staging_with_excludes_dry
export -f rsync_delete_prod_dry

export -f perform_hardlink
export -f createrepo_only
export -f createrepo_update
export -f createrepo_comps
export -f modifyrepo_module
export -f sign_data
export -f createrepo_updates