From 6451509161c658ba53a7e4b5031760a0e99bb1d4 Mon Sep 17 00:00:00 2001 From: nazunalika Date: Fri, 9 Jul 2021 11:30:49 -0700 Subject: [PATCH 1/5] simple update for fullfilelist --- sync/sync-to-prod.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sync/sync-to-prod.sh b/sync/sync-to-prod.sh index ca26a59..70d6d8e 100644 --- a/sync/sync-to-prod.sh +++ b/sync/sync-to-prod.sh @@ -18,4 +18,9 @@ if [ $ret_val -eq "0" ]; then # shellcheck disable=SC2035 sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ {} "${TARGET}" + + # Full file list update + cd "${PRODUCTION_ROOT}/${CATEGORY_STUB}/" || { echo "Failed to change directory"; exit 1; } + find . > fullfilelist fi + From 2afb5e7c1c2ed13ac4220b11b22df03d368eabf6 Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Fri, 9 Jul 2021 21:57:20 -0400 Subject: [PATCH 2/5] feat: torrent generation --- sync/common | 9 +++++ sync/gen-torrents.sh | 91 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100755 sync/gen-torrents.sh diff --git a/sync/common b/sync/common index 698fde5..87d632d 100644 --- a/sync/common +++ b/sync/common @@ -28,3 +28,12 @@ fi # Combined variables based on common data REV="${REVISION}${APPEND_TO_DIR}" + +# These repos have modules +MODS=( + AppStream + PowerTools +) + +# Used to iterate over ISOs? +VARIANTS=(boot minimal dvd1) diff --git a/sync/gen-torrents.sh b/sync/gen-torrents.sh new file mode 100755 index 0000000..022f6ca --- /dev/null +++ b/sync/gen-torrents.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# + +# Source common variables +# shellcheck disable=SC2046,1091,1090 +source "$(dirname "$0")/common" + +NAME=gen-torrents + +USAGE="usage: $NAME " +ISODIR=${1} + +if [[ -z "${ISODIR}" || $# == 0 ]]; then + echo $USAGE + exit +fi + +# Setup a lock? +LOCKFILE="/tmp/${NAME}.lock" +if [ -f "$LOCKFILE" ]; then + echo "Script is already running" + exit +fi + +trap "rm -f $LOCKFILE" EXIT +touch $LOCKFILE + +# stamp the email + +# Where to put torrent data +TORRENT_DOWNLOAD_DIR="/opt/rtorrent/download" +# Where to drop created torrents +TORRENT_START_DIR="/opt/rtorrent/watch/start" +# What trackers should be used +TORRENT_TRACKERS=( + "udp://tracker.opentrackr.org:1337/announce" + "udp://tracker.openbittorrent.com:80/announce" +) +# Regex of paths to exclude +TORRENT_EXCLUDES='.*\/CHECKSUM.asc' +TORRENT_COMMENT="https://docs.rockylinux.org/release_notes/${REVISION}/" +THREADS=10 + +printf "* Step 1: Create scaffolding and link\n" +cd "${TORRENT_DOWNLOAD_DIR}" || exit 1 +for variant in "${VARIANTS[@]}"; do + for arch in "${ARCHES[@]}"; do + # Skip this architecture if it's not there + if [[ ! -d "${ISODIR}/${arch}" ]]; then + printf "** %s - Does not exist. Skipping.\n" "${ISODIR}/${arch}" + continue + fi + + name_template="Rocky-${REVISION}-${arch}-${variant}" + + if [[ ! -f "${ISODIR}/${arch}/${name_template}.iso" ]]; then + printf "** %s - Does not exist. Skipping.\n" "${ISODIR}/${arch}/${name_template}.iso" + continue + fi + + printf "** Making directory: %s/%s\n" "${TORRENT_DOWNLOAD_DIR}" "${name_template}" + mkdir "${name_template}" || exit 2 + + printf "** Linking Version: %s; Arch: %s; Variant: %s\n" "${REVISION}" "${arch}" "${variant}" + ln -sv \ + ${ISODIR}/${arch}/{CHECKSUM*,${name_template}.iso*} \ + ${name_template}/ + done +done + + +printf "* Step 2: Generate torrents\n" +for torrent_directory in ${TORRENT_DOWNLOAD_DIR}/Rocky-${REVISION}-*; do + name="$(basename "${torrent_directory}")" + + if [[ -d "${torrent_directory}" ]]; then + printf "** Creating torrent for ${torrent_directory}\n" + else + continue + fi + torrenttools create \ + --announce "${TORRENT_TRACKERS[@]}" --name "${name}" \ + --exclude "${TORRENT_EXCLUDES}" --output "${TORRENT_START_DIR}" \ + --threads "${THREADS}" --comment "${TORRENT_COMMENT}" \ + "${torrent_directory}" + res=$? + if [[ $res -ne 0 ]]; then + printf "**[ERROR] Failed to create torrent." + exit "$res" + fi +done From 683c8d505ff88d8d446eb2c8bc6cdaa03e9c6350 Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Fri, 9 Jul 2021 22:03:32 -0400 Subject: [PATCH 3/5] fixup shellcheck --- sync/gen-torrents.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/sync/gen-torrents.sh b/sync/gen-torrents.sh index 022f6ca..28d2b3e 100755 --- a/sync/gen-torrents.sh +++ b/sync/gen-torrents.sh @@ -11,7 +11,7 @@ USAGE="usage: $NAME " ISODIR=${1} if [[ -z "${ISODIR}" || $# == 0 ]]; then - echo $USAGE + echo "$USAGE" exit fi @@ -22,7 +22,7 @@ if [ -f "$LOCKFILE" ]; then exit fi -trap "rm -f $LOCKFILE" EXIT +trap 'rm -f $LOCKFILE' EXIT touch $LOCKFILE # stamp the email @@ -63,25 +63,26 @@ for variant in "${VARIANTS[@]}"; do printf "** Linking Version: %s; Arch: %s; Variant: %s\n" "${REVISION}" "${arch}" "${variant}" ln -sv \ - ${ISODIR}/${arch}/{CHECKSUM*,${name_template}.iso*} \ - ${name_template}/ + "${ISODIR}"/"${arch}"/{CHECKSUM*,"${name_template}".iso*} \ + "${name_template}"/ done done printf "* Step 2: Generate torrents\n" -for torrent_directory in ${TORRENT_DOWNLOAD_DIR}/Rocky-${REVISION}-*; do +for torrent_directory in "${TORRENT_DOWNLOAD_DIR}"/Rocky-"${REVISION}"-*; do name="$(basename "${torrent_directory}")" if [[ -d "${torrent_directory}" ]]; then - printf "** Creating torrent for ${torrent_directory}\n" + printf "** Creating torrent for %s\n" "${torrent_directory}" else continue fi - torrenttools create \ - --announce "${TORRENT_TRACKERS[@]}" --name "${name}" \ - --exclude "${TORRENT_EXCLUDES}" --output "${TORRENT_START_DIR}" \ - --threads "${THREADS}" --comment "${TORRENT_COMMENT}" \ + + torrenttools create \ + --announce "${TORRENT_TRACKERS[@]}" --name "${name}" \ + --exclude "${TORRENT_EXCLUDES}" --output "${TORRENT_START_DIR}" \ + --threads "${THREADS}" --comment "${TORRENT_COMMENT}" \ "${torrent_directory}" res=$? if [[ $res -ne 0 ]]; then From 29500351a93a872e5c24220d3b058bd5c8d74012 Mon Sep 17 00:00:00 2001 From: nazunalika Date: Fri, 9 Jul 2021 21:17:40 -0700 Subject: [PATCH 4/5] add additional context for variable in comments --- sync/common | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/sync/common b/sync/common index 87d632d..0141d01 100644 --- a/sync/common +++ b/sync/common @@ -1,5 +1,13 @@ # To be sourced by scripts to use +# Variables that can be overriden should be noted with optional context. It is +# expected that these values are here in this file (per variable or per set): +# +# * Allowed +# * Allowed with caveats +# * Not Allowed +# * Required + # Temporary probably. This makes it so if RLVER=... is called before the script # it will set the version for the variables to call up. This was easier than # creating duplicates of a bunch of stuff. Default version is 8. @@ -7,19 +15,23 @@ if [ -z "$RLVER" ]; then export RLVER=8 fi +# Override: Not Allowed STAGING_ROOT="/mnt/repos-staging" PRODUCTION_ROOT="/mnt/repos-production" +# Override: Not Allowed # relative to ${ENV}_ROOT CATEGORY_STUB="mirror/pub/rocky" -# should be overriden in other commons +# Override: Required #RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}" # Define arches we support +# Override: Not Allowed ARCHES=(x86_64 aarch64) -#Source Major common +# Source Major common +# Override: Not Allowed test -f "$(dirname "$0")/common_${RLVER}" && source "$(dirname "$0")/common_${RLVER}" if [ "$?" -ne 0 ]; then echo "Could not source common_${RLVER}" @@ -27,13 +39,23 @@ if [ "$?" -ne 0 ]; then fi # Combined variables based on common data +# Override: Capable with caveats REV="${REVISION}${APPEND_TO_DIR}" # These repos have modules +# Override: Allowed +# This variable can (and probably should) be overriden by their common_X files +# that are sourced. The reason is because future EL's can potentially change up +# which repos are module based (whether adding/removing). This is something to +# keep in mind. For example, Fedora (sanely) has a "base" repo and then an +# updates repo for both their regular packages *and* their modularity repos. +# This obviously makes sense and I can only hope Red Hat decides to bring that +# back in some way. MODS=( AppStream PowerTools ) # Used to iterate over ISOs? +# Override: Allowed VARIANTS=(boot minimal dvd1) From 98eba8f20fd816ff57693e79d71444855d467429 Mon Sep 17 00:00:00 2001 From: nazunalika Date: Sat, 10 Jul 2021 16:09:34 -0700 Subject: [PATCH 5/5] quick fix --- sync/prep-staging-8.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/prep-staging-8.sh b/sync/prep-staging-8.sh index 90cbb7d..b0b7979 100644 --- a/sync/prep-staging-8.sh +++ b/sync/prep-staging-8.sh @@ -51,7 +51,7 @@ for x in "${ARCHES[@]}"; do test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree" ret_val=$? if [ "$ret_val" -eq 0 ]; then - createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \ + createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree" \ "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" else echo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree does not exist"