From a63e0d832eb4965c2d9a5d118d1fa560a723dab3 Mon Sep 17 00:00:00 2001 From: nazunalika Date: Tue, 10 May 2022 06:31:47 -0700 Subject: [PATCH 01/29] NFV is part of regular compose now --- sync/common_8 | 1 - 1 file changed, 1 deletion(-) diff --git a/sync/common_8 b/sync/common_8 index 374c22a..8431a0f 100644 --- a/sync/common_8 +++ b/sync/common_8 @@ -46,7 +46,6 @@ NONSIG_COMPOSE=( Rocky Extras Rocky-devel - NFV Plus rockyrpi ) From 671a02d60cc5b0b29d1acc9f45c611277f5898f1 Mon Sep 17 00:00:00 2001 From: nazunalika Date: Thu, 12 May 2022 00:18:34 -0700 Subject: [PATCH 02/29] fix up NFV a bit --- sync/common_8 | 7 ++++++- sync/minor-release-sync-to-staging.sh | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/sync/common_8 b/sync/common_8 index 8431a0f..2a8fe96 100644 --- a/sync/common_8 +++ b/sync/common_8 @@ -22,11 +22,15 @@ RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}" NONMODS_REPOS=( Extras Devel - nfv plus rockyrpi ) +# These repos were originally separate from the main compose and need symlinks +LINK_REPOS=( + [NFV]="nfv" +) + # These repos have comps/groups, except for debuginfo and sources MODS_REPOS=( BaseOS @@ -35,6 +39,7 @@ MODS_REPOS=( ResilientStorage PowerTools RT + NFV ) ALL_REPOS=( diff --git a/sync/minor-release-sync-to-staging.sh b/sync/minor-release-sync-to-staging.sh index 297363f..13a8cfa 100644 --- a/sync/minor-release-sync-to-staging.sh +++ b/sync/minor-release-sync-to-staging.sh @@ -41,6 +41,12 @@ for SIG in "${!SIG_COMPOSE[@]}"; do {} "${TARGET}" done +# Create symlinks for repos that were once separate from the main compose +for LINK in "${LINK_REPOS[@]}"; do + ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${LINK}" \ + "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${LINK_REPOS[$LINK]}" +done + # copy around the ISOs a bit, make things comfortable for ARCH in "${ARCHES[@]}"; do TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/isos/${ARCH}" From a40f49e2f2741addffe0ad53f9808b88e218642e Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Fri, 6 Aug 2021 16:45:54 -0400 Subject: [PATCH 03/29] Add really simple download stats script to parse fastly logs on disk --- analyze/common | 0 analyze/common_8 | 0 analyze/gen_download_statistics.sh | 81 ++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 analyze/common create mode 100644 analyze/common_8 create mode 100644 analyze/gen_download_statistics.sh diff --git a/analyze/common b/analyze/common new file mode 100644 index 0000000..e69de29 diff --git a/analyze/common_8 b/analyze/common_8 new file mode 100644 index 0000000..e69de29 diff --git a/analyze/gen_download_statistics.sh b/analyze/gen_download_statistics.sh new file mode 100644 index 0000000..de9a142 --- /dev/null +++ b/analyze/gen_download_statistics.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +log () { + printf "[LOG] %s\n" "$1" +} + + +log "Begin generation" +log "Generating intermediary ISO logfile" + +# Generate intermediary iso log +if [[ ! -f intermediary_isos.log ]]; then + awk '/Rocky-8.4-(aarch64|x86_64)-.*?\.iso/ {if ($12==200 && $4!="3.134.114.30") print $0}' *.log **/*.log > intermediary_isos.log +else + log "Skipped ISO intermediary" +fi + +log "Done" +log "Generating intermediary mirrorlist stats" + +# Generate intermediary mirrorlist stats +if [[ ! -f intermediary_mirrorlist.log || ! -f mirrorlist_parsed ]]; then + awk '/GET \/mirrorlist/ { if ($12==200 && $4!="3.134.114.30") print $0}' *.log **/*.log > intermediary_mirrorlist.log + awk '{ date=substr($7,2,11); ip=$4; path=$10; match(path, /arch=(x86_64|aarch64|source)/, arch_matches); match(path,/repo=([a-zA-z\-0-9]+)/, repo_matches); arch=arch_matches[1]; repository=repo_matches[1] } { print date, arch, repository, ip }' intermediary_mirrorlist.log > mirrorlist_parsed +else + log "Skipped mirrorlist intermediary" +fi + + +log "Done" + +log "Count unique and nonunique requests" + +# "Unique" count by IP addresses +totaldls_u=$(awk '{print $4}' intermediary_isos.log | sort | uniq | wc -l) + +# Total count +totaldls=$(wc -l intermediary_isos.log | awk '{print $1}') + + +log "Generate download stats for every date" + +# Get all the dates +declare -a dates +dates=( $(awk '{print substr($7,2,11)}' intermediary_isos.log | sort | uniq) ) +download_res="" +for date in "${dates[@]}"; do + total_count=$(grep "${date}" intermediary_isos.log | wc -l) + download_res="${download_res}${date} ${total_count}\n" +done + +log "Done" +log "Generate mirrorlist stats for every date" +dates=( $(awk '{print $1}' mirrorlist_parsed | sort | uniq) ) +#repositories=( $(awk '{print $3}' mirrorlist_parsed | sort | uniq ) ) +repositories=({AppStream,BaseOS,PowerTools,ResilientStorage,Minimal,Devel,HighAvailability,extras,rockyplus,NFV}-{8,8-source}) + + +mirror_res="Date Total x86_64 aarch64 source ${repositories[@]}\n" +for date in "${dates[@]}"; do + today=$(grep "${date}" mirrorlist_parsed) + total_count=$(echo "${today}" | wc -l) + arches=($(echo "${today}" | awk 'BEGIN {source=0; x86=0; a64=0; }{ if ($2=="x86_64") { x86+=1 } else if ($2=="aarch64") { a64+=1 } else if ($2=="source") { source+=1 } } END { print x86, a64, source }')) + declare -A repos + for repo in "${repositories[@]}"; do + repos["${repo}"]=$(echo "${today}" | grep "${repo}" | wc -l) + done + mirror_res="${mirror_res}${date} ${total_count} ${arches[@]} ${repos[@]}\n" +done + +log "Done" +log "End processing. Begin output" + +# Output shit +echo -e "Download Information\n------------------" +echo -e "Total: ${totaldls}\nUnique: ${totaldls_u}\n\n\n" +echo -e "Downloads by date\n------------------" +echo -e "${download_res}" | column -t +echo -e "Mirror requests by date\n------------------" +# Sort by date +echo -e "${mirror_res}" | column -t | sort -t'/' -Mk2 From d46e60dfb568dd7ba25bd808f32021e82cf00599 Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Thu, 12 May 2022 18:38:23 -0400 Subject: [PATCH 04/29] Update release notes url --- sync/gen-torrents.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sync/gen-torrents.sh b/sync/gen-torrents.sh index 28d2b3e..c80c90d 100755 --- a/sync/gen-torrents.sh +++ b/sync/gen-torrents.sh @@ -38,7 +38,7 @@ TORRENT_TRACKERS=( ) # Regex of paths to exclude TORRENT_EXCLUDES='.*\/CHECKSUM.asc' -TORRENT_COMMENT="https://docs.rockylinux.org/release_notes/${REVISION}/" +TORRENT_COMMENT="https://docs.rockylinux.org/release_notes/${REVISION//\./_}/" # dots are bad, mkay? THREADS=10 printf "* Step 1: Create scaffolding and link\n" From 7602ae1a44e6ea2b75b75c07fda3e7562b812738 Mon Sep 17 00:00:00 2001 From: nazunalika Date: Thu, 19 May 2022 00:46:16 -0700 Subject: [PATCH 05/29] begin ISO magic --- iso/common | 50 ++++++++ iso/common_8 | 4 + iso/common_9 | 4 + live/common | 34 ++++++ sync/common_8 | 5 +- sync/common_9 | 174 ++++++++++++++++++++++++++++ sync/symlink-images-and-checksum.sh | 25 ++++ 7 files changed, 294 insertions(+), 2 deletions(-) create mode 100644 iso/common create mode 100644 iso/common_8 create mode 100644 iso/common_9 create mode 100644 live/common create mode 100644 sync/common_9 create mode 100644 sync/symlink-images-and-checksum.sh diff --git a/iso/common b/iso/common new file mode 100644 index 0000000..55cab71 --- /dev/null +++ b/iso/common @@ -0,0 +1,50 @@ +# To be sourced by scripts to use + +# Variables that can be overriden should be noted with optional context. It is +# expected that these values are here in this file (per variable or per set): +# +# * Allowed +# * Allowed with caveats +# * Not Allowed +# * Required + +# Set the Rocky Linux version. +# Override: Required +if [ -z "$RLVER" ]; then + echo "RLVER is not defined." + exit 2 +fi + +# Architecture of the system - Overriding this would be a mistake. Lorax and +# other runroot-like operations should occur on their respective architectures. +# Override: Not Allowed +ARCH="$(uname -p)" + +# Source Major common +# Override: Not Allowed +test -f "$(dirname "$0")/common_${RLVER}" && source "$(dirname "$0")/common_${RLVER}" +if [ "$?" -ne 0 ]; then + echo "Could not source common_${RLVER}" + exit 1 +fi + +DATE_STAMP="$(date +%Y%m%d)" +COMPOSE_ROOT="/mnt/compose/${RLVER}" +COMPOSE_ISO_WORKDIR="${COMPOSE_ROOT}/work/${ARCH}/${DATE_STAMP}" + +# ISO Functions +function build_graft_points() { + echo "" +} + +function build_lorax_source_list() { + echo "" +} + +function build_lorax() { + echo "" +} + +function build_extra_iso() { + echo "" +} diff --git a/iso/common_8 b/iso/common_8 new file mode 100644 index 0000000..b0a0012 --- /dev/null +++ b/iso/common_8 @@ -0,0 +1,4 @@ +# To be sourced by scripts to use + +# These are the architectures supported for 8 +ARCHES=(x86_64 aarch64) diff --git a/iso/common_9 b/iso/common_9 new file mode 100644 index 0000000..d0cba4f --- /dev/null +++ b/iso/common_9 @@ -0,0 +1,4 @@ +# To be sourced by scripts to use + +# These are the architectures supported for 9 +ARCHES=(x86_64 aarch64 ppc64le s390x) diff --git a/live/common b/live/common new file mode 100644 index 0000000..7382a3f --- /dev/null +++ b/live/common @@ -0,0 +1,34 @@ +# To be sourced by scripts that build live images + +# Variables that can be overriden should be noted with optional context. It is +# expected that these values are here in this file (per variable or per set): +# +# * Allowed +# * Allowed with caveats +# * Not Allowed +# * Required + +# Temporary probably. This makes it so if RLVER=... is called before the script +# it will set the version for the variables to call up. This was easier than +# creating duplicates of a bunch of stuff. Default version is 8. + +# Override: Required +if [ -z "$RLVER" ]; then + echo "RLVER is not defined." + exit 2 +fi + +# Set git branch name scheme +# Override: Allowed with caveats +GIT_BRANCH="r${RLVER}" + +# Source Major common +# Override: Not Allowed +test -f "$(dirname "$0")/common_${RLVER}" && source "$(dirname "$0")/common_${RLVER}" +if [ "$?" -ne 0 ]; then + echo "Could not source common_${RLVER}" + exit 1 +fi + +# Used to iterate over types of live images +VARIANTS=(XFCE KDE Workstation Workstation-Lite) diff --git a/sync/common_8 b/sync/common_8 index 2a8fe96..b26db53 100644 --- a/sync/common_8 +++ b/sync/common_8 @@ -7,7 +7,7 @@ MAJOR="${REVISION:0:1}" MINOR="${REVISION:2:1}" # comment or blank if needed -APPEND_TO_DIR="-RC1" +APPEND_TO_DIR="-RC2" STAGING_ROOT="/mnt/repos-staging" PRODUCTION_ROOT="/mnt/repos-production" @@ -20,13 +20,14 @@ RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}" # cases where repos will not be available by normal means. It's just for # consistency. NONMODS_REPOS=( - Extras + extras Devel plus rockyrpi ) # These repos were originally separate from the main compose and need symlinks +declare -A LINK_REPOS LINK_REPOS=( [NFV]="nfv" ) diff --git a/sync/common_9 b/sync/common_9 new file mode 100644 index 0000000..a0e2262 --- /dev/null +++ b/sync/common_9 @@ -0,0 +1,174 @@ +# To be sourced by scripts to use + +# Revision must always start with a major number +REVISION=9.0 + +ARCHES=(x86_64 aarch64 ppc64le s390x) + +MAJOR="${REVISION:0:1}" +MINOR="${REVISION:2:1}" + +# comment or blank if needed +APPEND_TO_DIR="-RC1" + +STAGING_ROOT="/mnt/repos-staging" +PRODUCTION_ROOT="/mnt/repos-production" + +# relative to ${ENV}_ROOT +CATEGORY_STUB="mirror/pub/rocky" +RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}" + +# Set all repos that have no comps/groups associated with them. This is even in +# cases where repos will not be available by normal means. It's just for +# consistency. +NONMODS_REPOS=( + extras + Devel + plus + rockyrpi +) + +# These repos were originally separate from the main compose and need symlinks +declare -A LINK_REPOS +LINK_REPOS=( + [NFV]="nfv" +) + +# These repos have comps/groups, except for debuginfo and sources +MODS_REPOS=( + BaseOS + AppStream + HighAvailability + ResilientStorage + CRB + RT + NFV + SAP + SAPHANA +) + +ALL_REPOS=( + "${NONMODS_REPOS[@]}" + "${MODS_REPOS[@]}" +) + +NONSIG_COMPOSE=( + Rocky + Extras + Rocky-devel + Plus + rockyrpi +) + +declare -A SIG_COMPOSE +SIG_COMPOSE=( + [kernel]="kernel/kernel" + [kmod]="kernel/kmod" +) + +# These repos have modules +MODS=( + AppStream + PowerTools +) + +# functions +# Note, EL8 and EL9 may not be the same, do not put in 'common' +function treeinfoSaver() { + BaseOSArch="${1}" + TREEINFO_VAR="${STAGING_ROOT}/${RELEASE_DIR}/BaseOS/${BaseOSArch}/os/.treeinfo" + PRISTINE_TREE="${STAGING_ROOT}/${RELEASE_DIR}/BaseOS/${BaseOSArch}/os/.treeinfo-pristine" + /bin/cp "${TREEINFO_VAR}" "${PRISTINE_TREE}" +} + +function treeinfoFixer() { + BaseOSArch="${1}" + TREEINFO_VAR="${STAGING_ROOT}/${RELEASE_DIR}/BaseOS/${BaseOSArch}/os/.treeinfo" + PRISTINE_TREE="${STAGING_ROOT}/${RELEASE_DIR}/BaseOS/${BaseOSArch}/os/.treeinfo-pristine" + test -f "${PRISTINE_TREE}" + pris_retval=$? + if [ "$pris_retval" -eq 0 ]; then + /bin/cp "${PRISTINE_TREE}" "${TREEINFO_VAR}" + else + echo "WARNING: We do not have a pristine treeinfo to copy from" + fi +} + +# Note, EL8 and EL9 may not be the same, do not put in 'common' +function treeinfoModder() { + BaseOSArch="${1}" + TREEINFO_VAR="${STAGING_ROOT}/${RELEASE_DIR}/BaseOS/${BaseOSArch}/os/.treeinfo" + PRISTINE_TREE="${STAGING_ROOT}/${RELEASE_DIR}/BaseOS/${BaseOSArch}/os/.treeinfo-pristine" + test -f "${TREEINFO_VAR}" + treeinfo_retval=$? + test -x /usr/bin/python3 + python_retval=$? + # There is an awk way to do this, but it was easier to implement python and + # cat heredoc together. It felt cleaner. This was a trick I had used in a + # previous life when I had to admin Solaris systems, and I needed a way to + # add a solaris 10 system into FreeIPA (it was not fun, let me tell you). But + # the take away is I learned something kind of on the fly and well, it worked. + # Emails should have stamps. + if [ "$treeinfo_retval" -eq 0 ] && [ "$python_retval" -eq 0 ]; then + cat < Date: Fri, 20 May 2022 01:13:57 -0700 Subject: [PATCH 06/29] start python conversion --- iso/py/.gitignore | 4 +++ iso/py/README.md | 19 ++++++++++++ iso/py/common.py | 57 +++++++++++++++++++++++++++++++++++ iso/py/configs/el8.yaml | 45 ++++++++++++++++++++++++++++ iso/py/configs/el9.yaml | 66 +++++++++++++++++++++++++++++++++++++++++ iso/py/test.py | 6 ++++ iso/py/test2.py | 38 ++++++++++++++++++++++++ iso/{ => sh}/common | 0 iso/{ => sh}/common_8 | 0 iso/{ => sh}/common_9 | 0 10 files changed, 235 insertions(+) create mode 100644 iso/py/.gitignore create mode 100644 iso/py/README.md create mode 100644 iso/py/common.py create mode 100644 iso/py/configs/el8.yaml create mode 100644 iso/py/configs/el9.yaml create mode 100644 iso/py/test.py create mode 100644 iso/py/test2.py rename iso/{ => sh}/common (100%) rename iso/{ => sh}/common_8 (100%) rename iso/{ => sh}/common_9 (100%) diff --git a/iso/py/.gitignore b/iso/py/.gitignore new file mode 100644 index 0000000..961321b --- /dev/null +++ b/iso/py/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.py[cod] +*$py.class +*.so diff --git a/iso/py/README.md b/iso/py/README.md new file mode 100644 index 0000000..99f6dac --- /dev/null +++ b/iso/py/README.md @@ -0,0 +1,19 @@ +iso +=== + +scripts +------- + +* sync-variant-pungi +* sync-variant-peridot +* sync-all-pungi +* sync-all-peridot +* sync-sig +* build-all-iso +* sign-repos-only + +wrappers +-------- + +* lorax-generators +* sync-generators diff --git a/iso/py/common.py b/iso/py/common.py new file mode 100644 index 0000000..d8e59a4 --- /dev/null +++ b/iso/py/common.py @@ -0,0 +1,57 @@ +# All imports are here +import platform +import time +import glob +import rpm +import yaml + +# These are a bunch of colors we may use in terminal output +class Color: + RED = '\033[91m' + GREEN = '\033[92m' + PURPLE = '\033[95m' + CYAN = '\033[96m' + DARKCYAN = '\033[36m' + BLUE = '\033[94m' + YELLOW = '\033[93m' + UNDERLINE = '\033[4m' + BOLD = '\033[1m' + END = '\033[0m' + +# vars and additional checks +#RLVER = rpm.expandMacro('%rhel') +RLVER = '9' +rldict = {} +arch = platform.machine() + +# Importing the config from yaml +for conf in glob.iglob('configs/*.yaml'): + with open(conf, 'r', encoding="utf-8") as file: + rldict.update(yaml.safe_load(file)) + +# The system needs to be a RHEL-like system. It cannot be Fedora or SuSE. +#if "%rhel" in RLVER: +# raise SystemExit(Color.BOLD + 'This is not a RHEL-like system.' + Color.END +# + '\n\nPlease verify you are running on a RHEL-like system that is ' +# 'not Fedora nor SuSE. This means that the %rhel macro will be ' +# 'defined with a value equal to the version you are targetting. RHEL' +# ' and its derivatives have this set.') + +# Generic rlvars for the particular EL release we're on. This does not have to +# be used. A different one can be assigned based on script need. +rlvars = rldict[RLVER] + +# Is our arch allowed for this particular release? Some previous releases do +# not support ppc or s390x +if arch not in rlvars['allowed_arches']: + raise SystemExit(Color.BOLD + 'This architecture is not supported.' + + Color.END + '\n\nEnsure that the architecture you are building ' + 'for is supported for this compose process.') + +date_stamp = time.strftime("%Y%m%d", time.localtime()) +COMPOSE_ROOT = "/mnt/compose/" + RLVER +COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp +STAGING_ROOT = "/mnt/repos-staging" +PRODUCTION_ROOT = "/mnt/repos-production" +CATEGORY_STUB = "/mirror/pub/rocky" +REVISION = rlvars['revision'] + '-' + rlvars['rclvl'] diff --git a/iso/py/configs/el8.yaml b/iso/py/configs/el8.yaml new file mode 100644 index 0000000..8176f67 --- /dev/null +++ b/iso/py/configs/el8.yaml @@ -0,0 +1,45 @@ +--- +'8': + revision: '8.6' + rclvl: 'RC2' + allowed_arches: + - x86_64 + - aarch64 + provide_multilib: False + repo_url_list: [] + required_packages: + - 'lorax' + - 'genisoimage' + - 'isomd5sum' + symlink_refs: + devel: 'Devel' + NFV: 'nfv' + all_repos: + - 'BaseOS' + - 'AppStream' + - 'HighAvailability' + - 'ResilientStorage' + - 'PowerTools' + - 'RT' + - 'NFV' + - 'extras' + - 'devel' + - 'plus' + - 'rockyrpi' + no_comps_or_groups: + - 'extras' + - 'devel' + - 'plus' + - 'rockyrpi' + comps_or_groups: + - 'BaseOS' + - 'AppStream' + - 'HighAvailability' + - 'ResilientStorage' + - 'PowerTools' + - 'RT' + - 'NFV' + has_modules: + - 'AppStream' + - 'PowerTools' +... diff --git a/iso/py/configs/el9.yaml b/iso/py/configs/el9.yaml new file mode 100644 index 0000000..4672fdb --- /dev/null +++ b/iso/py/configs/el9.yaml @@ -0,0 +1,66 @@ +--- +'9': + revision: '9.0' + rclvl: 'RC1' + allowed_arches: + - x86_64 + - aarch64 + - ppc64le + - s390x + provide_multilib: True + repo_url_list: + - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/all' + - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/BaseOS' + - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/AppStream' + - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/CRB' + - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/HighAvailability' + - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/ResilientStorage' + - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/RT' + - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/NFV' + - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/SAP' + - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/SAPHANA' + - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/extras' + - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/devel' + - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/plus' + required_packages: + - 'lorax' + - 'genisoimage' + - 'isomd5sum' + symlink_refs: + devel: 'Devel' + NFV: 'nfv' + renames: + all: 'nplb' + all_repos: + - 'nplb' + - 'BaseOS' + - 'AppStream' + - 'HighAvailability' + - 'ResilientStorage' + - 'CRB' + - 'RT' + - 'NFV' + - 'SAP' + - 'SAPHANA' + - 'extras' + - 'devel' + - 'plus' + no_comps_or_groups: + - 'nplb' + - 'extras' + - 'devel' + - 'plus' + comps_or_groups: + - 'BaseOS' + - 'AppStream' + - 'HighAvailability' + - 'ResilientStorage' + - 'CRB' + - 'RT' + - 'NFV' + - 'SAP' + - 'SAPHANA' + has_modules: + - 'AppStream' + - 'CRB' +... diff --git a/iso/py/test.py b/iso/py/test.py new file mode 100644 index 0000000..29d2cd2 --- /dev/null +++ b/iso/py/test.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 + +from common import * + +print(platform.machine()) +print(arch) diff --git a/iso/py/test2.py b/iso/py/test2.py new file mode 100644 index 0000000..afda7dc --- /dev/null +++ b/iso/py/test2.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +import desert +from attrs import define, field +import typing as t + +CONFIG = { + "8": { + "allowed_arches": ["x86_64", "aarch64"], + "repo_url_list": ["some", "shit", "here"] + }, + "9": { + "allowed_arches": ["x86_64", "aarch64", "ppc64le", "s390x"], + "repo_url_list": ["some", "other", "shit", "here"] + } +} + +@define +class VersionConfig: + allowed_arches: t.List[str] = field() + repo_url_list: t.List[str] = field() + + @allowed_arches.validator + def check(self, attribute, value): + if not all(v in ["x86_64", "aarch64", "ppc64le", "s390x"] for v in value): + raise ValueError("Architecture list does not match") + +def new(version): + schema = desert.schema(VersionConfig) + config = CONFIG[str(version)] + return schema.load(config) + +eight = new(8) +nine = new(9) + +print(eight) +print(eight.allowed_arches) +print(nine) diff --git a/iso/common b/iso/sh/common similarity index 100% rename from iso/common rename to iso/sh/common diff --git a/iso/common_8 b/iso/sh/common_8 similarity index 100% rename from iso/common_8 rename to iso/sh/common_8 diff --git a/iso/common_9 b/iso/sh/common_9 similarity index 100% rename from iso/common_9 rename to iso/sh/common_9 From 4f9bfd44a04c2d3657363246507b9800eff7641b Mon Sep 17 00:00:00 2001 From: nazunalika Date: Sat, 21 May 2022 00:10:37 -0700 Subject: [PATCH 07/29] Final Commit for 5/21 Add better syncing for current bash scripts (thanks pgreco) Continued python scripts, creating classes for local module imports, with some inspiration coming from cobbler --- iso/py/README.md | 43 ++++++++++++++--- iso/py/common.py | 40 ++++++++-------- iso/py/configs/el8.yaml | 5 +- iso/py/configs/el9.yaml | 21 ++------ iso/py/sig/cloud.yaml | 10 ++++ iso/py/sig/core.yaml | 12 +++++ iso/py/sync-all-peridot | 13 +++++ iso/py/test.py | 7 ++- iso/py/util/__init__.py | 12 +++++ iso/py/util/check.py | 14 ++++++ iso/py/util/dnf_utils.py | 69 +++++++++++++++++++++++++++ sync/common_8 | 1 + sync/minor-release-sync-to-staging.sh | 9 ++-- sync/sync-to-prod.sh | 5 +- 14 files changed, 208 insertions(+), 53 deletions(-) create mode 100644 iso/py/sig/cloud.yaml create mode 100644 iso/py/sig/core.yaml create mode 100755 iso/py/sync-all-peridot create mode 100644 iso/py/util/__init__.py create mode 100644 iso/py/util/check.py create mode 100644 iso/py/util/dnf_utils.py diff --git a/iso/py/README.md b/iso/py/README.md index 99f6dac..90d0f63 100644 --- a/iso/py/README.md +++ b/iso/py/README.md @@ -1,8 +1,6 @@ -iso -=== +# iso -scripts -------- +## scripts * sync-variant-pungi * sync-variant-peridot @@ -12,8 +10,41 @@ scripts * build-all-iso * sign-repos-only -wrappers --------- +## wrappers * lorax-generators * sync-generators + +## rules + +### imports + +When making a script, you *must* import common. This is insanely bad practice, +but we would prefer if we started out this way: + +``` +from common import * +import argparse +``` + +Whatever is imported in common will effectively be imported in your scripts as +well, but there is nothing stopping you from defining them again, even out of +habit. `argparse` is there because you better have a very, *very* good reason +to not be writing scripts that are major version specific. + +If you are writing something that could be arch specific based on the major +version (which is likely), make sure to import the util module and use it arch +checker appropriately. Small (but weak) example. + +``` +from util import Checks + +rlvars = rldict['9'] +r = Checks(rlvars, arch) +r.check_valid_arch() +``` + +### script names and permissions + +* Callable scripts should *not* end in `.py` +* They should have at least `775` or `+x` permissions diff --git a/iso/py/common.py b/iso/py/common.py index d8e59a4..3a2a6d4 100644 --- a/iso/py/common.py +++ b/iso/py/common.py @@ -1,9 +1,11 @@ # All imports are here +import os import platform import time import glob import rpm import yaml +import logging # These are a bunch of colors we may use in terminal output class Color: @@ -19,10 +21,17 @@ class Color: END = '\033[0m' # vars and additional checks -#RLVER = rpm.expandMacro('%rhel') -RLVER = '9' rldict = {} -arch = platform.machine() +config = { + "rlmacro": rpm.expandMacro('%rhel'), + "arch": platform.machine(), + "date_stamp": time.strftime("%Y%m%d", time.localtime()), + "staging_root": "/mnt/repos-staging", + "production_root": "/mnt/repos-production", + "category_stub": "/mirror/pub/rocky", + "sig_category_stub": "/mirror/pub/sig", + "repo_base_url": "https://yumrepofs.build.resf.org/v1/projects/" +} # Importing the config from yaml for conf in glob.iglob('configs/*.yaml'): @@ -30,28 +39,17 @@ for conf in glob.iglob('configs/*.yaml'): rldict.update(yaml.safe_load(file)) # The system needs to be a RHEL-like system. It cannot be Fedora or SuSE. -#if "%rhel" in RLVER: +#if "%rhel" in config['RLMACRO']: # raise SystemExit(Color.BOLD + 'This is not a RHEL-like system.' + Color.END # + '\n\nPlease verify you are running on a RHEL-like system that is ' # 'not Fedora nor SuSE. This means that the %rhel macro will be ' # 'defined with a value equal to the version you are targetting. RHEL' # ' and its derivatives have this set.') -# Generic rlvars for the particular EL release we're on. This does not have to -# be used. A different one can be assigned based on script need. -rlvars = rldict[RLVER] -# Is our arch allowed for this particular release? Some previous releases do -# not support ppc or s390x -if arch not in rlvars['allowed_arches']: - raise SystemExit(Color.BOLD + 'This architecture is not supported.' - + Color.END + '\n\nEnsure that the architecture you are building ' - 'for is supported for this compose process.') - -date_stamp = time.strftime("%Y%m%d", time.localtime()) -COMPOSE_ROOT = "/mnt/compose/" + RLVER -COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp -STAGING_ROOT = "/mnt/repos-staging" -PRODUCTION_ROOT = "/mnt/repos-production" -CATEGORY_STUB = "/mirror/pub/rocky" -REVISION = rlvars['revision'] + '-' + rlvars['rclvl'] +# These will be set in their respective var files +#REVISION = rlvars['revision'] + '-' + rlvars['rclvl'] +#rlvars = rldict[RLVER] +#rlvars = rldict[RLMACRO] +#COMPOSE_ROOT = "/mnt/compose/" + RLVER +#COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp diff --git a/iso/py/configs/el8.yaml b/iso/py/configs/el8.yaml index 8176f67..67e0701 100644 --- a/iso/py/configs/el8.yaml +++ b/iso/py/configs/el8.yaml @@ -6,14 +6,15 @@ - x86_64 - aarch64 provide_multilib: False - repo_url_list: [] + project_id: '' required_packages: - 'lorax' - 'genisoimage' - 'isomd5sum' - symlink_refs: + repo_symlinks: devel: 'Devel' NFV: 'nfv' + renames: {} all_repos: - 'BaseOS' - 'AppStream' diff --git a/iso/py/configs/el9.yaml b/iso/py/configs/el9.yaml index 4672fdb..c43ae84 100644 --- a/iso/py/configs/el9.yaml +++ b/iso/py/configs/el9.yaml @@ -8,31 +8,18 @@ - ppc64le - s390x provide_multilib: True - repo_url_list: - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/all' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/BaseOS' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/AppStream' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/CRB' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/HighAvailability' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/ResilientStorage' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/RT' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/NFV' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/SAP' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/SAPHANA' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/extras' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/devel' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/plus' + project_id: '55b17281-bc54-4929-8aca-a8a11d628738' required_packages: - 'lorax' - 'genisoimage' - 'isomd5sum' - symlink_refs: + repo_symlinks: devel: 'Devel' NFV: 'nfv' renames: all: 'nplb' all_repos: - - 'nplb' + - 'all' - 'BaseOS' - 'AppStream' - 'HighAvailability' @@ -46,7 +33,7 @@ - 'devel' - 'plus' no_comps_or_groups: - - 'nplb' + - 'all' - 'extras' - 'devel' - 'plus' diff --git a/iso/py/sig/cloud.yaml b/iso/py/sig/cloud.yaml new file mode 100644 index 0000000..985a405 --- /dev/null +++ b/iso/py/sig/cloud.yaml @@ -0,0 +1,10 @@ +--- +'8': + cloud-kernel: + project_id: 'f91da90d-5bdb-4cf2-80ea-e07f8dae5a5c' + cloud-common: + project_id: '' +'9': + cloud-common: + project_id: '' +... diff --git a/iso/py/sig/core.yaml b/iso/py/sig/core.yaml new file mode 100644 index 0000000..0047ac0 --- /dev/null +++ b/iso/py/sig/core.yaml @@ -0,0 +1,12 @@ +--- +'8': + core-common: + project_id: '' + core-infra: + project_id: '' +'9': + core-common: + project_id: '' + core-infra: + project_id: '' +... diff --git a/iso/py/sync-all-peridot b/iso/py/sync-all-peridot new file mode 100755 index 0000000..97820be --- /dev/null +++ b/iso/py/sync-all-peridot @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +from common import * +import argparse +from util import Checks +from util import RepoSync + +rlvars = rldict['9'] +r = Checks(rlvars, config['arch']) +r.check_valid_arch() + +a = RepoSync(rlvars, config, repo="BaseOS") +print(a.sync()) diff --git a/iso/py/test.py b/iso/py/test.py index 29d2cd2..871e88e 100644 --- a/iso/py/test.py +++ b/iso/py/test.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 from common import * +import argparse +from util import Checks -print(platform.machine()) -print(arch) +rlvars = rldict['9'] +r = Checks(rlvars, arch) +r.check_valid_arch() diff --git a/iso/py/util/__init__.py b/iso/py/util/__init__.py new file mode 100644 index 0000000..817cb0d --- /dev/null +++ b/iso/py/util/__init__.py @@ -0,0 +1,12 @@ +from .check import ( + Checks, +) + +from .dnf_utils import ( + RepoSync, +) + +__all__ = [ + 'Checks', + 'RepoSync' +] diff --git a/iso/py/util/check.py b/iso/py/util/check.py new file mode 100644 index 0000000..a87d86f --- /dev/null +++ b/iso/py/util/check.py @@ -0,0 +1,14 @@ +# Is our arch allowed for this particular release? Some previous releases do +# not support ppc or s390x +from common import Color +class Checks: + """This class helps check some things""" + def __init__(self, rlvars, arch): + self.arches = rlvars['allowed_arches'] + self.arch = arch + + def check_valid_arch(self): + if self.arch not in self.arches: + raise SystemExit(Color.BOLD + 'This architecture is not supported.' + + Color.END + '\n\nEnsure that the architecture you are ' + 'building for is supported for this compose process.') diff --git a/iso/py/util/dnf_utils.py b/iso/py/util/dnf_utils.py new file mode 100644 index 0000000..e509044 --- /dev/null +++ b/iso/py/util/dnf_utils.py @@ -0,0 +1,69 @@ +HAS_LIBREPO = True +import os +import os.path +import pipes +import shutil + +try: + import librepo +except: + HAS_LIBREPO = False + +class RepoSync: + """ + This helps us do reposync operations for the base system. SIG syncs are a + different class entirely. This is on purpose. Please use the SigRepoSync + class for SIG syncs. + """ + def __init__(self, rlvars, config, repo=None, nofail: bool = False): + self.nofail = nofail + # Relevant config items + self.major_version = config['rlmacro'] + self.date_stamp = config['date_stamp'] + self.staging_dir = config['staging_root'] + config['category_stub'] + self.major_version + self.repo_base_url = config['repo_base_url'] + + # Relevant major version items + self.revision = rlvars['revision'] + "-" + rlvars['rclvl'] + self.arches = rlvars['allowed_arches'] + self.project_id = rlvars['project_id'] + self.repo_renames = rlvars['renames'] + self.repos = rlvars['all_repos'] + self.repo = repo + + def run(self): + pass + + def sync(self): + cmd = self.reposync_cmd() + print(self.revision) + + def generate_conf(self, dest_path: str, repo): + """ + Generates the necessary repo conf file for the operation. This repo + file should be temporary in nature. This will generate a repo file + with all repos by default. If a repo is chosen for sync, that will be + the only one synced. + + :param dest_path: The destination where the temporary conf goes + :param repo: The repo object to create a file for + """ + pass + + def reposync_cmd(self) -> str: + """ + This generates the reposync command. We don't support reposync by + itself and will raise an error. + + :return: The path to the reposync command. If dnf exists, we'll use + that. + """ + cmd = None + if os.path.exists("/usr/bin/dnf"): + cmd = "/usr/bin/dnf reposync" + else: + raise SystemExit("/usr/bin/dnf was not found. /usr/bin/reposync is " + "is not sufficient and you are likely running on an el7 " + "system, which tells us that you made changes to these " + "tools.") + return cmd diff --git a/sync/common_8 b/sync/common_8 index b26db53..3dd5a37 100644 --- a/sync/common_8 +++ b/sync/common_8 @@ -30,6 +30,7 @@ NONMODS_REPOS=( declare -A LINK_REPOS LINK_REPOS=( [NFV]="nfv" + [Devel]="devel" ) # These repos have comps/groups, except for debuginfo and sources diff --git a/sync/minor-release-sync-to-staging.sh b/sync/minor-release-sync-to-staging.sh index 13a8cfa..d21f921 100644 --- a/sync/minor-release-sync-to-staging.sh +++ b/sync/minor-release-sync-to-staging.sh @@ -56,11 +56,12 @@ for ARCH in "${ARCHES[@]}"; do # Hardcoding this for now SOURCE="/mnt/compose/${MAJ}/latest-Rocky-${MAJ}/compose/${x}/${ARCH}/iso" TARGET_ARCH="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${x}/${ARCH}/iso" - mkdir -p "${SOURCE}" "${TARGET}" "${TARGET_ARCH}" + mkdir -p "${TARGET}" + #mkdir -p "${SOURCE}" "${TARGET}" "${TARGET_ARCH}" # Copy the ISO and manifests into their target architecture - cp -n "${SOURCE}"/*.iso "${TARGET_ARCH}/" - cp -n "${SOURCE}"/*.iso.manifest "${TARGET_ARCH}/" - cp -n "${SOURCE}/CHECKSUM" "${TARGET_ARCH}/" + #cp -n "${SOURCE}"/*.iso "${TARGET_ARCH}/" + #cp -n "${SOURCE}"/*.iso.manifest "${TARGET_ARCH}/" + #cp -n "${SOURCE}/CHECKSUM" "${TARGET_ARCH}/" # Copy the ISO and manifests into the main isos target cp "${SOURCE}"/*.iso "${TARGET}/" cp "${SOURCE}"/*.iso.manifest "${TARGET}/" diff --git a/sync/sync-to-prod.sh b/sync/sync-to-prod.sh index e14a49a..76045c7 100644 --- a/sync/sync-to-prod.sh +++ b/sync/sync-to-prod.sh @@ -16,7 +16,10 @@ if [ $ret_val -eq "0" ]; then # disabling because none of our files should be starting with dashes. If they # are something is *seriously* wrong here. # shellcheck disable=SC2035 - sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ + sudo -l && find ./ -mindepth 1 -maxdepth 1 -type d -exec find {}/ -mindepth 1 -maxdepth 1 -type d \;|sed 's/^..//g' | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ + {} "${TARGET}" + # shellcheck disable=SC2035 + sudo -l && find ** -maxdepth 0 -type l | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ {} "${TARGET}" # Full file list update From 420aa02bdbfae0dd97c711e7d5137c59c13f9437 Mon Sep 17 00:00:00 2001 From: nazunalika Date: Sat, 21 May 2022 23:20:15 -0700 Subject: [PATCH 08/29] Final commit for 5/21 --- iso/py/README.md | 4 +- iso/py/common.py | 16 +-- iso/py/configs/el8.yaml | 4 +- iso/py/configs/el9.yaml | 4 +- iso/py/sync-all-peridot | 13 --- iso/py/sync-from-peridot | 18 ++++ iso/py/util/dnf_utils.py | 221 +++++++++++++++++++++++++++++++++++---- 7 files changed, 233 insertions(+), 47 deletions(-) delete mode 100755 iso/py/sync-all-peridot create mode 100755 iso/py/sync-from-peridot diff --git a/iso/py/README.md b/iso/py/README.md index 90d0f63..2e90592 100644 --- a/iso/py/README.md +++ b/iso/py/README.md @@ -4,8 +4,8 @@ * sync-variant-pungi * sync-variant-peridot -* sync-all-pungi -* sync-all-peridot +* sync-from-pungi +* sync-from-peridot * sync-sig * build-all-iso * sign-repos-only diff --git a/iso/py/common.py b/iso/py/common.py index 3a2a6d4..b36c728 100644 --- a/iso/py/common.py +++ b/iso/py/common.py @@ -25,12 +25,13 @@ rldict = {} config = { "rlmacro": rpm.expandMacro('%rhel'), "arch": platform.machine(), - "date_stamp": time.strftime("%Y%m%d", time.localtime()), + "date_stamp": time.strftime("%Y%m%d.%H%M%S", time.localtime()), + "compose_root": "/mnt/compose", "staging_root": "/mnt/repos-staging", "production_root": "/mnt/repos-production", - "category_stub": "/mirror/pub/rocky", - "sig_category_stub": "/mirror/pub/sig", - "repo_base_url": "https://yumrepofs.build.resf.org/v1/projects/" + "category_stub": "mirror/pub/rocky", + "sig_category_stub": "mirror/pub/sig", + "repo_base_url": "https://yumrepofs.build.resf.org/v1/projects" } # Importing the config from yaml @@ -39,7 +40,7 @@ for conf in glob.iglob('configs/*.yaml'): rldict.update(yaml.safe_load(file)) # The system needs to be a RHEL-like system. It cannot be Fedora or SuSE. -#if "%rhel" in config['RLMACRO']: +#if "%rhel" in config['rlmacro']: # raise SystemExit(Color.BOLD + 'This is not a RHEL-like system.' + Color.END # + '\n\nPlease verify you are running on a RHEL-like system that is ' # 'not Fedora nor SuSE. This means that the %rhel macro will be ' @@ -49,7 +50,6 @@ for conf in glob.iglob('configs/*.yaml'): # These will be set in their respective var files #REVISION = rlvars['revision'] + '-' + rlvars['rclvl'] -#rlvars = rldict[RLVER] -#rlvars = rldict[RLMACRO] -#COMPOSE_ROOT = "/mnt/compose/" + RLVER +#rlvars = rldict[rlver] +#rlvars = rldict[rlmacro] #COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp diff --git a/iso/py/configs/el8.yaml b/iso/py/configs/el8.yaml index 67e0701..fa070d2 100644 --- a/iso/py/configs/el8.yaml +++ b/iso/py/configs/el8.yaml @@ -18,9 +18,9 @@ all_repos: - 'BaseOS' - 'AppStream' + - 'PowerTools' - 'HighAvailability' - 'ResilientStorage' - - 'PowerTools' - 'RT' - 'NFV' - 'extras' @@ -35,9 +35,9 @@ comps_or_groups: - 'BaseOS' - 'AppStream' + - 'PowerTools' - 'HighAvailability' - 'ResilientStorage' - - 'PowerTools' - 'RT' - 'NFV' has_modules: diff --git a/iso/py/configs/el9.yaml b/iso/py/configs/el9.yaml index c43ae84..fa4911d 100644 --- a/iso/py/configs/el9.yaml +++ b/iso/py/configs/el9.yaml @@ -22,9 +22,9 @@ - 'all' - 'BaseOS' - 'AppStream' + - 'CRB' - 'HighAvailability' - 'ResilientStorage' - - 'CRB' - 'RT' - 'NFV' - 'SAP' @@ -40,9 +40,9 @@ comps_or_groups: - 'BaseOS' - 'AppStream' + - 'CRB' - 'HighAvailability' - 'ResilientStorage' - - 'CRB' - 'RT' - 'NFV' - 'SAP' diff --git a/iso/py/sync-all-peridot b/iso/py/sync-all-peridot deleted file mode 100755 index 97820be..0000000 --- a/iso/py/sync-all-peridot +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python3 - -from common import * -import argparse -from util import Checks -from util import RepoSync - -rlvars = rldict['9'] -r = Checks(rlvars, config['arch']) -r.check_valid_arch() - -a = RepoSync(rlvars, config, repo="BaseOS") -print(a.sync()) diff --git a/iso/py/sync-from-peridot b/iso/py/sync-from-peridot new file mode 100755 index 0000000..a883ce2 --- /dev/null +++ b/iso/py/sync-from-peridot @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +# This is intended for doing "full" syncs, not periodic update syncs. + +from common import * +import argparse +from util import Checks +from util import RepoSync + +rlvars = rldict['9'] +r = Checks(rlvars, config['arch']) +r.check_valid_arch() + +a = RepoSync(rlvars, config, major="9", repo="BaseOS") +a.run() +#a.generate_conf() +#somedir = a.generate_compose_dirs() +#print(a.sync()) diff --git a/iso/py/util/dnf_utils.py b/iso/py/util/dnf_utils.py index e509044..37a7359 100644 --- a/iso/py/util/dnf_utils.py +++ b/iso/py/util/dnf_utils.py @@ -1,13 +1,21 @@ -HAS_LIBREPO = True +""" +Syncs yum repos for mirroring and composing. + +Louis Abel