From 4f9bfd44a04c2d3657363246507b9800eff7641b Mon Sep 17 00:00:00 2001 From: nazunalika Date: Sat, 21 May 2022 00:10:37 -0700 Subject: [PATCH] Final Commit for 5/21 Add better syncing for current bash scripts (thanks pgreco) Continued python scripts, creating classes for local module imports, with some inspiration coming from cobbler --- iso/py/README.md | 43 ++++++++++++++--- iso/py/common.py | 40 ++++++++-------- iso/py/configs/el8.yaml | 5 +- iso/py/configs/el9.yaml | 21 ++------ iso/py/sig/cloud.yaml | 10 ++++ iso/py/sig/core.yaml | 12 +++++ iso/py/sync-all-peridot | 13 +++++ iso/py/test.py | 7 ++- iso/py/util/__init__.py | 12 +++++ iso/py/util/check.py | 14 ++++++ iso/py/util/dnf_utils.py | 69 +++++++++++++++++++++++++++ sync/common_8 | 1 + sync/minor-release-sync-to-staging.sh | 9 ++-- sync/sync-to-prod.sh | 5 +- 14 files changed, 208 insertions(+), 53 deletions(-) create mode 100644 iso/py/sig/cloud.yaml create mode 100644 iso/py/sig/core.yaml create mode 100755 iso/py/sync-all-peridot create mode 100644 iso/py/util/__init__.py create mode 100644 iso/py/util/check.py create mode 100644 iso/py/util/dnf_utils.py diff --git a/iso/py/README.md b/iso/py/README.md index 99f6dac..90d0f63 100644 --- a/iso/py/README.md +++ b/iso/py/README.md @@ -1,8 +1,6 @@ -iso -=== +# iso -scripts -------- +## scripts * sync-variant-pungi * sync-variant-peridot @@ -12,8 +10,41 @@ scripts * build-all-iso * sign-repos-only -wrappers --------- +## wrappers * lorax-generators * sync-generators + +## rules + +### imports + +When making a script, you *must* import common. This is insanely bad practice, +but we would prefer if we started out this way: + +``` +from common import * +import argparse +``` + +Whatever is imported in common will effectively be imported in your scripts as +well, but there is nothing stopping you from defining them again, even out of +habit. `argparse` is there because you better have a very, *very* good reason +to not be writing scripts that are major version specific. + +If you are writing something that could be arch specific based on the major +version (which is likely), make sure to import the util module and use it arch +checker appropriately. Small (but weak) example. + +``` +from util import Checks + +rlvars = rldict['9'] +r = Checks(rlvars, arch) +r.check_valid_arch() +``` + +### script names and permissions + +* Callable scripts should *not* end in `.py` +* They should have at least `775` or `+x` permissions diff --git a/iso/py/common.py b/iso/py/common.py index d8e59a4..3a2a6d4 100644 --- a/iso/py/common.py +++ b/iso/py/common.py @@ -1,9 +1,11 @@ # All imports are here +import os import platform import time import glob import rpm import yaml +import logging # These are a bunch of colors we may use in terminal output class Color: @@ -19,10 +21,17 @@ class Color: END = '\033[0m' # vars and additional checks -#RLVER = rpm.expandMacro('%rhel') -RLVER = '9' rldict = {} -arch = platform.machine() +config = { + "rlmacro": rpm.expandMacro('%rhel'), + "arch": platform.machine(), + "date_stamp": time.strftime("%Y%m%d", time.localtime()), + "staging_root": "/mnt/repos-staging", + "production_root": "/mnt/repos-production", + "category_stub": "/mirror/pub/rocky", + "sig_category_stub": "/mirror/pub/sig", + "repo_base_url": "https://yumrepofs.build.resf.org/v1/projects/" +} # Importing the config from yaml for conf in glob.iglob('configs/*.yaml'): @@ -30,28 +39,17 @@ for conf in glob.iglob('configs/*.yaml'): rldict.update(yaml.safe_load(file)) # The system needs to be a RHEL-like system. It cannot be Fedora or SuSE. -#if "%rhel" in RLVER: +#if "%rhel" in config['RLMACRO']: # raise SystemExit(Color.BOLD + 'This is not a RHEL-like system.' + Color.END # + '\n\nPlease verify you are running on a RHEL-like system that is ' # 'not Fedora nor SuSE. This means that the %rhel macro will be ' # 'defined with a value equal to the version you are targetting. RHEL' # ' and its derivatives have this set.') -# Generic rlvars for the particular EL release we're on. This does not have to -# be used. A different one can be assigned based on script need. -rlvars = rldict[RLVER] -# Is our arch allowed for this particular release? Some previous releases do -# not support ppc or s390x -if arch not in rlvars['allowed_arches']: - raise SystemExit(Color.BOLD + 'This architecture is not supported.' - + Color.END + '\n\nEnsure that the architecture you are building ' - 'for is supported for this compose process.') - -date_stamp = time.strftime("%Y%m%d", time.localtime()) -COMPOSE_ROOT = "/mnt/compose/" + RLVER -COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp -STAGING_ROOT = "/mnt/repos-staging" -PRODUCTION_ROOT = "/mnt/repos-production" -CATEGORY_STUB = "/mirror/pub/rocky" -REVISION = rlvars['revision'] + '-' + rlvars['rclvl'] +# These will be set in their respective var files +#REVISION = rlvars['revision'] + '-' + rlvars['rclvl'] +#rlvars = rldict[RLVER] +#rlvars = rldict[RLMACRO] +#COMPOSE_ROOT = "/mnt/compose/" + RLVER +#COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp diff --git a/iso/py/configs/el8.yaml b/iso/py/configs/el8.yaml index 8176f67..67e0701 100644 --- a/iso/py/configs/el8.yaml +++ b/iso/py/configs/el8.yaml @@ -6,14 +6,15 @@ - x86_64 - aarch64 provide_multilib: False - repo_url_list: [] + project_id: '' required_packages: - 'lorax' - 'genisoimage' - 'isomd5sum' - symlink_refs: + repo_symlinks: devel: 'Devel' NFV: 'nfv' + renames: {} all_repos: - 'BaseOS' - 'AppStream' diff --git a/iso/py/configs/el9.yaml b/iso/py/configs/el9.yaml index 4672fdb..c43ae84 100644 --- a/iso/py/configs/el9.yaml +++ b/iso/py/configs/el9.yaml @@ -8,31 +8,18 @@ - ppc64le - s390x provide_multilib: True - repo_url_list: - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/all' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/BaseOS' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/AppStream' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/CRB' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/HighAvailability' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/ResilientStorage' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/RT' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/NFV' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/SAP' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/SAPHANA' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/extras' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/devel' - - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/plus' + project_id: '55b17281-bc54-4929-8aca-a8a11d628738' required_packages: - 'lorax' - 'genisoimage' - 'isomd5sum' - symlink_refs: + repo_symlinks: devel: 'Devel' NFV: 'nfv' renames: all: 'nplb' all_repos: - - 'nplb' + - 'all' - 'BaseOS' - 'AppStream' - 'HighAvailability' @@ -46,7 +33,7 @@ - 'devel' - 'plus' no_comps_or_groups: - - 'nplb' + - 'all' - 'extras' - 'devel' - 'plus' diff --git a/iso/py/sig/cloud.yaml b/iso/py/sig/cloud.yaml new file mode 100644 index 0000000..985a405 --- /dev/null +++ b/iso/py/sig/cloud.yaml @@ -0,0 +1,10 @@ +--- +'8': + cloud-kernel: + project_id: 'f91da90d-5bdb-4cf2-80ea-e07f8dae5a5c' + cloud-common: + project_id: '' +'9': + cloud-common: + project_id: '' +... diff --git a/iso/py/sig/core.yaml b/iso/py/sig/core.yaml new file mode 100644 index 0000000..0047ac0 --- /dev/null +++ b/iso/py/sig/core.yaml @@ -0,0 +1,12 @@ +--- +'8': + core-common: + project_id: '' + core-infra: + project_id: '' +'9': + core-common: + project_id: '' + core-infra: + project_id: '' +... diff --git a/iso/py/sync-all-peridot b/iso/py/sync-all-peridot new file mode 100755 index 0000000..97820be --- /dev/null +++ b/iso/py/sync-all-peridot @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +from common import * +import argparse +from util import Checks +from util import RepoSync + +rlvars = rldict['9'] +r = Checks(rlvars, config['arch']) +r.check_valid_arch() + +a = RepoSync(rlvars, config, repo="BaseOS") +print(a.sync()) diff --git a/iso/py/test.py b/iso/py/test.py index 29d2cd2..871e88e 100644 --- a/iso/py/test.py +++ b/iso/py/test.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 from common import * +import argparse +from util import Checks -print(platform.machine()) -print(arch) +rlvars = rldict['9'] +r = Checks(rlvars, arch) +r.check_valid_arch() diff --git a/iso/py/util/__init__.py b/iso/py/util/__init__.py new file mode 100644 index 0000000..817cb0d --- /dev/null +++ b/iso/py/util/__init__.py @@ -0,0 +1,12 @@ +from .check import ( + Checks, +) + +from .dnf_utils import ( + RepoSync, +) + +__all__ = [ + 'Checks', + 'RepoSync' +] diff --git a/iso/py/util/check.py b/iso/py/util/check.py new file mode 100644 index 0000000..a87d86f --- /dev/null +++ b/iso/py/util/check.py @@ -0,0 +1,14 @@ +# Is our arch allowed for this particular release? Some previous releases do +# not support ppc or s390x +from common import Color +class Checks: + """This class helps check some things""" + def __init__(self, rlvars, arch): + self.arches = rlvars['allowed_arches'] + self.arch = arch + + def check_valid_arch(self): + if self.arch not in self.arches: + raise SystemExit(Color.BOLD + 'This architecture is not supported.' + + Color.END + '\n\nEnsure that the architecture you are ' + 'building for is supported for this compose process.') diff --git a/iso/py/util/dnf_utils.py b/iso/py/util/dnf_utils.py new file mode 100644 index 0000000..e509044 --- /dev/null +++ b/iso/py/util/dnf_utils.py @@ -0,0 +1,69 @@ +HAS_LIBREPO = True +import os +import os.path +import pipes +import shutil + +try: + import librepo +except: + HAS_LIBREPO = False + +class RepoSync: + """ + This helps us do reposync operations for the base system. SIG syncs are a + different class entirely. This is on purpose. Please use the SigRepoSync + class for SIG syncs. + """ + def __init__(self, rlvars, config, repo=None, nofail: bool = False): + self.nofail = nofail + # Relevant config items + self.major_version = config['rlmacro'] + self.date_stamp = config['date_stamp'] + self.staging_dir = config['staging_root'] + config['category_stub'] + self.major_version + self.repo_base_url = config['repo_base_url'] + + # Relevant major version items + self.revision = rlvars['revision'] + "-" + rlvars['rclvl'] + self.arches = rlvars['allowed_arches'] + self.project_id = rlvars['project_id'] + self.repo_renames = rlvars['renames'] + self.repos = rlvars['all_repos'] + self.repo = repo + + def run(self): + pass + + def sync(self): + cmd = self.reposync_cmd() + print(self.revision) + + def generate_conf(self, dest_path: str, repo): + """ + Generates the necessary repo conf file for the operation. This repo + file should be temporary in nature. This will generate a repo file + with all repos by default. If a repo is chosen for sync, that will be + the only one synced. + + :param dest_path: The destination where the temporary conf goes + :param repo: The repo object to create a file for + """ + pass + + def reposync_cmd(self) -> str: + """ + This generates the reposync command. We don't support reposync by + itself and will raise an error. + + :return: The path to the reposync command. If dnf exists, we'll use + that. + """ + cmd = None + if os.path.exists("/usr/bin/dnf"): + cmd = "/usr/bin/dnf reposync" + else: + raise SystemExit("/usr/bin/dnf was not found. /usr/bin/reposync is " + "is not sufficient and you are likely running on an el7 " + "system, which tells us that you made changes to these " + "tools.") + return cmd diff --git a/sync/common_8 b/sync/common_8 index b26db53..3dd5a37 100644 --- a/sync/common_8 +++ b/sync/common_8 @@ -30,6 +30,7 @@ NONMODS_REPOS=( declare -A LINK_REPOS LINK_REPOS=( [NFV]="nfv" + [Devel]="devel" ) # These repos have comps/groups, except for debuginfo and sources diff --git a/sync/minor-release-sync-to-staging.sh b/sync/minor-release-sync-to-staging.sh index 13a8cfa..d21f921 100644 --- a/sync/minor-release-sync-to-staging.sh +++ b/sync/minor-release-sync-to-staging.sh @@ -56,11 +56,12 @@ for ARCH in "${ARCHES[@]}"; do # Hardcoding this for now SOURCE="/mnt/compose/${MAJ}/latest-Rocky-${MAJ}/compose/${x}/${ARCH}/iso" TARGET_ARCH="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${x}/${ARCH}/iso" - mkdir -p "${SOURCE}" "${TARGET}" "${TARGET_ARCH}" + mkdir -p "${TARGET}" + #mkdir -p "${SOURCE}" "${TARGET}" "${TARGET_ARCH}" # Copy the ISO and manifests into their target architecture - cp -n "${SOURCE}"/*.iso "${TARGET_ARCH}/" - cp -n "${SOURCE}"/*.iso.manifest "${TARGET_ARCH}/" - cp -n "${SOURCE}/CHECKSUM" "${TARGET_ARCH}/" + #cp -n "${SOURCE}"/*.iso "${TARGET_ARCH}/" + #cp -n "${SOURCE}"/*.iso.manifest "${TARGET_ARCH}/" + #cp -n "${SOURCE}/CHECKSUM" "${TARGET_ARCH}/" # Copy the ISO and manifests into the main isos target cp "${SOURCE}"/*.iso "${TARGET}/" cp "${SOURCE}"/*.iso.manifest "${TARGET}/" diff --git a/sync/sync-to-prod.sh b/sync/sync-to-prod.sh index e14a49a..76045c7 100644 --- a/sync/sync-to-prod.sh +++ b/sync/sync-to-prod.sh @@ -16,7 +16,10 @@ if [ $ret_val -eq "0" ]; then # disabling because none of our files should be starting with dashes. If they # are something is *seriously* wrong here. # shellcheck disable=SC2035 - sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ + sudo -l && find ./ -mindepth 1 -maxdepth 1 -type d -exec find {}/ -mindepth 1 -maxdepth 1 -type d \;|sed 's/^..//g' | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ + {} "${TARGET}" + # shellcheck disable=SC2035 + sudo -l && find ** -maxdepth 0 -type l | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ {} "${TARGET}" # Full file list update