From fc227c947e75311d60a7e4db4e6ae19d9700e0e4 Mon Sep 17 00:00:00 2001 From: nazunalika Date: Wed, 25 May 2022 21:39:26 -0700 Subject: [PATCH] finish wrapper, start repoclosure --- iso/py/configs/el9.yaml | 27 ++++++++++++++++++ iso/py/sync-from-peridot | 53 ++++++++++++++++++++++++++++++----- iso/py/sync-from-peridot-test | 16 +++++++++++ iso/py/util/dnf_utils.py | 39 ++++++++++++++++++++------ sync/prep-staging-8.sh | 2 +- 5 files changed, 121 insertions(+), 16 deletions(-) create mode 100755 iso/py/sync-from-peridot-test diff --git a/iso/py/configs/el9.yaml b/iso/py/configs/el9.yaml index fa4911d..030f124 100644 --- a/iso/py/configs/el9.yaml +++ b/iso/py/configs/el9.yaml @@ -50,4 +50,31 @@ has_modules: - 'AppStream' - 'CRB' + repoclosure_map: + BaseOS: [] + AppStream: + - BaseOS + CRB: + - BaseOS + - AppStream + HighAvailability: + - BaseOS + - AppStream + ResilientStorage: + - BaseOS + - AppStream + RT: + - BaseOS + - AppStream + NFV: + - BaseOS + - AppStream + SAP: + - BaseOS + - AppStream + - HighAvailability + SAPHANA: + - BaseOS + - AppStream + - HighAvailability ... diff --git a/iso/py/sync-from-peridot b/iso/py/sync-from-peridot index fb2a7cd..56ba74d 100755 --- a/iso/py/sync-from-peridot +++ b/iso/py/sync-from-peridot @@ -1,18 +1,57 @@ #!/usr/bin/env python3 -# This is intended for doing "full" syncs, not periodic update syncs. +# This script can be called to do single syncs or full on syncs. -from common import * import argparse +from common import * from util import Checks from util import RepoSync -rlvars = rldict['9'] +#rlvars = rldict['9'] +#r = Checks(rlvars, config['arch']) +#r.check_valid_arch() + +# Start up the parser baby +parser = argparse.ArgumentParser(description="Peridot Sync and Compose") + +# All of our options +parser.add_argument('--release', type=str, help="Major Release Version", required=True) +parser.add_argument('--repo', type=str, help="Repository name") +parser.add_argument('--arch', type=str, help="Architecture") +parser.add_argument('--ignore-debug', action='store_true') +parser.add_argument('--ignore-source', action='store_true') +parser.add_argument('--repoclosure', action='store_true') +parser.add_argument('--skip-all', action='store_true') +parser.add_argument('--dry-run', action='store_true') +parser.add_argument('--full-run', action='store_true') +parser.add_argument('--no-fail', action='store_true') +# I am aware this is confusing, I want podman to be the default option +parser.add_argument('--simple', action='store_false') +parser.add_argument('--logger', type=str) + +# Parse them +results = parser.parse_args() + +rlvars = rldict[results.release] r = Checks(rlvars, config['arch']) r.check_valid_arch() -a = RepoSync(rlvars, config, major="9", repo="ResilientStorage", parallel=True, ignore_debug=False, ignore_source=False) +# Send them and do whatever I guess +a = RepoSync( + rlvars, + config, + major=results.release, + repo=results.repo, + arch=results.arch, + ignore_debug=results.ignore_debug, + ignore_source=results.ignore_source, + repoclosure=results.repoclosure, + skip_all=results.skip_all, + parallel=results.simple, + dryrun=results.dry_run, + fullrun=results.full_run, + nofail=results.no_fail, + logger=results.logger +) + a.run() -#a.generate_conf() -#somedir = a.generate_compose_dirs() -#print(a.sync()) diff --git a/iso/py/sync-from-peridot-test b/iso/py/sync-from-peridot-test new file mode 100755 index 0000000..f4ca170 --- /dev/null +++ b/iso/py/sync-from-peridot-test @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + +# This is a testing script to ensure the RepoSync class is working as intended. + +from common import * +import argparse +from util import Checks +from util import RepoSync + +rlvars = rldict['9'] +r = Checks(rlvars, config['arch']) +r.check_valid_arch() + +a = RepoSync(rlvars, config, major="9", repo="ResilientStorage", parallel=True, ignore_debug=False, ignore_source=False) +#a = RepoSync(rlvars, config, major="9", repo="ResilientStorage", parallel=True, ignore_debug=False, ignore_source=False, fullrun=True) +a.run() diff --git a/iso/py/util/dnf_utils.py b/iso/py/util/dnf_utils.py index 43a9913..537c774 100644 --- a/iso/py/util/dnf_utils.py +++ b/iso/py/util/dnf_utils.py @@ -34,10 +34,11 @@ class RepoSync: major, repo=None, arch=None, - ignore_debug=False, - ignore_source=False, - skip_all=False, - parallel=False, + ignore_debug: bool = False, + ignore_source: bool = False, + repoclosure: bool = False, + skip_all: bool = False, + parallel: bool = False, dryrun: bool = False, fullrun: bool = False, nofail: bool = False, @@ -50,6 +51,7 @@ class RepoSync: self.ignore_debug = ignore_debug self.ignore_source = ignore_source self.skip_all = skip_all + self.repoclosure = repoclosure # Enables podman syncing, which should effectively speed up operations self.parallel = parallel # Relevant config items @@ -74,6 +76,9 @@ class RepoSync: if 'container' in rlvars and len(rlvars['container']) > 0: self.container = rlvars['container'] + if 'repoclosure_map' in rlvars and len(rlvars['repoclosure_map']) > 0: + self.repoclosure_map = rlvars['repoclosure_map'] + self.staging_dir = os.path.join( config['staging_root'], config['category_stub'], @@ -153,6 +158,12 @@ class RepoSync: ) sync_root = self.compose_latest_sync + # Verify if the link even exists + if not os.path.exists(self.compose_latest_dir): + self.log.error('!! Latest compose link is broken does not exist: %s' % self.compose_latest_dir) + self.log.error('!! Please perform a full run if you have not done so.') + raise SystemExit() + log_root = os.path.join( work_root, "logs" @@ -167,6 +178,9 @@ class RepoSync: if self.fullrun: self.symlink_to_latest() + if self.repoclosure: + self.repoclosure_work(sync_root, work_root, log_root) + def sync(self, repo, sync_root, work_root, log_root, arch=None): """ Calls out syncing of the repos. We generally sync each component of a @@ -369,13 +383,19 @@ class RepoSync: for r in repos_to_sync: entry_name_list = [] repo_name = r + arch_sync = arches_to_sync + if r in self.repo_renames: repo_name = self.repo_renames[r] - for a in arches_to_sync: - # There should be a check here that if it's "all" and multilib - # is on, i686 should get synced too. + if 'all' in r and 'x86_64' in arches_to_sync and self.multilib: + arch_sync.append('i686') + + # There should be a check here that if it's "all" and multilib + # is on, i686 should get synced too. + + for a in arch_sync: entry_name = '{}-{}'.format(r, a) debug_entry_name = '{}-debug-{}'.format(r, a) @@ -471,7 +491,7 @@ class RepoSync: "--download-metadata --repoid={}-source -p {} " "--norepopath | tee -a {}/{}-source-{}.log").format( self.dnf_config, - repo_name, + r, source_sync_path, log_root, repo_name, @@ -704,6 +724,9 @@ class RepoSync: ) return cmd + def repoclosure_work(self, sync_root, work_root, log_root): + pass + class SigRepoSync: """ This helps us do reposync operations for SIG's. Do not use this for the diff --git a/sync/prep-staging-8.sh b/sync/prep-staging-8.sh index 10becea..c5c2c22 100644 --- a/sync/prep-staging-8.sh +++ b/sync/prep-staging-8.sh @@ -44,7 +44,7 @@ for x in "${ARCHES[@]}"; do test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" ret_val=$? if [ "$ret_val" -eq 0 ]; then - createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \ + createrepo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \ "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \ --workers 8 sed -i '/<\/open-size>/d' \