From d845b95e80343609d8d6f05b959c36c5f650cadc Mon Sep 17 00:00:00 2001 From: nazunalika Date: Mon, 23 May 2022 00:23:53 -0700 Subject: [PATCH] Final commit for 5/22 Finish regular reposync functionality, leave notes about i686 Start podman reposync functionality --- iso/py/README.md | 6 ++ iso/py/common.py | 3 +- iso/py/sync-from-peridot | 2 +- iso/py/util/dnf_utils.py | 206 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 208 insertions(+), 9 deletions(-) diff --git a/iso/py/README.md b/iso/py/README.md index 2e90592..41f3a41 100644 --- a/iso/py/README.md +++ b/iso/py/README.md @@ -1,5 +1,11 @@ # iso +## TODO + +Verbose mode should exist to output everything that's being called or ran. + +There should be additional logging regardless, not just to stdout, but also to a file. + ## scripts * sync-variant-pungi diff --git a/iso/py/common.py b/iso/py/common.py index b36c728..1d07574 100644 --- a/iso/py/common.py +++ b/iso/py/common.py @@ -31,7 +31,8 @@ config = { "production_root": "/mnt/repos-production", "category_stub": "mirror/pub/rocky", "sig_category_stub": "mirror/pub/sig", - "repo_base_url": "https://yumrepofs.build.resf.org/v1/projects" + "repo_base_url": "https://yumrepofs.build.resf.org/v1/projects", + "container": "centos:stream9" } # Importing the config from yaml diff --git a/iso/py/sync-from-peridot b/iso/py/sync-from-peridot index a883ce2..0e895f8 100755 --- a/iso/py/sync-from-peridot +++ b/iso/py/sync-from-peridot @@ -11,7 +11,7 @@ rlvars = rldict['9'] r = Checks(rlvars, config['arch']) r.check_valid_arch() -a = RepoSync(rlvars, config, major="9", repo="BaseOS") +a = RepoSync(rlvars, config, major="9", repo="ResilientStorage", parallel=False, ignore_debug=True, ignore_source=True) a.run() #a.generate_conf() #somedir = a.generate_compose_dirs() diff --git a/iso/py/util/dnf_utils.py b/iso/py/util/dnf_utils.py index 37a7359..c1954fd 100644 --- a/iso/py/util/dnf_utils.py +++ b/iso/py/util/dnf_utils.py @@ -8,6 +8,8 @@ import logging import sys import os import os.path +import subprocess +import shlex #import pipes from common import Color @@ -32,6 +34,7 @@ class RepoSync: arch=None, ignore_debug=False, ignore_source=False, + parallel=False, dryrun: bool = False, fullrun: bool = False, nofail: bool = False, @@ -43,6 +46,8 @@ class RepoSync: self.arch = arch self.ignore_debug = ignore_debug self.ignore_source = ignore_source + # Enables podman syncing, which should effectively speed up operations + self.parallel = parallel # Relevant config items self.major_version = major self.date_stamp = config['date_stamp'] @@ -55,6 +60,7 @@ class RepoSync: self.project_id = rlvars['project_id'] self.repo_renames = rlvars['renames'] self.repos = rlvars['all_repos'] + self.multilib = rlvars['provide_multilib'] self.repo = repo self.staging_dir = os.path.join( @@ -89,6 +95,8 @@ class RepoSync: self.log.info('reposync init') self.log.info(self.revision) + self.dnf_config = self.generate_conf() + def run(self): """ @@ -109,28 +117,194 @@ class RepoSync: self.log.error('A full and dry run is currently not supported.') raise SystemExit('\nA full and dry run is currently not supported.') - self.generate_conf() - + # This should create the initial compose dir and set the path. + # Otherwise, just use the latest link. if self.fullrun: sync_root = os.path.join( self.generate_compose_dirs(), 'compose' ) else: + # Put in a verification here. sync_root = self.compose_latest_sync + self.sync(self.repo, sync_root, self.arch) + + if self.fullrun: + self.symlink_to_latest() + def sync(self, repo, sync_root, arch=None): """ - Does the actual syncing of the repo. We generally sync each component - of a repo: + Calls out syncing of the repos. We generally sync each component of a + repo: * each architecture * each architecture debug * each source + + If parallel is true, we will run in podman. """ + if self.parallel: + self.podman_sync(repo, sync_root, arch) + else: + self.dnf_sync(repo, sync_root, arch) + + def dnf_sync(self, repo, sync_root, arch): + """ + This is for normal dnf syncs + """ + cmd = self.reposync_cmd() + + sync_single_arch = False + arches_to_sync = self.arches + if arch: + sync_single_arch = True + arches_to_sync = [arch] + + sync_single_repo = False + repos_to_sync = self.repos + if repo and not self.fullrun: + sync_single_repo = True + repos_to_sync = [repo] + # dnf reposync --download-metadata \ # --repoid fedora -p /tmp/test \ # --forcearch aarch64 --norepopath - cmd = self.reposync_cmd() + + self.log.info( + Color.BOLD + '!! WARNING !! ' + Color.END + 'You are performing a ' + 'local reposync, which may incur delays in your compose.' + ) + + if self.fullrun: + self.log.info( + Color.BOLD + '!! WARNING !! ' + Color.END + 'This is a full ' + 'run! This will take a LONG TIME.' + ) + + for r in repos_to_sync: + for a in arches_to_sync: + repo_name = r + if r in self.repo_renames: + repo_name = self.repo_renames[r] + + os_sync_path = os.path.join( + sync_root, + repo_name, + a, + 'os' + ) + + debug_sync_path = os.path.join( + sync_root, + repo_name, + a, + 'debug/tree' + ) + + sync_cmd = "{} -c {} --download-metadata --repoid={} -p {} --forcearch {} --norepopath".format( + cmd, + self.dnf_config, + r, + os_sync_path, + a + ) + + debug_sync_cmd = "{} -c {} --download-metadata --repoid={}-debug -p {} --forcearch {} --norepopath".format( + cmd, + self.dnf_config, + r, + debug_sync_path, + a + ) + + self.log.info('Syncing {} {}'.format(r, a)) + #self.log.info(sync_cmd) + # Try to figure out where to send the actual output of this... + # Also consider on running a try/except here? Basically if + # something happens (like a repo doesn't exist for some arch, + # eg RT for aarch64), make a note of it somehow (but don't + # break the entire sync). As it stands with this + # implementation, if something fails, it just continues on. + process = subprocess.call( + shlex.split(sync_cmd), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + if not self.ignore_debug: + self.log.info('Syncing {} {} (debug)'.format(r, a)) + process_debug = subprocess.call( + shlex.split(debug_sync_cmd), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + # There should be a check here that if it's "all" and multilib + # is on, i686 should get synced too. + + if not self.ignore_source: + source_sync_path = os.path.join( + sync_root, + repo_name, + 'source/tree' + ) + + source_sync_cmd = "{} -c {} --download-metadata --repoid={}-source -p {} --norepopath".format( + cmd, + self.dnf_config, + r, + source_sync_path + ) + + + self.log.info('Syncing {} source'.format(r)) + process_source = subprocess.call( + shlex.split(source_sync_cmd), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + self.log.info('Syncing complete') + + def podman_sync(self, repo, sync_root, arch): + """ + This is for podman syncs + + Create sync_root/work/entries + Generate scripts as needed into dir + Each container runs their own script + wait till all is finished + """ + cmd = self.podman_cmd() + contrunlist = [] + self.log.info('Generating container entries') + entries_dir = os.path.join(sync_root, "work", "entries") + if not os.path.exists(entries_dir): + os.makedirs(entries_dir, exist_ok=True) + + sync_single_arch = False + arches_to_sync = self.arches + if arch: + sync_single_arch = True + arches_to_sync = [arch] + + sync_single_repo = False + repos_to_sync = self.repos + if repo and not self.fullrun: + sync_single_repo = True + repos_to_sync = [repo] + + for r in repos_to_sync: + for a in arches_to_sync: + repo_name = r + if r in self.repo_renames: + repo_name = self.repo_renames[r] + + # There should be a check here that if it's "all" and multilib + # is on, i686 should get synced too. + + entry_name = '{}-{}'.format(r, a) + debug_entry_name = '{}-debug-{}'.format(r, a) def generate_compose_dirs(self) -> str: """ @@ -156,7 +330,7 @@ class RepoSync: """ pass - def generate_conf(self, dest_path='/var/tmp'): + def generate_conf(self, dest_path='/var/tmp') -> str: """ Generates the necessary repo conf file for the operation. This repo file should be temporary in nature. This will generate a repo file @@ -221,7 +395,7 @@ class RepoSync: config_file.write("enabled=1\n") config_file.write("gpgcheck=0\n\n") - + return fname def reposync_cmd(self) -> str: """ @@ -243,6 +417,24 @@ class RepoSync: "expecting them to work and got to this point." + Color.END) return cmd + def podman_cmd(self) -> str: + """ + This generates the podman run command. This is in the case that we want + to do reposyncs in parallel as we cannot reasonably run multiple + instances of dnf reposync on a single system. + """ + cmd = None + if os.path.exists("/usr/bin/podman"): + cmd = "/usr/bin/podman run" + else: + self.log.error('/usr/bin/podman was not found. Good bye.') + raise SystemExit("\n\n/usr/bin/podman was not found.\n\nPlease " + " ensure that you have installed the necessary packages on " + " this system. " + Color.BOLD + "Note that docker is not " + "supported." + Color.END + ) + return cmd + class SigRepoSync: """ This helps us do reposync operations for SIG's. Do not use this for the