From 9bfe6f505e08da12480fcba67561e0926f3bf534 Mon Sep 17 00:00:00 2001 From: Louis Abel Date: Wed, 5 Oct 2022 16:15:08 -0700 Subject: [PATCH] Add support for repoclosure at peridot level --- .../empanadas/scripts/peridot_repoclosure.py | 36 ++++ iso/empanadas/empanadas/util/dnf_utils.py | 180 ++++++++++++++++++ iso/empanadas/pyproject.toml | 1 + 3 files changed, 217 insertions(+) create mode 100644 iso/empanadas/empanadas/scripts/peridot_repoclosure.py diff --git a/iso/empanadas/empanadas/scripts/peridot_repoclosure.py b/iso/empanadas/empanadas/scripts/peridot_repoclosure.py new file mode 100644 index 0000000..dae02b7 --- /dev/null +++ b/iso/empanadas/empanadas/scripts/peridot_repoclosure.py @@ -0,0 +1,36 @@ +# This is for doing repoclosures upstream + +import argparse + +from empanadas.common import * +from empanadas.util import Checks +from empanadas.util import RepoSync + +# Start up the parser baby +parser = argparse.ArgumentParser(description="Peridot Upstream Repoclosure") + +# All of our options +parser.add_argument('--release', type=str, help="Major Release Version or major-type (eg 9-beta)", required=True) +parser.add_argument('--simple', action='store_false') +parser.add_argument('--enable-repo-gpg-check', action='store_true') +parser.add_argument('--logger', type=str) + +# Parse them +results = parser.parse_args() +rlvars = rldict[results.release] +major = rlvars['major'] + +r = Checks(rlvars, config['arch']) +r.check_valid_arch() + +a = RepoSync( + rlvars, + config, + major=major, + parallel=results.simple, + repo_gpg_check=results.enable_repo_gpg_check, + logger=results.logger, +) + +def run(): + a.run_upstream_repoclosure() diff --git a/iso/empanadas/empanadas/util/dnf_utils.py b/iso/empanadas/empanadas/util/dnf_utils.py index a10a19c..a706c38 100644 --- a/iso/empanadas/empanadas/util/dnf_utils.py +++ b/iso/empanadas/empanadas/util/dnf_utils.py @@ -1487,6 +1487,186 @@ class RepoSync: # Deploy final metadata for a close out self.deploy_metadata(sync_root) + def run_upstream_repoclosure(self): + """ + This does a repoclosure check in peridot + """ + work_root = os.path.join( + self.compose_latest_dir, + 'work' + ) + # Verify if the link even exists + if not os.path.exists(self.compose_latest_dir): + self.log.error( + '!! Latest compose link is broken does not exist: %s' % self.compose_latest_dir + ) + self.log.error( + '!! Please perform a full run if you have not done so.' + ) + raise SystemExit() + + log_root = os.path.join( + work_root, + "logs", + self.date_stamp + ) + + if not os.path.exists(log_root): + os.makedirs(log_root, exist_ok=True) + + cmd = Shared.podman_cmd(self.log) + entries_dir = os.path.join(work_root, "entries") + bad_exit_list = [] + dnf_config = Shared.generate_conf( + self.shortname, + self.major_version, + self.repos, + self.repo_base_url, + self.project_id, + self.hashed, + self.extra_files, + self.gpgkey, + self.gpg_check, + self.repo_gpg_check, + self.tmplenv, + self.log + ) + + + if not self.parallel: + self.log.error('repoclosure is too slow to run one by one. enable parallel mode.') + raise SystemExit() + + self.log.info('Beginning upstream repoclosure') + for repo in self.repoclosure_map['repos']: + if self.repo and repo not in self.repo: + continue + + repoclosure_entry_name_list = [] + self.log.info('Setting up repoclosure for {}'.format(repo)) + + for arch in self.repoclosure_map['arches']: + repo_combination = [] + repoclosure_entry_name = 'peridot-repoclosure-{}-{}'.format(repo, arch) + repoclosure_entry_name_list.append(repoclosure_entry_name) + repoclosure_arch_list = self.repoclosure_map['arches'][arch] + + # Some repos will have additional repos to close against - this + # helps append + if len(self.repoclosure_map['repos'][repo]) > 0: + for l in self.repoclosure_map['repos'][repo]: + stretch = '--repo={}'.format(l) + repo_combination.append(stretch) + + join_repo_comb = ' '.join(repo_combination) + + repoclosure_entry_point_sh = os.path.join( + entries_dir, + repoclosure_entry_name + ) + repoclosure_entry_point_sh = os.path.join( + entries_dir, + repoclosure_entry_name + ) + repoclosure_cmd = ('/usr/bin/dnf repoclosure {} ' + '--repo={} --check={} {} -c {} -y ' + '| tee -a {}/peridot-{}-repoclosure-{}.log').format( + repoclosure_arch_list, + repo, + repo, + join_repo_comb, + dnf_config, + log_root, + repo, + arch + ) + repoclosure_entry_point_open = open(repoclosure_entry_point_sh, "w+") + repoclosure_entry_point_open.write('#!/bin/bash\n') + repoclosure_entry_point_open.write('set -o pipefail\n') + repoclosure_entry_point_open.write('/usr/bin/dnf install dnf-plugins-core -y\n') + repoclosure_entry_point_open.write('/usr/bin/dnf clean all\n') + repoclosure_entry_point_open.write(repoclosure_cmd + '\n') + repoclosure_entry_point_open.close() + os.chmod(repoclosure_entry_point_sh, 0o755) + repo_combination.clear() + + self.log.info('Spawning pods for %s' % repo) + for pod in repoclosure_entry_name_list: + podman_cmd_entry = '{} run -d -it -v "{}:{}" -v "{}:{}:z" -v "{}:{}" --name {} --entrypoint {}/{} {}'.format( + cmd, + self.compose_root, + self.compose_root, + dnf_config, + dnf_config, + entries_dir, + entries_dir, + pod, + entries_dir, + pod, + self.container + ) + #print(podman_cmd_entry) + process = subprocess.call( + shlex.split(podman_cmd_entry), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + join_all_pods = ' '.join(repoclosure_entry_name_list) + time.sleep(3) + self.log.info('Performing repoclosure on %s ... ' % repo) + pod_watcher = '{} wait {}'.format( + cmd, + join_all_pods + ) + + watch_man = subprocess.call( + shlex.split(pod_watcher), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + for pod in repoclosure_entry_name_list: + checkcmd = '{} ps -f status=exited -f name={}'.format( + cmd, + pod + ) + podcheck = subprocess.Popen( + checkcmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True + ) + + output, errors = podcheck.communicate() + if 'Exited (0)' not in output.decode(): + self.log.error(Color.FAIL + pod) + bad_exit_list.append(pod) + + rmcmd = '{} rm {}'.format( + cmd, + join_all_pods + ) + + rmpod = subprocess.Popen( + rmcmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + shell=True + ) + + repoclosure_entry_name_list.clear() + self.log.info('Syncing %s completed' % repo) + + if len(bad_exit_list) > 0: + self.log.error( + Color.BOLD + Color.RED + 'There were issues closing these ' + 'repositories:' + Color.END + ) + for issue in bad_exit_list: + self.log.error(issue) + + class SigRepoSync: """ This helps us do reposync operations for SIG's. Do not use this for the diff --git a/iso/empanadas/pyproject.toml b/iso/empanadas/pyproject.toml index b00e60e..7dcc940 100644 --- a/iso/empanadas/pyproject.toml +++ b/iso/empanadas/pyproject.toml @@ -34,6 +34,7 @@ build-image = "empanadas.scripts.build_image:run" finalize_compose = "empanadas.scripts.finalize_compose:run" pull-cloud-image = "empanadas.scripts.pull_cloud_image:run" generate_compose = "empanadas.scripts.generate_compose:run" +peridot_repoclosure = "empanadas.scripts.peridot_repoclosure:run" [build-system] requires = ["poetry-core>=1.0.0"]