From 364e92dcb8302e1347f34f59998b02ef725dd616 Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Thu, 3 Oct 2024 17:04:00 -0400 Subject: [PATCH] WIP: support running toolkit against staging in addition to Peridot --- iso/empanadas/empanadas/common.py | 1 + iso/empanadas/empanadas/scripts/build_iso.py | 4 +- .../empanadas/scripts/sync_from_peridot.py | 2 + iso/empanadas/empanadas/util/dnf_utils.py | 14 ++++-- iso/empanadas/empanadas/util/shared.py | 44 +++++++++++++------ 5 files changed, 46 insertions(+), 19 deletions(-) diff --git a/iso/empanadas/empanadas/common.py b/iso/empanadas/empanadas/common.py index 41ca803..f2a43e5 100644 --- a/iso/empanadas/empanadas/common.py +++ b/iso/empanadas/empanadas/common.py @@ -65,6 +65,7 @@ config = { "category_stub": "mirror/pub/rocky", "sig_category_stub": "mirror/pub/sig", "repo_base_url": "https://yumrepofs.build.resf.org/v1/projects", + "staging_base_url": "https://dl.rockylinux.org/stg/rocky", "mock_work_root": "/builddir", "container": "centos:stream9", "distname": "Rocky Linux", diff --git a/iso/empanadas/empanadas/scripts/build_iso.py b/iso/empanadas/empanadas/scripts/build_iso.py index 6da2e4d..6f6372a 100755 --- a/iso/empanadas/empanadas/scripts/build_iso.py +++ b/iso/empanadas/empanadas/scripts/build_iso.py @@ -2,8 +2,7 @@ import argparse -from empanadas.common import * -from empanadas.util import Checks +from empanadas.common import config, rldict from empanadas.util import IsoBuild parser = argparse.ArgumentParser(description="ISO Compose") @@ -29,5 +28,6 @@ a = IsoBuild( logger=results.logger, ) + def run(): a.run() diff --git a/iso/empanadas/empanadas/scripts/sync_from_peridot.py b/iso/empanadas/empanadas/scripts/sync_from_peridot.py index 482a72a..0942b9b 100755 --- a/iso/empanadas/empanadas/scripts/sync_from_peridot.py +++ b/iso/empanadas/empanadas/scripts/sync_from_peridot.py @@ -33,6 +33,7 @@ parser.add_argument('--logger', type=str) parser.add_argument('--disable-gpg-check', action='store_false') parser.add_argument('--disable-repo-gpg-check', action='store_false') parser.add_argument('--clean-old-packages', action='store_true') +parser.add_argument('--use-staging', action='store_true') # Parse them results = parser.parse_args() @@ -64,6 +65,7 @@ a = RepoSync( gpg_check=results.disable_gpg_check, repo_gpg_check=results.disable_repo_gpg_check, reposync_clean_old=results.clean_old_packages, + use_staging=results.use_staging, ) def run(): diff --git a/iso/empanadas/empanadas/util/dnf_utils.py b/iso/empanadas/empanadas/util/dnf_utils.py index b737f8b..0c7ab30 100644 --- a/iso/empanadas/empanadas/util/dnf_utils.py +++ b/iso/empanadas/empanadas/util/dnf_utils.py @@ -62,7 +62,9 @@ class RepoSync: fpsync: bool = False, logger=None, log_level='INFO', - ): + use_staging: bool = False, + ): + self.nofail = nofail self.dryrun = dryrun self.fullrun = fullrun @@ -80,11 +82,14 @@ class RepoSync: # This makes it so every repo is synced at the same time. # This is EXTREMELY dangerous. self.just_pull_everything = just_pull_everything + # Use staging url instead of pulling from peridot (or, for EL8) + self.use_staging = use_staging # Relevant config items self.major_version = major self.date_stamp = config['date_stamp'] self.timestamp = time.time() self.repo_base_url = config['repo_base_url'] + self.staging_base_url = config['staging_base_url'] self.compose_root = config['compose_root'] self.compose_base = config['compose_root'] + "/" + major self.profile = rlvars['profile'] @@ -271,7 +276,9 @@ class RepoSync: self.gpg_check, self.repo_gpg_check, self.tmplenv, - self.log + self.log, + staging_base_url=self.staging_base_url, + use_staging=self.use_staging, ) if self.dryrun: @@ -1447,7 +1454,8 @@ class RepoSync: self.gpg_check, self.repo_gpg_check, self.tmplenv, - self.log + self.log, + staging_base_url=self.staging_base_url, ) diff --git a/iso/empanadas/empanadas/util/shared.py b/iso/empanadas/empanadas/util/shared.py index 6406c8d..021abd6 100644 --- a/iso/empanadas/empanadas/util/shared.py +++ b/iso/empanadas/empanadas/util/shared.py @@ -446,8 +446,10 @@ class Shared: repo_gpg_check, templates, logger, - dest_path='/var/tmp' - ) -> str: + dest_path='/var/tmp', + staging_base_url='https://dl.rockylinux.org/stg', + use_staging=False, + ) -> str: """ Generates the necessary repo conf file for the operation. This repo file should be temporary in nature. This will generate a repo file @@ -475,22 +477,36 @@ class Shared: if not os.path.exists(dest_path): os.makedirs(dest_path, exist_ok=True) config_file = open(fname, "w+") + repolist = [] for repo in repos: - constructed_url = '{}/{}/repo/{}{}/$basearch'.format( - repo_base_url, - project_id, - prehashed, - repo, - ) + if use_staging: + constructed_url = '{}/{}/{}/$basearch/os'.format( + staging_base_url, + major_version, + repo, + ) - constructed_url_src = '{}/{}/repo/{}{}/src'.format( - repo_base_url, - project_id, - prehashed, - repo, - ) + constructed_url_src = '{}/{}/{}/source/tree'.format( + staging_base_url, + major_version, + repo, + ) + else: + constructed_url = '{}/{}/repo/{}{}/$basearch'.format( + repo_base_url, + project_id, + prehashed, + repo, + ) + + constructed_url_src = '{}/{}/repo/{}{}/src'.format( + repo_base_url, + project_id, + prehashed, + repo, + ) repodata = { 'name': repo,