Final commit for 5/21

This commit is contained in:
Louis Abel 2022-05-21 23:20:15 -07:00
parent 4f9bfd44a0
commit 420aa02bdb
Signed by untrusted user: label
GPG Key ID: B37E62D143879B36
7 changed files with 233 additions and 47 deletions

View File

@ -4,8 +4,8 @@
* sync-variant-pungi * sync-variant-pungi
* sync-variant-peridot * sync-variant-peridot
* sync-all-pungi * sync-from-pungi
* sync-all-peridot * sync-from-peridot
* sync-sig * sync-sig
* build-all-iso * build-all-iso
* sign-repos-only * sign-repos-only

View File

@ -25,12 +25,13 @@ rldict = {}
config = { config = {
"rlmacro": rpm.expandMacro('%rhel'), "rlmacro": rpm.expandMacro('%rhel'),
"arch": platform.machine(), "arch": platform.machine(),
"date_stamp": time.strftime("%Y%m%d", time.localtime()), "date_stamp": time.strftime("%Y%m%d.%H%M%S", time.localtime()),
"compose_root": "/mnt/compose",
"staging_root": "/mnt/repos-staging", "staging_root": "/mnt/repos-staging",
"production_root": "/mnt/repos-production", "production_root": "/mnt/repos-production",
"category_stub": "/mirror/pub/rocky", "category_stub": "mirror/pub/rocky",
"sig_category_stub": "/mirror/pub/sig", "sig_category_stub": "mirror/pub/sig",
"repo_base_url": "https://yumrepofs.build.resf.org/v1/projects/" "repo_base_url": "https://yumrepofs.build.resf.org/v1/projects"
} }
# Importing the config from yaml # Importing the config from yaml
@ -39,7 +40,7 @@ for conf in glob.iglob('configs/*.yaml'):
rldict.update(yaml.safe_load(file)) rldict.update(yaml.safe_load(file))
# The system needs to be a RHEL-like system. It cannot be Fedora or SuSE. # The system needs to be a RHEL-like system. It cannot be Fedora or SuSE.
#if "%rhel" in config['RLMACRO']: #if "%rhel" in config['rlmacro']:
# raise SystemExit(Color.BOLD + 'This is not a RHEL-like system.' + Color.END # raise SystemExit(Color.BOLD + 'This is not a RHEL-like system.' + Color.END
# + '\n\nPlease verify you are running on a RHEL-like system that is ' # + '\n\nPlease verify you are running on a RHEL-like system that is '
# 'not Fedora nor SuSE. This means that the %rhel macro will be ' # 'not Fedora nor SuSE. This means that the %rhel macro will be '
@ -49,7 +50,6 @@ for conf in glob.iglob('configs/*.yaml'):
# These will be set in their respective var files # These will be set in their respective var files
#REVISION = rlvars['revision'] + '-' + rlvars['rclvl'] #REVISION = rlvars['revision'] + '-' + rlvars['rclvl']
#rlvars = rldict[RLVER] #rlvars = rldict[rlver]
#rlvars = rldict[RLMACRO] #rlvars = rldict[rlmacro]
#COMPOSE_ROOT = "/mnt/compose/" + RLVER
#COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp #COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp

View File

@ -18,9 +18,9 @@
all_repos: all_repos:
- 'BaseOS' - 'BaseOS'
- 'AppStream' - 'AppStream'
- 'PowerTools'
- 'HighAvailability' - 'HighAvailability'
- 'ResilientStorage' - 'ResilientStorage'
- 'PowerTools'
- 'RT' - 'RT'
- 'NFV' - 'NFV'
- 'extras' - 'extras'
@ -35,9 +35,9 @@
comps_or_groups: comps_or_groups:
- 'BaseOS' - 'BaseOS'
- 'AppStream' - 'AppStream'
- 'PowerTools'
- 'HighAvailability' - 'HighAvailability'
- 'ResilientStorage' - 'ResilientStorage'
- 'PowerTools'
- 'RT' - 'RT'
- 'NFV' - 'NFV'
has_modules: has_modules:

View File

@ -22,9 +22,9 @@
- 'all' - 'all'
- 'BaseOS' - 'BaseOS'
- 'AppStream' - 'AppStream'
- 'CRB'
- 'HighAvailability' - 'HighAvailability'
- 'ResilientStorage' - 'ResilientStorage'
- 'CRB'
- 'RT' - 'RT'
- 'NFV' - 'NFV'
- 'SAP' - 'SAP'
@ -40,9 +40,9 @@
comps_or_groups: comps_or_groups:
- 'BaseOS' - 'BaseOS'
- 'AppStream' - 'AppStream'
- 'CRB'
- 'HighAvailability' - 'HighAvailability'
- 'ResilientStorage' - 'ResilientStorage'
- 'CRB'
- 'RT' - 'RT'
- 'NFV' - 'NFV'
- 'SAP' - 'SAP'

View File

@ -1,13 +0,0 @@
#!/usr/bin/env python3
from common import *
import argparse
from util import Checks
from util import RepoSync
rlvars = rldict['9']
r = Checks(rlvars, config['arch'])
r.check_valid_arch()
a = RepoSync(rlvars, config, repo="BaseOS")
print(a.sync())

18
iso/py/sync-from-peridot Executable file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env python3
# This is intended for doing "full" syncs, not periodic update syncs.
from common import *
import argparse
from util import Checks
from util import RepoSync
rlvars = rldict['9']
r = Checks(rlvars, config['arch'])
r.check_valid_arch()
a = RepoSync(rlvars, config, major="9", repo="BaseOS")
a.run()
#a.generate_conf()
#somedir = a.generate_compose_dirs()
#print(a.sync())

View File

@ -1,13 +1,21 @@
HAS_LIBREPO = True """
Syncs yum repos for mirroring and composing.
Louis Abel <label AT rockylinux.org>
"""
#import shutil
import logging
import sys
import os import os
import os.path import os.path
import pipes #import pipes
import shutil from common import Color
try: #HAS_LIBREPO = True
import librepo #try:
except: # import librepo
HAS_LIBREPO = False #except:
# HAS_LIBREPO = False
class RepoSync: class RepoSync:
""" """
@ -15,13 +23,31 @@ class RepoSync:
different class entirely. This is on purpose. Please use the SigRepoSync different class entirely. This is on purpose. Please use the SigRepoSync
class for SIG syncs. class for SIG syncs.
""" """
def __init__(self, rlvars, config, repo=None, nofail: bool = False): def __init__(
self,
rlvars,
config,
major,
repo=None,
arch=None,
ignore_debug=False,
ignore_source=False,
dryrun: bool = False,
fullrun: bool = False,
nofail: bool = False,
logger=None
):
self.nofail = nofail self.nofail = nofail
self.dryrun = dryrun
self.fullrun = fullrun
self.arch = arch
self.ignore_debug = ignore_debug
self.ignore_source = ignore_source
# Relevant config items # Relevant config items
self.major_version = config['rlmacro'] self.major_version = major
self.date_stamp = config['date_stamp'] self.date_stamp = config['date_stamp']
self.staging_dir = config['staging_root'] + config['category_stub'] + self.major_version
self.repo_base_url = config['repo_base_url'] self.repo_base_url = config['repo_base_url']
self.compose_base = config['compose_root'] + "/" + major
# Relevant major version items # Relevant major version items
self.revision = rlvars['revision'] + "-" + rlvars['rclvl'] self.revision = rlvars['revision'] + "-" + rlvars['rclvl']
@ -31,14 +57,106 @@ class RepoSync:
self.repos = rlvars['all_repos'] self.repos = rlvars['all_repos']
self.repo = repo self.repo = repo
self.staging_dir = os.path.join(
config['staging_root'],
config['category_stub'],
self.revision
)
self.compose_latest_dir = os.path.join(
config['compose_root'],
major,
"latest-Rocky-{}".format(major)
)
self.compose_latest_sync = os.path.join(
self.compose_latest_dir,
"compose"
)
# This is temporary for now.
if logger is None:
self.log = logging.getLogger("reposync")
self.log.setLevel(logging.INFO)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.INFO)
formatter = logging.Formatter(
'%(asctime)s :: %(name)s :: %(message)s',
'%Y-%m-%d %H:%M:%S'
)
handler.setFormatter(formatter)
self.log.addHandler(handler)
self.log.info('reposync init')
self.log.info(self.revision)
def run(self): def run(self):
"""
This must be called to perform the sync. This will run through, create
the configuration file as required, and try to do a sync of every repo
applicable or the repo actually specified. If self.repo is None, it
will be assumed all repos are synced as dictated by rlvars.
* Dry runs only create initial directories and structure
* Full runs sync everything from the top and setup structure,
including creating a symlink to latest-Rocky-X
* self.repo is ignored during full runs (noted in stdout)
* self.arch being set will force only that arch to sync
"""
if self.fullrun and self.repo:
self.log.error('WARNING: repo ignored when doing a full sync')
if self.fullrun and self.dryrun:
self.log.error('A full and dry run is currently not supported.')
raise SystemExit('\nA full and dry run is currently not supported.')
self.generate_conf()
if self.fullrun:
sync_root = os.path.join(
self.generate_compose_dirs(),
'compose'
)
else:
sync_root = self.compose_latest_sync
def sync(self, repo, sync_root, arch=None):
"""
Does the actual syncing of the repo. We generally sync each component
of a repo:
* each architecture
* each architecture debug
* each source
"""
# dnf reposync --download-metadata \
# --repoid fedora -p /tmp/test \
# --forcearch aarch64 --norepopath
cmd = self.reposync_cmd()
def generate_compose_dirs(self) -> str:
"""
Generate compose dirs for full runs
"""
compose_base_dir = os.path.join(
self.compose_base,
"Rocky-{}-{}".format(self.major_version, self.date_stamp)
)
self.log.info('Creating compose directory %s' % compose_base_dir)
if not os.path.exists(compose_base_dir):
os.makedirs(compose_base_dir)
return compose_base_dir
def symlink_to_latest(self):
"""
Emulates pungi and symlinks latest-Rocky-X
This link will be what is updated in full runs. Whatever is in this
'latest' directory is what is rsynced on to staging after completion.
This link should not change often.
"""
pass pass
def sync(self): def generate_conf(self, dest_path='/var/tmp'):
cmd = self.reposync_cmd()
print(self.revision)
def generate_conf(self, dest_path: str, repo):
""" """
Generates the necessary repo conf file for the operation. This repo Generates the necessary repo conf file for the operation. This repo
file should be temporary in nature. This will generate a repo file file should be temporary in nature. This will generate a repo file
@ -48,7 +166,62 @@ class RepoSync:
:param dest_path: The destination where the temporary conf goes :param dest_path: The destination where the temporary conf goes
:param repo: The repo object to create a file for :param repo: The repo object to create a file for
""" """
pass fname = os.path.join(
dest_path,
"{}-config.repo".format(self.major_version)
)
self.log.info('Generating the repo configuration: %s' % fname)
if self.repo_base_url.startswith("/"):
self.log.error("Local file syncs are not supported.")
raise SystemExit(Color.BOLD + "Local file syncs are not "
"supported." + Color.END)
# create dest_path
if not os.path.exists(dest_path):
os.makedirs(dest_path, exist_ok=True)
config_file = open(fname, "w+")
for repo in self.repos:
constructed_url = '{}/{}/repo/{}/$basearch'.format(
self.repo_base_url,
self.project_id,
repo,
)
constructed_url_debug = '{}/{}/repo/{}/$basearch-debug'.format(
self.repo_base_url,
self.project_id,
repo,
)
constructed_url_src = '{}/{}/repo/{}/src'.format(
self.repo_base_url,
self.project_id,
repo,
)
# normal
config_file.write('[%s]\n' % repo)
config_file.write('name=%s\n' % repo)
config_file.write('baseurl=%s\n' % constructed_url)
config_file.write("enabled=1\n")
config_file.write("gpgcheck=0\n\n")
# debug
config_file.write('[%s-debug]\n' % repo)
config_file.write('name=%s debug\n' % repo)
config_file.write('baseurl=%s\n' % constructed_url_debug)
config_file.write("enabled=1\n")
config_file.write("gpgcheck=0\n\n")
# src
config_file.write('[%s-source]\n' % repo)
config_file.write('name=%s source\n' % repo)
config_file.write('baseurl=%s\n' % constructed_url_src)
config_file.write("enabled=1\n")
config_file.write("gpgcheck=0\n\n")
def reposync_cmd(self) -> str: def reposync_cmd(self) -> str:
""" """
@ -56,14 +229,22 @@ class RepoSync:
itself and will raise an error. itself and will raise an error.
:return: The path to the reposync command. If dnf exists, we'll use :return: The path to the reposync command. If dnf exists, we'll use
that. that. Otherwise, fail immediately.
""" """
cmd = None cmd = None
if os.path.exists("/usr/bin/dnf"): if os.path.exists("/usr/bin/dnf"):
cmd = "/usr/bin/dnf reposync" cmd = "/usr/bin/dnf reposync"
else: else:
raise SystemExit("/usr/bin/dnf was not found. /usr/bin/reposync is " self.log.error('/usr/bin/dnf was not found. Good bye.')
raise SystemExit("/usr/bin/dnf was not found. \n\n/usr/bin/reposync "
"is not sufficient and you are likely running on an el7 " "is not sufficient and you are likely running on an el7 "
"system, which tells us that you made changes to these " "system or a grossly modified EL8+ system, " + Color.BOLD +
"tools.") "which tells us that you probably made changes to these tools "
"expecting them to work and got to this point." + Color.END)
return cmd return cmd
class SigRepoSync:
"""
This helps us do reposync operations for SIG's. Do not use this for the
base system. Use RepoSync for that.
"""