Final commit for 5/21
This commit is contained in:
parent
4f9bfd44a0
commit
420aa02bdb
@ -4,8 +4,8 @@
|
||||
|
||||
* sync-variant-pungi
|
||||
* sync-variant-peridot
|
||||
* sync-all-pungi
|
||||
* sync-all-peridot
|
||||
* sync-from-pungi
|
||||
* sync-from-peridot
|
||||
* sync-sig
|
||||
* build-all-iso
|
||||
* sign-repos-only
|
||||
|
@ -25,12 +25,13 @@ rldict = {}
|
||||
config = {
|
||||
"rlmacro": rpm.expandMacro('%rhel'),
|
||||
"arch": platform.machine(),
|
||||
"date_stamp": time.strftime("%Y%m%d", time.localtime()),
|
||||
"date_stamp": time.strftime("%Y%m%d.%H%M%S", time.localtime()),
|
||||
"compose_root": "/mnt/compose",
|
||||
"staging_root": "/mnt/repos-staging",
|
||||
"production_root": "/mnt/repos-production",
|
||||
"category_stub": "/mirror/pub/rocky",
|
||||
"sig_category_stub": "/mirror/pub/sig",
|
||||
"repo_base_url": "https://yumrepofs.build.resf.org/v1/projects/"
|
||||
"category_stub": "mirror/pub/rocky",
|
||||
"sig_category_stub": "mirror/pub/sig",
|
||||
"repo_base_url": "https://yumrepofs.build.resf.org/v1/projects"
|
||||
}
|
||||
|
||||
# Importing the config from yaml
|
||||
@ -39,7 +40,7 @@ for conf in glob.iglob('configs/*.yaml'):
|
||||
rldict.update(yaml.safe_load(file))
|
||||
|
||||
# The system needs to be a RHEL-like system. It cannot be Fedora or SuSE.
|
||||
#if "%rhel" in config['RLMACRO']:
|
||||
#if "%rhel" in config['rlmacro']:
|
||||
# raise SystemExit(Color.BOLD + 'This is not a RHEL-like system.' + Color.END
|
||||
# + '\n\nPlease verify you are running on a RHEL-like system that is '
|
||||
# 'not Fedora nor SuSE. This means that the %rhel macro will be '
|
||||
@ -49,7 +50,6 @@ for conf in glob.iglob('configs/*.yaml'):
|
||||
|
||||
# These will be set in their respective var files
|
||||
#REVISION = rlvars['revision'] + '-' + rlvars['rclvl']
|
||||
#rlvars = rldict[RLVER]
|
||||
#rlvars = rldict[RLMACRO]
|
||||
#COMPOSE_ROOT = "/mnt/compose/" + RLVER
|
||||
#rlvars = rldict[rlver]
|
||||
#rlvars = rldict[rlmacro]
|
||||
#COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp
|
||||
|
@ -18,9 +18,9 @@
|
||||
all_repos:
|
||||
- 'BaseOS'
|
||||
- 'AppStream'
|
||||
- 'PowerTools'
|
||||
- 'HighAvailability'
|
||||
- 'ResilientStorage'
|
||||
- 'PowerTools'
|
||||
- 'RT'
|
||||
- 'NFV'
|
||||
- 'extras'
|
||||
@ -35,9 +35,9 @@
|
||||
comps_or_groups:
|
||||
- 'BaseOS'
|
||||
- 'AppStream'
|
||||
- 'PowerTools'
|
||||
- 'HighAvailability'
|
||||
- 'ResilientStorage'
|
||||
- 'PowerTools'
|
||||
- 'RT'
|
||||
- 'NFV'
|
||||
has_modules:
|
||||
|
@ -22,9 +22,9 @@
|
||||
- 'all'
|
||||
- 'BaseOS'
|
||||
- 'AppStream'
|
||||
- 'CRB'
|
||||
- 'HighAvailability'
|
||||
- 'ResilientStorage'
|
||||
- 'CRB'
|
||||
- 'RT'
|
||||
- 'NFV'
|
||||
- 'SAP'
|
||||
@ -40,9 +40,9 @@
|
||||
comps_or_groups:
|
||||
- 'BaseOS'
|
||||
- 'AppStream'
|
||||
- 'CRB'
|
||||
- 'HighAvailability'
|
||||
- 'ResilientStorage'
|
||||
- 'CRB'
|
||||
- 'RT'
|
||||
- 'NFV'
|
||||
- 'SAP'
|
||||
|
@ -1,13 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from common import *
|
||||
import argparse
|
||||
from util import Checks
|
||||
from util import RepoSync
|
||||
|
||||
rlvars = rldict['9']
|
||||
r = Checks(rlvars, config['arch'])
|
||||
r.check_valid_arch()
|
||||
|
||||
a = RepoSync(rlvars, config, repo="BaseOS")
|
||||
print(a.sync())
|
18
iso/py/sync-from-peridot
Executable file
18
iso/py/sync-from-peridot
Executable file
@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# This is intended for doing "full" syncs, not periodic update syncs.
|
||||
|
||||
from common import *
|
||||
import argparse
|
||||
from util import Checks
|
||||
from util import RepoSync
|
||||
|
||||
rlvars = rldict['9']
|
||||
r = Checks(rlvars, config['arch'])
|
||||
r.check_valid_arch()
|
||||
|
||||
a = RepoSync(rlvars, config, major="9", repo="BaseOS")
|
||||
a.run()
|
||||
#a.generate_conf()
|
||||
#somedir = a.generate_compose_dirs()
|
||||
#print(a.sync())
|
@ -1,13 +1,21 @@
|
||||
HAS_LIBREPO = True
|
||||
"""
|
||||
Syncs yum repos for mirroring and composing.
|
||||
|
||||
Louis Abel <label AT rockylinux.org>
|
||||
"""
|
||||
#import shutil
|
||||
import logging
|
||||
import sys
|
||||
import os
|
||||
import os.path
|
||||
import pipes
|
||||
import shutil
|
||||
#import pipes
|
||||
from common import Color
|
||||
|
||||
try:
|
||||
import librepo
|
||||
except:
|
||||
HAS_LIBREPO = False
|
||||
#HAS_LIBREPO = True
|
||||
#try:
|
||||
# import librepo
|
||||
#except:
|
||||
# HAS_LIBREPO = False
|
||||
|
||||
class RepoSync:
|
||||
"""
|
||||
@ -15,13 +23,31 @@ class RepoSync:
|
||||
different class entirely. This is on purpose. Please use the SigRepoSync
|
||||
class for SIG syncs.
|
||||
"""
|
||||
def __init__(self, rlvars, config, repo=None, nofail: bool = False):
|
||||
def __init__(
|
||||
self,
|
||||
rlvars,
|
||||
config,
|
||||
major,
|
||||
repo=None,
|
||||
arch=None,
|
||||
ignore_debug=False,
|
||||
ignore_source=False,
|
||||
dryrun: bool = False,
|
||||
fullrun: bool = False,
|
||||
nofail: bool = False,
|
||||
logger=None
|
||||
):
|
||||
self.nofail = nofail
|
||||
self.dryrun = dryrun
|
||||
self.fullrun = fullrun
|
||||
self.arch = arch
|
||||
self.ignore_debug = ignore_debug
|
||||
self.ignore_source = ignore_source
|
||||
# Relevant config items
|
||||
self.major_version = config['rlmacro']
|
||||
self.major_version = major
|
||||
self.date_stamp = config['date_stamp']
|
||||
self.staging_dir = config['staging_root'] + config['category_stub'] + self.major_version
|
||||
self.repo_base_url = config['repo_base_url']
|
||||
self.compose_base = config['compose_root'] + "/" + major
|
||||
|
||||
# Relevant major version items
|
||||
self.revision = rlvars['revision'] + "-" + rlvars['rclvl']
|
||||
@ -31,14 +57,106 @@ class RepoSync:
|
||||
self.repos = rlvars['all_repos']
|
||||
self.repo = repo
|
||||
|
||||
self.staging_dir = os.path.join(
|
||||
config['staging_root'],
|
||||
config['category_stub'],
|
||||
self.revision
|
||||
)
|
||||
|
||||
self.compose_latest_dir = os.path.join(
|
||||
config['compose_root'],
|
||||
major,
|
||||
"latest-Rocky-{}".format(major)
|
||||
)
|
||||
|
||||
self.compose_latest_sync = os.path.join(
|
||||
self.compose_latest_dir,
|
||||
"compose"
|
||||
)
|
||||
|
||||
# This is temporary for now.
|
||||
if logger is None:
|
||||
self.log = logging.getLogger("reposync")
|
||||
self.log.setLevel(logging.INFO)
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setLevel(logging.INFO)
|
||||
formatter = logging.Formatter(
|
||||
'%(asctime)s :: %(name)s :: %(message)s',
|
||||
'%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
handler.setFormatter(formatter)
|
||||
self.log.addHandler(handler)
|
||||
|
||||
self.log.info('reposync init')
|
||||
self.log.info(self.revision)
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
This must be called to perform the sync. This will run through, create
|
||||
the configuration file as required, and try to do a sync of every repo
|
||||
applicable or the repo actually specified. If self.repo is None, it
|
||||
will be assumed all repos are synced as dictated by rlvars.
|
||||
|
||||
* Dry runs only create initial directories and structure
|
||||
* Full runs sync everything from the top and setup structure,
|
||||
including creating a symlink to latest-Rocky-X
|
||||
* self.repo is ignored during full runs (noted in stdout)
|
||||
* self.arch being set will force only that arch to sync
|
||||
"""
|
||||
if self.fullrun and self.repo:
|
||||
self.log.error('WARNING: repo ignored when doing a full sync')
|
||||
if self.fullrun and self.dryrun:
|
||||
self.log.error('A full and dry run is currently not supported.')
|
||||
raise SystemExit('\nA full and dry run is currently not supported.')
|
||||
|
||||
self.generate_conf()
|
||||
|
||||
if self.fullrun:
|
||||
sync_root = os.path.join(
|
||||
self.generate_compose_dirs(),
|
||||
'compose'
|
||||
)
|
||||
else:
|
||||
sync_root = self.compose_latest_sync
|
||||
|
||||
def sync(self, repo, sync_root, arch=None):
|
||||
"""
|
||||
Does the actual syncing of the repo. We generally sync each component
|
||||
of a repo:
|
||||
* each architecture
|
||||
* each architecture debug
|
||||
* each source
|
||||
"""
|
||||
# dnf reposync --download-metadata \
|
||||
# --repoid fedora -p /tmp/test \
|
||||
# --forcearch aarch64 --norepopath
|
||||
cmd = self.reposync_cmd()
|
||||
|
||||
def generate_compose_dirs(self) -> str:
|
||||
"""
|
||||
Generate compose dirs for full runs
|
||||
"""
|
||||
compose_base_dir = os.path.join(
|
||||
self.compose_base,
|
||||
"Rocky-{}-{}".format(self.major_version, self.date_stamp)
|
||||
)
|
||||
self.log.info('Creating compose directory %s' % compose_base_dir)
|
||||
if not os.path.exists(compose_base_dir):
|
||||
os.makedirs(compose_base_dir)
|
||||
|
||||
return compose_base_dir
|
||||
|
||||
def symlink_to_latest(self):
|
||||
"""
|
||||
Emulates pungi and symlinks latest-Rocky-X
|
||||
|
||||
This link will be what is updated in full runs. Whatever is in this
|
||||
'latest' directory is what is rsynced on to staging after completion.
|
||||
This link should not change often.
|
||||
"""
|
||||
pass
|
||||
|
||||
def sync(self):
|
||||
cmd = self.reposync_cmd()
|
||||
print(self.revision)
|
||||
|
||||
def generate_conf(self, dest_path: str, repo):
|
||||
def generate_conf(self, dest_path='/var/tmp'):
|
||||
"""
|
||||
Generates the necessary repo conf file for the operation. This repo
|
||||
file should be temporary in nature. This will generate a repo file
|
||||
@ -48,7 +166,62 @@ class RepoSync:
|
||||
:param dest_path: The destination where the temporary conf goes
|
||||
:param repo: The repo object to create a file for
|
||||
"""
|
||||
pass
|
||||
fname = os.path.join(
|
||||
dest_path,
|
||||
"{}-config.repo".format(self.major_version)
|
||||
)
|
||||
self.log.info('Generating the repo configuration: %s' % fname)
|
||||
|
||||
if self.repo_base_url.startswith("/"):
|
||||
self.log.error("Local file syncs are not supported.")
|
||||
raise SystemExit(Color.BOLD + "Local file syncs are not "
|
||||
"supported." + Color.END)
|
||||
|
||||
# create dest_path
|
||||
if not os.path.exists(dest_path):
|
||||
os.makedirs(dest_path, exist_ok=True)
|
||||
config_file = open(fname, "w+")
|
||||
for repo in self.repos:
|
||||
constructed_url = '{}/{}/repo/{}/$basearch'.format(
|
||||
self.repo_base_url,
|
||||
self.project_id,
|
||||
repo,
|
||||
)
|
||||
|
||||
constructed_url_debug = '{}/{}/repo/{}/$basearch-debug'.format(
|
||||
self.repo_base_url,
|
||||
self.project_id,
|
||||
repo,
|
||||
)
|
||||
|
||||
constructed_url_src = '{}/{}/repo/{}/src'.format(
|
||||
self.repo_base_url,
|
||||
self.project_id,
|
||||
repo,
|
||||
)
|
||||
|
||||
# normal
|
||||
config_file.write('[%s]\n' % repo)
|
||||
config_file.write('name=%s\n' % repo)
|
||||
config_file.write('baseurl=%s\n' % constructed_url)
|
||||
config_file.write("enabled=1\n")
|
||||
config_file.write("gpgcheck=0\n\n")
|
||||
|
||||
# debug
|
||||
config_file.write('[%s-debug]\n' % repo)
|
||||
config_file.write('name=%s debug\n' % repo)
|
||||
config_file.write('baseurl=%s\n' % constructed_url_debug)
|
||||
config_file.write("enabled=1\n")
|
||||
config_file.write("gpgcheck=0\n\n")
|
||||
|
||||
# src
|
||||
config_file.write('[%s-source]\n' % repo)
|
||||
config_file.write('name=%s source\n' % repo)
|
||||
config_file.write('baseurl=%s\n' % constructed_url_src)
|
||||
config_file.write("enabled=1\n")
|
||||
config_file.write("gpgcheck=0\n\n")
|
||||
|
||||
|
||||
|
||||
def reposync_cmd(self) -> str:
|
||||
"""
|
||||
@ -56,14 +229,22 @@ class RepoSync:
|
||||
itself and will raise an error.
|
||||
|
||||
:return: The path to the reposync command. If dnf exists, we'll use
|
||||
that.
|
||||
that. Otherwise, fail immediately.
|
||||
"""
|
||||
cmd = None
|
||||
if os.path.exists("/usr/bin/dnf"):
|
||||
cmd = "/usr/bin/dnf reposync"
|
||||
else:
|
||||
raise SystemExit("/usr/bin/dnf was not found. /usr/bin/reposync is "
|
||||
self.log.error('/usr/bin/dnf was not found. Good bye.')
|
||||
raise SystemExit("/usr/bin/dnf was not found. \n\n/usr/bin/reposync "
|
||||
"is not sufficient and you are likely running on an el7 "
|
||||
"system, which tells us that you made changes to these "
|
||||
"tools.")
|
||||
"system or a grossly modified EL8+ system, " + Color.BOLD +
|
||||
"which tells us that you probably made changes to these tools "
|
||||
"expecting them to work and got to this point." + Color.END)
|
||||
return cmd
|
||||
|
||||
class SigRepoSync:
|
||||
"""
|
||||
This helps us do reposync operations for SIG's. Do not use this for the
|
||||
base system. Use RepoSync for that.
|
||||
"""
|
||||
|
Loading…
Reference in New Issue
Block a user