Final commit for 5/21
This commit is contained in:
parent
4f9bfd44a0
commit
420aa02bdb
@ -4,8 +4,8 @@
|
|||||||
|
|
||||||
* sync-variant-pungi
|
* sync-variant-pungi
|
||||||
* sync-variant-peridot
|
* sync-variant-peridot
|
||||||
* sync-all-pungi
|
* sync-from-pungi
|
||||||
* sync-all-peridot
|
* sync-from-peridot
|
||||||
* sync-sig
|
* sync-sig
|
||||||
* build-all-iso
|
* build-all-iso
|
||||||
* sign-repos-only
|
* sign-repos-only
|
||||||
|
@ -25,12 +25,13 @@ rldict = {}
|
|||||||
config = {
|
config = {
|
||||||
"rlmacro": rpm.expandMacro('%rhel'),
|
"rlmacro": rpm.expandMacro('%rhel'),
|
||||||
"arch": platform.machine(),
|
"arch": platform.machine(),
|
||||||
"date_stamp": time.strftime("%Y%m%d", time.localtime()),
|
"date_stamp": time.strftime("%Y%m%d.%H%M%S", time.localtime()),
|
||||||
|
"compose_root": "/mnt/compose",
|
||||||
"staging_root": "/mnt/repos-staging",
|
"staging_root": "/mnt/repos-staging",
|
||||||
"production_root": "/mnt/repos-production",
|
"production_root": "/mnt/repos-production",
|
||||||
"category_stub": "/mirror/pub/rocky",
|
"category_stub": "mirror/pub/rocky",
|
||||||
"sig_category_stub": "/mirror/pub/sig",
|
"sig_category_stub": "mirror/pub/sig",
|
||||||
"repo_base_url": "https://yumrepofs.build.resf.org/v1/projects/"
|
"repo_base_url": "https://yumrepofs.build.resf.org/v1/projects"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Importing the config from yaml
|
# Importing the config from yaml
|
||||||
@ -39,7 +40,7 @@ for conf in glob.iglob('configs/*.yaml'):
|
|||||||
rldict.update(yaml.safe_load(file))
|
rldict.update(yaml.safe_load(file))
|
||||||
|
|
||||||
# The system needs to be a RHEL-like system. It cannot be Fedora or SuSE.
|
# The system needs to be a RHEL-like system. It cannot be Fedora or SuSE.
|
||||||
#if "%rhel" in config['RLMACRO']:
|
#if "%rhel" in config['rlmacro']:
|
||||||
# raise SystemExit(Color.BOLD + 'This is not a RHEL-like system.' + Color.END
|
# raise SystemExit(Color.BOLD + 'This is not a RHEL-like system.' + Color.END
|
||||||
# + '\n\nPlease verify you are running on a RHEL-like system that is '
|
# + '\n\nPlease verify you are running on a RHEL-like system that is '
|
||||||
# 'not Fedora nor SuSE. This means that the %rhel macro will be '
|
# 'not Fedora nor SuSE. This means that the %rhel macro will be '
|
||||||
@ -49,7 +50,6 @@ for conf in glob.iglob('configs/*.yaml'):
|
|||||||
|
|
||||||
# These will be set in their respective var files
|
# These will be set in their respective var files
|
||||||
#REVISION = rlvars['revision'] + '-' + rlvars['rclvl']
|
#REVISION = rlvars['revision'] + '-' + rlvars['rclvl']
|
||||||
#rlvars = rldict[RLVER]
|
#rlvars = rldict[rlver]
|
||||||
#rlvars = rldict[RLMACRO]
|
#rlvars = rldict[rlmacro]
|
||||||
#COMPOSE_ROOT = "/mnt/compose/" + RLVER
|
|
||||||
#COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp
|
#COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp
|
||||||
|
@ -18,9 +18,9 @@
|
|||||||
all_repos:
|
all_repos:
|
||||||
- 'BaseOS'
|
- 'BaseOS'
|
||||||
- 'AppStream'
|
- 'AppStream'
|
||||||
|
- 'PowerTools'
|
||||||
- 'HighAvailability'
|
- 'HighAvailability'
|
||||||
- 'ResilientStorage'
|
- 'ResilientStorage'
|
||||||
- 'PowerTools'
|
|
||||||
- 'RT'
|
- 'RT'
|
||||||
- 'NFV'
|
- 'NFV'
|
||||||
- 'extras'
|
- 'extras'
|
||||||
@ -35,9 +35,9 @@
|
|||||||
comps_or_groups:
|
comps_or_groups:
|
||||||
- 'BaseOS'
|
- 'BaseOS'
|
||||||
- 'AppStream'
|
- 'AppStream'
|
||||||
|
- 'PowerTools'
|
||||||
- 'HighAvailability'
|
- 'HighAvailability'
|
||||||
- 'ResilientStorage'
|
- 'ResilientStorage'
|
||||||
- 'PowerTools'
|
|
||||||
- 'RT'
|
- 'RT'
|
||||||
- 'NFV'
|
- 'NFV'
|
||||||
has_modules:
|
has_modules:
|
||||||
|
@ -22,9 +22,9 @@
|
|||||||
- 'all'
|
- 'all'
|
||||||
- 'BaseOS'
|
- 'BaseOS'
|
||||||
- 'AppStream'
|
- 'AppStream'
|
||||||
|
- 'CRB'
|
||||||
- 'HighAvailability'
|
- 'HighAvailability'
|
||||||
- 'ResilientStorage'
|
- 'ResilientStorage'
|
||||||
- 'CRB'
|
|
||||||
- 'RT'
|
- 'RT'
|
||||||
- 'NFV'
|
- 'NFV'
|
||||||
- 'SAP'
|
- 'SAP'
|
||||||
@ -40,9 +40,9 @@
|
|||||||
comps_or_groups:
|
comps_or_groups:
|
||||||
- 'BaseOS'
|
- 'BaseOS'
|
||||||
- 'AppStream'
|
- 'AppStream'
|
||||||
|
- 'CRB'
|
||||||
- 'HighAvailability'
|
- 'HighAvailability'
|
||||||
- 'ResilientStorage'
|
- 'ResilientStorage'
|
||||||
- 'CRB'
|
|
||||||
- 'RT'
|
- 'RT'
|
||||||
- 'NFV'
|
- 'NFV'
|
||||||
- 'SAP'
|
- 'SAP'
|
||||||
|
@ -1,13 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
from common import *
|
|
||||||
import argparse
|
|
||||||
from util import Checks
|
|
||||||
from util import RepoSync
|
|
||||||
|
|
||||||
rlvars = rldict['9']
|
|
||||||
r = Checks(rlvars, config['arch'])
|
|
||||||
r.check_valid_arch()
|
|
||||||
|
|
||||||
a = RepoSync(rlvars, config, repo="BaseOS")
|
|
||||||
print(a.sync())
|
|
18
iso/py/sync-from-peridot
Executable file
18
iso/py/sync-from-peridot
Executable file
@ -0,0 +1,18 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
# This is intended for doing "full" syncs, not periodic update syncs.
|
||||||
|
|
||||||
|
from common import *
|
||||||
|
import argparse
|
||||||
|
from util import Checks
|
||||||
|
from util import RepoSync
|
||||||
|
|
||||||
|
rlvars = rldict['9']
|
||||||
|
r = Checks(rlvars, config['arch'])
|
||||||
|
r.check_valid_arch()
|
||||||
|
|
||||||
|
a = RepoSync(rlvars, config, major="9", repo="BaseOS")
|
||||||
|
a.run()
|
||||||
|
#a.generate_conf()
|
||||||
|
#somedir = a.generate_compose_dirs()
|
||||||
|
#print(a.sync())
|
@ -1,13 +1,21 @@
|
|||||||
HAS_LIBREPO = True
|
"""
|
||||||
|
Syncs yum repos for mirroring and composing.
|
||||||
|
|
||||||
|
Louis Abel <label AT rockylinux.org>
|
||||||
|
"""
|
||||||
|
#import shutil
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
import os
|
import os
|
||||||
import os.path
|
import os.path
|
||||||
import pipes
|
#import pipes
|
||||||
import shutil
|
from common import Color
|
||||||
|
|
||||||
try:
|
#HAS_LIBREPO = True
|
||||||
import librepo
|
#try:
|
||||||
except:
|
# import librepo
|
||||||
HAS_LIBREPO = False
|
#except:
|
||||||
|
# HAS_LIBREPO = False
|
||||||
|
|
||||||
class RepoSync:
|
class RepoSync:
|
||||||
"""
|
"""
|
||||||
@ -15,13 +23,31 @@ class RepoSync:
|
|||||||
different class entirely. This is on purpose. Please use the SigRepoSync
|
different class entirely. This is on purpose. Please use the SigRepoSync
|
||||||
class for SIG syncs.
|
class for SIG syncs.
|
||||||
"""
|
"""
|
||||||
def __init__(self, rlvars, config, repo=None, nofail: bool = False):
|
def __init__(
|
||||||
|
self,
|
||||||
|
rlvars,
|
||||||
|
config,
|
||||||
|
major,
|
||||||
|
repo=None,
|
||||||
|
arch=None,
|
||||||
|
ignore_debug=False,
|
||||||
|
ignore_source=False,
|
||||||
|
dryrun: bool = False,
|
||||||
|
fullrun: bool = False,
|
||||||
|
nofail: bool = False,
|
||||||
|
logger=None
|
||||||
|
):
|
||||||
self.nofail = nofail
|
self.nofail = nofail
|
||||||
|
self.dryrun = dryrun
|
||||||
|
self.fullrun = fullrun
|
||||||
|
self.arch = arch
|
||||||
|
self.ignore_debug = ignore_debug
|
||||||
|
self.ignore_source = ignore_source
|
||||||
# Relevant config items
|
# Relevant config items
|
||||||
self.major_version = config['rlmacro']
|
self.major_version = major
|
||||||
self.date_stamp = config['date_stamp']
|
self.date_stamp = config['date_stamp']
|
||||||
self.staging_dir = config['staging_root'] + config['category_stub'] + self.major_version
|
|
||||||
self.repo_base_url = config['repo_base_url']
|
self.repo_base_url = config['repo_base_url']
|
||||||
|
self.compose_base = config['compose_root'] + "/" + major
|
||||||
|
|
||||||
# Relevant major version items
|
# Relevant major version items
|
||||||
self.revision = rlvars['revision'] + "-" + rlvars['rclvl']
|
self.revision = rlvars['revision'] + "-" + rlvars['rclvl']
|
||||||
@ -31,14 +57,106 @@ class RepoSync:
|
|||||||
self.repos = rlvars['all_repos']
|
self.repos = rlvars['all_repos']
|
||||||
self.repo = repo
|
self.repo = repo
|
||||||
|
|
||||||
|
self.staging_dir = os.path.join(
|
||||||
|
config['staging_root'],
|
||||||
|
config['category_stub'],
|
||||||
|
self.revision
|
||||||
|
)
|
||||||
|
|
||||||
|
self.compose_latest_dir = os.path.join(
|
||||||
|
config['compose_root'],
|
||||||
|
major,
|
||||||
|
"latest-Rocky-{}".format(major)
|
||||||
|
)
|
||||||
|
|
||||||
|
self.compose_latest_sync = os.path.join(
|
||||||
|
self.compose_latest_dir,
|
||||||
|
"compose"
|
||||||
|
)
|
||||||
|
|
||||||
|
# This is temporary for now.
|
||||||
|
if logger is None:
|
||||||
|
self.log = logging.getLogger("reposync")
|
||||||
|
self.log.setLevel(logging.INFO)
|
||||||
|
handler = logging.StreamHandler(sys.stdout)
|
||||||
|
handler.setLevel(logging.INFO)
|
||||||
|
formatter = logging.Formatter(
|
||||||
|
'%(asctime)s :: %(name)s :: %(message)s',
|
||||||
|
'%Y-%m-%d %H:%M:%S'
|
||||||
|
)
|
||||||
|
handler.setFormatter(formatter)
|
||||||
|
self.log.addHandler(handler)
|
||||||
|
|
||||||
|
self.log.info('reposync init')
|
||||||
|
self.log.info(self.revision)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
|
"""
|
||||||
|
This must be called to perform the sync. This will run through, create
|
||||||
|
the configuration file as required, and try to do a sync of every repo
|
||||||
|
applicable or the repo actually specified. If self.repo is None, it
|
||||||
|
will be assumed all repos are synced as dictated by rlvars.
|
||||||
|
|
||||||
|
* Dry runs only create initial directories and structure
|
||||||
|
* Full runs sync everything from the top and setup structure,
|
||||||
|
including creating a symlink to latest-Rocky-X
|
||||||
|
* self.repo is ignored during full runs (noted in stdout)
|
||||||
|
* self.arch being set will force only that arch to sync
|
||||||
|
"""
|
||||||
|
if self.fullrun and self.repo:
|
||||||
|
self.log.error('WARNING: repo ignored when doing a full sync')
|
||||||
|
if self.fullrun and self.dryrun:
|
||||||
|
self.log.error('A full and dry run is currently not supported.')
|
||||||
|
raise SystemExit('\nA full and dry run is currently not supported.')
|
||||||
|
|
||||||
|
self.generate_conf()
|
||||||
|
|
||||||
|
if self.fullrun:
|
||||||
|
sync_root = os.path.join(
|
||||||
|
self.generate_compose_dirs(),
|
||||||
|
'compose'
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
sync_root = self.compose_latest_sync
|
||||||
|
|
||||||
|
def sync(self, repo, sync_root, arch=None):
|
||||||
|
"""
|
||||||
|
Does the actual syncing of the repo. We generally sync each component
|
||||||
|
of a repo:
|
||||||
|
* each architecture
|
||||||
|
* each architecture debug
|
||||||
|
* each source
|
||||||
|
"""
|
||||||
|
# dnf reposync --download-metadata \
|
||||||
|
# --repoid fedora -p /tmp/test \
|
||||||
|
# --forcearch aarch64 --norepopath
|
||||||
|
cmd = self.reposync_cmd()
|
||||||
|
|
||||||
|
def generate_compose_dirs(self) -> str:
|
||||||
|
"""
|
||||||
|
Generate compose dirs for full runs
|
||||||
|
"""
|
||||||
|
compose_base_dir = os.path.join(
|
||||||
|
self.compose_base,
|
||||||
|
"Rocky-{}-{}".format(self.major_version, self.date_stamp)
|
||||||
|
)
|
||||||
|
self.log.info('Creating compose directory %s' % compose_base_dir)
|
||||||
|
if not os.path.exists(compose_base_dir):
|
||||||
|
os.makedirs(compose_base_dir)
|
||||||
|
|
||||||
|
return compose_base_dir
|
||||||
|
|
||||||
|
def symlink_to_latest(self):
|
||||||
|
"""
|
||||||
|
Emulates pungi and symlinks latest-Rocky-X
|
||||||
|
|
||||||
|
This link will be what is updated in full runs. Whatever is in this
|
||||||
|
'latest' directory is what is rsynced on to staging after completion.
|
||||||
|
This link should not change often.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def sync(self):
|
def generate_conf(self, dest_path='/var/tmp'):
|
||||||
cmd = self.reposync_cmd()
|
|
||||||
print(self.revision)
|
|
||||||
|
|
||||||
def generate_conf(self, dest_path: str, repo):
|
|
||||||
"""
|
"""
|
||||||
Generates the necessary repo conf file for the operation. This repo
|
Generates the necessary repo conf file for the operation. This repo
|
||||||
file should be temporary in nature. This will generate a repo file
|
file should be temporary in nature. This will generate a repo file
|
||||||
@ -48,7 +166,62 @@ class RepoSync:
|
|||||||
:param dest_path: The destination where the temporary conf goes
|
:param dest_path: The destination where the temporary conf goes
|
||||||
:param repo: The repo object to create a file for
|
:param repo: The repo object to create a file for
|
||||||
"""
|
"""
|
||||||
pass
|
fname = os.path.join(
|
||||||
|
dest_path,
|
||||||
|
"{}-config.repo".format(self.major_version)
|
||||||
|
)
|
||||||
|
self.log.info('Generating the repo configuration: %s' % fname)
|
||||||
|
|
||||||
|
if self.repo_base_url.startswith("/"):
|
||||||
|
self.log.error("Local file syncs are not supported.")
|
||||||
|
raise SystemExit(Color.BOLD + "Local file syncs are not "
|
||||||
|
"supported." + Color.END)
|
||||||
|
|
||||||
|
# create dest_path
|
||||||
|
if not os.path.exists(dest_path):
|
||||||
|
os.makedirs(dest_path, exist_ok=True)
|
||||||
|
config_file = open(fname, "w+")
|
||||||
|
for repo in self.repos:
|
||||||
|
constructed_url = '{}/{}/repo/{}/$basearch'.format(
|
||||||
|
self.repo_base_url,
|
||||||
|
self.project_id,
|
||||||
|
repo,
|
||||||
|
)
|
||||||
|
|
||||||
|
constructed_url_debug = '{}/{}/repo/{}/$basearch-debug'.format(
|
||||||
|
self.repo_base_url,
|
||||||
|
self.project_id,
|
||||||
|
repo,
|
||||||
|
)
|
||||||
|
|
||||||
|
constructed_url_src = '{}/{}/repo/{}/src'.format(
|
||||||
|
self.repo_base_url,
|
||||||
|
self.project_id,
|
||||||
|
repo,
|
||||||
|
)
|
||||||
|
|
||||||
|
# normal
|
||||||
|
config_file.write('[%s]\n' % repo)
|
||||||
|
config_file.write('name=%s\n' % repo)
|
||||||
|
config_file.write('baseurl=%s\n' % constructed_url)
|
||||||
|
config_file.write("enabled=1\n")
|
||||||
|
config_file.write("gpgcheck=0\n\n")
|
||||||
|
|
||||||
|
# debug
|
||||||
|
config_file.write('[%s-debug]\n' % repo)
|
||||||
|
config_file.write('name=%s debug\n' % repo)
|
||||||
|
config_file.write('baseurl=%s\n' % constructed_url_debug)
|
||||||
|
config_file.write("enabled=1\n")
|
||||||
|
config_file.write("gpgcheck=0\n\n")
|
||||||
|
|
||||||
|
# src
|
||||||
|
config_file.write('[%s-source]\n' % repo)
|
||||||
|
config_file.write('name=%s source\n' % repo)
|
||||||
|
config_file.write('baseurl=%s\n' % constructed_url_src)
|
||||||
|
config_file.write("enabled=1\n")
|
||||||
|
config_file.write("gpgcheck=0\n\n")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def reposync_cmd(self) -> str:
|
def reposync_cmd(self) -> str:
|
||||||
"""
|
"""
|
||||||
@ -56,14 +229,22 @@ class RepoSync:
|
|||||||
itself and will raise an error.
|
itself and will raise an error.
|
||||||
|
|
||||||
:return: The path to the reposync command. If dnf exists, we'll use
|
:return: The path to the reposync command. If dnf exists, we'll use
|
||||||
that.
|
that. Otherwise, fail immediately.
|
||||||
"""
|
"""
|
||||||
cmd = None
|
cmd = None
|
||||||
if os.path.exists("/usr/bin/dnf"):
|
if os.path.exists("/usr/bin/dnf"):
|
||||||
cmd = "/usr/bin/dnf reposync"
|
cmd = "/usr/bin/dnf reposync"
|
||||||
else:
|
else:
|
||||||
raise SystemExit("/usr/bin/dnf was not found. /usr/bin/reposync is "
|
self.log.error('/usr/bin/dnf was not found. Good bye.')
|
||||||
|
raise SystemExit("/usr/bin/dnf was not found. \n\n/usr/bin/reposync "
|
||||||
"is not sufficient and you are likely running on an el7 "
|
"is not sufficient and you are likely running on an el7 "
|
||||||
"system, which tells us that you made changes to these "
|
"system or a grossly modified EL8+ system, " + Color.BOLD +
|
||||||
"tools.")
|
"which tells us that you probably made changes to these tools "
|
||||||
|
"expecting them to work and got to this point." + Color.END)
|
||||||
return cmd
|
return cmd
|
||||||
|
|
||||||
|
class SigRepoSync:
|
||||||
|
"""
|
||||||
|
This helps us do reposync operations for SIG's. Do not use this for the
|
||||||
|
base system. Use RepoSync for that.
|
||||||
|
"""
|
||||||
|
Loading…
Reference in New Issue
Block a user