sig_core
/
toolkit
7
2
Fork 1
Code Issues 5 Pull Requests Actions Projects Releases Wiki Activity

finish wrapper, start repoclosure

This commit is contained in:
Louis Abel 2022-05-25 21:39:26 -07:00
parent 550526f535
commit fc227c947e
Signed by: label
GPG Key ID: B37E62D143879B36
5 changed files with 121 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
iso/py/configs/el9.yaml
View File

@ -50,4 +50,31 @@
has_modules:
- 'AppStream'
- 'CRB'
repoclosure_map:
BaseOS: []
AppStream:
- BaseOS
CRB:
- BaseOS
- AppStream
HighAvailability:
- BaseOS
- AppStream
ResilientStorage:
- BaseOS
- AppStream
RT:
- BaseOS
- AppStream
NFV:
- BaseOS
- AppStream
SAP:
- BaseOS
- AppStream
- HighAvailability
SAPHANA:
- BaseOS
- AppStream
- HighAvailability
...

53
iso/py/sync-from-peridot
View File

@ -1,18 +1,57 @@
#!/usr/bin/env python3
# This is intended for doing "full" syncs, not periodic update syncs.
# This script can be called to do single syncs or full on syncs.
from common import *
import argparse
from common import *
from util import Checks
from util import RepoSync
rlvars = rldict['9']
#rlvars = rldict['9']
#r = Checks(rlvars, config['arch'])
#r.check_valid_arch()
# Start up the parser baby
parser = argparse.ArgumentParser(description="Peridot Sync and Compose")
# All of our options
parser.add_argument('--release', type=str, help="Major Release Version", required=True)
parser.add_argument('--repo', type=str, help="Repository name")
parser.add_argument('--arch', type=str, help="Architecture")
parser.add_argument('--ignore-debug', action='store_true')
parser.add_argument('--ignore-source', action='store_true')
parser.add_argument('--repoclosure', action='store_true')
parser.add_argument('--skip-all', action='store_true')
parser.add_argument('--dry-run', action='store_true')
parser.add_argument('--full-run', action='store_true')
parser.add_argument('--no-fail', action='store_true')
# I am aware this is confusing, I want podman to be the default option
parser.add_argument('--simple', action='store_false')
parser.add_argument('--logger', type=str)
# Parse them
results = parser.parse_args()
rlvars = rldict[results.release]
r = Checks(rlvars, config['arch'])
r.check_valid_arch()
a = RepoSync(rlvars, config, major="9", repo="ResilientStorage", parallel=True, ignore_debug=False, ignore_source=False)
# Send them and do whatever I guess
a = RepoSync(
rlvars,
config,
major=results.release,
repo=results.repo,
arch=results.arch,
ignore_debug=results.ignore_debug,
ignore_source=results.ignore_source,
repoclosure=results.repoclosure,
skip_all=results.skip_all,
parallel=results.simple,
dryrun=results.dry_run,
fullrun=results.full_run,
nofail=results.no_fail,
logger=results.logger
)
a.run()
#a.generate_conf()
#somedir = a.generate_compose_dirs()
#print(a.sync())

16
iso/py/sync-from-peridot-test Executable file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env python3
# This is a testing script to ensure the RepoSync class is working as intended.
from common import *
import argparse
from util import Checks
from util import RepoSync
rlvars = rldict['9']
r = Checks(rlvars, config['arch'])
r.check_valid_arch()
a = RepoSync(rlvars, config, major="9", repo="ResilientStorage", parallel=True, ignore_debug=False, ignore_source=False)
#a = RepoSync(rlvars, config, major="9", repo="ResilientStorage", parallel=True, ignore_debug=False, ignore_source=False, fullrun=True)
a.run()

39
iso/py/util/dnf_utils.py
View File

@ -34,10 +34,11 @@ class RepoSync:
major,
repo=None,
arch=None,
ignore_debug=False,
ignore_source=False,
skip_all=False,
parallel=False,
ignore_debug: bool = False,
ignore_source: bool = False,
repoclosure: bool = False,
skip_all: bool = False,
parallel: bool = False,
dryrun: bool = False,
fullrun: bool = False,
nofail: bool = False,
@ -50,6 +51,7 @@ class RepoSync:
self.ignore_debug = ignore_debug
self.ignore_source = ignore_source
self.skip_all = skip_all
self.repoclosure = repoclosure
# Enables podman syncing, which should effectively speed up operations
self.parallel = parallel
# Relevant config items
@ -74,6 +76,9 @@ class RepoSync:
if 'container' in rlvars and len(rlvars['container']) > 0:
self.container = rlvars['container']
if 'repoclosure_map' in rlvars and len(rlvars['repoclosure_map']) > 0:
self.repoclosure_map = rlvars['repoclosure_map']
self.staging_dir = os.path.join(
config['staging_root'],
config['category_stub'],
@ -153,6 +158,12 @@ class RepoSync:
)
sync_root = self.compose_latest_sync
# Verify if the link even exists
if not os.path.exists(self.compose_latest_dir):
self.log.error('!! Latest compose link is broken does not exist: %s' % self.compose_latest_dir)
self.log.error('!! Please perform a full run if you have not done so.')
raise SystemExit()
log_root = os.path.join(
work_root,
"logs"
@ -167,6 +178,9 @@ class RepoSync:
if self.fullrun:
self.symlink_to_latest()
if self.repoclosure:
self.repoclosure_work(sync_root, work_root, log_root)
def sync(self, repo, sync_root, work_root, log_root, arch=None):
"""
Calls out syncing of the repos. We generally sync each component of a
@ -369,13 +383,19 @@ class RepoSync:
for r in repos_to_sync:
entry_name_list = []
repo_name = r
arch_sync = arches_to_sync
if r in self.repo_renames:
repo_name = self.repo_renames[r]
for a in arches_to_sync:
# There should be a check here that if it's "all" and multilib
# is on, i686 should get synced too.
if 'all' in r and 'x86_64' in arches_to_sync and self.multilib:
arch_sync.append('i686')
# There should be a check here that if it's "all" and multilib
# is on, i686 should get synced too.
for a in arch_sync:
entry_name = '{}-{}'.format(r, a)
debug_entry_name = '{}-debug-{}'.format(r, a)
@ -471,7 +491,7 @@ class RepoSync:
"--download-metadata --repoid={}-source -p {} "
"--norepopath | tee -a {}/{}-source-{}.log").format(
self.dnf_config,
repo_name,
r,
source_sync_path,
log_root,
repo_name,
@ -704,6 +724,9 @@ class RepoSync:
)
return cmd
def repoclosure_work(self, sync_root, work_root, log_root):
pass
class SigRepoSync:
"""
This helps us do reposync operations for SIG's. Do not use this for the

2
sync/prep-staging-8.sh
View File

@ -44,7 +44,7 @@ for x in "${ARCHES[@]}"; do
test -d "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}"
ret_val=$?
if [ "$ret_val" -eq 0 ]; then
createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \
createrepo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \
"--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \
--workers 8
sed -i '/<open-size><\/open-size>/d' \