Final commit for 5/22
Finish regular reposync functionality, leave notes about i686 Start podman reposync functionality
This commit is contained in:
parent
56ae6fc4dc
commit
d845b95e80
GPG Key ID:
B37E62D143879B36
@ -1,5 +1,11 @@
|
||||
# iso
|
||||
|
||||
## TODO
|
||||
|
||||
Verbose mode should exist to output everything that's being called or ran.
|
||||
|
||||
There should be additional logging regardless, not just to stdout, but also to a file.
|
||||
|
||||
## scripts
|
||||
|
||||
* sync-variant-pungi
|
||||
|
||||
@ -31,7 +31,8 @@ config = {
|
||||
"production_root": "/mnt/repos-production",
|
||||
"category_stub": "mirror/pub/rocky",
|
||||
"sig_category_stub": "mirror/pub/sig",
|
||||
"repo_base_url": "https://yumrepofs.build.resf.org/v1/projects"
|
||||
"repo_base_url": "https://yumrepofs.build.resf.org/v1/projects",
|
||||
"container": "centos:stream9"
|
||||
}
|
||||
|
||||
# Importing the config from yaml
|
||||
|
||||
@ -11,7 +11,7 @@ rlvars = rldict['9']
|
||||
r = Checks(rlvars, config['arch'])
|
||||
r.check_valid_arch()
|
||||
|
||||
a = RepoSync(rlvars, config, major="9", repo="BaseOS")
|
||||
a = RepoSync(rlvars, config, major="9", repo="ResilientStorage", parallel=False, ignore_debug=True, ignore_source=True)
|
||||
a.run()
|
||||
#a.generate_conf()
|
||||
#somedir = a.generate_compose_dirs()
|
||||
|
||||
@ -8,6 +8,8 @@ import logging
|
||||
import sys
|
||||
import os
|
||||
import os.path
|
||||
import subprocess
|
||||
import shlex
|
||||
#import pipes
|
||||
from common import Color
|
||||
|
||||
@ -32,6 +34,7 @@ class RepoSync:
|
||||
arch=None,
|
||||
ignore_debug=False,
|
||||
ignore_source=False,
|
||||
parallel=False,
|
||||
dryrun: bool = False,
|
||||
fullrun: bool = False,
|
||||
nofail: bool = False,
|
||||
@ -43,6 +46,8 @@ class RepoSync:
|
||||
self.arch = arch
|
||||
self.ignore_debug = ignore_debug
|
||||
self.ignore_source = ignore_source
|
||||
# Enables podman syncing, which should effectively speed up operations
|
||||
self.parallel = parallel
|
||||
# Relevant config items
|
||||
self.major_version = major
|
||||
self.date_stamp = config['date_stamp']
|
||||
@ -55,6 +60,7 @@ class RepoSync:
|
||||
self.project_id = rlvars['project_id']
|
||||
self.repo_renames = rlvars['renames']
|
||||
self.repos = rlvars['all_repos']
|
||||
self.multilib = rlvars['provide_multilib']
|
||||
self.repo = repo
|
||||
|
||||
self.staging_dir = os.path.join(
|
||||
@ -89,6 +95,8 @@ class RepoSync:
|
||||
|
||||
self.log.info('reposync init')
|
||||
self.log.info(self.revision)
|
||||
self.dnf_config = self.generate_conf()
|
||||
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
@ -109,28 +117,194 @@ class RepoSync:
|
||||
self.log.error('A full and dry run is currently not supported.')
|
||||
raise SystemExit('\nA full and dry run is currently not supported.')
|
||||
|
||||
self.generate_conf()
|
||||
|
||||
# This should create the initial compose dir and set the path.
|
||||
# Otherwise, just use the latest link.
|
||||
if self.fullrun:
|
||||
sync_root = os.path.join(
|
||||
self.generate_compose_dirs(),
|
||||
'compose'
|
||||
)
|
||||
else:
|
||||
# Put in a verification here.
|
||||
sync_root = self.compose_latest_sync
|
||||
|
||||
self.sync(self.repo, sync_root, self.arch)
|
||||
|
||||
if self.fullrun:
|
||||
self.symlink_to_latest()
|
||||
|
||||
def sync(self, repo, sync_root, arch=None):
|
||||
"""
|
||||
Does the actual syncing of the repo. We generally sync each component
|
||||
of a repo:
|
||||
Calls out syncing of the repos. We generally sync each component of a
|
||||
repo:
|
||||
* each architecture
|
||||
* each architecture debug
|
||||
* each source
|
||||
|
||||
If parallel is true, we will run in podman.
|
||||
"""
|
||||
if self.parallel:
|
||||
self.podman_sync(repo, sync_root, arch)
|
||||
else:
|
||||
self.dnf_sync(repo, sync_root, arch)
|
||||
|
||||
def dnf_sync(self, repo, sync_root, arch):
|
||||
"""
|
||||
This is for normal dnf syncs
|
||||
"""
|
||||
cmd = self.reposync_cmd()
|
||||
|
||||
sync_single_arch = False
|
||||
arches_to_sync = self.arches
|
||||
if arch:
|
||||
sync_single_arch = True
|
||||
arches_to_sync = [arch]
|
||||
|
||||
sync_single_repo = False
|
||||
repos_to_sync = self.repos
|
||||
if repo and not self.fullrun:
|
||||
sync_single_repo = True
|
||||
repos_to_sync = [repo]
|
||||
|
||||
# dnf reposync --download-metadata \
|
||||
# --repoid fedora -p /tmp/test \
|
||||
# --forcearch aarch64 --norepopath
|
||||
cmd = self.reposync_cmd()
|
||||
|
||||
self.log.info(
|
||||
Color.BOLD + '!! WARNING !! ' + Color.END + 'You are performing a '
|
||||
'local reposync, which may incur delays in your compose.'
|
||||
)
|
||||
|
||||
if self.fullrun:
|
||||
self.log.info(
|
||||
Color.BOLD + '!! WARNING !! ' + Color.END + 'This is a full '
|
||||
'run! This will take a LONG TIME.'
|
||||
)
|
||||
|
||||
for r in repos_to_sync:
|
||||
for a in arches_to_sync:
|
||||
repo_name = r
|
||||
if r in self.repo_renames:
|
||||
repo_name = self.repo_renames[r]
|
||||
|
||||
os_sync_path = os.path.join(
|
||||
sync_root,
|
||||
repo_name,
|
||||
a,
|
||||
'os'
|
||||
)
|
||||
|
||||
debug_sync_path = os.path.join(
|
||||
sync_root,
|
||||
repo_name,
|
||||
a,
|
||||
'debug/tree'
|
||||
)
|
||||
|
||||
sync_cmd = "{} -c {} --download-metadata --repoid={} -p {} --forcearch {} --norepopath".format(
|
||||
cmd,
|
||||
self.dnf_config,
|
||||
r,
|
||||
os_sync_path,
|
||||
a
|
||||
)
|
||||
|
||||
debug_sync_cmd = "{} -c {} --download-metadata --repoid={}-debug -p {} --forcearch {} --norepopath".format(
|
||||
cmd,
|
||||
self.dnf_config,
|
||||
r,
|
||||
debug_sync_path,
|
||||
a
|
||||
)
|
||||
|
||||
self.log.info('Syncing {} {}'.format(r, a))
|
||||
#self.log.info(sync_cmd)
|
||||
# Try to figure out where to send the actual output of this...
|
||||
# Also consider on running a try/except here? Basically if
|
||||
# something happens (like a repo doesn't exist for some arch,
|
||||
# eg RT for aarch64), make a note of it somehow (but don't
|
||||
# break the entire sync). As it stands with this
|
||||
# implementation, if something fails, it just continues on.
|
||||
process = subprocess.call(
|
||||
shlex.split(sync_cmd),
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL
|
||||
)
|
||||
|
||||
if not self.ignore_debug:
|
||||
self.log.info('Syncing {} {} (debug)'.format(r, a))
|
||||
process_debug = subprocess.call(
|
||||
shlex.split(debug_sync_cmd),
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL
|
||||
)
|
||||
|
||||
# There should be a check here that if it's "all" and multilib
|
||||
# is on, i686 should get synced too.
|
||||
|
||||
if not self.ignore_source:
|
||||
source_sync_path = os.path.join(
|
||||
sync_root,
|
||||
repo_name,
|
||||
'source/tree'
|
||||
)
|
||||
|
||||
source_sync_cmd = "{} -c {} --download-metadata --repoid={}-source -p {} --norepopath".format(
|
||||
cmd,
|
||||
self.dnf_config,
|
||||
r,
|
||||
source_sync_path
|
||||
)
|
||||
|
||||
|
||||
self.log.info('Syncing {} source'.format(r))
|
||||
process_source = subprocess.call(
|
||||
shlex.split(source_sync_cmd),
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL
|
||||
)
|
||||
|
||||
self.log.info('Syncing complete')
|
||||
|
||||
def podman_sync(self, repo, sync_root, arch):
|
||||
"""
|
||||
This is for podman syncs
|
||||
|
||||
Create sync_root/work/entries
|
||||
Generate scripts as needed into dir
|
||||
Each container runs their own script
|
||||
wait till all is finished
|
||||
"""
|
||||
cmd = self.podman_cmd()
|
||||
contrunlist = []
|
||||
self.log.info('Generating container entries')
|
||||
entries_dir = os.path.join(sync_root, "work", "entries")
|
||||
if not os.path.exists(entries_dir):
|
||||
os.makedirs(entries_dir, exist_ok=True)
|
||||
|
||||
sync_single_arch = False
|
||||
arches_to_sync = self.arches
|
||||
if arch:
|
||||
sync_single_arch = True
|
||||
arches_to_sync = [arch]
|
||||
|
||||
sync_single_repo = False
|
||||
repos_to_sync = self.repos
|
||||
if repo and not self.fullrun:
|
||||
sync_single_repo = True
|
||||
repos_to_sync = [repo]
|
||||
|
||||
for r in repos_to_sync:
|
||||
for a in arches_to_sync:
|
||||
repo_name = r
|
||||
if r in self.repo_renames:
|
||||
repo_name = self.repo_renames[r]
|
||||
|
||||
# There should be a check here that if it's "all" and multilib
|
||||
# is on, i686 should get synced too.
|
||||
|
||||
entry_name = '{}-{}'.format(r, a)
|
||||
debug_entry_name = '{}-debug-{}'.format(r, a)
|
||||
|
||||
def generate_compose_dirs(self) -> str:
|
||||
"""
|
||||
@ -156,7 +330,7 @@ class RepoSync:
|
||||
"""
|
||||
pass
|
||||
|
||||
def generate_conf(self, dest_path='/var/tmp'):
|
||||
def generate_conf(self, dest_path='/var/tmp') -> str:
|
||||
"""
|
||||
Generates the necessary repo conf file for the operation. This repo
|
||||
file should be temporary in nature. This will generate a repo file
|
||||
@ -221,7 +395,7 @@ class RepoSync:
|
||||
config_file.write("enabled=1\n")
|
||||
config_file.write("gpgcheck=0\n\n")
|
||||
|
||||
|
||||
return fname
|
||||
|
||||
def reposync_cmd(self) -> str:
|
||||
"""
|
||||
@ -243,6 +417,24 @@ class RepoSync:
|
||||
"expecting them to work and got to this point." + Color.END)
|
||||
return cmd
|
||||
|
||||
def podman_cmd(self) -> str:
|
||||
"""
|
||||
This generates the podman run command. This is in the case that we want
|
||||
to do reposyncs in parallel as we cannot reasonably run multiple
|
||||
instances of dnf reposync on a single system.
|
||||
"""
|
||||
cmd = None
|
||||
if os.path.exists("/usr/bin/podman"):
|
||||
cmd = "/usr/bin/podman run"
|
||||
else:
|
||||
self.log.error('/usr/bin/podman was not found. Good bye.')
|
||||
raise SystemExit("\n\n/usr/bin/podman was not found.\n\nPlease "
|
||||
" ensure that you have installed the necessary packages on "
|
||||
" this system. " + Color.BOLD + "Note that docker is not "
|
||||
"supported." + Color.END
|
||||
)
|
||||
return cmd
|
||||
|
||||
class SigRepoSync:
|
||||
"""
|
||||
This helps us do reposync operations for SIG's. Do not use this for the
|
||||
|
||||
Loading…
Reference in New Issue
Block a user