sig_core/toolkit
sig_core/toolkit
7
2
Fork 3
Code Issues 6 Pull Requests 2 Actions Projects Releases Wiki Activity

Simplify colors and start fleshing out cloud image pulls

Browse Source
This commit is contained in:
Louis Abel 2022-07-05 14:47:38 -07:00
parent a7cf5db050
commit 6946b737fc
Signed by: label
GPG Key ID: B37E62D143879B36
5 changed files with 183 additions and 298 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
iso/empanadas/empanadas/common.py
View File

@ -38,6 +38,10 @@ class Color:
UNDERLINE = '\033[4m' UNDERLINE = '\033[4m'
BOLD = '\033[1m' BOLD = '\033[1m'
END = '\033[0m' END = '\033[0m'
INFO = '[' + BOLD + GREEN + 'INFO' + END + '] '
WARN = '[' + BOLD + YELLOW + 'WARN' + END + '] '
FAIL = '[' + BOLD + RED + 'FAIL' + END + '] '
STAT = '[' + BOLD + CYAN + 'STAT' + END + '] '
# vars and additional checks # vars and additional checks
rldict = AttributeDict() rldict = AttributeDict()

3
iso/empanadas/empanadas/configs/el9.yaml
View File

@ -72,6 +72,9 @@
- 'lorax-templates-rhel' - 'lorax-templates-rhel'
- 'lorax-templates-generic' - 'lorax-templates-generic'
- 'xorriso' - 'xorriso'
cloudimages:
- EC2
- GenericCloud
repoclosure_map: repoclosure_map:
arches: arches:
x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch' x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'

233
iso/empanadas/empanadas/util/dnf_utils.py
View File

@ -235,10 +235,7 @@ class RepoSync:
raise SystemExit() raise SystemExit()
if self.fullrun and self.refresh_extra_files: if self.fullrun and self.refresh_extra_files:
self.log.warn( self.log.warn(Color.WARN + 'A full run implies extra files are also deployed.')
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
'A full run implies extra files are also deployed.'
)
self.sync(self.repo, sync_root, work_root, log_root, global_work_root, self.arch) self.sync(self.repo, sync_root, work_root, log_root, global_work_root, self.arch)
@ -575,10 +572,7 @@ class RepoSync:
join_all_pods = ' '.join(entry_name_list) join_all_pods = ' '.join(entry_name_list)
time.sleep(3) time.sleep(3)
self.log.info( self.log.info(Color.INFO + 'Syncing ' + r + ' ...')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Syncing ' + r + ' ...'
)
pod_watcher = '{} wait {}'.format( pod_watcher = '{} wait {}'.format(
cmd, cmd,
join_all_pods join_all_pods
@ -608,9 +602,7 @@ class RepoSync:
output, errors = podcheck.communicate() output, errors = podcheck.communicate()
if 'Exited (0)' not in output.decode(): if 'Exited (0)' not in output.decode():
self.log.error( self.log.error(Color.FAIL + pod)
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + pod
)
bad_exit_list.append(pod) bad_exit_list.append(pod)
rmcmd = '{} rm {}'.format( rmcmd = '{} rm {}'.format(
@ -626,10 +618,7 @@ class RepoSync:
) )
entry_name_list.clear() entry_name_list.clear()
self.log.info( self.log.info(Color.INFO + 'Syncing ' + r + ' completed')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Syncing ' + r + ' completed'
)
if len(bad_exit_list) > 0: if len(bad_exit_list) > 0:
self.log.error( self.log.error(
@ -849,9 +838,7 @@ class RepoSync:
output, errors = podcheck.communicate() output, errors = podcheck.communicate()
if 'Exited (0)' not in output.decode(): if 'Exited (0)' not in output.decode():
self.log.error( self.log.error(Color.FAIL + pod)
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + pod
)
bad_exit_list.append(pod) bad_exit_list.append(pod)
rmcmd = '{} rm {}'.format( rmcmd = '{} rm {}'.format(
@ -885,10 +872,7 @@ class RepoSync:
might also deploy COMPOSE_ID and maybe in the future a metadata dir with might also deploy COMPOSE_ID and maybe in the future a metadata dir with
a bunch of compose-esque stuff. a bunch of compose-esque stuff.
""" """
self.log.info( self.log.info(Color.INFO + 'Deploying treeinfo, discinfo, and media.repo')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Deploying treeinfo, discinfo, and media.repo'
)
cmd = Shared.git_cmd(self.log) cmd = Shared.git_cmd(self.log)
tmpclone = '/tmp/clone' tmpclone = '/tmp/clone'
@ -919,10 +903,7 @@ class RepoSync:
stderr=subprocess.DEVNULL stderr=subprocess.DEVNULL
) )
self.log.info( self.log.info(Color.INFO + 'Deploying extra files to work and metadata directories ...')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Deploying extra files to work and metadata directories ...'
)
# Copy files to work root # Copy files to work root
for extra in self.extra_files['list']: for extra in self.extra_files['list']:
@ -934,18 +915,13 @@ class RepoSync:
shutil.copy2(src, extra_files_dir) shutil.copy2(src, extra_files_dir)
shutil.copy2(src, metadata_dir) shutil.copy2(src, metadata_dir)
except: except:
self.log.warn( self.log.warn(Color.WARN + 'Extra file not copied: ' + src)
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
'Extra file not copied: ' + src
)
try: try:
shutil.rmtree(tmpclone) shutil.rmtree(tmpclone)
except OSError as e: except OSError as e:
self.log.error( self.log.error(Color.FAIL + 'Directory ' + tmpclone +
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + ' could not be removed: ' + e.strerror
'Directory ' + tmpclone + ' could not be removed: ' +
e.strerror
) )
def deploy_metadata(self, sync_root): def deploy_metadata(self, sync_root):
@ -954,10 +930,7 @@ class RepoSync:
will be close to how pungi produces it, but it won't be exact nor a will be close to how pungi produces it, but it won't be exact nor a
perfect replica. perfect replica.
""" """
self.log.info( self.log.info(Color.INFO + 'Deploying metadata for this compose')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Deploying metadata for this compose'
)
# Create metadata here # Create metadata here
# Create COMPOSE_ID here (this doesn't necessarily match anything, it's # Create COMPOSE_ID here (this doesn't necessarily match anything, it's
# just an indicator) # just an indicator)
@ -984,10 +957,7 @@ class RepoSync:
metadata_dir + '/metadata' metadata_dir + '/metadata'
) )
self.log.info( self.log.info(Color.INFO + 'Metadata files phase completed.')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Metadata files phase completed.'
)
# Deploy README to metadata directory # Deploy README to metadata directory
readme_template = self.tmplenv.get_template('README.tmpl') readme_template = self.tmplenv.get_template('README.tmpl')
@ -1007,10 +977,7 @@ class RepoSync:
overwritten by our ISO process, which is fine. If there is a treeinfo overwritten by our ISO process, which is fine. If there is a treeinfo
found, it will be skipped. found, it will be skipped.
""" """
self.log.info( self.log.info(Color.INFO + 'Deploying treeinfo, discinfo, and media.repo')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Deploying treeinfo, discinfo, and media.repo'
)
arches_to_tree = self.arches arches_to_tree = self.arches
if arch: if arch:
@ -1093,16 +1060,12 @@ class RepoSync:
repo_name repo_name
) )
except Exception as e: except Exception as e:
self.log.error( self.log.error(Color.FAIL + repo_name + ' ' +
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + a + ' os .treeinfo could not be written'
repo_name + ' ' + a + ' os .treeinfo could not be written'
) )
self.log.error(e) self.log.error(e)
else: else:
self.log.warn( self.log.warn(Color.WARN + repo_name + ' ' + a + ' os .treeinfo already exists')
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
repo_name + ' ' + a + ' os .treeinfo already exists'
)
if not os.path.exists(os_disc_path): if not os.path.exists(os_disc_path):
try: try:
@ -1113,15 +1076,13 @@ class RepoSync:
os_disc_path os_disc_path
) )
except Exception as e: except Exception as e:
self.log.error( self.log.error(Color.FAIL + repo_name + ' ' +
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + a + ' os .discinfo could not be written'
repo_name + ' ' + a + ' os .discinfo could not be written'
) )
self.log.error(e) self.log.error(e)
else: else:
self.log.warn( self.log.warn(Color.WARN + repo_name + ' ' + a +
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' + ' os .discinfo already exists'
repo_name + ' ' + a + ' os .discinfo already exists'
) )
if not os.path.exists(os_media_path): if not os.path.exists(os_media_path):
@ -1132,15 +1093,13 @@ class RepoSync:
os_media_path os_media_path
) )
except Exception as e: except Exception as e:
self.log.error( self.log.error(Color.FAIL + repo_name + ' ' + a +
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + ' os media.repo could not be written'
repo_name + ' ' + a + ' os media.repo could not be written'
) )
self.log.error(e) self.log.error(e)
else: else:
self.log.warn( self.log.warn(Color.WARN + repo_name + ' ' + a +
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' + ' os media.repo already exists'
repo_name + ' ' + a + ' os media.repo already exists'
) )
# Kickstart part of the repos # Kickstart part of the repos
@ -1156,15 +1115,13 @@ class RepoSync:
repo_name repo_name
) )
except Exception as e: except Exception as e:
self.log.error( self.log.error(Color.FAIL + repo_name + ' ' + a +
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + ' kickstart .treeinfo could not be written'
repo_name + ' ' + a + ' kickstart .treeinfo could not be written'
) )
self.log.error(e) self.log.error(e)
else: else:
self.log.warn( self.log.warn(Color.WARN + repo_name + ' ' + a +
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' + ' kickstart .treeinfo already exists'
repo_name + ' ' + a + ' kickstart .treeinfo already exists'
) )
if not os.path.exists(ks_disc_path): if not os.path.exists(ks_disc_path):
@ -1176,15 +1133,13 @@ class RepoSync:
ks_disc_path ks_disc_path
) )
except Exception as e: except Exception as e:
self.log.error( self.log.error(Color.FAIL + repo_name + ' ' + a +
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + ' kickstart .discinfo could not be written'
repo_name + ' ' + a + ' kickstart .discinfo could not be written'
) )
self.log.error(e) self.log.error(e)
else: else:
self.log.warn( self.log.warn(Color.FAIL + repo_name + ' ' + a +
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' + ' kickstart .discinfo already exists'
repo_name + ' ' + a + ' kickstart .discinfo already exists'
) )
if not os.path.exists(ks_media_path): if not os.path.exists(ks_media_path):
@ -1195,15 +1150,13 @@ class RepoSync:
ks_media_path ks_media_path
) )
except Exception as e: except Exception as e:
self.log.error( self.log.error(Color.FAIL + repo_name + ' ' + a +
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + ' kickstart media.repo could not be written'
repo_name + ' ' + a + ' kickstart media.repo could not be written'
) )
self.log.error(e) self.log.error(e)
else: else:
self.log.warn( self.log.warn(Color.WARN + repo_name + ' ' + a +
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' + ' kickstart media.repo already exists'
repo_name + ' ' + a + ' kickstart media.repo already exists'
) )
if not self.ignore_debug and not a == 'source': if not self.ignore_debug and not a == 'source':
@ -1240,15 +1193,13 @@ class RepoSync:
repo_name repo_name
) )
except Exception as e: except Exception as e:
self.log.error( self.log.error(Color.FAIL + repo_name + ' ' + a +
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + ' debug .treeinfo could not be written'
repo_name + ' ' + a + ' debug .treeinfo could not be written'
) )
self.log.error(e) self.log.error(e)
else: else:
self.log.warn( self.log.warn(Color.WARN + r + ' ' + a +
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' + ' debug .treeinfo already exists'
r + ' ' + a + ' debug .treeinfo already exists'
) )
if not os.path.exists(debug_disc_path): if not os.path.exists(debug_disc_path):
@ -1260,15 +1211,13 @@ class RepoSync:
debug_disc_path debug_disc_path
) )
except Exception as e: except Exception as e:
self.log.error( self.log.error(Color.FAIL + repo_name + ' ' + a +
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + ' debug .discinfo could not be written'
repo_name + ' ' + a + ' debug .discinfo could not be written'
) )
self.log.error(e) self.log.error(e)
else: else:
self.log.warn( self.log.warn(Color.WARN + r + ' ' + a +
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' + ' debug .discinfo already exists'
r + ' ' + a + ' debug .discinfo already exists'
) )
if not os.path.exists(debug_media_path): if not os.path.exists(debug_media_path):
@ -1279,15 +1228,13 @@ class RepoSync:
debug_media_path debug_media_path
) )
except Exception as e: except Exception as e:
self.log.error( self.log.error(Color.FAIL + repo_name + ' ' + a +
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + ' debug media.repo could not be written'
repo_name + ' ' + a + ' debug media.repo could not be written'
) )
self.log.error(e) self.log.error(e)
else: else:
self.log.warn( self.log.warn(Color.WARN + repo_name + ' ' + a +
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' + ' debug media.repo already exists'
repo_name + ' ' + a + ' debug media.repo already exists'
) )
@ -1322,16 +1269,10 @@ class RepoSync:
repo_name repo_name
) )
except Exception as e: except Exception as e:
self.log.error( self.log.error(Color.FAIL + repo_name + ' source os .treeinfo could not be written')
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
repo_name + ' source os .treeinfo could not be written'
)
self.log.error(e) self.log.error(e)
else: else:
self.log.warn( self.log.warn(Color.WARN + repo_name + ' source os .treeinfo already exists')
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
repo_name + ' source os .treeinfo already exists'
)
if not os.path.exists(source_disc_path): if not os.path.exists(source_disc_path):
try: try:
@ -1342,16 +1283,10 @@ class RepoSync:
source_disc_path source_disc_path
) )
except Exception as e: except Exception as e:
self.log.error( self.log.error(Color.FAIL + repo_name + ' source os .discinfo could not be written')
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
repo_name + ' source os .discinfo could not be written'
)
self.log.error(e) self.log.error(e)
else: else:
self.log.warn( self.log.warn(Color.WARN + repo_name + ' source .discinfo already exists')
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
repo_name + ' source .discinfo already exists'
)
if not os.path.exists(source_media_path): if not os.path.exists(source_media_path):
try: try:
@ -1361,16 +1296,10 @@ class RepoSync:
source_media_path source_media_path
) )
except Exception as e: except Exception as e:
self.log.error( self.log.error(Color.FAIL + repo_name + ' source os media.repo could not be written')
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
repo_name + ' source os media.repo could not be written'
)
self.log.error(e) self.log.error(e)
else: else:
self.log.warn( self.log.warn(Color.WARN + repo_name + ' source media.repo already exists')
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
repo_name + ' source media.repo already exists'
)
def tweak_treeinfo(self, repo, sync_root, arch): def tweak_treeinfo(self, repo, sync_root, arch):
""" """
@ -1396,18 +1325,12 @@ class RepoSync:
variants_to_tweak.append(r) variants_to_tweak.append(r)
if not len(variants_to_tweak) > 0: if not len(variants_to_tweak) > 0:
self.log.info( self.log.info(Color.INFO + 'No treeinfo to tweak.')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'No treeinfo to tweak.'
)
return return
for a in arches_to_tree: for a in arches_to_tree:
for v in variants_to_tweak: for v in variants_to_tweak:
self.log.info( self.log.info(Color.INFO + 'Tweaking treeinfo for ' + a + ' ' + v)
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Tweaking treeinfo for ' + a + ' ' + v
)
image = os.path.join(sync_root, v, a, 'os') image = os.path.join(sync_root, v, a, 'os')
imagemap = self.iso_map['images'][v] imagemap = self.iso_map['images'][v]
data = { data = {
@ -1425,10 +1348,7 @@ class RepoSync:
try: try:
Shared.treeinfo_modify_write(data, imagemap, self.log) Shared.treeinfo_modify_write(data, imagemap, self.log)
except Exception as e: except Exception as e:
self.log.error( self.log.error(Color.FAIL + 'There was an error writing os treeinfo.')
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
'There was an error writing os treeinfo.'
)
self.log.error(e) self.log.error(e)
if self.fullrun: if self.fullrun:
@ -1448,10 +1368,7 @@ class RepoSync:
try: try:
Shared.treeinfo_modify_write(ksdata, imagemap, self.log) Shared.treeinfo_modify_write(ksdata, imagemap, self.log)
except Exception as e: except Exception as e:
self.log.error( self.log.error(Color.FAIL + 'There was an error writing kickstart treeinfo.')
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
'There was an error writing kickstart treeinfo.'
)
self.log.error(e) self.log.error(e)
def run_compose_closeout(self): def run_compose_closeout(self):
@ -1513,18 +1430,17 @@ class RepoSync:
"global", "global",
) )
self.log.info( self.log.info(Color.INFO + 'Starting to sync ISOs to compose')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Starting to sync ISOs to compose'
)
if os.path.exists('/usr/bin/fpsync'): if os.path.exists('/usr/bin/fpsync'):
self.log.info(Color.INFO + 'Starting up fpsync')
message, ret = Shared.fpsync_method(iso_root, sync_iso_root, tmp_dir) message, ret = Shared.fpsync_method(iso_root, sync_iso_root, tmp_dir)
elif os.path.exists('/usr/bin/parallel') and os.path.exists('/usr/bin/rsync'): elif os.path.exists('/usr/bin/parallel') and os.path.exists('/usr/bin/rsync'):
self.log.info(Color.INFO + 'Starting up parallel | rsync')
message, ret = Shared.rsync_method(iso_root, sync_iso_root) message, ret = Shared.rsync_method(iso_root, sync_iso_root)
else: else:
self.log.error( self.log.error(
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + Color.FAIL +
'fpsync nor parallel + rsync were found on this system. ' + 'fpsync nor parallel + rsync were found on this system. ' +
'There is also no built-in parallel rsync method at this ' + 'There is also no built-in parallel rsync method at this ' +
'time.' 'time.'
@ -1532,21 +1448,12 @@ class RepoSync:
raise SystemExit() raise SystemExit()
if ret != 0: if ret != 0:
self.log.error( self.log.error(Color.FAIL + message)
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
message
)
else: else:
self.log.info( self.log.info(Color.INFO + message)
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
message
)
if os.path.exists(live_root): if os.path.exists(live_root):
self.log.info( self.log.info(Color.INFO + 'Starting to sync live images to compose')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Starting to sync live images to compose'
)
if os.path.exists('/usr/bin/fpsync'): if os.path.exists('/usr/bin/fpsync'):
message, ret = Shared.fpsync_method(live_root, sync_live_root, tmp_dir) message, ret = Shared.fpsync_method(live_root, sync_live_root, tmp_dir)
@ -1554,15 +1461,9 @@ class RepoSync:
message, ret = Shared.rsync_method(live_root, sync_live_root) message, ret = Shared.rsync_method(live_root, sync_live_root)
if ret != 0: if ret != 0:
self.log.error( self.log.error(Color.FAIL + message)
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
message
)
else: else:
self.log.info( self.log.info(Color.INFO + message)
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
message
)
# Combine all checksums here # Combine all checksums here
for arch in self.arches: for arch in self.arches:

233
iso/empanadas/empanadas/util/iso_utils.py
View File

@ -83,6 +83,7 @@ class IsoBuild:
self.lorax_result_root = config['mock_work_root'] + "/" + "lorax" self.lorax_result_root = config['mock_work_root'] + "/" + "lorax"
self.mock_isolation = isolation self.mock_isolation = isolation
self.iso_map = rlvars['iso_map'] self.iso_map = rlvars['iso_map']
self.cloudimages = rlvars['cloudimages']
self.release_candidate = rc self.release_candidate = rc
self.s3 = s3 self.s3 = s3
self.force_unpack = force_unpack self.force_unpack = force_unpack
@ -156,6 +157,16 @@ class IsoBuild:
"work/isos" "work/isos"
) )
self.live_work_dir = os.path.join(
self.compose_latest_dir,
"work/live"
)
self.image_work_dir = os.path.join(
self.compose_latest_dir,
"work/images"
)
self.lorax_work_dir = os.path.join( self.lorax_work_dir = os.path.join(
self.compose_latest_dir, self.compose_latest_dir,
"work/lorax" "work/lorax"
@ -347,16 +358,14 @@ class IsoBuild:
unpack_single_arch = True unpack_single_arch = True
arches_to_unpack = [self.arch] arches_to_unpack = [self.arch]
self.log.info( self.log.info(Color.INFO + 'Determining the latest pulls...')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Determining the latest pulls...'
)
if self.s3: if self.s3:
latest_artifacts = Shared.s3_determine_latest( latest_artifacts = Shared.s3_determine_latest(
self.s3_bucket, self.s3_bucket,
self.release, self.release,
self.arches, self.arches,
'tar.gz', 'tar.gz',
'lorax',
self.log self.log
) )
else: else:
@ -365,13 +374,11 @@ class IsoBuild:
self.release, self.release,
self.arches, self.arches,
'tar.gz', 'tar.gz',
'lorax',
self.log self.log
) )
self.log.info( self.log.info(Color.INFO + 'Downloading requested artifact(s)')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Downloading requested artifact(s)'
)
for arch in arches_to_unpack: for arch in arches_to_unpack:
lorax_arch_dir = os.path.join( lorax_arch_dir = os.path.join(
self.lorax_work_dir, self.lorax_work_dir,
@ -408,14 +415,8 @@ class IsoBuild:
full_drop, full_drop,
self.log self.log
) )
self.log.info( self.log.info(Color.INFO + 'Download phase completed')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' + self.log.info(Color.INFO + 'Beginning unpack phase...')
'Download phase completed'
)
self.log.info(
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Beginning unpack phase...'
)
for arch in arches_to_unpack: for arch in arches_to_unpack:
tarname = 'lorax-{}-{}.tar.gz'.format( tarname = 'lorax-{}-{}.tar.gz'.format(
@ -430,22 +431,13 @@ class IsoBuild:
) )
if not os.path.exists(tarball): if not os.path.exists(tarball):
self.log.error( self.log.error(Color.FAIL + 'Artifact does not exist: ' + tarball)
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
'Artifact does not exist: ' + tarball
)
continue continue
self._unpack_artifacts(self.force_unpack, arch, tarball) self._unpack_artifacts(self.force_unpack, arch, tarball)
self.log.info( self.log.info(Color.INFO + 'Unpack phase completed')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' + self.log.info(Color.INFO + 'Beginning image variant phase')
'Unpack phase completed'
)
self.log.info(
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Beginning image variant phase'
)
for arch in arches_to_unpack: for arch in arches_to_unpack:
self.log.info( self.log.info(
@ -456,15 +448,9 @@ class IsoBuild:
self._copy_boot_to_work(self.force_unpack, arch) self._copy_boot_to_work(self.force_unpack, arch)
self.log.info( self.log.info(Color.INFO + 'Image variant phase completed')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Image variant phase completed'
)
self.log.info( self.log.info(Color.INFO + 'Beginning treeinfo phase')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Beginning treeinfo phase'
)
for arch in arches_to_unpack: for arch in arches_to_unpack:
for variant in self.iso_map['images']: for variant in self.iso_map['images']:
@ -488,10 +474,7 @@ class IsoBuild:
if not force_unpack: if not force_unpack:
file_check = os.path.join(unpack_dir, 'lorax/.treeinfo') file_check = os.path.join(unpack_dir, 'lorax/.treeinfo')
if os.path.exists(file_check): if os.path.exists(file_check):
self.log.warn( self.log.warn(Color.WARN + 'Artifact (' + arch + ') already unpacked')
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
'Artifact (' + arch + ') already unpacked'
)
return return
self.log.info('Unpacking %s' % tarball) self.log.info('Unpacking %s' % tarball)
@ -515,10 +498,7 @@ class IsoBuild:
) )
if not os.path.exists(os.path.join(src_to_image, '.treeinfo')): if not os.path.exists(os.path.join(src_to_image, '.treeinfo')):
self.log.error( self.log.error(Color.FAIL + 'Lorax base image does not exist')
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
'Lorax base image does not exist'
)
return return
path_to_image = os.path.join( path_to_image = os.path.join(
@ -530,10 +510,7 @@ class IsoBuild:
if not force_unpack: if not force_unpack:
file_check = os.path.join(path_to_image, '.treeinfo') file_check = os.path.join(path_to_image, '.treeinfo')
if os.path.exists(file_check): if os.path.exists(file_check):
self.log.warn( self.log.warn(Color.WARN + 'Lorax image for ' + image + ' already exists')
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
'Lorax image for ' + image + ' already exists'
)
return return
self.log.info('Copying base lorax to %s directory...' % image) self.log.info('Copying base lorax to %s directory...' % image)
@ -595,10 +572,7 @@ class IsoBuild:
if not force_unpack: if not force_unpack:
file_check = isobootpath file_check = isobootpath
if os.path.exists(file_check): if os.path.exists(file_check):
self.log.warn( self.log.warn(Color.WARN + 'Boot image (' + discname + ') already exists')
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
'Boot image (' + discname + ') already exists'
)
return return
self.log.info('Copying %s boot iso to work directory...' % arch) self.log.info('Copying %s boot iso to work directory...' % arch)
@ -610,10 +584,7 @@ class IsoBuild:
self.log.info('Creating checksum for %s boot iso...' % arch) self.log.info('Creating checksum for %s boot iso...' % arch)
checksum = Shared.get_checksum(isobootpath, self.checksum, self.log) checksum = Shared.get_checksum(isobootpath, self.checksum, self.log)
if not checksum: if not checksum:
self.log.error( self.log.error(Color.FAIL + isobootpath + ' not found! Are you sure we copied it?')
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
isobootpath + ' not found! Are you sure we copied it?'
)
return return
with open(isobootpath + '.CHECKSUM', "w+") as c: with open(isobootpath + '.CHECKSUM', "w+") as c:
c.write(checksum) c.write(checksum)
@ -638,8 +609,7 @@ class IsoBuild:
) )
if not os.path.exists(pathway): if not os.path.exists(pathway):
self.log.error( self.log.error(Color.FAIL +
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
'Repo and Image variant either does NOT match or does ' + 'Repo and Image variant either does NOT match or does ' +
'NOT exist. Are you sure you have synced the repository?' 'NOT exist. Are you sure you have synced the repository?'
) )
@ -658,16 +628,10 @@ class IsoBuild:
found_files.append('/images/boot.iso') found_files.append('/images/boot.iso')
if len(found_files) > 0: if len(found_files) > 0:
self.log.warn( self.log.warn(Color.WARN + 'Images and data for ' + repo + ' and ' + arch + ' already exists.')
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
'Images and data for ' + repo + ' and ' + arch + ' already exists.'
)
return return
self.log.info( self.log.info(Color.INFO + 'Copying images and data for ' + repo + ' ' + arch)
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Copying images and data for ' + repo + ' ' + arch
)
try: try:
shutil.copytree(src_to_image, pathway, copy_function=shutil.copy2, dirs_exist_ok=True) shutil.copytree(src_to_image, pathway, copy_function=shutil.copy2, dirs_exist_ok=True)
@ -727,10 +691,7 @@ class IsoBuild:
try: try:
Shared.treeinfo_modify_write(data, imagemap, self.log) Shared.treeinfo_modify_write(data, imagemap, self.log)
except Exception as e: except Exception as e:
self.log.error( self.log.error(Color.FAIL + 'There was an error writing treeinfo.')
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
'There was an error writing treeinfo.'
)
self.log.error(e) self.log.error(e)
# Next set of functions are loosely borrowed (in concept) from pungi. Some # Next set of functions are loosely borrowed (in concept) from pungi. Some
@ -743,26 +704,17 @@ class IsoBuild:
""" """
sync_root = self.compose_latest_sync sync_root = self.compose_latest_sync
self.log.info( self.log.info(Color.INFO + 'Starting Extra ISOs phase')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Starting Extra ISOs phase'
)
if not os.path.exists(self.compose_base): if not os.path.exists(self.compose_base):
self.log.info( self.log.info(Color.FAIL + 'The compose directory MUST be here. Cannot continue.')
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
'The compose directory MUST be here. Cannot continue.'
)
raise SystemExit() raise SystemExit()
self._extra_iso_build_wrap() self._extra_iso_build_wrap()
self.log.info('Compose repo directory: %s' % sync_root) self.log.info('Compose repo directory: %s' % sync_root)
self.log.info('ISO result directory: %s/$arch' % self.lorax_work_dir) self.log.info('ISO result directory: %s/$arch' % self.lorax_work_dir)
self.log.info( self.log.info(Color.INFO + 'Extra ISO phase completed.')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Extra ISO phase completed.'
)
def _extra_iso_build_wrap(self): def _extra_iso_build_wrap(self):
""" """
@ -784,26 +736,17 @@ class IsoBuild:
for y in images_to_build: for y in images_to_build:
if 'isoskip' in self.iso_map['images'][y] and self.iso_map['images'][y]['isoskip']: if 'isoskip' in self.iso_map['images'][y] and self.iso_map['images'][y]['isoskip']:
self.log.info( self.log.info(Color.WARN + 'Skipping ' + y + ' image')
'[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' +
'Skipping ' + y + ' image'
)
continue continue
for a in arches_to_build: for a in arches_to_build:
lorax_path = os.path.join(self.lorax_work_dir, a, 'lorax', '.treeinfo') lorax_path = os.path.join(self.lorax_work_dir, a, 'lorax', '.treeinfo')
image_path = os.path.join(self.lorax_work_dir, a, y, '.treeinfo') image_path = os.path.join(self.lorax_work_dir, a, y, '.treeinfo')
if not os.path.exists(image_path): if not os.path.exists(image_path):
self.log.error( self.log.error(Color.FAIL + 'Lorax data not found for ' + y + '. Skipping.')
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
'Lorax data not found for ' + y + '. Skipping.'
)
if not os.path.exists(lorax_path): if not os.path.exists(lorax_path):
self.log.error( self.log.error(Color.FAIL + 'Lorax not found at all. This is considered fatal.')
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
'Lorax not found at all. This is considered fatal.'
)
raise SystemExit() raise SystemExit()
@ -819,10 +762,7 @@ class IsoBuild:
elif self.extra_iso_mode == 'podman': elif self.extra_iso_mode == 'podman':
continue continue
else: else:
self.log.info( self.log.info(Color.FAIL + 'Mode specified is not valid.')
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
'Mode specified is not valid.'
)
raise SystemExit() raise SystemExit()
if self.extra_iso_mode == 'podman': if self.extra_iso_mode == 'podman':
@ -1053,10 +993,7 @@ class IsoBuild:
join_all_pods = ' '.join(entry_name_list) join_all_pods = ' '.join(entry_name_list)
time.sleep(3) time.sleep(3)
self.log.info( self.log.info(Color.INFO + 'Building ' + i + ' ...')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Building ' + i + ' ...'
)
pod_watcher = '{} wait {}'.format( pod_watcher = '{} wait {}'.format(
cmd, cmd,
join_all_pods join_all_pods
@ -1085,9 +1022,7 @@ class IsoBuild:
output, errors = podcheck.communicate() output, errors = podcheck.communicate()
if 'Exited (0)' not in output.decode(): if 'Exited (0)' not in output.decode():
self.log.error( self.log.error(Color.FAIL + pod)
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + pod
)
bad_exit_list.append(pod) bad_exit_list.append(pod)
rmcmd = '{} rm {}'.format( rmcmd = '{} rm {}'.format(
@ -1106,34 +1041,21 @@ class IsoBuild:
for p in checksum_list: for p in checksum_list:
path = os.path.join(isos_dir, p) path = os.path.join(isos_dir, p)
if os.path.exists(path): if os.path.exists(path):
self.log.info( self.log.info(Color.INFO + 'Performing checksum for ' + p)
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Performing checksum for ' + p
)
checksum = Shared.get_checksum(path, self.checksum, self.log) checksum = Shared.get_checksum(path, self.checksum, self.log)
if not checksum: if not checksum:
self.log.error( self.log.error(Color.FAIL + path + ' not found! Are you sure it was built?')
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
path + ' not found! Are you sure it was built?'
)
with open(path + '.CHECKSUM', "w+") as c: with open(path + '.CHECKSUM', "w+") as c:
c.write(checksum) c.write(checksum)
c.close() c.close()
self.log.info( self.log.info(Color.INFO + 'Building ' + i + ' completed')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Building ' + i + ' completed'
)
if len(bad_exit_list) == 0: if len(bad_exit_list) == 0:
self.log.info( self.log.info(Color.INFO + 'Copying ISOs over to compose directory...')
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Copying ISOs over to compose directory...'
)
print()
else: else:
self.log.error( self.log.error(
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + Color.FAIL +
'There were issues with the work done. As a result, ' + 'There were issues with the work done. As a result, ' +
'the ISOs will not be copied.' 'the ISOs will not be copied.'
) )
@ -1152,17 +1074,14 @@ class IsoBuild:
lorax_base_dir = os.path.join(self.lorax_work_dir, arch) lorax_base_dir = os.path.join(self.lorax_work_dir, arch)
global_work_dir = os.path.join(self.compose_latest_dir, "work/global") global_work_dir = os.path.join(self.compose_latest_dir, "work/global")
self.log.info( self.log.info(Color.INFO + 'Generating graft points for extra iso: (' + arch + ') ' + iso)
'[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
'Generating graft points for extra iso: (' + arch + ') ' + iso
)
files = {} files = {}
# This is the data we need to actually boot # This is the data we need to actually boot
lorax_for_var = os.path.join(lorax_base_dir, iso) lorax_for_var = os.path.join(lorax_base_dir, iso)
if not os.path.exists(lorax_for_var + '/.treeinfo'): if not os.path.exists(lorax_for_var + '/.treeinfo'):
self.log.info( self.log.info(
'[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + Color.FAIL +
'!! .treeinfo is missing, does this variant actually exist? !!' '!! .treeinfo is missing, does this variant actually exist? !!'
) )
return return
@ -1612,6 +1531,64 @@ class IsoBuild:
returned_cmd = ' '.join(cmd) returned_cmd = ' '.join(cmd)
return returned_cmd return returned_cmd
def run_pull_generic_images(self):
"""
Pulls generic images built in peridot and places them where they need
to be. This relies on a list called "cloudimages" in the version
configuration.
"""
unpack_single_arch = False
arches_to_unpack = self.arches
if self.arch:
unpack_single_arch = True
arches_to_unpack = [self.arch]
for imagename in self.cloudimages:
self.log.info(Color.INFO + 'Determining the latest images for ' + imagename + ' ...')
if self.s3:
latest_artifacts = Shared.s3_determine_latest(
self.s3_bucket,
self.release,
self.arches,
'qcow2',
imagename,
self.log
)
else:
latest_artifacts = Shared.reqs_determine_latest(
self.s3_bucket_url,
self.release,
self.arches,
'qcow2',
imagename,
self.log
)
def run_build_live_iso(self):
"""
Builds DVD images based on the data created from the initial lorax on
each arch. This should NOT be called during the usual run() section.
"""
sync_root = self.compose_latest_sync
self.log.info(Color.INFO + 'Starting Live ISOs phase')
self._live_iso_build_wrap()
self.log.info('Compose repo directory: %s' % sync_root)
self.log.info('ISO result directory: %s/$arch' % self.lorax_work_dir)
self.log.info(Color.INFO + 'Extra ISO phase completed.')
def _live_iso_build_wrap(self):
"""
Prepare and actually build the live images. Based on arguments in self,
we'll either do it on mock in a loop or in podman, just like with the
extra iso phase.
"""
class LiveBuild: class LiveBuild:
""" """
This helps us build the live images for Rocky Linux. This helps us build the live images for Rocky Linux.

8
iso/empanadas/empanadas/util/shared.py
View File

@ -513,7 +513,7 @@ class Shared:
return 'Not available', 1 return 'Not available', 1
@staticmethod @staticmethod
def s3_determine_latest(s3_bucket, release, arches, filetype, logger): def s3_determine_latest(s3_bucket, release, arches, filetype, name, logger):
""" """
Using native s3, determine the latest artifacts and return a dict Using native s3, determine the latest artifacts and return a dict
""" """
@ -531,7 +531,7 @@ class Shared:
raise SystemExit() raise SystemExit()
for y in s3.list_objects(Bucket=s3_bucket)['Contents']: for y in s3.list_objects(Bucket=s3_bucket)['Contents']:
if filetype in y['Key'] and release in y['Key']: if filetype in y['Key'] and release in y['Key'] and name in y['Key']:
temp.append(y['Key']) temp.append(y['Key'])
for arch in arches: for arch in arches:
@ -569,7 +569,7 @@ class Shared:
logger.error('There was an issue downloading from %s' % s3_bucket) logger.error('There was an issue downloading from %s' % s3_bucket)
@staticmethod @staticmethod
def reqs_determine_latest(s3_bucket_url, release, arches, filetype, logger): def reqs_determine_latest(s3_bucket_url, release, arches, filetype, name, logger):
""" """
Using requests, determine the latest artifacts and return a list Using requests, determine the latest artifacts and return a list
""" """
@ -585,7 +585,7 @@ class Shared:
resp = xmltodict.parse(bucket_data.content) resp = xmltodict.parse(bucket_data.content)
for y in resp['ListBucketResult']['Contents']: for y in resp['ListBucketResult']['Contents']:
if filetype in y['Key'] and release in y['Key']: if filetype in y['Key'] and release in y['Key'] and name in y['Key']:
temp.append(y['Key']) temp.append(y['Key'])
for arch in arches: for arch in arches: