From d028b28d63d6a594d33946a9ff9a843e58c8a98c Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Sat, 12 Nov 2022 12:04:52 -0500 Subject: [PATCH 1/4] beginning work on downloading variant images --- iso/empanadas/empanadas/common.py | 6 +-- iso/empanadas/empanadas/configs/el8.yaml | 10 ++++ iso/empanadas/empanadas/util/iso_utils.py | 56 +++++++++++++---------- 3 files changed, 45 insertions(+), 27 deletions(-) diff --git a/iso/empanadas/empanadas/common.py b/iso/empanadas/empanadas/common.py index 56c0942..71d6e0f 100644 --- a/iso/empanadas/empanadas/common.py +++ b/iso/empanadas/empanadas/common.py @@ -103,10 +103,10 @@ for conf in glob.iglob(f"{_rootdir}/sig/*.yaml"): ALLOWED_TYPE_VARIANTS = { - "Azure": None, + "Azure": ["Base", "LVM"], "Container": ["Base", "Minimal", "UBI"], - "EC2": None, - "GenericCloud": None, + "EC2": ["Base", "LVM"], + "GenericCloud": ["Base", "LVM"], "Vagrant": ["Libvirt", "Vbox"], "OCP": None } diff --git a/iso/empanadas/empanadas/configs/el8.yaml b/iso/empanadas/empanadas/configs/el8.yaml index 0a0f3bf..4b11031 100644 --- a/iso/empanadas/empanadas/configs/el8.yaml +++ b/iso/empanadas/empanadas/configs/el8.yaml @@ -74,10 +74,20 @@ - 'xorriso' cloudimages: images: + Azure: + format: vhd + variants: [Base, LVM] EC2: format: raw + variants: [Base, LVM] GenericCloud: format: qcow2 + variants: [Base, LVM] + Container: + format: tar.xz + variants: [Base, Minimal, UBI] + OCP: + format: qcow2 livemap: git_repo: 'https://git.resf.org/sig_core/kickstarts.git' branch: 'r8' diff --git a/iso/empanadas/empanadas/util/iso_utils.py b/iso/empanadas/empanadas/util/iso_utils.py index 85757b0..ea56ce3 100644 --- a/iso/empanadas/empanadas/util/iso_utils.py +++ b/iso/empanadas/empanadas/util/iso_utils.py @@ -1361,29 +1361,37 @@ class IsoBuild: unpack_single_arch = True arches_to_unpack = [self.arch] - for imagename in self.cloudimages['images']: - self.log.info(Color.INFO + 'Determining the latest images for ' + imagename + ' ...') - formattype = self.cloudimages['images'][imagename]['format'] + for name, extra in self.cloudimages['images'].items(): + self.log.info(Color.INFO + 'Determining the latest images for ' + name + ' ...') + formattype = extra['format'] - if self.s3: - latest_artifacts = Shared.s3_determine_latest( - self.s3_bucket, - self.release, - arches_to_unpack, - formattype, - imagename, - self.log - ) + variants = extra['variants'] if 'variants' in extra.keys() else [None] # need to loop once + latest_artifacts = [] - else: - latest_artifacts = Shared.reqs_determine_latest( - self.s3_bucket_url, - self.release, - arches_to_unpack, - formattype, - imagename, - self.log - ) + for variant in variants: + if variant: + name = f"{name}-{variant}" + if self.s3: + latest_artifacts.append(Shared.s3_determine_latest( + self.s3_bucket, + self.release, + arches_to_unpack, + formattype, + name, + self.log + )) + + else: + latest_artifacts.append(Shared.reqs_determine_latest( + self.s3_bucket_url, + self.release, + arches_to_unpack, + formattype, + name, + self.log + )) + + # latest_artifacts should have at least 1 result if has_variants, else == 1 if not len(latest_artifacts) > 0: self.log.warn(Color.WARN + 'No images found.') @@ -1397,7 +1405,7 @@ class IsoBuild: ) if arch not in latest_artifacts.keys(): - self.log.warn(Color.WARN + 'Artifact for ' + imagename + + self.log.warn(Color.WARN + 'Artifact for ' + name + ' ' + arch + ' (' + formattype + ') does not exist.') continue @@ -1449,7 +1457,7 @@ class IsoBuild: image_arch_dir, self.shortname, self.major_version, - imagename, + name, arch, formattype ) @@ -1458,7 +1466,7 @@ class IsoBuild: image_arch_dir, self.shortname, self.major_version, - imagename, + name, arch, formattype ) From d98c4ee01bec047b393615ac0e4eee9c7ef1c4ef Mon Sep 17 00:00:00 2001 From: Louis Abel Date: Sat, 12 Nov 2022 19:34:53 -0700 Subject: [PATCH 2/4] add in primary_variant to do further linking if necessary --- iso/empanadas/empanadas/configs/el8.yaml | 3 + iso/empanadas/empanadas/util/iso_utils.py | 200 +++++++++++++--------- 2 files changed, 121 insertions(+), 82 deletions(-) diff --git a/iso/empanadas/empanadas/configs/el8.yaml b/iso/empanadas/empanadas/configs/el8.yaml index 4b11031..0b027a4 100644 --- a/iso/empanadas/empanadas/configs/el8.yaml +++ b/iso/empanadas/empanadas/configs/el8.yaml @@ -77,12 +77,15 @@ Azure: format: vhd variants: [Base, LVM] + primary_variant: 'Base' EC2: format: raw variants: [Base, LVM] + primary_variant: 'Base' GenericCloud: format: qcow2 variants: [Base, LVM] + primary_variant: 'Base' Container: format: tar.xz variants: [Base, Minimal, UBI] diff --git a/iso/empanadas/empanadas/util/iso_utils.py b/iso/empanadas/empanadas/util/iso_utils.py index ea56ce3..7f7151d 100644 --- a/iso/empanadas/empanadas/util/iso_utils.py +++ b/iso/empanadas/empanadas/util/iso_utils.py @@ -1366,6 +1366,8 @@ class IsoBuild: formattype = extra['format'] variants = extra['variants'] if 'variants' in extra.keys() else [None] # need to loop once + imagename = name + primary_variant = extra['primary_variant'] if 'primary_variant' in extra else None latest_artifacts = [] for variant in variants: @@ -1398,95 +1400,129 @@ class IsoBuild: continue self.log.info(Color.INFO + 'Attempting to download requested artifacts') - for arch in arches_to_unpack: - image_arch_dir = os.path.join( - self.image_work_dir, - arch - ) - - if arch not in latest_artifacts.keys(): - self.log.warn(Color.WARN + 'Artifact for ' + name + - ' ' + arch + ' (' + formattype + ') does not exist.') - continue - - source_path = latest_artifacts[arch] - drop_name = source_path.split('/')[-1] - checksum_name = drop_name + '.CHECKSUM' - full_drop = '{}/{}'.format( - image_arch_dir, - drop_name - ) - - checksum_drop = '{}/{}.CHECKSUM'.format( - image_arch_dir, - drop_name - ) - - if not os.path.exists(image_arch_dir): - os.makedirs(image_arch_dir, exist_ok=True) - - self.log.info('Downloading artifact for ' + Color.BOLD + arch + Color.END) - if self.s3: - Shared.s3_download_artifacts( - self.force_download, - self.s3_bucket, - source_path, - full_drop, - self.log - ) - else: - Shared.reqs_download_artifacts( - self.force_download, - self.s3_bucket_url, - source_path, - full_drop, - self.log + for item in latest_artifacts: + for arch in arches_to_unpack: + image_arch_dir = os.path.join( + self.image_work_dir, + arch ) - self.log.info('Creating checksum ...') - checksum = Shared.get_checksum(full_drop, self.checksum, self.log) - if not checksum: - self.log.error(Color.FAIL + full_drop + ' not found! Are you sure we copied it?') - continue - with open(checksum_drop, 'w+') as c: - c.write(checksum) - c.close() + if arch not in item.keys(): + self.log.warn(Color.WARN + 'Artifact for ' + name + + ' ' + arch + ' (' + formattype + ') does not exist.') + continue - self.log.info('Creating a symlink to latest image...') - latest_name = '{}/{}-{}-{}.latest.{}.{}'.format( - image_arch_dir, - self.shortname, - self.major_version, - name, - arch, - formattype - ) - latest_path = latest_name.split('/')[-1] - latest_checksum = '{}/{}-{}-{}.latest.{}.{}.CHECKSUM'.format( - image_arch_dir, - self.shortname, - self.major_version, - name, - arch, - formattype - ) - # For some reason python doesn't have a "yeah just change this - # link" part of the function - if os.path.exists(latest_name): - os.remove(latest_name) + source_path = item[arch] + drop_name = source_path.split('/')[-1] + checksum_name = drop_name + '.CHECKSUM' + full_drop = '{}/{}'.format( + image_arch_dir, + drop_name + ) - os.symlink(drop_name, latest_name) + checksum_drop = '{}/{}.CHECKSUM'.format( + image_arch_dir, + drop_name + ) - self.log.info('Creating checksum for latest symlinked image...') - shutil.copy2(checksum_drop, latest_checksum) - with open(latest_checksum, 'r') as link: - checkdata = link.read() + if not os.path.exists(image_arch_dir): + os.makedirs(image_arch_dir, exist_ok=True) - checkdata = checkdata.replace(drop_name, latest_path) + self.log.info('Downloading artifact for ' + Color.BOLD + arch + Color.END) + if self.s3: + Shared.s3_download_artifacts( + self.force_download, + self.s3_bucket, + source_path, + full_drop, + self.log + ) + else: + Shared.reqs_download_artifacts( + self.force_download, + self.s3_bucket_url, + source_path, + full_drop, + self.log + ) - with open(latest_checksum, 'w+') as link: - link.write(checkdata) - link.close() + self.log.info('Creating checksum ...') + checksum = Shared.get_checksum(full_drop, self.checksum, self.log) + if not checksum: + self.log.error(Color.FAIL + full_drop + ' not found! Are you sure we copied it?') + continue + with open(checksum_drop, 'w+') as c: + c.write(checksum) + c.close() + + self.log.info('Creating a symlink to latest image...') + latest_name = '{}/{}-{}-{}.latest.{}.{}'.format( + image_arch_dir, + self.shortname, + self.major_version, + name, + arch, + formattype + ) + latest_path = latest_name.split('/')[-1] + latest_checksum = '{}/{}-{}-{}.latest.{}.{}.CHECKSUM'.format( + image_arch_dir, + self.shortname, + self.major_version, + name, + arch, + formattype + ) + # If an image is the primary, we set this. + latest_primary_name = '{}/{}-{}-{}.latest.{}.{}'.format( + image_arch_dir, + self.shortname, + self.major_version, + imagename, + arch, + formattype + ) + latest_primary_checksum = '{}/{}-{}-{}.latest.{}.{}.CHECKSUM'.format( + image_arch_dir, + self.shortname, + self.major_version, + imagename, + arch, + formattype + ) + latest_primary_path = latest_name.split('/')[-1] + # For some reason python doesn't have a "yeah just change this + # link" part of the function + if os.path.exists(latest_name): + os.remove(latest_name) + + os.symlink(drop_name, latest_name) + + self.log.info('Creating checksum for latest symlinked image...') + shutil.copy2(checksum_drop, latest_checksum) + with open(latest_checksum, 'r') as link: + checkdata = link.read() + + checkdata = checkdata.replace(drop_name, latest_path) + + with open(latest_checksum, 'w+') as link: + link.write(checkdata) + link.close() + + # If this is the primary image, set the appropriate symlink + # and checksum + if primary_variant and primary_variant in drop_name: + self.log.info('This is the primary image, setting link and checksum') + if os.path.exists(latest_primary_name): + os.remove(latest_primary_name) + os.symlink(drop_name, latest_primary_name) + shutil.copy2(checksum_drop, latest_primary_checksum) + with open(latest_primary_checksum) as link: + checkpdata = link.read() + checkpdata = checkpdata.replace(drop_name, latest_primary_path) + with open(latest_primary_checksum, 'w+') as link: + link.write(checkpdata) + link.close() self.log.info(Color.INFO + 'Image download phase completed') From dba0d00e17dad7188cc78d2adc01eccc9c54b9d1 Mon Sep 17 00:00:00 2001 From: Louis Abel Date: Sun, 13 Nov 2022 10:29:21 -0700 Subject: [PATCH 3/4] update all configs to use new cloud image format --- iso/empanadas/empanadas/configs/el8-beta.yaml | 13 +++++++++++++ iso/empanadas/empanadas/configs/el8lh.yaml | 13 +++++++++++++ iso/empanadas/empanadas/configs/el9-beta.yaml | 13 +++++++++++++ iso/empanadas/empanadas/configs/el9.yaml | 13 +++++++++++++ iso/empanadas/empanadas/configs/el9lh.yaml | 13 +++++++++++++ iso/empanadas/empanadas/configs/rln.yaml | 13 +++++++++++++ 6 files changed, 78 insertions(+) diff --git a/iso/empanadas/empanadas/configs/el8-beta.yaml b/iso/empanadas/empanadas/configs/el8-beta.yaml index 607cc81..e20b4dd 100644 --- a/iso/empanadas/empanadas/configs/el8-beta.yaml +++ b/iso/empanadas/empanadas/configs/el8-beta.yaml @@ -74,10 +74,23 @@ - 'xorriso' cloudimages: images: + Azure: + format: vhd + variants: [Base, LVM] + primary_variant: 'Base' EC2: format: raw + variants: [Base, LVM] + primary_variant: 'Base' GenericCloud: format: qcow2 + variants: [Base, LVM] + primary_variant: 'Base' + Container: + format: tar.xz + variants: [Base, Minimal, UBI] + OCP: + format: qcow2 livemap: git_repo: 'https://git.resf.org/sig_core/kickstarts.git' branch: 'r8' diff --git a/iso/empanadas/empanadas/configs/el8lh.yaml b/iso/empanadas/empanadas/configs/el8lh.yaml index 5de2c9e..cf099db 100644 --- a/iso/empanadas/empanadas/configs/el8lh.yaml +++ b/iso/empanadas/empanadas/configs/el8lh.yaml @@ -74,10 +74,23 @@ - 'xorriso' cloudimages: images: + Azure: + format: vhd + variants: [Base, LVM] + primary_variant: 'Base' EC2: format: raw + variants: [Base, LVM] + primary_variant: 'Base' GenericCloud: format: qcow2 + variants: [Base, LVM] + primary_variant: 'Base' + Container: + format: tar.xz + variants: [Base, Minimal, UBI] + OCP: + format: qcow2 livemap: git_repo: 'https://git.resf.org/sig_core/kickstarts.git' branch: 'r8' diff --git a/iso/empanadas/empanadas/configs/el9-beta.yaml b/iso/empanadas/empanadas/configs/el9-beta.yaml index e0da593..e971314 100644 --- a/iso/empanadas/empanadas/configs/el9-beta.yaml +++ b/iso/empanadas/empanadas/configs/el9-beta.yaml @@ -77,10 +77,23 @@ - 'xorriso' cloudimages: images: + Azure: + format: vhd + variants: [Base, LVM] + primary_variant: 'Base' EC2: format: raw + variants: [Base, LVM] + primary_variant: 'Base' GenericCloud: format: qcow2 + variants: [Base, LVM] + primary_variant: 'Base' + Container: + format: tar.xz + variants: [Base, Minimal, UBI] + OCP: + format: qcow2 livemap: git_repo: 'https://git.resf.org/sig_core/kickstarts.git' branch: 'r9-beta' diff --git a/iso/empanadas/empanadas/configs/el9.yaml b/iso/empanadas/empanadas/configs/el9.yaml index 1243997..034ffb7 100644 --- a/iso/empanadas/empanadas/configs/el9.yaml +++ b/iso/empanadas/empanadas/configs/el9.yaml @@ -78,10 +78,23 @@ - 'xorriso' cloudimages: images: + Azure: + format: vhd + variants: [Base, LVM] + primary_variant: 'Base' EC2: format: raw + variants: [Base, LVM] + primary_variant: 'Base' GenericCloud: format: qcow2 + variants: [Base, LVM] + primary_variant: 'Base' + Container: + format: tar.xz + variants: [Base, Minimal, UBI] + OCP: + format: qcow2 livemap: git_repo: 'https://git.resf.org/sig_core/kickstarts.git' branch: 'r9' diff --git a/iso/empanadas/empanadas/configs/el9lh.yaml b/iso/empanadas/empanadas/configs/el9lh.yaml index 7749b93..a6d5a51 100644 --- a/iso/empanadas/empanadas/configs/el9lh.yaml +++ b/iso/empanadas/empanadas/configs/el9lh.yaml @@ -77,10 +77,23 @@ - 'xorriso' cloudimages: images: + Azure: + format: vhd + variants: [Base, LVM] + primary_variant: 'Base' EC2: format: raw + variants: [Base, LVM] + primary_variant: 'Base' GenericCloud: format: qcow2 + variants: [Base, LVM] + primary_variant: 'Base' + Container: + format: tar.xz + variants: [Base, Minimal, UBI] + OCP: + format: qcow2 livemap: git_repo: 'https://git.resf.org/sig_core/kickstarts.git' branch: 'r9lh' diff --git a/iso/empanadas/empanadas/configs/rln.yaml b/iso/empanadas/empanadas/configs/rln.yaml index 8f5feed..c5ddf70 100644 --- a/iso/empanadas/empanadas/configs/rln.yaml +++ b/iso/empanadas/empanadas/configs/rln.yaml @@ -74,10 +74,23 @@ - 'xorriso' cloudimages: images: + Azure: + format: vhd + variants: [Base, LVM] + primary_variant: 'Base' EC2: format: raw + variants: [Base, LVM] + primary_variant: 'Base' GenericCloud: format: qcow2 + variants: [Base, LVM] + primary_variant: 'Base' + Container: + format: tar.xz + variants: [Base, Minimal, UBI] + OCP: + format: qcow2 livemap: git_repo: 'https://git.resf.org/sig_core/kickstarts.git' branch: 'rln' From 7b9db7d5477a8cf55d4483a824c8c5f67d908e97 Mon Sep 17 00:00:00 2001 From: Louis Abel Date: Sun, 13 Nov 2022 13:16:38 -0700 Subject: [PATCH 4/4] utilize dictionaries to pull images --- iso/empanadas/empanadas/util/iso_utils.py | 106 +++++++++++++--------- 1 file changed, 64 insertions(+), 42 deletions(-) diff --git a/iso/empanadas/empanadas/util/iso_utils.py b/iso/empanadas/empanadas/util/iso_utils.py index 7f7151d..103ed8c 100644 --- a/iso/empanadas/empanadas/util/iso_utils.py +++ b/iso/empanadas/empanadas/util/iso_utils.py @@ -1357,6 +1357,7 @@ class IsoBuild: """ unpack_single_arch = False arches_to_unpack = self.arches + latest_artifacts = {} if self.arch: unpack_single_arch = True arches_to_unpack = [self.arch] @@ -1364,56 +1365,76 @@ class IsoBuild: for name, extra in self.cloudimages['images'].items(): self.log.info(Color.INFO + 'Determining the latest images for ' + name + ' ...') formattype = extra['format'] + latest_artifacts[name] = {} + primary_variant = extra['primary_variant'] if 'primary_variant' in extra else None + latest_artifacts[name]['primary'] = primary_variant variants = extra['variants'] if 'variants' in extra.keys() else [None] # need to loop once imagename = name - primary_variant = extra['primary_variant'] if 'primary_variant' in extra else None - latest_artifacts = [] + variantname = name for variant in variants: if variant: - name = f"{name}-{variant}" + variantname = f"{name}-{variant}" + self.log.info(Color.INFO + 'Getting latest for variant ' + variant + ' ...') if self.s3: - latest_artifacts.append(Shared.s3_determine_latest( + latest_artifacts[name][variantname] = Shared.s3_determine_latest( self.s3_bucket, self.release, arches_to_unpack, formattype, - name, + variantname, self.log - )) + ) else: - latest_artifacts.append(Shared.reqs_determine_latest( + latest_artifacts[name][variantname] = Shared.reqs_determine_latest( self.s3_bucket_url, self.release, arches_to_unpack, formattype, - name, + variantname, self.log - )) + ) - # latest_artifacts should have at least 1 result if has_variants, else == 1 + # latest_artifacts should have at least 1 result if has_variants, else == 1 + if not len(latest_artifacts[name][variantname]) > 0: + self.log.warn(Color.WARN + 'No images found for ' + variantname + + '. This means it will be skipped.') - if not len(latest_artifacts) > 0: - self.log.warn(Color.WARN + 'No images found.') - continue + del imagename + del variantname + del variants + + #print(latest_artifacts) + for keyname in latest_artifacts.keys(): + primary = latest_artifacts[keyname]['primary'] + for imgname in latest_artifacts[keyname]: + keysect = latest_artifacts[keyname][imgname] + if imgname == 'primary': + continue + + if not keysect: + continue + + self.log.info(Color.INFO + 'Attempting to download requested ' + + 'artifacts (' + keyname + ')') - self.log.info(Color.INFO + 'Attempting to download requested artifacts') - for item in latest_artifacts: for arch in arches_to_unpack: image_arch_dir = os.path.join( self.image_work_dir, arch ) - if arch not in item.keys(): - self.log.warn(Color.WARN + 'Artifact for ' + name + - ' ' + arch + ' (' + formattype + ') does not exist.') - continue - - source_path = item[arch] + source_path = keysect[arch] drop_name = source_path.split('/')[-1] + + # Docker containers get a "layer" name, this hack gets + # around it. I didn't feel like adding another config opt. + if 'layer' in drop_name: + fsuffix = drop_name.replace('layer', '') + drop_name = source_path.split('/')[-3] + fsuffix + checksum_name = drop_name + '.CHECKSUM' full_drop = '{}/{}'.format( image_arch_dir, @@ -1460,7 +1481,7 @@ class IsoBuild: image_arch_dir, self.shortname, self.major_version, - name, + imgname, arch, formattype ) @@ -1469,28 +1490,10 @@ class IsoBuild: image_arch_dir, self.shortname, self.major_version, - name, + imgname, arch, formattype ) - # If an image is the primary, we set this. - latest_primary_name = '{}/{}-{}-{}.latest.{}.{}'.format( - image_arch_dir, - self.shortname, - self.major_version, - imagename, - arch, - formattype - ) - latest_primary_checksum = '{}/{}-{}-{}.latest.{}.{}.CHECKSUM'.format( - image_arch_dir, - self.shortname, - self.major_version, - imagename, - arch, - formattype - ) - latest_primary_path = latest_name.split('/')[-1] # For some reason python doesn't have a "yeah just change this # link" part of the function if os.path.exists(latest_name): @@ -1511,7 +1514,26 @@ class IsoBuild: # If this is the primary image, set the appropriate symlink # and checksum - if primary_variant and primary_variant in drop_name: + if primary and primary in drop_name: + # If an image is the primary, we set this. + latest_primary_name = '{}/{}-{}-{}.latest.{}.{}'.format( + image_arch_dir, + self.shortname, + self.major_version, + keyname, + arch, + formattype + ) + latest_primary_checksum = '{}/{}-{}-{}.latest.{}.{}.CHECKSUM'.format( + image_arch_dir, + self.shortname, + self.major_version, + keyname, + arch, + formattype + ) + latest_primary_path = latest_name.split('/')[-1] + self.log.info('This is the primary image, setting link and checksum') if os.path.exists(latest_primary_name): os.remove(latest_primary_name)