From a4ee9ecc02506c6c2595f6fd859507bc2c135220 Mon Sep 17 00:00:00 2001 From: Louis Abel Date: Fri, 8 Jul 2022 09:54:43 -0700 Subject: [PATCH] images should be pulled based on format and name --- iso/empanadas/README.md | 20 ++- iso/empanadas/empanadas/configs/el8.yaml | 9 +- iso/empanadas/empanadas/configs/el9-beta.yaml | 9 +- iso/empanadas/empanadas/configs/el9.yaml | 9 +- iso/empanadas/empanadas/configs/el9lh.yaml | 9 +- iso/empanadas/empanadas/configs/rln.yaml | 9 +- iso/empanadas/empanadas/util/.shared.py.swp | Bin 0 -> 4096 bytes iso/empanadas/empanadas/util/iso_utils.py | 130 ++++++++++-------- 8 files changed, 108 insertions(+), 87 deletions(-) create mode 100644 iso/empanadas/empanadas/util/.shared.py.swp diff --git a/iso/empanadas/README.md b/iso/empanadas/README.md index 0c90442..b7ec7ef 100644 --- a/iso/empanadas/README.md +++ b/iso/empanadas/README.md @@ -1,12 +1,21 @@ # iso - ## Setup / Install 1. Install [Poetry](https://python-poetry.org/docs/) 2. Setup: `poetry install` +3. Install dependencies: `dnf install podman mock` 3. Have fun +## Reliance on podman and mock + +### Why podman? + +Podman is a requirement for performing reposyncs. This was done because it was found to be easier to spin up several podman containers than several mock chroots and it was faster than doing one at a time in a loop. Podman is also used to parallelize ISO builds. + +### Why mock? + +There are cases where running `mock` is the preferred go-to: For example, building lorax images. Since you cannot build a lorax image for an architecture your system does not support, trying to "parallelize" it was out of the question. Adding this support in was not only for local testing without podman, it was also done so it can be run in our peridot kube cluster for each architecture. ## Updating dependencies @@ -16,9 +25,8 @@ Changes to the poetry.lock should be commited if dependencies are added or updat ## TODO -Verbose mode should exist to output everything that's being called or ran. - -There should be additional logging regardless, not just to stdout, but also to a file. +* Verbose mode should exist to output everything that's being called or ran. +* There should be additional logging regardless, not just to stdout, but also to a file. ## scripts @@ -27,6 +35,10 @@ There should be additional logging regardless, not just to stdout, but also to a * sync_sig -> Syncs SIG repositories from Peridot * build-iso -> Builds initial ISO's using Lorax * build-iso-extra -> Builds DVD's and other images based on Lorax data +* build-iso-live -> Builds live images +* pull-unpack-tree -> Pulls the latest lorax data from an S3 bucket and configures treeinfo +* pull-cloud-image -> Pulls the latest cloud images from an S3 bucket +* finalize_compose -> Finalizes a compose with metadata and checksums, as well as copies images * launch-builds -> Creates a kube config to run build-iso * build-image -> Runs build-iso ``` diff --git a/iso/empanadas/empanadas/configs/el8.yaml b/iso/empanadas/empanadas/configs/el8.yaml index 85e7c0a..3764f51 100644 --- a/iso/empanadas/empanadas/configs/el8.yaml +++ b/iso/empanadas/empanadas/configs/el8.yaml @@ -86,11 +86,10 @@ - 'xorriso' cloudimages: images: - - EC2 - - GenericCloud - formats: - - qcow2 - - raw + EC2: + format: raw + GenericCloud: + format: qcow2 livemap: git_repo: 'https://git.resf.org/sig_core/kickstarts.git' branch: 'r9' diff --git a/iso/empanadas/empanadas/configs/el9-beta.yaml b/iso/empanadas/empanadas/configs/el9-beta.yaml index 67b9b6d..b18186c 100644 --- a/iso/empanadas/empanadas/configs/el9-beta.yaml +++ b/iso/empanadas/empanadas/configs/el9-beta.yaml @@ -74,11 +74,10 @@ - 'xorriso' cloudimages: images: - - EC2 - - GenericCloud - formats: - - qcow2 - - raw + EC2: + format: raw + GenericCloud: + format: qcow2 livemap: git_repo: 'https://git.resf.org/sig_core/kickstarts.git' branch: 'r9-beta' diff --git a/iso/empanadas/empanadas/configs/el9.yaml b/iso/empanadas/empanadas/configs/el9.yaml index 0e42697..7d16f9b 100644 --- a/iso/empanadas/empanadas/configs/el9.yaml +++ b/iso/empanadas/empanadas/configs/el9.yaml @@ -74,11 +74,10 @@ - 'xorriso' cloudimages: images: - - EC2 - - GenericCloud - formats: - - qcow2 - - raw + EC2: + format: raw + GenericCloud: + format: qcow2 livemap: git_repo: 'https://git.resf.org/sig_core/kickstarts.git' branch: 'r9' diff --git a/iso/empanadas/empanadas/configs/el9lh.yaml b/iso/empanadas/empanadas/configs/el9lh.yaml index 817b033..e7500e4 100644 --- a/iso/empanadas/empanadas/configs/el9lh.yaml +++ b/iso/empanadas/empanadas/configs/el9lh.yaml @@ -74,11 +74,10 @@ - 'xorriso' cloudimages: images: - - EC2 - - GenericCloud - formats: - - qcow2 - - raw + EC2: + format: raw + GenericCloud: + format: qcow2 livemap: git_repo: 'https://git.resf.org/sig_core/kickstarts.git' branch: 'r9lh' diff --git a/iso/empanadas/empanadas/configs/rln.yaml b/iso/empanadas/empanadas/configs/rln.yaml index 22b0578..3df3104 100644 --- a/iso/empanadas/empanadas/configs/rln.yaml +++ b/iso/empanadas/empanadas/configs/rln.yaml @@ -74,11 +74,10 @@ - 'xorriso' cloudimages: images: - - EC2 - - GenericCloud - formats: - - qcow2 - - raw + EC2: + format: raw + GenericCloud: + format: qcow2 livemap: git_repo: 'https://git.resf.org/sig_core/kickstarts.git' branch: 'rln' diff --git a/iso/empanadas/empanadas/util/.shared.py.swp b/iso/empanadas/empanadas/util/.shared.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..344c9d057297e69acd8ab9b75837c3ea792a3df9 GIT binary patch literal 4096 zcmYc?2=nw+u+TGN00IF9hCi>5C&{RHF_^8AV#rBMO3eX@69C1Dd6{~NdFiP+#rbK; zIr+)i<@rU~dU>fOs2c0Q2I{A0mgpB}rt2r?7p3Z# 0: - self.log.warn(Color.WARN + 'No images found.') + if not len(latest_artifacts) > 0: + self.log.warn(Color.WARN + 'No images found.') + continue + + self.log.info(Color.INFO + 'Attempting to download requested artifacts') + for arch in arches_to_unpack: + image_arch_dir = os.path.join( + self.image_work_dir, + arch + ) + + if arch not in latest_artifacts.keys(): + self.log.warn(Color.WARN + 'Artifact for ' + imagename + + ' ' + arch + ' (' + formattype + ') does not exist.') continue - self.log.info(Color.INFO + 'Attempting to download requested artifacts') - for arch in arches_to_unpack: - image_arch_dir = os.path.join( - self.image_work_dir, - arch + source_path = latest_artifacts[arch] + drop_name = source_path.split('/')[-1] + full_drop = '{}/{}'.format( + image_arch_dir, + drop_name + ) + + checksum_drop = '{}/{}.CHECKSUM'.format( + image_arch_dir, + drop_name + ) + + if not os.path.exists(image_arch_dir): + os.makedirs(image_arch_dir, exist_ok=True) + + self.log.info('Downloading artifact for ' + Color.BOLD + arch + Color.END) + if self.s3: + Shared.s3_download_artifacts( + self.force_download, + self.s3_bucket, + source_path, + full_drop, + self.log + ) + else: + Shared.reqs_download_artifacts( + self.force_download, + self.s3_bucket_url, + source_path, + full_drop, + self.log ) - if arch not in latest_artifacts.keys(): - self.log.warn(Color.WARN + 'Artifact for ' + imagename + - ' ' + arch + ' (' + formattype + ') does not exist.') - continue - - source_path = latest_artifacts[arch] - drop_name = source_path.split('/')[-1] - full_drop = '{}/{}'.format( - image_arch_dir, - drop_name - ) - - if not os.path.exists(image_arch_dir): - os.makedirs(image_arch_dir, exist_ok=True) - - self.log.info('Downloading artifact for ' + Color.BOLD + arch + Color.END) - if self.s3: - Shared.s3_download_artifacts( - self.force_download, - self.s3_bucket, - source_path, - full_drop, - self.log - ) - else: - Shared.reqs_download_artifacts( - self.force_download, - self.s3_bucket_url, - source_path, - full_drop, - self.log - ) + self.log.info('Creating checksum ...') + checksum = Shared.get_checksum(full_drop, self.checksum, self.log) + if not checksum: + self.log.error(Color.FAIL + full_drop + ' not found! Are you sure we copied it?') + continue + with open(checksum_drop, 'w+') as c: + c.write(checksum) + c.close() self.log.info(Color.INFO + 'Image download phase completed')