From 37c0be3fd06a0d9e177c427d0c36fb8a6ded7008 Mon Sep 17 00:00:00 2001 From: nazunalika Date: Sun, 26 Jun 2022 21:29:55 -0700 Subject: [PATCH] attempt to build ISO in both mock and podman --- iso/empanadas/empanadas/common.py | 42 ++++ iso/empanadas/empanadas/configs/el9-beta.yaml | 1 + iso/empanadas/empanadas/configs/el9.yaml | 1 + iso/empanadas/empanadas/configs/el9lh.yaml | 1 + iso/empanadas/empanadas/configs/rln.yaml | 1 - .../empanadas/templates/buildImage.tmpl.sh | 12 + .../empanadas/templates/extraisobuild.tmpl.sh | 2 +- iso/empanadas/empanadas/util/iso_utils.py | 225 ++++++++++++++++-- 8 files changed, 262 insertions(+), 23 deletions(-) diff --git a/iso/empanadas/empanadas/common.py b/iso/empanadas/empanadas/common.py index bf081ab..e445caf 100644 --- a/iso/empanadas/empanadas/common.py +++ b/iso/empanadas/empanadas/common.py @@ -6,6 +6,7 @@ import glob import rpm import yaml import logging +import hashlib # These are a bunch of colors we may use in terminal output class Color: @@ -20,6 +21,47 @@ class Color: BOLD = '\033[1m' END = '\033[0m' +class Utils: + """ + Quick utilities that may be commonly used + """ + @staticmethod + def get_checksum(path, hashtype, logger): + """ + Generates a checksum from the provided path by doing things in chunks. + This way we don't do it in memory. + """ + try: + checksum = hashlib.new(hashtype) + except ValueError: + logger.error("Invalid hash type: %s" % hashtype) + return False + + try: + input_file = open(path, "rb") + except IOError as e: + logger.error("Could not open file %s: %s" % (path, e)) + return False + + while True: + chunk = input_file.read(8192) + if not chunk: + break + checksum.update(chunk) + + input_file.close() + stat = os.stat(path) + base = os.path.basename(path) + # This emulates our current syncing scripts that runs stat and + # sha256sum and what not with a very specific output. + return "%s: %s bytes\n%s (%s) = %s" % ( + base, + stat.st_size, + hashtype.upper(), + base, + checksum.hexdigest() + ) + # vars and additional checks rldict = {} sigdict = {} diff --git a/iso/empanadas/empanadas/configs/el9-beta.yaml b/iso/empanadas/empanadas/configs/el9-beta.yaml index 6de4bbe..0ca78a8 100644 --- a/iso/empanadas/empanadas/configs/el9-beta.yaml +++ b/iso/empanadas/empanadas/configs/el9-beta.yaml @@ -70,6 +70,7 @@ - 'isomd5sum' - 'lorax-templates-rhel' - 'lorax-templates-generic' + - 'xorriso' repoclosure_map: arches: x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch' diff --git a/iso/empanadas/empanadas/configs/el9.yaml b/iso/empanadas/empanadas/configs/el9.yaml index b71a1ef..1918b68 100644 --- a/iso/empanadas/empanadas/configs/el9.yaml +++ b/iso/empanadas/empanadas/configs/el9.yaml @@ -70,6 +70,7 @@ - 'isomd5sum' - 'lorax-templates-rhel' - 'lorax-templates-generic' + - 'xorriso' repoclosure_map: arches: x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch' diff --git a/iso/empanadas/empanadas/configs/el9lh.yaml b/iso/empanadas/empanadas/configs/el9lh.yaml index 621e758..78de775 100644 --- a/iso/empanadas/empanadas/configs/el9lh.yaml +++ b/iso/empanadas/empanadas/configs/el9lh.yaml @@ -70,6 +70,7 @@ - 'isomd5sum' - 'lorax-templates-rhel' - 'lorax-templates-generic' + - 'xorriso' repoclosure_map: arches: x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch' diff --git a/iso/empanadas/empanadas/configs/rln.yaml b/iso/empanadas/empanadas/configs/rln.yaml index 1bd52d6..c36fcb1 100644 --- a/iso/empanadas/empanadas/configs/rln.yaml +++ b/iso/empanadas/empanadas/configs/rln.yaml @@ -65,7 +65,6 @@ - 'libreport-rhel-anaconda-bugzilla' required_pkgs: - 'lorax' - - 'genisoimage' - 'isomd5sum' - 'lorax-templates-rhel' - 'lorax-templates-generic' diff --git a/iso/empanadas/empanadas/templates/buildImage.tmpl.sh b/iso/empanadas/empanadas/templates/buildImage.tmpl.sh index d7b7708..a17c853 100644 --- a/iso/empanadas/empanadas/templates/buildImage.tmpl.sh +++ b/iso/empanadas/empanadas/templates/buildImage.tmpl.sh @@ -39,6 +39,18 @@ fi # If we didn't fail, let's pack up everything! cd "${MOCKBLD}" + +# Get ISO manifest +if [ -f "/usr/bin/xorriso" ]; then + /usr/bin/xorriso -dev lorax/images/boot.iso --find | + tail -n+2 | + tr -d "'" | + cut -c2- sort >> lorax/images/boot.iso.manifest +elif [ -f "/usr/bin/isoinfo" ]; then + /usr/bin/isoinfo -R -f -i lorax/images/boot.iso | + grep -v '/TRANS.TBL$' | sort >> lorax/images/boot.iso.manifest +fi + tar czf "${LORAX_TAR}" lorax "${LOGFILE}" tar_ret_val=$? diff --git a/iso/empanadas/empanadas/templates/extraisobuild.tmpl.sh b/iso/empanadas/empanadas/templates/extraisobuild.tmpl.sh index 7dc6584..892c5c9 100644 --- a/iso/empanadas/empanadas/templates/extraisobuild.tmpl.sh +++ b/iso/empanadas/empanadas/templates/extraisobuild.tmpl.sh @@ -40,7 +40,7 @@ mock_ret_val=$? if [ $mock_ret_val -eq 0 ]; then # Copy resulting data to /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result mkdir -p "${MOCK_RESL}" - cp "${MOCK_CHRO}${BUILDDIR}/${IMAGE_ISO}" "${MOCK_RESL}" + cp "${MOCK_CHRO}${BUILDDIR}/${IMAGE_ISO}*" "${MOCK_RESL}" else echo "!! EXTRA ISO RUN FAILED !!" exit 1 diff --git a/iso/empanadas/empanadas/util/iso_utils.py b/iso/empanadas/empanadas/util/iso_utils.py index 2784e77..23c06ad 100644 --- a/iso/empanadas/empanadas/util/iso_utils.py +++ b/iso/empanadas/empanadas/util/iso_utils.py @@ -35,7 +35,7 @@ import productmd.treeinfo from jinja2 import Environment, FileSystemLoader -from empanadas.common import Color, _rootdir +from empanadas.common import Color, _rootdir, Utils class IsoBuild: """ @@ -105,6 +105,10 @@ class IsoBuild: self.extra_files = rlvars['extra_files'] + self.container = config['container'] + if 'container' in rlvars and len(rlvars['container']) > 0: + self.container = rlvars['container'] + self.staging_dir = os.path.join( config['staging_root'], config['category_stub'], @@ -416,6 +420,8 @@ class IsoBuild: for variant in self.iso_map['images']: self._copy_lorax_to_variant(self.force_unpack, arch, variant) + self._copy_boot_to_work(self.force_unpack, arch) + self.log.info( '[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' + 'Image variant phase completed' @@ -595,25 +601,6 @@ class IsoBuild: ) return - self.log.info('Copying %s boot iso to work directory...' % arch) - os.makedirs(iso_to_go, exist_ok=True) - - rclevel = '' - if self.release_candidate: - rclevel = '-' + self.rclvl - - isobootpath = '{}/{}-{}.{}{}-{}-{}.iso'.format( - iso_to_go, - self.shortname, - self.major_version, - self.minor_version, - rclevel, - arch, - image - ) - - shutil.copy2(src_to_image + '/images/boot.iso', isobootpath) - self.log.info('Copying base lorax to %s directory...' % image) try: shutil.copytree(src_to_image, path_to_image, copy_function=shutil.copy2, dirs_exist_ok=True) @@ -630,6 +617,73 @@ class IsoBuild: 'Cannot remove boot.iso' ) + def _copy_boot_to_work(self, force_unpack, arch): + src_to_image = os.path.join( + self.lorax_work_dir, + arch, + 'lorax' + ) + + iso_to_go = os.path.join( + self.iso_work_dir, + arch + ) + + path_to_src_image = '{}/{}'.format( + src_to_image, + '/images/boot.iso' + ) + + rclevel = '' + if self.release_candidate: + rclevel = '-' + self.rclvl + + discname = '{}-{}.{}{}-{}-{}.iso'.format( + self.shortname, + self.major_version, + self.minor_version, + rclevel, + arch, + 'boot' + ) + + isobootpath = '{}/{}'.format( + iso_to_go, + discname + ) + + manifest = '{}.{}'.format( + isobootpath, + 'manifest' + ) + + if not force_unpack: + file_check = isobootpath + if os.path.exists(file_check): + self.log.warn( + '[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' + + 'Boot image (' + discname + ') already exists' + ) + return + + self.log.info('Copying %s boot iso to work directory...' % arch) + os.makedirs(iso_to_go, exist_ok=True) + shutil.copy2(path_to_src_image, isobootpath) + if os.path.exists(path_to_src_image + '.manifest'): + shutil.copy2(path_to_src_image + '.manifest', manifest) + + self.log.info('Creating checksum for %s boot iso...' % arch) + checksum = Utils.get_checksum(isobootpath, self.checksum, self.log) + if not checksum: + self.log.error( + '[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + + isobootpath + ' not found! Are you sure we copied it?' + ) + return + with open(isobootpath + '.CHECKSUM', "w+") as c: + c.write(checksum) + c.close() + def run_boot_sync(self): """ This unpacks into BaseOS/$arch/os, assuming there's no data actually @@ -866,9 +920,13 @@ class IsoBuild: ) raise SystemExit() + if self.extra_iso_mode == 'podman': + self._extra_iso_podman_run(arches_to_build, images_to_build, work_root) + def _extra_iso_local_config(self, arch, image, grafts, work_root): """ - Local ISO build mode - this should build in mock + Local ISO build configuration - This generates the configuration for + both mock and podman entries """ self.log.info('Generating Extra ISO configuration and script') @@ -997,6 +1055,114 @@ class IsoBuild: """ Runs the actual local process using mock """ + entries_dir = os.path.join(work_root, "entries") + extra_iso_cmd = '/bin/bash {}/extraisobuild-{}-{}.sh'.format(entries_dir, arch, image) + self.log.info('Starting mock build...') + p = subprocess.call(shlex.split(extra_iso_cmd)) + if p != 0: + self.log.error('An error occured during execution.') + self.log.error('See the logs for more information.') + raise SystemExit() + # Copy it if the compose dir is here? + + def _extra_iso_podman_run(self, arches, images, work_root): + """ + Does all the image building in podman containers to parallelize the + builds. This is a case where you can call this instead of looping mock, + or not run it in peridot. This gives the Release Engineer a little more + flexibility if they care enough. + + This honestly assumes you are running this on a machine that has access + to the compose directories. It's the same as if you were doing a + reposync of the repositories. + """ + cmd = self.podman_cmd() + entries_dir = os.path.join(work_root, "entries") + for i in images: + entry_name_list = [] + image_name = i + arch_sync = arches.copy() + + for a in arch_sync: + entry_name = 'buildExtraImage-{}-{}.sh'.format(a, i) + entry_name_list.append(entry_name) + + for pod in entry_name_list: + podman_cmd_entry = '{} run -d -it -v "{}:{}" -v "{}:{}" --name {} --entrypoint {}/{} {}'.format( + cmd, + self.compose_root, + self.compose_root, + entries_dir, + entries_dir, + pod, + entries_dir, + pod, + self.container + ) + + process = subprocess.call( + shlex.split(podman_cmd_entry), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + join_all_pods = ' '.join(entry_name_list) + time.sleep(3) + self.log.info( + '[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' + + 'Building ' + i + ' ...' + ) + pod_watcher = '{} wait {}'.format( + cmd, + join_all_pods + ) + + watch_man = subprocess.call( + shlex.split(pod_watcher), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + # After the above is done, we'll check each pod process for an exit + # code. + pattern = "Exited (0)" + for pod in entry_name_list: + checkcmd = '{} ps -f status=exited -f name={}'.format( + cmd, + pod + ) + podcheck = subprocess.Popen( + checkcmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True + ) + + output, errors = podcheck.communicate() + if 'Exited (0)' not in output.decode(): + self.log.error( + '[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + pod + ) + bad_exit_list.append(pod) + + rmcmd = '{} rm {}'.format( + cmd, + join_all_pods + ) + + rmpod = subprocess.Popen( + rmcmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + shell=True + ) + + entry_name_list.clear() + self.log.info( + '[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' + + 'Building ' + i + ' completed' + ) + def _generate_graft_points( self, @@ -1471,6 +1637,23 @@ class IsoBuild: returned_cmd = ' '.join(cmd) return returned_cmd + def podman_cmd(self) -> str: + """ + This generates the podman run command. This is in the case that we want + to do reposyncs in parallel as we cannot reasonably run multiple + instances of dnf reposync on a single system. + """ + cmd = None + if os.path.exists("/usr/bin/podman"): + cmd = "/usr/bin/podman" + else: + self.log.error('/usr/bin/podman was not found. Good bye.') + raise SystemExit("\n\n/usr/bin/podman was not found.\n\nPlease " + " ensure that you have installed the necessary packages on " + " this system. " + Color.BOLD + "Note that docker is not " + "supported." + Color.END + ) + return cmd class LiveBuild: """