From 00f330226d4f3f982a87229bd409773c72980bc8 Mon Sep 17 00:00:00 2001 From: nazunalika Date: Tue, 21 Jun 2022 17:21:37 -0700 Subject: [PATCH] update poetry and iso utilities --- .../empanadas/templates/mkiso.tmpl.sh | 16 ++ iso/empanadas/empanadas/util/iso_utils.py | 259 +++++++++++++++++- iso/empanadas/poetry.lock | 30 +- iso/empanadas/pyproject.toml | 1 + 4 files changed, 295 insertions(+), 11 deletions(-) create mode 100644 iso/empanadas/empanadas/templates/mkiso.tmpl.sh diff --git a/iso/empanadas/empanadas/templates/mkiso.tmpl.sh b/iso/empanadas/empanadas/templates/mkiso.tmpl.sh new file mode 100644 index 0000000..7f78fc2 --- /dev/null +++ b/iso/empanadas/empanadas/templates/mkiso.tmpl.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -ex + +cd /builddir + +if ! TEMPLATE="$($(head -n1 $(which lorax) | cut -c3-) -c 'import pylorax; print(pylorax.find_templates())')"; then + TEMPLATE="/usr/share/lorax" +fi + +{{ make_image }} + +{{ isohybrid }} + +{{ implantmd5 }} + +{{ make_manifest }} diff --git a/iso/empanadas/empanadas/util/iso_utils.py b/iso/empanadas/empanadas/util/iso_utils.py index ed0f7e5..c0524b6 100644 --- a/iso/empanadas/empanadas/util/iso_utils.py +++ b/iso/empanadas/empanadas/util/iso_utils.py @@ -13,6 +13,7 @@ import shlex import time import tarfile import shutil +import hashlib # lazy person's s3 parser import requests @@ -20,6 +21,7 @@ import json import xmltodict # if we can access s3 import boto3 +import kobo.shortcuts # This is for treeinfo from configparser import ConfigParser @@ -48,6 +50,7 @@ class IsoBuild: config, major, arch=None, + hfs_compat: bool = False, rc: bool = False, s3: bool = False, force_download: bool = False, @@ -108,6 +111,9 @@ class IsoBuild: if s3: self.s3 = boto3.client('s3') + # arch specific + self.hfs_compat = hfs_compat + # Templates file_loader = FileSystemLoader(f"{_rootdir}/templates") self.tmplenv = Environment(loader=file_loader) @@ -608,7 +614,7 @@ class IsoBuild: unpack_single_arch = True arches_to_unpack = [self.arch] - self._sync_boot(force_unpack=self.force_unpack, arch=self.arch) + self._sync_boot(force_unpack=self.force_unpack, arch=self.arch, image=None) self.treeinfo_write(arch=self.arch) def _sync_boot(self, force_unpack, arch, image): @@ -656,19 +662,266 @@ class IsoBuild: "", ] + # Next set of functions are loosely borrowed (in concept) from pungi. Some + # stuff may be combined/mixed together, other things may be simplified or + # reduced in nature. def build_extra_iso(self): """ Builds DVD images based on the data created from the initial lorax on each arch. This should NOT be called during the usual run() section. """ - print() def _generate_graft_points(self): """ Get a list of packages for an extras ISO. This should NOT be called during the usual run() section. """ - print() + + def _get_grafts(self): + """ + Actually get some grafts (get_iso_contents), called by generate grafts + """ + + def _write_grafts(self): + """ + Write out the graft points, called by get_grafts + """ + + def _scanning(self): + """ + Scan tree + """ + + def _merging(self): + """ + Merge tree + """ + + def _sorting(self): + """ + Sorting using the is_rpm and is_image funcs + """ + + def _is_rpm(self): + """ + Is this an RPM? :o + """ + + def _is_image(self): + """ + Is this an image? :o + """ + + def _get_vol_id(self): + """ + Gets a volume ID + """ + + def _get_boot_options(self, arch, createfrom, efi=True, hfs_compat=False): + """ + Gets boot options based on architecture, the iso commands are not + universal. + """ + if arch in ("armhfp",): + result = [] + return result + + if arch in ("aarch64",): + result = [ + "-eltorito-alt-boot", + "-e", + "images/efiboot.img", + "-no-emul-boot", + ] + return result + + if arch in ("i386", "i686", "x86_64"): + result = [ + "-b", + "isolinux/isolinux.bin", + "-c", + "isolinux/boot.cat", + "-no-emul-boot", + "-boot-load-size", + "4", + "-boot-info-table", + ] + + # EFI args + if arch == "x86_64": + result.extend( + [ + "-eltorito-alt-boot", + "-e", + "images/efiboot.img", + "-no-emul-boot" + ] + ) + return result + + # need to go double check if this is needed with stream 9 + if arch == "ppc64le" and hfs_compat: + result = [ + "-part", + "-hfs", + "-r", + "-l", + "-sysid", + "PPC", + "-no-desktop", + "-allow-multidot", + "-chrp-boot", + "-map", + os.path.join(createfrom, "mapping"), + "-hfs-bless", + "/ppc/mac" + ] + return result + + if arch == "ppc64le" and not hfs_compat: + result = [ + "-r", + "-l", + "-sysid", + "PPC", + "-chrp-boot", + ] + return result + + if arch in ("s390x",): + result = [ + "-eltorito-boot", + "images/cdboot.img", + "-no-emul-boot", + ] + return result + + raise ValueError("Architecture %s%s%s is NOT known" % (Color.BOLD, arch, Color.END)) + + # ALL COMMANDS # + def _get_mkisofs_cmd( + self, + iso, + paths, + appid=None, + volid=None, + volset=None, + exclude=None, + boot_args=None, + input_charset="utf-8", + grafts=None, + use_xorrisofs=False, + iso_level=None + ): + # I should hardcode this I think + #untranslated_filenames = True + #translation_table = True + #joliet = True + #joliet_long = True + #rock = True + + cmd = ["/usr/bin/xorrisofs" if use_xorrisofs else "/usr/bin/genisoimage"] + + if iso_level: + cmd.extend(["-iso-level", str(iso_level)]) + + if appid: + cmd.extend(["-appid", appid]) + + #if untranslated_filenames: + cmd.append("-untranslated-filenames") + + if volid: + cmd.extend(["-volid", volid]) + + #if joliet: + cmd.append("-J") + + #if joliet_long: + cmd.append("-joliet-long") + + if volset: + cmd.extend(["-volset", volset]) + + #if rock: + cmd.append("-rational-rock") + + #if not use_xorrisofs and translation_table: + if not use_xorrisofs: + cmd.append("-translation-table") + + if input_charset: + cmd.extend(["-input-charset", input_charset]) + + if exclude: + for i in kobo.shortcuts.force_list(exclude): + cmd.extend(["-x", i]) + + if boot_args: + cmd.extend(boot_args) + + cmd.extend(["-o", iso]) + + if grafts: + cmd.append("-graft-points") + cmd.extend(["-path-list", grafts]) + + return cmd + + def _get_implantisomd5_cmd(self, opts): + """ + Implants md5 into iso + """ + cmd = ["/usr/bin/implantisomd5", "--supported-iso", opts['iso_path']] + return cmd + + def _get_manifest_cmd(self, opts): + """ + Gets an ISO manifest + """ + return "/usr/bin/isoinfo -R -f -i %s | grep -v '/TRANS.TBL$' | sort >> %s.manifest" % ( + shlex.quote(opts['iso_name']), + shlex.quote(opts['iso_name']), + ) + + def _get_isohybrid_cmd(self, opts): + cmd = [] + if opts['arch'] == "x86_64": + cmd = ["/usr/bin/isohybrid"] + cmd.append("--uefi") + cmd.append(opts['iso_path']) + return cmd + + def _get_make_image_cmd(self, opts): + """ + Generates the command to actually make the image in the first place + """ + isokwargs = {} + isokwargs["boot_args"] = self._get_boot_options( + opts['arch'], + os.path.join("$TEMPLATE", "config_files/ppc"), + hfs_compat=self.hfs_compat, + ) + + if opts['arch'] in ("ppc64", "ppc64le"): + isokwargs["input_charset"] = None + + cmd = self._get_mkisofs_cmd( + opts['iso_name'], + volid=opts['volid'], + exclude=["./lost+found"], + grafts=opts['graft_points'], + use_xorrisofs=opts['use_xorrisofs'], + iso_level=opts['iso_level'], + **isokwargs + ) + return cmd + + + def _write_script(self, opts): + """ + Writes out the script to make the DVD + """ class LiveBuild: """ diff --git a/iso/empanadas/poetry.lock b/iso/empanadas/poetry.lock index 716b15a..35ab49d 100644 --- a/iso/empanadas/poetry.lock +++ b/iso/empanadas/poetry.lock @@ -22,14 +22,14 @@ tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (> [[package]] name = "boto3" -version = "1.24.12" +version = "1.24.14" description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" [package.dependencies] -botocore = ">=1.27.12,<1.28.0" +botocore = ">=1.27.14,<1.28.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -38,7 +38,7 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.27.12" +version = "1.27.14" description = "Low-level, data-driven core of boto 3." category = "main" optional = false @@ -141,6 +141,17 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "kobo" +version = "0.24.1" +description = "A pile of python modules used by Red Hat release engineering to build their tools" +category = "main" +optional = false +python-versions = ">2.6" + +[package.dependencies] +six = "*" + [[package]] name = "markupsafe" version = "2.0.1" @@ -354,7 +365,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = ">=3.7,<4" -content-hash = "d011f4622c248f6aa107fd679616eaa19a897147398c6f52dd0dea0ab1d74486" +content-hash = "ccd47ad1b0819968dbad34b68c3f9afd98bd657ee639f9037731fd2a0746bd16" [metadata.files] atomicwrites = [ @@ -366,12 +377,12 @@ attrs = [ {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, ] boto3 = [ - {file = "boto3-1.24.12-py3-none-any.whl", hash = "sha256:0b9757575b8003928defc5fb6e816936fa1bdb1384d0edec6622bb9fb104e96c"}, - {file = "boto3-1.24.12.tar.gz", hash = "sha256:f39b91a4c3614db8e44912ee82426fb4b16d5df2cd66883f3aff6f76d7f5d310"}, + {file = "boto3-1.24.14-py3-none-any.whl", hash = "sha256:490f5e88f5551b33ae3019a37412158b76426d63d1fb910968ade9b6a024e5fe"}, + {file = "boto3-1.24.14.tar.gz", hash = "sha256:e284705da36faa668c715ae1f74ebbff4320dbfbe3a733df3a8ab076d1ed1226"}, ] botocore = [ - {file = "botocore-1.27.12-py3-none-any.whl", hash = "sha256:b8ac156e55267da6e728ea0b806bfcd97adf882801cffe7849c4b88ce4780326"}, - {file = "botocore-1.27.12.tar.gz", hash = "sha256:17d3ec9f684d21e06b64d9cb224934557bcd95031e2ecb551bf16271e8722fec"}, + {file = "botocore-1.27.14-py3-none-any.whl", hash = "sha256:df1e9b208ff93daac7c645b0b04fb6dccd7f20262eae24d87941727025cbeece"}, + {file = "botocore-1.27.14.tar.gz", hash = "sha256:bb56fa77b8fa1ec367c2e16dee62d60000451aac5140dcce3ebddc167fd5c593"}, ] certifi = [ {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, @@ -405,6 +416,9 @@ jmespath = [ {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] +kobo = [ + {file = "kobo-0.24.1.tar.gz", hash = "sha256:d5a30cc20c323f3e9d9b4b2e511650c4b98929b88859bd8cf57463876686e407"}, +] markupsafe = [ {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53"}, {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38"}, diff --git a/iso/empanadas/pyproject.toml b/iso/empanadas/pyproject.toml index fb44274..8755e18 100644 --- a/iso/empanadas/pyproject.toml +++ b/iso/empanadas/pyproject.toml @@ -15,6 +15,7 @@ importlib-resources = "^5.8.0" boto3 = "^1.24.12" xmltodict = "^0.13.0" requests = "^2.28.0" +kobo = "^0.24.1" [tool.poetry.dev-dependencies] pytest = "~5"