From 3d161df451fa798fdfd70b0a6ea3d29fef476a6b Mon Sep 17 00:00:00 2001 From: nazunalika Date: Wed, 15 Jun 2022 13:53:12 -0700 Subject: [PATCH] update iso utils and prep --- iso/py/common.py | 5 +- iso/py/configs/el8.yaml | 16 +- iso/py/configs/el9.yaml | 13 +- iso/py/templates/buildImage.tmpl.sh | 19 ++ iso/py/templates/isobuild.tmpl.sh | 23 ++- iso/py/templates/isolorax.tmpl.sh | 6 + .../{isomock.tmpl => isomock.tmpl.cfg} | 16 +- iso/py/util/dnf_utils.py | 7 +- iso/py/util/iso_utils.py | 176 +++++++++++++----- sync/common_9 | 6 +- 10 files changed, 219 insertions(+), 68 deletions(-) create mode 100644 iso/py/templates/buildImage.tmpl.sh rename iso/py/templates/{isomock.tmpl => isomock.tmpl.cfg} (73%) diff --git a/iso/py/common.py b/iso/py/common.py index 8032efd..509a622 100644 --- a/iso/py/common.py +++ b/iso/py/common.py @@ -34,7 +34,10 @@ config = { "category_stub": "mirror/pub/rocky", "sig_category_stub": "mirror/pub/sig", "repo_base_url": "https://yumrepofs.build.resf.org/v1/projects", - "container": "centos:stream9" + "mock_work_root": "/builddir", + "container": "centos:stream9", + "distname": "Rocky Linux", + "shortname": "Rocky" } # Importing the config from yaml diff --git a/iso/py/configs/el8.yaml b/iso/py/configs/el8.yaml index f7f64d2..93bcdf3 100644 --- a/iso/py/configs/el8.yaml +++ b/iso/py/configs/el8.yaml @@ -8,10 +8,6 @@ - aarch64 provide_multilib: False project_id: '' - required_packages: - - 'lorax' - - 'genisoimage' - - 'isomd5sum' repo_symlinks: devel: 'Devel' NFV: 'nfv' @@ -54,6 +50,18 @@ - dvd1 - minimal - boot + repos: + - 'BaseOS' + - 'AppStream' + variant: 'BaseOS' + lorax_removes: + - 'libreport-rhel-anaconda-bugzilla' + required_packages: + - 'lorax' + - 'genisoimage' + - 'isomd5sum' + - 'lorax-templates-rhel' + - 'lorax-templates-generic' repoclosure_map: arches: x86_64: '--arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch' diff --git a/iso/py/configs/el9.yaml b/iso/py/configs/el9.yaml index cf5c49e..b0a371b 100644 --- a/iso/py/configs/el9.yaml +++ b/iso/py/configs/el9.yaml @@ -3,6 +3,7 @@ fullname: 'Rocky Linux 9' revision: '9.0' rclvl: 'RC1' + bugurl: 'https://bugs.rockylinux.org' allowed_arches: - x86_64 - aarch64 @@ -10,10 +11,6 @@ - s390x provide_multilib: True project_id: '55b17281-bc54-4929-8aca-a8a11d628738' - required_packages: - - 'lorax' - - 'genisoimage' - - 'isomd5sum' repo_symlinks: NFV: 'nfv' renames: @@ -62,6 +59,14 @@ - 'BaseOS' - 'AppStream' variant: 'BaseOS' + lorax_removes: + - 'libreport-rhel-anaconda-bugzilla' + required_pkgs: + - 'lorax' + - 'genisoimage' + - 'isomd5sum' + - 'lorax-templates-rhel' + - 'lorax-templates-generic' repoclosure_map: arches: x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch' diff --git a/iso/py/templates/buildImage.tmpl.sh b/iso/py/templates/buildImage.tmpl.sh new file mode 100644 index 0000000..efe097d --- /dev/null +++ b/iso/py/templates/buildImage.tmpl.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +{% for pkg in lorax %} +sed -i '/{{ pkg }}/ s/^/#/' /usr/share/lorax/templates.d/80-rhel/runtime-install.tmpl +{% endfor %} + +lorax --product='{{ distname }}' \ + --version='{{ revision }}' \ + --release='{{ revision }}' \ +{%- for repo in repos %} + --source={{ repo.url }} \ +{%- endfor %} + --variant={{ variant }} \ + --nomacboot \ + --buildarch={{ arch }} \ + --volid={{ shortname }}-{{ major }}-{{ minor }}-{{ arch }}-dvd1 \ + --logfile={{ mock_work_root }}/lorax.log \ + --rootfs-size=3 \ + {{ lorax_work_root }} diff --git a/iso/py/templates/isobuild.tmpl.sh b/iso/py/templates/isobuild.tmpl.sh index 880aa16..21b03f1 100644 --- a/iso/py/templates/isobuild.tmpl.sh +++ b/iso/py/templates/isobuild.tmpl.sh @@ -4,12 +4,29 @@ # Init the container mock \ - -r /mnt/compose/9/latest-Rocky-9/work/entries/lorax-{{ major }}-{{ arch }}.cfg \ + -r /var/tmp/lorax-{{ major }}.cfg \ + --isolation={{ isolation }} \ + --enable-network \ --init +cp /var/tmp/buildImage.sh \ + /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/root/var/tmp + mock \ - -r /mnt/compose/9/latest-Rocky-9/work/entries/lorax-{{ major }}-{{ arch }}.cfg \ + -r /var/tmp/lorax-{{ major }}.cfg \ --shell \ - --enable-network -- /bin/bash /mnt/compose/9/latest-Rocky-9/work/entries/runLorax-{{ arch }}.sh + --isolation={{ isolation }} \ + --enable-network -- /bin/bash /var/tmp/buildImage.sh + +ret_val=$? +if [ $ret_val -eq 0 ]; then + # Copy resulting data to /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result + mkdir /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result + cp /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/root/{{ builddir }}/lorax-{{ major }}-{{ arch }}.tar.gz \ + /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result +else + echo "!! LORAX RUN FAILED !!" + exit 1 +fi # Clean up? diff --git a/iso/py/templates/isolorax.tmpl.sh b/iso/py/templates/isolorax.tmpl.sh index 5dba11d..f08b123 100644 --- a/iso/py/templates/isolorax.tmpl.sh +++ b/iso/py/templates/isolorax.tmpl.sh @@ -1,2 +1,8 @@ #!/bin/bash # You should not be running this manually. + +# everything should be logged like | tee -a {{ log_path }}/lorax-{{ arch }}-{{ date_stamp }}.log +# for the dvd, we need to rely on pulling from {{ entries_root }}/dvd-{{ arch }}-list + +# Run the base lorax steps into a work dir specific to its arch +# copy everything into BaseOS/arch/os diff --git a/iso/py/templates/isomock.tmpl b/iso/py/templates/isomock.tmpl.cfg similarity index 73% rename from iso/py/templates/isomock.tmpl rename to iso/py/templates/isomock.tmpl.cfg index a07589c..69f84e3 100644 --- a/iso/py/templates/isomock.tmpl +++ b/iso/py/templates/isomock.tmpl.cfg @@ -3,16 +3,18 @@ config_opts['description'] = '{{ fullname }}' config_opts['target_arch'] = '{{ arch }}' config_opts['legal_host_arches'] = ('{{ arch }}',) -config_opts['chroot_setup_cmd'] = 'install bash bzip2 coreutils cpio diffutils redhat-release findutils gawk glibc-minimal-langpack grep gzip info patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz genisoimage isomd5sum lorax lorax-templates-rhel lorax-templates-generic' +config_opts['chroot_setup_cmd'] = 'install bash bzip2 coreutils cpio diffutils redhat-release findutils gawk glibc-minimal-langpack grep gzip info patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz {{ required_pkgs|join(' ') }}' config_opts['dist'] = '{{ dist }}' # only useful for --resultdir variable subst config_opts['releasever'] = '{{ major }}' -config_opts['package_manager'] = 'dnf' +config_opts['package_manager'] = '{{ pkgmanager|default("dnf") }}' config_opts['extra_chroot_dirs'] = [ '/run/lock', ] # config_opts['bootstrap_image'] = 'quay.io/rockylinux/rockylinux:{{ major }}' +# If compose is local, the bind mounts will be here +{% if compose_dir_is_here %} config_opts['plugin_conf']['bind_mount_enable'] = True -config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/mnt/compose', '/mnt/compose')) - +config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('{{ compose_dir }}', '{{ compose_dir }}')) +{% endif %} config_opts['dnf.conf'] = """ [main] @@ -35,9 +37,9 @@ module_platform_id=platform:{{ dist }} user_agent={{ user_agent }} {% for repo in repos %} -[{{ repo }}] -name={{ repo }} -baseurl=file:///mnt/compose/{{ major }}/latest-Rocky-{{ major }}/compose/{{ repo }}/{{ arch }}/os +[{{ repo.name }}] +name={{ repo.name }} +baseurl={{ repo.url }} enabled=1 gpgcheck=0 diff --git a/iso/py/util/dnf_utils.py b/iso/py/util/dnf_utils.py index ecc75b9..6f6ec5e 100644 --- a/iso/py/util/dnf_utils.py +++ b/iso/py/util/dnf_utils.py @@ -12,6 +12,7 @@ import subprocess import shlex import time import re +import json #import pipes from common import Color from jinja2 import Environment, FileSystemLoader @@ -793,7 +794,11 @@ class RepoSync: def deploy_extra_files(self): """ - deploys extra files based on info of rlvars + deploys extra files based on info of rlvars including a + extra_files.json + + also deploys COMPOSE_ID and maybe in the future a metadata dir with a + bunch of compose-esque stuff. """ pass diff --git a/iso/py/util/iso_utils.py b/iso/py/util/iso_utils.py index 6bc2c19..1461019 100644 --- a/iso/py/util/iso_utils.py +++ b/iso/py/util/iso_utils.py @@ -14,30 +14,34 @@ import time import re from productmd.common import SortedConfigParser from common import Color +from jinja2 import Environment, FileSystemLoader class IsoBuild: """ This helps us build the generic ISO's for a Rocky Linux release. In - particular, this is for the boot and dvd images. + particular, this is for the boot images. - Live images are built in another class. + While there is a function for building the DVD and live images, this not + the main design of this class. The other functions can be called on their + own to facilitate those particular builds. """ def __init__( self, rlvars, config, major, - host=None, + isolation: str = 'auto', + compose_dir_is_here: bool = False, image=None, - arch=None, logger=None ): - self.arch = arch self.image = image - self.host = host self.fullname = rlvars['fullname'] + self.distname = config['distname'] + self.shortname = config['shortname'] # Relevant config items self.major_version = major + self.compose_dir_is_here = compose_dir_is_here self.disttag = config['dist'] self.date_stamp = config['date_stamp'] self.timestamp = time.time() @@ -45,11 +49,21 @@ class IsoBuild: self.compose_base = config['compose_root'] + "/" + major self.iso_drop = config['compose_root'] + "/" + major + "/isos" self.current_arch = config['arch'] - self.extra_files = rlvars['extra_files'] + self.required_pkgs = rlvars['iso_map']['required_pkgs'] + self.mock_work_root = config['mock_work_root'] + self.lorax_result_root = config['mock_work_root'] + "/" + "lorax" + self.mock_isolation = isolation + self.iso_map = rlvars['iso_map'] # Relevant major version items + self.release = rlvars['revision'] + self.minor_version = rlvars['revision'].split('.')[1] self.revision = rlvars['revision'] + "-" + rlvars['rclvl'] - self.arches = rlvars['allowed_arches'] + self.repos = rlvars['iso_map']['repos'] + self.repo_base_url = config['repo_base_url'] + self.project_id = rlvars['project_id'] + + self.extra_files = rlvars['extra_files'] self.staging_dir = os.path.join( config['staging_root'], @@ -57,6 +71,10 @@ class IsoBuild: self.revision ) + # Templates + file_loader = FileSystemLoader('templates') + self.tmplenv = Environment(loader=file_loader) + self.compose_latest_dir = os.path.join( config['compose_root'], major, @@ -73,6 +91,12 @@ class IsoBuild: "work/logs" ) + self.iso_work_dir = os.path.join( + self.compose_latest_dir, + "work/iso", + config['arch'] + ) + # This is temporary for now. if logger is None: self.log = logging.getLogger("iso") @@ -87,6 +111,7 @@ class IsoBuild: self.log.addHandler(handler) self.log.info('iso build init') + self.repolist = self.build_repo_list() self.log.info(self.revision) def run(self): @@ -101,66 +126,121 @@ class IsoBuild: "logs" ) - self.iso_build( - sync_root, - work_root, - log_root, - self.arch, - self.host - ) + self.iso_build() self.log.info('Compose repo directory: %s' % sync_root) self.log.info('ISO Build Logs: %s' % log_root) self.log.info('ISO Build completed.') - def iso_build(self, sync_root, work_root, log_root, arch, host): + def build_repo_list(self): """ - Calls out the ISO builds to the individual hosts listed in the map. - Each architecture is expected to build their own ISOs, similar to - runroot operations of koji and pungi. + Builds the repo dictionary + """ + repolist = [] + for name in self.repos: + if not self.compose_dir_is_here: + constructed_url = '{}/{}/repo/hashed-{}/{}'.format( + self.repo_base_url, + self.project_id, + name, + self.current_arch + ) + else: + constructed_url = 'file://{}/{}/{}/os'.format( + self.compose_latest_sync, + name, + self.current_arch + ) - It IS possible to run locally, but that would mean this only builds - ISOs for the architecture of the running machine. Please keep this in - mind when stating host=local. + + repodata = { + 'name': name, + 'url': constructed_url + } + + repolist.append(repodata) + + return repolist + + def iso_build(self): + """ + This does the general ISO building for the current running + architecture. This generates the mock config and the general script + needed to get this part running. """ # Check for local build, build accordingly # Check for arch specific build, build accordingly # local AND arch cannot be used together, local supersedes. print # warning. - local_only = False - if 'local' in self.host: - local_only = True - - arch = self.arch.copy() - if local_only and self.arch: - self.log.warn('You cannot set local build AND an architecture.') - self.log.warn('The architecture %s will be set' % self.current_arch) - arch = self.current_arch - - def iso_build_local(self, sync_root, work_root, log_root): - """ - Local iso builds only. Architecture is locked. - """ - print() - - def iso_build_remote(self, sync_root, work_root, log_root, arch): - """ - Remote ISO builds. Architecture is all or single. - """ + self.log.info('Generating ISO configuration and scripts') + self.generate_iso_scripts() print() def generate_iso_scripts(self): """ Generates the scripts needed to be ran in the mock roots """ + mock_iso_template = self.tmplenv.get_template('isomock.tmpl.cfg') + mock_sh_template = self.tmplenv.get_template('isobuild.tmpl.sh') + iso_template = self.tmplenv.get_template('buildImage.tmpl.sh') + + mock_iso_path = '/var/tmp/lorax-' + self.major_version + '.cfg' + mock_sh_path = '/var/tmp/isobuild.sh' + iso_template_path = '/var/tmp/buildImage.sh' + + mock_iso_template_output = mock_iso_template.render( + arch=self.current_arch, + major=self.major_version, + fullname=self.fullname, + required_pkgs=self.required_pkgs, + dist=self.disttag, + repos=self.repolist, + user_agent='{{ user_agent }}', + ) + + mock_sh_template_output = mock_sh_template.render( + arch=self.current_arch, + major=self.major_version, + isolation=self.mock_isolation, + builddir=self.mock_work_root, + shortname=self.shortname, + ) + + iso_template_output = iso_template.render( + arch=self.current_arch, + major=self.major_version, + minor=self.minor_version, + shortname=self.shortname, + repos=self.repolist, + variant=self.iso_map['variant'], + lorax=self.iso_map['lorax_removes'], + distname=self.distname, + revision=self.release, + ) + + mock_iso_entry = open(mock_iso_path, "w+") + mock_iso_entry.write(mock_iso_template_output) + mock_iso_entry.close() + + mock_sh_entry = open(mock_sh_path, "w+") + mock_sh_entry.write(mock_sh_template_output) + mock_sh_entry.close() + + iso_template_entry = open(iso_template_path, "w+") + iso_template_entry.write(iso_template_output) + iso_template_entry.close() print() + # !!! Send help, we would prefer to do this using the productmd python + # !!! library. If you are reading this and you can help us, please do so! def treeinfo_write(self): """ Ensure treeinfo is written correctly """ print() + # !!! Send help, we would prefer to do this using the productmd python + # !!! library. If you are reading this and you can help us, please do so! def discinfo_write(self): """ Ensure discinfo is written correctly @@ -187,13 +267,19 @@ class IsoBuild: "", ] - - def generate_graft_points(self): + def build_extra_iso(self): """ - Get a list of packages for an ISO + Builds DVD images based on the data created from the initial lorax on + each arch. This should NOT be called during the usual run() section. """ print() + def generate_graft_points(self): + """ + Get a list of packages for an extras ISO. This should NOT be called + during the usual run() section. + """ + print() class LiveBuild: """ diff --git a/sync/common_9 b/sync/common_9 index a0e2262..8b1cbb8 100644 --- a/sync/common_9 +++ b/sync/common_9 @@ -169,6 +169,6 @@ EOF /bin/cp "${TREEINFO_VAR}" "${PRISTINE_TREE}" } -export -f treeinfoFixer -export -f treeinfoModder -export -f treeinfoModderKickstart +#export -f treeinfoFixer +#export -f treeinfoModder +#export -f treeinfoModderKickstart