update iso utils and prep

iso/py/common.py

@@ -34,7 +34,10 @@ config = {
     "category_stub": "mirror/pub/rocky",
     "sig_category_stub": "mirror/pub/sig",
     "repo_base_url": "https://yumrepofs.build.resf.org/v1/projects",
-    "container": "centos:stream9"
+    "mock_work_root": "/builddir",
+    "container": "centos:stream9",
+    "distname": "Rocky Linux",
+    "shortname": "Rocky"
 }
 
 # Importing the config from yaml

iso/py/configs/el8.yaml

@@ -8,10 +8,6 @@
     - aarch64
   provide_multilib: False
   project_id: ''
-  required_packages:
-    - 'lorax'
-    - 'genisoimage'
-    - 'isomd5sum'
   repo_symlinks:
     devel: 'Devel'
     NFV: 'nfv'
@@ -54,6 +50,18 @@
       - dvd1
       - minimal
       - boot
+    repos:
+      - 'BaseOS'
+      - 'AppStream'
+    variant: 'BaseOS'
+    lorax_removes:
+      - 'libreport-rhel-anaconda-bugzilla'
+    required_packages:
+      - 'lorax'
+      - 'genisoimage'
+      - 'isomd5sum'
+      - 'lorax-templates-rhel'
+      - 'lorax-templates-generic'
   repoclosure_map:
     arches:
       x86_64: '--arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'

iso/py/configs/el9.yaml

@@ -3,6 +3,7 @@
   fullname: 'Rocky Linux 9'
   revision: '9.0'
   rclvl: 'RC1'
+  bugurl: 'https://bugs.rockylinux.org'
   allowed_arches:
     - x86_64
     - aarch64
@@ -10,10 +11,6 @@
     - s390x
   provide_multilib: True
   project_id: '55b17281-bc54-4929-8aca-a8a11d628738'
-  required_packages:
-    - 'lorax'
-    - 'genisoimage'
-    - 'isomd5sum'
   repo_symlinks:
     NFV: 'nfv'
   renames:
@@ -62,6 +59,14 @@
       - 'BaseOS'
       - 'AppStream'
     variant: 'BaseOS'
+    lorax_removes:
+      - 'libreport-rhel-anaconda-bugzilla'
+    required_pkgs:
+      - 'lorax'
+      - 'genisoimage'
+      - 'isomd5sum'
+      - 'lorax-templates-rhel'
+      - 'lorax-templates-generic'
   repoclosure_map:
     arches:
       x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'

iso/py/templates/buildImage.tmpl.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+{% for pkg in lorax %}
+sed -i '/{{ pkg }}/ s/^/#/' /usr/share/lorax/templates.d/80-rhel/runtime-install.tmpl
+{% endfor %}
+
+lorax --product='{{ distname }}' \
+  --version='{{ revision }}' \
+  --release='{{ revision }}' \
+{%- for repo in repos %}
+  --source={{ repo.url }} \
+{%- endfor %}
+  --variant={{ variant }} \
+  --nomacboot \
+  --buildarch={{ arch }} \
+  --volid={{ shortname }}-{{ major }}-{{ minor }}-{{ arch }}-dvd1 \
+  --logfile={{ mock_work_root }}/lorax.log \
+  --rootfs-size=3 \
+  {{ lorax_work_root }}

iso/py/templates/isobuild.tmpl.sh

@@ -4,12 +4,29 @@
 
 # Init the container
 mock \
-  -r /mnt/compose/9/latest-Rocky-9/work/entries/lorax-{{ major }}-{{ arch }}.cfg \
+  -r /var/tmp/lorax-{{ major }}.cfg \
+  --isolation={{ isolation }} \
+  --enable-network \
   --init
 
+cp /var/tmp/buildImage.sh \
+  /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/root/var/tmp
+
 mock \
-  -r /mnt/compose/9/latest-Rocky-9/work/entries/lorax-{{ major }}-{{ arch }}.cfg \
+  -r /var/tmp/lorax-{{ major }}.cfg \
   --shell \
-  --enable-network -- /bin/bash /mnt/compose/9/latest-Rocky-9/work/entries/runLorax-{{ arch }}.sh
+  --isolation={{ isolation }} \
+  --enable-network -- /bin/bash /var/tmp/buildImage.sh
+
+ret_val=$?
+if [ $ret_val -eq 0 ]; then
+  # Copy resulting data to /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result
+  mkdir /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result
+  cp /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/root/{{ builddir }}/lorax-{{ major }}-{{ arch }}.tar.gz \
+    /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result
+else
+  echo "!! LORAX RUN FAILED !!"
+  exit 1
+fi
 
 # Clean up?

iso/py/templates/isolorax.tmpl.sh

@@ -1,2 +1,8 @@
 #!/bin/bash
 # You should not be running this manually.
+
+# everything should be logged like | tee -a {{ log_path }}/lorax-{{ arch }}-{{ date_stamp }}.log
+# for the dvd, we need to rely on pulling from {{ entries_root }}/dvd-{{ arch }}-list
+
+# Run the base lorax steps into a work dir specific to its arch
+# copy everything into BaseOS/arch/os

iso/py/templates/isomock.tmpl.cfg

@@ -3,16 +3,18 @@ config_opts['description'] = '{{ fullname }}'
 config_opts['target_arch'] = '{{ arch }}'
 config_opts['legal_host_arches'] = ('{{ arch }}',)
 
-config_opts['chroot_setup_cmd'] = 'install bash bzip2 coreutils cpio diffutils redhat-release findutils gawk glibc-minimal-langpack grep gzip info patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz genisoimage isomd5sum lorax lorax-templates-rhel lorax-templates-generic'
+config_opts['chroot_setup_cmd'] = 'install bash bzip2 coreutils cpio diffutils redhat-release findutils gawk glibc-minimal-langpack grep gzip info patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz {{ required_pkgs|join(' ') }}'
 config_opts['dist'] = '{{ dist }}'  # only useful for --resultdir variable subst
 config_opts['releasever'] = '{{ major }}'
-config_opts['package_manager'] = 'dnf'
+config_opts['package_manager'] = '{{ pkgmanager|default("dnf") }}'
 config_opts['extra_chroot_dirs'] = [ '/run/lock', ]
 # config_opts['bootstrap_image'] = 'quay.io/rockylinux/rockylinux:{{ major }}'
 
+# If compose is local, the bind mounts will be here
+{% if compose_dir_is_here %}
 config_opts['plugin_conf']['bind_mount_enable'] = True
-config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/mnt/compose', '/mnt/compose'))
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('{{ compose_dir }}', '{{ compose_dir }}'))
-
+{% endif %}
 
 config_opts['dnf.conf'] = """
 [main]
@@ -35,9 +37,9 @@ module_platform_id=platform:{{ dist }}
 user_agent={{ user_agent }}
 
 {% for repo in repos %}
-[{{ repo }}]
+[{{ repo.name }}]
-name={{ repo }}
+name={{ repo.name }}
-baseurl=file:///mnt/compose/{{ major }}/latest-Rocky-{{ major }}/compose/{{ repo }}/{{ arch }}/os
+baseurl={{ repo.url }}
 enabled=1
 gpgcheck=0
 

iso/py/util/dnf_utils.py

@@ -12,6 +12,7 @@ import subprocess
 import shlex
 import time
 import re
+import json
 #import pipes
 from common import Color
 from jinja2 import Environment, FileSystemLoader
@@ -793,7 +794,11 @@ class RepoSync:
 
     def deploy_extra_files(self):
         """
-        deploys extra files based on info of rlvars
+        deploys extra files based on info of rlvars including a
+        extra_files.json
+
+        also deploys COMPOSE_ID and maybe in the future a metadata dir with a
+        bunch of compose-esque stuff.
         """
         pass
 

iso/py/util/iso_utils.py

@@ -14,30 +14,34 @@ import time
 import re
 from productmd.common import SortedConfigParser
 from common import Color
+from jinja2 import Environment, FileSystemLoader
 
 class IsoBuild:
     """
     This helps us build the generic ISO's for a Rocky Linux release. In
-    particular, this is for the boot and dvd images.
+    particular, this is for the boot images.
 
-    Live images are built in another class.
+    While there is a function for building the DVD and live images, this not
+    the main design of this class. The other functions can be called on their
+    own to facilitate those particular builds.
     """
     def __init__(
             self,
             rlvars,
             config,
             major,
-            host=None,
+            isolation: str = 'auto',
+            compose_dir_is_here: bool = False,
             image=None,
-            arch=None,
             logger=None
     ):
-        self.arch = arch
         self.image = image
-        self.host = host
         self.fullname = rlvars['fullname']
+        self.distname = config['distname']
+        self.shortname = config['shortname']
         # Relevant config items
         self.major_version = major
+        self.compose_dir_is_here = compose_dir_is_here
         self.disttag = config['dist']
         self.date_stamp = config['date_stamp']
         self.timestamp = time.time()
@@ -45,11 +49,21 @@ class IsoBuild:
         self.compose_base = config['compose_root'] + "/" + major
         self.iso_drop = config['compose_root'] + "/" + major + "/isos"
         self.current_arch = config['arch']
-        self.extra_files = rlvars['extra_files']
+        self.required_pkgs = rlvars['iso_map']['required_pkgs']
+        self.mock_work_root = config['mock_work_root']
+        self.lorax_result_root = config['mock_work_root'] + "/" + "lorax"
+        self.mock_isolation = isolation
+        self.iso_map = rlvars['iso_map']
 
         # Relevant major version items
+        self.release = rlvars['revision']
+        self.minor_version = rlvars['revision'].split('.')[1]
         self.revision = rlvars['revision'] + "-" + rlvars['rclvl']
-        self.arches = rlvars['allowed_arches']
+        self.repos = rlvars['iso_map']['repos']
+        self.repo_base_url = config['repo_base_url']
+        self.project_id = rlvars['project_id']
+
+        self.extra_files = rlvars['extra_files']
 
         self.staging_dir = os.path.join(
                     config['staging_root'],
@@ -57,6 +71,10 @@ class IsoBuild:
                     self.revision
         )
 
+        # Templates
+        file_loader = FileSystemLoader('templates')
+        self.tmplenv = Environment(loader=file_loader)
+
         self.compose_latest_dir = os.path.join(
                 config['compose_root'],
                 major,
@@ -73,6 +91,12 @@ class IsoBuild:
                 "work/logs"
         )
 
+        self.iso_work_dir = os.path.join(
+                self.compose_latest_dir,
+                "work/iso",
+                config['arch']
+        )
+
         # This is temporary for now.
         if logger is None:
             self.log = logging.getLogger("iso")
@@ -87,6 +111,7 @@ class IsoBuild:
             self.log.addHandler(handler)
 
         self.log.info('iso build init')
+        self.repolist = self.build_repo_list()
         self.log.info(self.revision)
 
     def run(self):
@@ -101,66 +126,121 @@ class IsoBuild:
                 "logs"
         )
 
-        self.iso_build(
+        self.iso_build()
-                sync_root,
-                work_root,
-                log_root,
-                self.arch,
-                self.host
-        )
 
         self.log.info('Compose repo directory: %s' % sync_root)
         self.log.info('ISO Build Logs: %s' % log_root)
         self.log.info('ISO Build completed.')
 
-    def iso_build(self, sync_root, work_root, log_root, arch, host):
+    def build_repo_list(self):
         """
-        Calls out the ISO builds to the individual hosts listed in the map.
+        Builds the repo dictionary
-        Each architecture is expected to build their own ISOs, similar to
+        """
-        runroot operations of koji and pungi.
+        repolist = []
+        for name in self.repos:
+            if not self.compose_dir_is_here:
+                constructed_url = '{}/{}/repo/hashed-{}/{}'.format(
+                        self.repo_base_url,
+                        self.project_id,
+                        name,
+                        self.current_arch
+                )
+            else:
+                constructed_url = 'file://{}/{}/{}/os'.format(
+                        self.compose_latest_sync,
+                        name,
+                        self.current_arch
+                )
 
-        It IS possible to run locally, but that would mean this only builds
+
-        ISOs for the architecture of the running machine. Please keep this in
+            repodata = {
-        mind when stating host=local.
+                'name': name,
+                'url': constructed_url
+            }
+
+            repolist.append(repodata)
+
+        return repolist
+
+    def iso_build(self):
+        """
+        This does the general ISO building for the current running
+        architecture. This generates the mock config and the general script
+        needed to get this part running.
         """
         # Check for local build, build accordingly
         # Check for arch specific build, build accordingly
         # local AND arch cannot be used together, local supersedes. print
         # warning.
-        local_only = False
+        self.log.info('Generating ISO configuration and scripts')
-        if 'local' in self.host:
+        self.generate_iso_scripts()
-            local_only = True
-
-        arch = self.arch.copy()
-        if local_only and self.arch:
-            self.log.warn('You cannot set local build AND an architecture.')
-            self.log.warn('The architecture %s will be set' % self.current_arch)
-            arch = self.current_arch
-
-    def iso_build_local(self, sync_root, work_root, log_root):
-        """
-        Local iso builds only. Architecture is locked.
-        """
-        print()
-
-    def iso_build_remote(self, sync_root, work_root, log_root, arch):
-        """
-        Remote ISO builds. Architecture is all or single.
-        """
         print()
 
     def generate_iso_scripts(self):
         """
         Generates the scripts needed to be ran in the mock roots
         """
+        mock_iso_template = self.tmplenv.get_template('isomock.tmpl.cfg')
+        mock_sh_template = self.tmplenv.get_template('isobuild.tmpl.sh')
+        iso_template = self.tmplenv.get_template('buildImage.tmpl.sh')
+
+        mock_iso_path = '/var/tmp/lorax-' + self.major_version + '.cfg'
+        mock_sh_path = '/var/tmp/isobuild.sh'
+        iso_template_path = '/var/tmp/buildImage.sh'
+
+        mock_iso_template_output = mock_iso_template.render(
+                arch=self.current_arch,
+                major=self.major_version,
+                fullname=self.fullname,
+                required_pkgs=self.required_pkgs,
+                dist=self.disttag,
+                repos=self.repolist,
+                user_agent='{{ user_agent }}',
+        )
+
+        mock_sh_template_output = mock_sh_template.render(
+                arch=self.current_arch,
+                major=self.major_version,
+                isolation=self.mock_isolation,
+                builddir=self.mock_work_root,
+                shortname=self.shortname,
+        )
+
+        iso_template_output = iso_template.render(
+                arch=self.current_arch,
+                major=self.major_version,
+                minor=self.minor_version,
+                shortname=self.shortname,
+                repos=self.repolist,
+                variant=self.iso_map['variant'],
+                lorax=self.iso_map['lorax_removes'],
+                distname=self.distname,
+                revision=self.release,
+        )
+
+        mock_iso_entry = open(mock_iso_path, "w+")
+        mock_iso_entry.write(mock_iso_template_output)
+        mock_iso_entry.close()
+
+        mock_sh_entry = open(mock_sh_path, "w+")
+        mock_sh_entry.write(mock_sh_template_output)
+        mock_sh_entry.close()
+
+        iso_template_entry = open(iso_template_path, "w+")
+        iso_template_entry.write(iso_template_output)
+        iso_template_entry.close()
         print()
 
+    # !!! Send help, we would prefer to do this using the productmd python
+    # !!! library. If you are reading this and you can help us, please do so!
     def treeinfo_write(self):
         """
         Ensure treeinfo is written correctly
         """
         print()
 
+    # !!! Send help, we would prefer to do this using the productmd python
+    # !!! library. If you are reading this and you can help us, please do so!
     def discinfo_write(self):
         """
         Ensure discinfo is written correctly
@@ -187,13 +267,19 @@ class IsoBuild:
             "",
         ]
 
-
+    def build_extra_iso(self):
-    def generate_graft_points(self):
         """
-        Get a list of packages for an ISO
+        Builds DVD images based on the data created from the initial lorax on
+        each arch. This should NOT be called during the usual run() section.
         """
         print()
 
+    def generate_graft_points(self):
+        """
+        Get a list of packages for an extras ISO. This should NOT be called
+        during the usual run() section.
+        """
+        print()
 
 class LiveBuild:
     """

sync/common_9

@@ -169,6 +169,6 @@ EOF
   /bin/cp "${TREEINFO_VAR}" "${PRISTINE_TREE}"
 }
 
-export -f treeinfoFixer
+#export -f treeinfoFixer
-export -f treeinfoModder
+#export -f treeinfoModder
-export -f treeinfoModderKickstart
+#export -f treeinfoModderKickstart