From f4f29d97ccfe6ac5027c342e9575985ce3f46c73 Mon Sep 17 00:00:00 2001
From: nazunalika <label@rockylinux.org>
Date: Mon, 27 Jun 2022 17:59:21 -0700
Subject: [PATCH] Expand and Generalize ISO Functions

* Utils is now empanadas.util.Shared
* Each config now has a profile name to determine latest-Rocky-{} link
  (Results RLBT#0000131)
* Check added to see if an ISO was built at some point, and forcefully
  exit if so
* Lorax tar ball should be in the format of of lorax-X.Y-ARCH to ensure
  there's no collisions between stable, beta, and lh builds
---
 iso/empanadas/empanadas/common.py             | 41 ----------
 iso/empanadas/empanadas/configs/el8.yaml      |  1 +
 iso/empanadas/empanadas/configs/el9-beta.yaml |  3 +-
 iso/empanadas/empanadas/configs/el9.yaml      |  3 +-
 iso/empanadas/empanadas/configs/el9lh.yaml    |  3 +-
 iso/empanadas/empanadas/configs/rln.yaml      |  3 +-
 iso/empanadas/empanadas/scripts/build_iso.py  |  2 +-
 .../empanadas/scripts/build_iso_extra.py      |  4 +-
 .../empanadas/scripts/sync_from_peridot.py    |  5 +-
 .../scripts/sync_from_peridot_test.py         |  3 +-
 .../templates/buildExtraImage.tmpl.sh         |  1 +
 .../empanadas/templates/buildImage.tmpl.sh    |  6 +-
 .../empanadas/templates/isobuild.tmpl.sh      |  2 +-
 iso/empanadas/empanadas/util/__init__.py      |  7 +-
 iso/empanadas/empanadas/util/dnf_utils.py     |  4 +-
 iso/empanadas/empanadas/util/iso_utils.py     | 50 ++++--------
 iso/empanadas/empanadas/util/shared.py        | 79 +++++++++++++++++++
 17 files changed, 126 insertions(+), 91 deletions(-)
 create mode 100644 iso/empanadas/empanadas/util/shared.py

diff --git a/iso/empanadas/empanadas/common.py b/iso/empanadas/empanadas/common.py
index b1b4d25..c3619ce 100644
--- a/iso/empanadas/empanadas/common.py
+++ b/iso/empanadas/empanadas/common.py
@@ -21,47 +21,6 @@ class Color:
     BOLD = '\033[1m'
     END = '\033[0m'
 
-class Utils:
-    """
-    Quick utilities that may be commonly used
-    """
-    @staticmethod
-    def get_checksum(path, hashtype, logger):
-        """
-        Generates a checksum from the provided path by doing things in chunks.
-        This way we don't do it in memory.
-        """
-        try:
-            checksum = hashlib.new(hashtype)
-        except ValueError:
-            logger.error("Invalid hash type: %s" % hashtype)
-            return False
-
-        try:
-            input_file = open(path, "rb")
-        except IOError as e:
-            logger.error("Could not open file %s: %s" % (path, e))
-            return False
-
-        while True:
-            chunk = input_file.read(8192)
-            if not chunk:
-                break
-            checksum.update(chunk)
-
-        input_file.close()
-        stat = os.stat(path)
-        base = os.path.basename(path)
-        # This emulates our current syncing scripts that runs stat and
-        # sha256sum and what not with a very specific output.
-        return "%s: %s bytes\n%s (%s) = %s\n" % (
-                base,
-                stat.st_size,
-                hashtype.upper(),
-                base,
-                checksum.hexdigest()
-        )
-
 # vars and additional checks
 rldict = {}
 sigdict = {}
diff --git a/iso/empanadas/empanadas/configs/el8.yaml b/iso/empanadas/empanadas/configs/el8.yaml
index f6c1a67..eb80aff 100644
--- a/iso/empanadas/empanadas/configs/el8.yaml
+++ b/iso/empanadas/empanadas/configs/el8.yaml
@@ -5,6 +5,7 @@
   rclvl: 'RC2'
   major: '8'
   minor: '6'
+  profile: '8'
   bugurl: 'https://bugs.rockylinux.org'
   allowed_arches:
     - x86_64
diff --git a/iso/empanadas/empanadas/configs/el9-beta.yaml b/iso/empanadas/empanadas/configs/el9-beta.yaml
index f5f3a07..19d6cd5 100644
--- a/iso/empanadas/empanadas/configs/el9-beta.yaml
+++ b/iso/empanadas/empanadas/configs/el9-beta.yaml
@@ -1,10 +1,11 @@
 ---
 '9-beta':
-  fullname: 'Rocky Linux 9'
+  fullname: 'Rocky Linux 9.1'
   revision: '9.1'
   rclvl: 'BETA1'
   major: '9'
   minor: '1'
+  profile: '9-beta'
   bugurl: 'https://bugs.rockylinux.org'
   checksum: 'sha256'
   allowed_arches:
diff --git a/iso/empanadas/empanadas/configs/el9.yaml b/iso/empanadas/empanadas/configs/el9.yaml
index fe393a8..88a978b 100644
--- a/iso/empanadas/empanadas/configs/el9.yaml
+++ b/iso/empanadas/empanadas/configs/el9.yaml
@@ -1,10 +1,11 @@
 ---
 '9':
-  fullname: 'Rocky Linux 9'
+  fullname: 'Rocky Linux 9.0'
   revision: '9.0'
   rclvl: 'RC1'
   major: '9'
   minor: '0'
+  profile: '9'
   bugurl: 'https://bugs.rockylinux.org'
   checksum: 'sha256'
   allowed_arches:
diff --git a/iso/empanadas/empanadas/configs/el9lh.yaml b/iso/empanadas/empanadas/configs/el9lh.yaml
index 9583886..4176f66 100644
--- a/iso/empanadas/empanadas/configs/el9lh.yaml
+++ b/iso/empanadas/empanadas/configs/el9lh.yaml
@@ -1,10 +1,11 @@
 ---
 '9-lookahead':
-  fullname: 'Rocky Linux 9'
+  fullname: 'Rocky Linux 9.1'
   revision: '9.1'
   rclvl: 'LH1'
   major: '9'
   minor: '1'
+  profile: '9-lookahead'
   bugurl: 'https://bugs.rockylinux.org'
   checksum: 'sha256'
   allowed_arches:
diff --git a/iso/empanadas/empanadas/configs/rln.yaml b/iso/empanadas/empanadas/configs/rln.yaml
index c36fcb1..7544ce4 100644
--- a/iso/empanadas/empanadas/configs/rln.yaml
+++ b/iso/empanadas/empanadas/configs/rln.yaml
@@ -2,9 +2,10 @@
 'rln':
   fullname: 'Rocky Linux New'
   revision: '10'
-  rclvl: 'RC1'
+  rclvl: 'RLN120'
   major: '10'
   minor: '0'
+  profile: 'rln'
   bugurl: 'https://bugs.rockylinux.org'
   checksum: 'sha256'
   allowed_arches:
diff --git a/iso/empanadas/empanadas/scripts/build_iso.py b/iso/empanadas/empanadas/scripts/build_iso.py
index c5907f1..b35cdf2 100755
--- a/iso/empanadas/empanadas/scripts/build_iso.py
+++ b/iso/empanadas/empanadas/scripts/build_iso.py
@@ -8,7 +8,7 @@ from empanadas.util import IsoBuild
 
 parser = argparse.ArgumentParser(description="ISO Compose")
 
-parser.add_argument('--release', type=str, help="Major Release Version", required=True)
+parser.add_argument('--release', type=str, help="Major Release Version or major-type (eg 9-beta)", required=True)
 parser.add_argument('--isolation', type=str, help="mock isolation mode")
 parser.add_argument('--rc', action='store_true', help="Release Candidate")
 parser.add_argument('--local-compose', action='store_true', help="Compose Directory is Here")
diff --git a/iso/empanadas/empanadas/scripts/build_iso_extra.py b/iso/empanadas/empanadas/scripts/build_iso_extra.py
index 074dba4..a646d12 100755
--- a/iso/empanadas/empanadas/scripts/build_iso_extra.py
+++ b/iso/empanadas/empanadas/scripts/build_iso_extra.py
@@ -8,7 +8,7 @@ from empanadas.util import IsoBuild
 
 parser = argparse.ArgumentParser(description="ISO Compose")
 
-parser.add_argument('--release', type=str, help="Major Release Version", required=True)
+parser.add_argument('--release', type=str, help="Major Release Version or major-type (eg 9-beta)", required=True)
 parser.add_argument('--rc', action='store_true', help="Release Candidate")
 parser.add_argument('--arch', type=str, help="Architecture")
 parser.add_argument('--isolation', type=str, help="Mock Isolation")
@@ -30,7 +30,7 @@ a = IsoBuild(
         extra_iso=results.extra_iso,
         extra_iso_mode=results.extra_iso_mode,
         compose_dir_is_here=results.local_compose,
-        logger=results.logger,
+        logger=results.logger
 )
 
 def run():
diff --git a/iso/empanadas/empanadas/scripts/sync_from_peridot.py b/iso/empanadas/empanadas/scripts/sync_from_peridot.py
index e2481d7..1e40ed8 100755
--- a/iso/empanadas/empanadas/scripts/sync_from_peridot.py
+++ b/iso/empanadas/empanadas/scripts/sync_from_peridot.py
@@ -28,8 +28,9 @@ parser.add_argument('--logger', type=str)
 
 # Parse them
 results = parser.parse_args()
-
 rlvars = rldict[results.release]
+major = rlvars['major']
+
 r = Checks(rlvars, config['arch'])
 r.check_valid_arch()
 
@@ -37,7 +38,7 @@ r.check_valid_arch()
 a = RepoSync(
         rlvars,
         config,
-        major=rlvars['major'],
+        major=major,
         repo=results.repo,
         arch=results.arch,
         ignore_debug=results.ignore_debug,
diff --git a/iso/empanadas/empanadas/scripts/sync_from_peridot_test.py b/iso/empanadas/empanadas/scripts/sync_from_peridot_test.py
index 023ba42..5057753 100755
--- a/iso/empanadas/empanadas/scripts/sync_from_peridot_test.py
+++ b/iso/empanadas/empanadas/scripts/sync_from_peridot_test.py
@@ -14,4 +14,5 @@ r.check_valid_arch()
 a = RepoSync(rlvars, config, major="9", repo="BaseOS", parallel=True, ignore_debug=False, ignore_source=False, hashed=True)
 
 def run():
-    a.run()
+    print(rlvars.keys())
+    print(rlvars)
diff --git a/iso/empanadas/empanadas/templates/buildExtraImage.tmpl.sh b/iso/empanadas/empanadas/templates/buildExtraImage.tmpl.sh
index 0fedfca..8f145de 100644
--- a/iso/empanadas/empanadas/templates/buildExtraImage.tmpl.sh
+++ b/iso/empanadas/empanadas/templates/buildExtraImage.tmpl.sh
@@ -5,6 +5,7 @@ set -ex
 {{ lorax_pkg_cmd }}
 mkdir -p {{ compose_work_iso_dir }}/{{ arch }}
 cd  {{ compose_work_iso_dir  }}/{{ arch }}
+test -f {{ isoname }} || { echo "!! ISO ALREDY EXISTS !!"; exit 1;  }
 {% else %}
 cd /builddir
 
diff --git a/iso/empanadas/empanadas/templates/buildImage.tmpl.sh b/iso/empanadas/empanadas/templates/buildImage.tmpl.sh
index bdf062b..817d73b 100644
--- a/iso/empanadas/empanadas/templates/buildImage.tmpl.sh
+++ b/iso/empanadas/empanadas/templates/buildImage.tmpl.sh
@@ -7,8 +7,9 @@ VERSION="{{ revision }}"
 PRODUCT="{{ distname }}"
 MOCKBLD="{{ builddir }}"
 LORAXRES="{{ lorax_work_root }}"
-LORAX_TAR="lorax-{{ major }}-{{ arch }}.tar.gz"
+LORAX_TAR="lorax-{{ revision }}-{{ arch }}.tar.gz"
 LOGFILE="lorax-{{ arch }}.log"
+BUGURL="{{ bugurl }}"
 
 {% for pkg in lorax %}
 sed -i '/{{ pkg }}/ s/^/#/' /usr/share/lorax/templates.d/80-rhel/runtime-install.tmpl
@@ -23,6 +24,7 @@ lorax --product="${PRODUCT}" \
 {%- for repo in repos %}
   --source={{ repo.url }} \
 {%- endfor %}
+  --bugurl="${BUGURL}" \
   --variant="${VARIANT}" \
   --nomacboot \
   --buildarch="${ARCH}" \
@@ -45,7 +47,7 @@ if [ -f "/usr/bin/xorriso" ]; then
     /usr/bin/xorriso -dev lorax/images/boot.iso --find |
       tail -n+2 |
       tr -d "'" |
-      cut -c2- sort >> lorax/images/boot.iso.manifest
+      cut -c2-  | sort >> lorax/images/boot.iso.manifest
 elif [ -f "/usr/bin/isoinfo" ]; then
     /usr/bin/isoinfo -R -f -i lorax/images/boot.iso |
       grep -v '/TRANS.TBL$' | sort >> lorax/images/boot.iso.manifest
diff --git a/iso/empanadas/empanadas/templates/isobuild.tmpl.sh b/iso/empanadas/empanadas/templates/isobuild.tmpl.sh
index d11a2f1..95184b6 100644
--- a/iso/empanadas/empanadas/templates/isobuild.tmpl.sh
+++ b/iso/empanadas/empanadas/templates/isobuild.tmpl.sh
@@ -9,7 +9,7 @@ MOCK_RESL="${MOCK_ROOT}/result"
 MOCK_CHRO="${MOCK_ROOT}/root"
 MOCK_LOG="${MOCK_RESL}/mock-output.log"
 LORAX_SCR="/var/tmp/buildImage.sh"
-LORAX_TAR="lorax-{{ major }}-{{ arch }}.tar.gz"
+LORAX_TAR="lorax-{{ revision }}-{{ arch }}.tar.gz"
 ISOLATION="{{ isolation }}"
 BUILDDIR="{{ builddir }}"
 
diff --git a/iso/empanadas/empanadas/util/__init__.py b/iso/empanadas/empanadas/util/__init__.py
index 495236c..f107a54 100644
--- a/iso/empanadas/empanadas/util/__init__.py
+++ b/iso/empanadas/empanadas/util/__init__.py
@@ -6,6 +6,10 @@ from empanadas.util.check import (
         Checks,
 )
 
+from empanadas.util.shared import (
+        Shared,
+)
+
 from empanadas.util.dnf_utils import (
         RepoSync,
         SigRepoSync
@@ -18,5 +22,6 @@ from empanadas.util.iso_utils import (
 
 __all__ = [
         'Checks',
-        'RepoSync'
+        'RepoSync',
+        'Shared'
 ]
diff --git a/iso/empanadas/empanadas/util/dnf_utils.py b/iso/empanadas/empanadas/util/dnf_utils.py
index ee5ca07..a54b4ee 100644
--- a/iso/empanadas/empanadas/util/dnf_utils.py
+++ b/iso/empanadas/empanadas/util/dnf_utils.py
@@ -19,6 +19,7 @@ import json
 from jinja2 import Environment, FileSystemLoader
 
 from empanadas.common import Color, _rootdir
+from empanadas.util import Shared
 
 # initial treeinfo data is made here
 import productmd.treeinfo
@@ -76,6 +77,7 @@ class RepoSync:
         self.repo_base_url = config['repo_base_url']
         self.compose_root = config['compose_root']
         self.compose_base = config['compose_root'] + "/" + major
+        self.profile = rlvars['profile']
 
         # Relevant major version items
         self.shortname = config['shortname']
@@ -112,7 +114,7 @@ class RepoSync:
         self.compose_latest_dir = os.path.join(
                 config['compose_root'],
                 major,
-                "latest-Rocky-{}".format(major)
+                "latest-Rocky-{}".format(self.profile)
         )
 
         self.compose_latest_sync = os.path.join(
diff --git a/iso/empanadas/empanadas/util/iso_utils.py b/iso/empanadas/empanadas/util/iso_utils.py
index 70244aa..5921df9 100644
--- a/iso/empanadas/empanadas/util/iso_utils.py
+++ b/iso/empanadas/empanadas/util/iso_utils.py
@@ -13,7 +13,6 @@ import shlex
 import time
 import tarfile
 import shutil
-import hashlib
 
 # lazy person's s3 parser
 import requests
@@ -35,7 +34,8 @@ import productmd.treeinfo
 
 from jinja2 import Environment, FileSystemLoader
 
-from empanadas.common import Color, _rootdir, Utils
+from empanadas.common import Color, _rootdir
+from empanadas.util import Shared
 
 class IsoBuild:
     """
@@ -90,6 +90,7 @@ class IsoBuild:
         self.extra_iso = extra_iso
         self.extra_iso_mode = extra_iso_mode
         self.checksum = rlvars['checksum']
+        self.profile = rlvars['profile']
 
         # Relevant major version items
         self.arch = arch
@@ -102,6 +103,7 @@ class IsoBuild:
         self.repo_base_url = config['repo_base_url']
         self.project_id = rlvars['project_id']
         self.structure = rlvars['structure']
+        self.bugurl = rlvars['bugurl']
 
         self.extra_files = rlvars['extra_files']
 
@@ -133,7 +135,7 @@ class IsoBuild:
         self.compose_latest_dir = os.path.join(
                 config['compose_root'],
                 major,
-                "latest-Rocky-{}".format(major)
+                "latest-Rocky-{}".format(self.profile)
         )
 
         self.compose_latest_sync = os.path.join(
@@ -189,7 +191,7 @@ class IsoBuild:
 
         self.log.info('Compose repo directory: %s' % sync_root)
         self.log.info('ISO Build Logs: /var/lib/mock/{}-{}-{}/result'.format(
-            self.shortname, self.major_version, self.current_arch)
+            self.shortname.lower(), self.major_version, self.current_arch)
         )
         self.log.info('ISO Build completed.')
 
@@ -282,6 +284,7 @@ class IsoBuild:
                 isolation=self.mock_isolation,
                 builddir=self.mock_work_root,
                 shortname=self.shortname,
+                revision=self.release,
         )
 
         iso_template_output = iso_template.render(
@@ -297,6 +300,7 @@ class IsoBuild:
                 rc=rclevel,
                 builddir=self.mock_work_root,
                 lorax_work_root=self.lorax_result_root,
+                bugurl=self.bugurl,
         )
 
         mock_iso_entry = open(mock_iso_path, "w+")
@@ -363,7 +367,7 @@ class IsoBuild:
 
             full_drop = '{}/lorax-{}-{}.tar.gz'.format(
                     lorax_arch_dir,
-                    self.major_version,
+                    self.release,
                     arch
             )
 
@@ -396,7 +400,7 @@ class IsoBuild:
 
         for arch in arches_to_unpack:
             tarname = 'lorax-{}-{}.tar.gz'.format(
-                    self.major_version,
+                    self.release,
                     arch
             )
 
@@ -474,7 +478,7 @@ class IsoBuild:
             raise SystemExit()
 
         for y in self.s3.list_objects(Bucket=self.s3_bucket)['Contents']:
-            if 'tar.gz' in y['Key']:
+            if 'tar.gz' in y['Key'] and self.release in y['Key']:
                 temp.append(y['Key'])
 
         for arch in self.arches:
@@ -525,7 +529,7 @@ class IsoBuild:
         resp = xmltodict.parse(bucket_data.content)
 
         for y in resp['ListBucketResult']['Contents']:
-            if 'tar.gz' in y['Key']:
+            if 'tar.gz' in y['Key'] and self.release in y['Key']:
                 temp.append(y['Key'])
 
         for arch in self.arches:
@@ -690,7 +694,7 @@ class IsoBuild:
             shutil.copy2(path_to_src_image + '.manifest', manifest)
 
         self.log.info('Creating checksum for %s boot iso...' % arch)
-        checksum = Utils.get_checksum(isobootpath, self.checksum, self.log)
+        checksum = Shared.get_checksum(isobootpath, self.checksum, self.log)
         if not checksum:
             self.log.error(
                     '[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
@@ -824,31 +828,6 @@ class IsoBuild:
         # Set default variant
         ti.dump(treeinfo, main_variant=primary)
 
-    def discinfo_write(self, file_path, arch):
-        """
-        Ensure discinfo is written correctly
-        """
-        with open(file_path, "w+") as f:
-            f.write("%s\n" % self.timestamp)
-            f.write("%s\n" % self.fullname)
-            f.write("%s\n" % arch)
-            f.write("ALL\n")
-            f.close()
-
-    def write_media_repo(self):
-        """
-        Ensure media.repo exists
-        """
-        data = [
-            "[InstallMedia]",
-            "name=%s" % self.fullname,
-            "mediaid=%s" % self.timestamp,
-            "metadata_expire=-1",
-            "gpgcheck=0",
-            "cost=500",
-            "",
-        ]
-
     # Next set of functions are loosely borrowed (in concept) from pungi. Some
     # stuff may be combined/mixed together, other things may be simplified or
     # reduced in nature.
@@ -1075,6 +1054,7 @@ class IsoBuild:
                 implantmd5=implantmd5,
                 make_manifest=make_manifest,
                 lorax_pkg_cmd=lorax_pkg_cmd,
+                isoname=isoname,
         )
 
         mock_iso_entry = open(mock_iso_path, "w+")
@@ -1225,7 +1205,7 @@ class IsoBuild:
                             '[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' +
                             'Performing checksum for ' + p
                     )
-                    checksum = Utils.get_checksum(path, self.checksum, self.log)
+                    checksum = Shared.get_checksum(path, self.checksum, self.log)
                     if not checksum:
                         self.log.error(
                                 '[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' +
diff --git a/iso/empanadas/empanadas/util/shared.py b/iso/empanadas/empanadas/util/shared.py
new file mode 100644
index 0000000..d9bb357
--- /dev/null
+++ b/iso/empanadas/empanadas/util/shared.py
@@ -0,0 +1,79 @@
+# These are shared utilities used
+
+import os
+import hashlib
+
+class Shared:
+    """
+    Quick utilities that may be commonly used
+    """
+    @staticmethod
+    def get_checksum(path, hashtype, logger):
+        """
+        Generates a checksum from the provided path by doing things in chunks.
+        This way we don't do it in memory.
+        """
+        try:
+            checksum = hashlib.new(hashtype)
+        except ValueError:
+            logger.error("Invalid hash type: %s" % hashtype)
+            return False
+
+        try:
+            input_file = open(path, "rb")
+        except IOError as e:
+            logger.error("Could not open file %s: %s" % (path, e))
+            return False
+
+        while True:
+            chunk = input_file.read(8192)
+            if not chunk:
+                break
+            checksum.update(chunk)
+
+        input_file.close()
+        stat = os.stat(path)
+        base = os.path.basename(path)
+        # This emulates our current syncing scripts that runs stat and
+        # sha256sum and what not with a very specific output.
+        return "%s: %s bytes\n%s (%s) = %s\n" % (
+                base,
+                stat.st_size,
+                hashtype.upper(),
+                base,
+                checksum.hexdigest()
+        )
+
+    @staticmethod
+    def discinfo_write(timestamp, fullname, arch, file_path):
+        """
+        Ensure discinfo is written correctly
+        """
+        data = [
+            "%s" % timestamp,
+            "%s" % fullname,
+            "%s" % arch,
+            "ALL"
+        ]
+
+        with open(file_path, "w+") as f:
+            f.write("\n".join(data))
+            f.close()
+
+    @staticmethod
+    def media_repo_write(timestamp, fullname, file_path):
+        """
+        Ensure media.repo exists
+        """
+        data = [
+            "[InstallMedia]",
+            "name=%s" % fullname,
+            "mediaid=%s" % timestamp,
+            "metadata_expire=-1",
+            "gpgcheck=0",
+            "cost=500",
+            "",
+        ]
+
+        with open(file_path, "w") as f:
+            f.write("\n".join(data))