Bump main to 0.6.0

iso/empanadas/empanadas/__init__.py

@@ -1 +1 @@
-__version__ = '0.5.0'
+__version__ = '0.6.0'

iso/empanadas/empanadas/configs/el9.yaml

@@ -109,6 +109,7 @@
       XFCE: rocky-live-xfce.ks
       KDE: rocky-live-kde.ks
       MATE: rocky-live-mate.ks
+      Cinnamon: rocky-live-cinnamon.ks
     allowed_arches:
       - x86_64
       - aarch64

iso/empanadas/empanadas/util/__init__.py

@@ -9,6 +9,7 @@ from empanadas.util.check import (
 from empanadas.util.shared import (
         Shared,
         ArchCheck,
+        Idents,
 )
 
 from empanadas.util.dnf_utils import (

iso/empanadas/empanadas/util/dnf_utils.py

@@ -289,7 +289,9 @@ class RepoSync:
             Shared.deploy_extra_files(self.extra_files, sync_root, global_work_root, self.log)
             self.deploy_treeinfo(self.repo, sync_root, self.arch)
             self.tweak_treeinfo(self.repo, sync_root, self.arch)
-            self.symlink_to_latest(generated_dir)
+            #self.symlink_to_latest(generated_dir)
+            Shared.symlink_to_latest(self.shortname, self.major_version,
+                    generated_dir, self.compose_latest_dir, self.log)
 
         if self.repoclosure:
             self.repoclosure_work(sync_root, work_root, log_root)
@@ -323,7 +325,7 @@ class RepoSync:
         if self.parallel:
             self.podman_sync(repo, sync_root, work_root, log_root, global_work_root, arch)
         else:
-            Shared.dnf_sync(repo, sync_root, work_root, arch, self.log)
+            Shared.norm_dnf_sync(self, repo, sync_root, work_root, arch, self.log)
 
     def podman_sync(
             self,
@@ -709,22 +711,6 @@ class RepoSync:
                     'No issues detected.'
             )
 
-    def symlink_to_latest(self, generated_dir):
-        """
-        Emulates pungi and symlinks latest-Rocky-X
-
-        This link will be what is updated in full runs. Whatever is in this
-        'latest' directory is what is rsynced on to staging after completion.
-        This link should not change often.
-        """
-        try:
-            os.remove(self.compose_latest_dir)
-        except:
-            pass
-
-        self.log.info('Symlinking to latest-{}-{}...'.format(self.shortname, self.major_version))
-        os.symlink(generated_dir, self.compose_latest_dir)
-
     def repoclosure_work(self, sync_root, work_root, log_root):
         """
         This is where we run repoclosures, based on the configuration of each

iso/empanadas/empanadas/util/iso_utils.py

@@ -35,7 +35,7 @@ import productmd.treeinfo
 from jinja2 import Environment, FileSystemLoader
 
 from empanadas.common import Color, _rootdir
-from empanadas.util import Shared, ArchCheck
+from empanadas.util import Shared, ArchCheck, Idents
 
 class IsoBuild:
     """
@@ -138,10 +138,7 @@ class IsoBuild:
         self.compose_latest_dir = os.path.join(
                 config['compose_root'],
                 major,
-                "latest-{}-{}".format(
+                f"latest-{self.shortname}-{self.profile}"
-                    self.shortname,
-                    self.profile
-                )
         )
 
         self.compose_latest_sync = os.path.join(
@@ -371,11 +368,7 @@ class IsoBuild:
 
             source_path = latest_artifacts[arch]
 
-            full_drop = '{}/lorax-{}-{}.tar.gz'.format(
+            full_drop = f'{lorax_arch_dir}/lorax-{self.release}-{arch}.tar.gz'
-                    lorax_arch_dir,
-                    self.release,
-                    arch
-            )
 
             if not os.path.exists(lorax_arch_dir):
                 os.makedirs(lorax_arch_dir, exist_ok=True)
@@ -403,10 +396,7 @@ class IsoBuild:
         self.log.info(Color.INFO + 'Beginning unpack phase...')
 
         for arch in arches_to_unpack:
-            tarname = 'lorax-{}-{}.tar.gz'.format(
+            tarname = f'lorax-{self.release}-{arch}.tar.gz'
-                    self.release,
-                    arch
-            )
 
             tarball = os.path.join(
                     self.lorax_work_dir,
@@ -523,22 +513,13 @@ class IsoBuild:
         if self.release_candidate:
             rclevel = '-' + self.rclvl
 
-        discname = '{}-{}.{}{}-{}-{}.iso'.format(
+        discname = f'{self.shortname}-{self.major_version}.{self.minor_version}{rclevel}-{arch}-boot.iso'
-                self.shortname,
-                self.major_version,
-                self.minor_version,
-                rclevel,
-                arch,
-                'boot'
-        )
 
         isobootpath = os.path.join(iso_to_go, discname)
-        manifest = '{}.manifest'.format(isobootpath)
+        manifest = f'{isobootpath}.manifest'
-        link_name = '{}-{}-boot.iso'.format(self.shortname, arch)
+        link_name = f'{self.shortname}-{arch}-boot.iso'
         link_manifest = link_name + '.manifest'
-        latest_link_name = '{}-{}-latest-{}-boot.iso'.format(self.shortname,
+        latest_link_name = f'{self.shortname}-{self.major_version}-latest-{arch}-boot.iso'
-                                                   self.major_version,
-                                                   arch)
         latest_link_manifest = latest_link_name + '.manifest'
         isobootpath = os.path.join(iso_to_go, discname)
         linkbootpath = os.path.join(iso_to_go, link_name)
@@ -813,11 +794,11 @@ class IsoBuild:
         xorriso_template = self.tmplenv.get_template('xorriso.tmpl.txt')
         iso_readme_template = self.tmplenv.get_template('ISOREADME.tmpl')
 
-        mock_iso_path = '/var/tmp/lorax-{}.cfg'.format(self.major_version)
+        mock_iso_path = f'/var/tmp/lorax-{self.major_version}.cfg'
-        mock_sh_path = '{}/extraisobuild-{}-{}.sh'.format(entries_dir, arch, image)
+        mock_sh_path = f'{entries_dir}/extraisobuild-{arch}-{image}.sh'
-        iso_template_path = '{}/buildExtraImage-{}-{}.sh'.format(entries_dir, arch, image)
+        iso_template_path = f'{entries_dir}/buildExtraImage-{arch}-{image}.sh'
-        xorriso_template_path = '{}/xorriso-{}-{}.txt'.format(entries_dir, arch, image)
+        xorriso_template_path = f'{entries_dir}/xorriso-{arch}-{image}.txt'
-        iso_readme_path = '{}/{}/README'.format(self.iso_work_dir, arch)
+        iso_readme_path = f'{self.iso_work_dir}/{arch}/README'
         #print(iso_readme_path)
 
         log_root = os.path.join(
@@ -829,7 +810,7 @@ class IsoBuild:
         if not os.path.exists(log_root):
             os.makedirs(log_root, exist_ok=True)
 
-        log_path_command = '| tee -a {}/{}-{}.log'.format(log_root, arch, image)
+        log_path_command = f'| tee -a {log_root}/{arch}-{image}.log'
 
         # This is kind of a hack. Installing xorrisofs sets the alternatives to
         # it, so backwards compatibility is sort of guaranteed. But we want to
@@ -850,31 +831,10 @@ class IsoBuild:
         if self.updated_image:
             datestamp = '-' + self.updated_image_date
 
-        volid = '{}-{}-{}{}-{}-{}'.format(
+        volid = f'{self.shortname}-{self.major_version}-{self.minor_version}{rclevel}-{arch}-{volname}'
-                self.shortname,
+        isoname = f'{self.shortname}-{self.revision}{rclevel}{datestamp}-{arch}-{image}.iso'
-                self.major_version,
+        generic_isoname = f'{self.shortname}-{arch}-{image}.iso'
-                self.minor_version,
+        latest_isoname = f'{self.shortname}-{self.major_version}-latest-{arch}-{image}.iso'
-                rclevel,
-                arch,
-                volname
-        )
-
-        isoname = '{}-{}{}{}-{}-{}.iso'.format(
-                self.shortname,
-                self.revision,
-                rclevel,
-                datestamp,
-                arch,
-                image
-        )
-
-        generic_isoname = '{}-{}-{}.iso'.format(self.shortname, arch, image)
-        latest_isoname = '{}-{}-latest-{}-{}.iso'.format(
-                                               self.shortname,
-                                               self.major_version,
-                                               arch,
-                                               image
-        )
 
         lorax_pkg_cmd = '/usr/bin/dnf install {} -y {}'.format(
                 ' '.join(required_pkgs),
@@ -986,7 +946,7 @@ class IsoBuild:
         have mock available.
         """
         entries_dir = os.path.join(work_root, "entries")
-        extra_iso_cmd = '/bin/bash {}/extraisobuild-{}-{}.sh'.format(entries_dir, arch, image)
+        extra_iso_cmd = f'/bin/bash {entries_dir}/extraisobuild-{arch}-{image}.sh'
         self.log.info('Starting mock build...')
         p = subprocess.call(shlex.split(extra_iso_cmd))
         if p != 0:
@@ -1022,7 +982,7 @@ class IsoBuild:
             arch_sync = arches.copy()
 
             for a in arch_sync:
-                entry_name = 'buildExtraImage-{}-{}.sh'.format(a, i)
+                entry_name = f'buildExtraImage-{a}-{i}.sh'
                 entry_name_list.append(entry_name)
 
                 rclevel = ''
@@ -1080,10 +1040,7 @@ class IsoBuild:
             join_all_pods = ' '.join(entry_name_list)
             time.sleep(3)
             self.log.info(Color.INFO + 'Building ' + i + ' ...')
-            pod_watcher = '{} wait {}'.format(
+            pod_watcher = f'{cmd} wait {join_all_pods}'
-                    cmd,
-                    join_all_pods
-            )
 
             watch_man = subprocess.call(
                     shlex.split(pod_watcher),
@@ -1095,10 +1052,7 @@ class IsoBuild:
             # code.
             pattern = "Exited (0)"
             for pod in entry_name_list:
-                checkcmd = '{} ps -f status=exited -f name={}'.format(
+                checkcmd = f'{cmd} ps -f status=exited -f name={pod}'
-                        cmd,
-                        pod
-                )
                 podcheck = subprocess.Popen(
                         checkcmd,
                         stdout=subprocess.PIPE,
@@ -1111,10 +1065,7 @@ class IsoBuild:
                     self.log.error(Color.FAIL + pod)
                     bad_exit_list.append(pod)
 
-            rmcmd = '{} rm {}'.format(
+            rmcmd = f'{cmd} rm {join_all_pods}'
-                    cmd,
-                    join_all_pods
-            )
 
             rmpod = subprocess.Popen(
                     rmcmd,
@@ -1202,32 +1153,41 @@ class IsoBuild:
             for k, v in self._get_grafts([rd_for_var]).items():
                 files[os.path.join(repo, "repodata", k)] = v
 
-        grafts = '{}/{}-{}-grafts'.format(
+        grafts = f'{lorax_base_dir}/{iso}-{arch}-grafts'
-                lorax_base_dir,
-                iso,
-                arch
-        )
 
-        xorrs = '{}/xorriso-{}.txt'.format(
+        xorrs = f'{lorax_base_dir}/xorriso-{iso}-{arch}.txt'
-                lorax_base_dir,
+
-                arch
+        # Generate exclusion list/dict from boot.iso manifest
-        )
+        boot_manifest = f'{lorax_base_dir}/lorax/images/boot.iso.manifest'
+        # Boot configs and images that may change
+        # It's unlikely these will be changed in empanadas, they're used as is
+        # and it works fine. This is a carry over from a recent pungi commit,
+        # based on an issue I had filed. The above was the original part, the
+        # below is a pungi "buildinstall" thing that we don't do, but may
+        # include as a feature if it ever happens.
+        updatable_files = set(ArchCheck.boot_configs + ArchCheck.boot_images + ['.discinfo'])
+        ignores = set()
+        updatables = set()
+
+        try:
+            with open(boot_manifest) as i:
+        #        ignores = set(line.lstrip("/").rstrip("\n") for line in i)
+                for line in i:
+                    path = line.lstrip("/").rstrip("\n")
+                    if path in updatable_files:
+                        updatables.add(path)
+                    else:
+                        ignores.add(path)
+        except Exception as e:
+            self.log.error(Color.FAIL + 'File was likely not found.')
+            raise SystemExit(e)
 
         self._write_grafts(
                 grafts,
                 xorrs,
                 files,
-                exclude=[
+                exclude=ignores,
-                    "*/lost+found",
+                update=updatables
-                    "*/boot.iso",
-                    "*/boot.iso.manifest",
-                    "EFI/*",
-                    "images/*",
-                    "isolinux/*",
-                    "boot/*",
-                    "ppc/*",
-                    "generic.ins"
-                ]
         )
 
         if self.iso_map['xorrisofs']:
@@ -1249,12 +1209,12 @@ class IsoBuild:
             if isinstance(p, dict):
                 tree = p
             else:
-                tree = self._scanning(p)
+                tree = Idents.scanning(p)
-            result = self._merging(result, tree)
+            result = Idents.merging(result, tree)
 
         for p in exclusive_paths:
-            tree = self._scanning(p)
+            tree = Idents.scanning(p)
-            result = self._merging(result, tree, exclusive=True)
+            result = Idents.merging(result, tree, exclusive=True)
 
         # Resolves possible symlinks
         for key in result.keys():
@@ -1267,12 +1227,17 @@ class IsoBuild:
 
         return result
 
-    def _write_grafts(self, filepath, xorrspath, u, exclude=None):
+    def _write_grafts(self, filepath, xorrspath, u, exclude=None, update=None):
         """
         Write out the graft points
         """
         seen = set()
+        # There are files that are on the exclude list typically.
         exclude = exclude or []
+        # There is a chance files may get updated before being placed in a
+        # variant ISO - it's rare though. most that will be different is
+        # .discinfo
+        update = update or []
         result = {}
         for zl in sorted(u, reverse=True):
             dirn = os.path.dirname(zl)
@@ -1291,119 +1256,43 @@ class IsoBuild:
                 result[zl] = u[zl]
             seen.add(dirn)
 
+        # We check first if a file needs to be updated first before relying on
+        # the boot.iso manifest to exclude a file
         if self.iso_map['xorrisofs']:
             fx = open(xorrspath, "w")
-            for zm in sorted(result, key=self._sorting):
+            for zm in sorted(result, key=Idents.sorting):
                 found = False
+                replace = False
+                for upda in update:
+                    if fnmatch(zm, upda):
+                        #print(f'updating: {zm} {upda}')
+                        replace = True
+                        break
                 for excl in exclude:
                     if fnmatch(zm, excl):
+                        #print(f'ignoring: {zm} {excl}')
                         found = True
                         break
                 if found:
                     continue
-                fx.write("-map %s %s\n" % (u[zm], zm))
+                mcmd = "-update" if replace else "-map"
+                fx.write("%s %s %s\n" % (mcmd, u[zm], zm))
             fx.close()
         else:
             fh = open(filepath, "w")
-            for zl in sorted(result, key=self._sorting):
+            self.log.info(Color.WARN + 'Nothing should be excluded in legacy ' +
-                found = False
+                          'genisoimage. Ignoring exclude list.')
-                for excl in exclude:
+            for zl in sorted(result, key=Idents.sorting):
-                    if fnmatch(zl, excl):
+                #found = False
-                        found = True
+                #for excl in exclude:
-                        break
+                #    if fnmatch(zl, excl):
-                if found:
+                #        found = True
-                    continue
+                #        break
+                #if found:
+                #    continue
                 fh.write("%s=%s\n" % (zl, u[zl]))
             fh.close()
 
-    def _scanning(self, p):
-        """
-        Scan tree
-        """
-        path = os.path.abspath(p)
-        result = {}
-        for root, dirs, files in os.walk(path):
-            for file in files:
-                abspath = os.path.join(root, file)
-                relpath = kobo.shortcuts.relative_path(abspath, path.rstrip("/") + "/")
-                result[relpath] = abspath
-
-            # Include empty directories too
-            if root != path:
-                abspath = os.path.join(root, "")
-                relpath = kobo.shortcuts.relative_path(abspath, path.rstrip("/") + "/")
-                result[relpath] = abspath
-
-        return result
-
-
-    def _merging(self, tree_a, tree_b, exclusive=False):
-        """
-        Merge tree
-        """
-        result = tree_b.copy()
-        all_dirs = set(
-            [os.path.dirname(dirn).rstrip("/") for dirn in result if os.path.dirname(dirn) != ""]
-        )
-
-        for dirn in tree_a:
-            dn = os.path.dirname(dirn)
-            if exclusive:
-                match = False
-                for x in all_dirs:
-                    if dn == x or dn.startswith("%s/" % x):
-                        match = True
-                        break
-                if match:
-                    continue
-
-            if dirn in result:
-                continue
-
-            result[dirn] = tree_a[dirn]
-        return result
-
-    def _sorting(self, k):
-        """
-        Sorting using the is_rpm and is_image funcs. Images are first, extras
-        next, rpm's last.
-        """
-        rolling = (0 if self._is_image(k) else 2 if self._is_rpm(k) else 1, k)
-        return rolling
-
-    def _is_rpm(self, k):
-        """
-        Is this an RPM? :o
-        """
-        result = k.endswith(".rpm")
-        return result
-
-    def _is_image(self, k):
-        """
-        Is this an image? :o
-        """
-        if (
-                k.startswith("images/") or
-                k.startswith("isolinux/") or
-                k.startswith("EFI/") or
-                k.startswith("etc/") or
-                k.startswith("ppc/")
-           ):
-            return True
-
-        if (
-                k.endswith(".img") or
-                k.endswith(".ins")
-           ):
-            return True
-
-        return False
-
-    def _get_vol_id(self):
-        """
-        Gets a volume ID
-        """
-
     def run_pull_generic_images(self):
         """
         Pulls generic images built in peridot and places them where they need
@@ -1500,15 +1389,9 @@ class IsoBuild:
                         drop_name = source_path.split('/')[-3] + fsuffix
 
                     checksum_name = drop_name + '.CHECKSUM'
-                    full_drop = '{}/{}'.format(
+                    full_drop = f'{image_arch_dir}/{drop_name}'
-                            image_arch_dir,
-                            drop_name
-                    )
 
-                    checksum_drop = '{}/{}.CHECKSUM'.format(
+                    checksum_drop = f'{image_arch_dir}/{drop_name}.CHECKSUM'
-                            image_arch_dir,
-                            drop_name
-                    )
 
                     if not os.path.exists(image_arch_dir):
                         os.makedirs(image_arch_dir, exist_ok=True)
@@ -1694,10 +1577,7 @@ class LiveBuild:
         self.compose_latest_dir = os.path.join(
                 config['compose_root'],
                 major,
-                "latest-{}-{}".format(
+                f"latest-{self.shortname}-{self.profile}"
-                    self.shortname,
-                    self.profile
-                )
         )
 
         self.compose_latest_sync = os.path.join(
@@ -1820,17 +1700,9 @@ class LiveBuild:
         if self.peridot:
             kloc = 'peridot'
 
-        mock_iso_path = '/var/tmp/live-{}.cfg'.format(self.major_version)
+        mock_iso_path = f'/var/tmp/live-{self.major_version}.cfg'
-        mock_sh_path = '{}/liveisobuild-{}-{}.sh'.format(
+        mock_sh_path = f'{entries_dir}/liveisobuild-{self.current_arch}-{image}.sh'
-                entries_dir,
+        iso_template_path = f'{entries_dir}/buildLiveImage-{self.current_arch}-{image}.sh'
-                self.current_arch,
-                image
-        )
-        iso_template_path = '{}/buildLiveImage-{}-{}.sh'.format(
-                entries_dir,
-                self.current_arch,
-                image
-        )
 
         log_root = os.path.join(
                 work_root,
@@ -1843,27 +1715,12 @@ class LiveBuild:
         if not os.path.exists(log_root):
             os.makedirs(log_root, exist_ok=True)
 
-        log_path_command = '| tee -a {}/{}-{}.log'.format(
+        log_path_command = f'| tee -a {log_root}/{self.current_arch}-{image}.log'
-                log_root,
-                self.current_arch,
-                image
-        )
         required_pkgs = self.livemap['required_pkgs']
 
-        volid = '{}-{}-{}-{}'.format(
+        volid = f'{self.shortname}-{self.major_version}-{self.minor_version}-{image}'
-                self.shortname,
-                self.major_version,
-                self.minor_version,
-                image
-        )
 
-        isoname = '{}-{}-{}-{}-{}.iso'.format(
+        isoname = f'{self.shortname}-{self.release}-{image}-{self.current_arch}-{self.date}.iso'
-                self.shortname,
-                self.release,
-                image,
-                self.current_arch,
-                self.date
-        )
 
         live_pkg_cmd = '/usr/bin/dnf install {} -y {}'.format(
                 ' '.join(required_pkgs),
@@ -1960,17 +1817,10 @@ class LiveBuild:
         self.log.warn(Color.WARN + 'This mode does not work properly. It will fail.')
         for i in images:
             image_name = i
-            entry_name = 'buildLiveImage-{}-{}.sh'.format(arch, i)
+            entry_name = f'buildLiveImage-{arch}-{i}.sh'
             entry_name_list.append(entry_name)
 
-            isoname = '{}/{}-{}-{}-{}-{}.iso'.format(
+            isoname = f'{arch}/{self.shortname}-{i}-{self.major_version}-{arch}-{self.date}.iso'
-                    arch,
-                    self.shortname,
-                    i,
-                    self.major_version,
-                    arch,
-                    self.date
-            )
 
             checksum_list.append(isoname)
 
@@ -1998,10 +1848,7 @@ class LiveBuild:
         time.sleep(3)
         self.log.info(Color.INFO + 'Building requested live images ...')
 
-        pod_watcher = '{} wait {}'.format(
+        pod_watcher = f'{cmd} wait {join_all_pods}'
-                cmd,
-                join_all_pods
-        )
 
         watch_man = subprocess.call(
                 shlex.split(pod_watcher),
@@ -2013,10 +1860,7 @@ class LiveBuild:
         # code.
         pattern = "Exited (0)"
         for pod in entry_name_list:
-            checkcmd = '{} ps -f status=exited -f name={}'.format(
+            checkcmd = f'{cmd} ps -f status=exited -f name={pod}'
-                    cmd,
-                    pod
-            )
             podcheck = subprocess.Popen(
                     checkcmd,
                     stdout=subprocess.PIPE,
@@ -2029,10 +1873,7 @@ class LiveBuild:
                 self.log.error(Color.FAIL + pod)
                 bad_exit_list.append(pod)
 
-        rmcmd = '{} rm {}'.format(
+        rmcmd = f'{cmd} rm {join_all_pods}'
-                cmd,
-                join_all_pods
-        )
 
         rmpod = subprocess.Popen(
                 rmcmd,
@@ -2072,25 +1913,9 @@ class LiveBuild:
         """
         entries_dir = os.path.join(work_root, "entries")
         live_dir_arch = os.path.join(self.live_work_dir, arch)
-        isoname = '{}-{}-{}-{}-{}.iso'.format(
+        isoname = f'{self.shortname}-{self.release}-{image}-{arch}-{self.date}.iso'
-                self.shortname,
+        isolink = f'{self.shortname}-{self.major_version}-{image}-{arch}-latest.iso'
-                self.release,
+        live_res_dir = f'/var/lib/mock/{self.shortname.lower()}-{self.major_version}-{arch}/result'
-                image,
-                arch,
-                self.date
-        )
-        isolink = '{}-{}-{}-{}-{}.iso'.format(
-                self.shortname,
-                self.major_version,
-                image,
-                arch,
-                'latest'
-        )
-        live_res_dir = '/var/lib/mock/{}-{}-{}/result'.format(
-                self.shortname.lower(),
-                self.major_version,
-                arch
-        )
 
         if self.justcopyit:
             if os.path.exists(os.path.join(live_dir_arch, isoname)):
@@ -2101,7 +1926,7 @@ class LiveBuild:
                     self.log.warn(Color.WARN + 'Skipping.')
                     return
 
-        live_iso_cmd = '/bin/bash {}/liveisobuild-{}-{}.sh'.format(entries_dir, arch, image)
+        live_iso_cmd = f'/bin/bash {entries_dir}/liveisobuild-{arch}-{image}.sh'
         self.log.info('Starting mock build...')
         p = subprocess.call(shlex.split(live_iso_cmd))
         if p != 0:

iso/empanadas/empanadas/util/shared.py

@@ -40,6 +40,18 @@ class ArchCheck:
         ]
     }
 
+    # These are files that can potentially change on an image.
+    boot_configs = [
+            "isolinux/isolinux.cfg",
+            "etc/yaboot.conf",
+            "ppc/ppc64/yaboot.conf",
+            "EFI/BOOT/BOOTX64.conf",
+            "EFI/BOOT/grub.cfg"
+    ]
+    boot_images = [
+            "images/efiboot.img"
+    ]
+
 class Shared:
     """
     Quick utilities that may be commonly used
@@ -73,7 +85,7 @@ class Shared:
         base = os.path.basename(path)
         # This emulates our current syncing scripts that runs stat and
         # sha256sum and what not with a very specific output.
-        return "%s: %s bytes\n%s (%s) = %s\n" % (
+        return "# %s: %s bytes\n%s (%s) = %s\n" % (
                 base,
                 stat.st_size,
                 hashtype.upper(),
@@ -1141,3 +1153,208 @@ class Shared:
         logger.error('DNF syncing has been removed.')
         logger.error('Please install podman and enable parallel')
         raise SystemExit()
+
+    @staticmethod
+    def norm_dnf_sync(data, repo, sync_root, work_root, arch, logger):
+        """
+        This is for normal dnf syncs. This is very slow.
+        """
+        cmd = Shared.reposync_cmd(logger)
+        sync_single_arch = False
+        arches_to_sync = data.arches
+        if arch:
+            sync_single_arch = True
+            arches_to_sync = [arch]
+
+        logger.info(
+                Color.BOLD + '!! WARNING !! ' + Color.END + 'You are performing a '
+                'local reposync, which will incur delays in your compose.'
+        )
+
+        if data.fullrun:
+            logger.info(
+                    Color.BOLD + '!! WARNING !! ' + Color.END + 'This is a full '
+                    'sync. Expect a few days for it to complete.'
+            )
+
+        for r in repos_to_sync:
+            for a in arches_to_sync:
+                repo_name = r
+                if r in data.repo_renames:
+                    repo_name = data.repo_renames[r]
+
+                os_sync_path = os.path.join(
+                        sync_root,
+                        repo_name,
+                        a,
+                        'os'
+                )
+
+                debug_sync_path = os.path.join(
+                        sync_root,
+                        repo_name,
+                        a,
+                        'debug/tree'
+                )
+
+                sync_cmd = "{} -c {} --download-metadata --repoid={} -p {} --forcearch {} --norepopath".format(
+                        cmd,
+                        data.dnf_config,
+                        r,
+                        os_sync_path,
+                        a
+                )
+
+                debug_sync_cmd = "{} -c {} --download-metadata --repoid={}-debug -p {} --forcearch {} --norepopath".format(
+                        cmd,
+                        data.dnf_config,
+                        r,
+                        debug_sync_path,
+                        a
+                )
+
+                logger.info('Syncing {} {}'.format(r, a))
+                #logger.info(sync_cmd)
+                # Try to figure out where to send the actual output of this...
+                # Also consider on running a try/except here? Basically if
+                # something happens (like a repo doesn't exist for some arch,
+                # eg RT for aarch64), make a note of it somehow (but don't
+                # break the entire sync). As it stands with this
+                # implementation, if something fails, it just continues on.
+                process = subprocess.call(
+                        shlex.split(sync_cmd),
+                        stdout=subprocess.DEVNULL,
+                        stderr=subprocess.DEVNULL
+                )
+
+                if not data.ignore_debug:
+                    logger.info('Syncing {} {} (debug)'.format(r, a))
+                    process_debug = subprocess.call(
+                            shlex.split(debug_sync_cmd),
+                            stdout=subprocess.DEVNULL,
+                            stderr=subprocess.DEVNULL
+                    )
+
+                # There should be a check here that if it's "all" and multilib
+                # is on, i686 should get synced too.
+
+            if not data.ignore_source:
+                source_sync_path = os.path.join(
+                    sync_root,
+                    repo_name,
+                    'source/tree'
+                )
+
+                source_sync_cmd = "{} -c {} --download-metadata --repoid={}-source -p {} --norepopath".format(
+                    cmd,
+                    data.dnf_config,
+                    r,
+                    source_sync_path
+                )
+
+
+                logger.info('Syncing {} source'.format(r))
+                process_source = subprocess.call(
+                        shlex.split(source_sync_cmd),
+                        stdout=subprocess.DEVNULL,
+                        stderr=subprocess.DEVNULL
+                )
+
+        logger.info('Syncing complete')
+
+class Idents:
+    """
+    Identifiers or locators
+    """
+    @staticmethod
+    def scanning(p):
+        """
+        Scan tree
+        """
+        path = os.path.abspath(p)
+        result = {}
+        for root, dirs, files in os.walk(path):
+            for file in files:
+                abspath = os.path.join(root, file)
+                relpath = kobo.shortcuts.relative_path(abspath, path.rstrip("/") + "/")
+                result[relpath] = abspath
+
+            # Include empty directories too
+            if root != path:
+                abspath = os.path.join(root, "")
+                relpath = kobo.shortcuts.relative_path(abspath, path.rstrip("/") + "/")
+                result[relpath] = abspath
+
+        return result
+
+    @staticmethod
+    def merging(tree_a, tree_b, exclusive=False):
+        """
+        Merge tree
+        """
+        result = tree_b.copy()
+        all_dirs = set(
+            [os.path.dirname(dirn).rstrip("/") for dirn in result if os.path.dirname(dirn) != ""]
+        )
+
+        for dirn in tree_a:
+            dn = os.path.dirname(dirn)
+            if exclusive:
+                match = False
+                for x in all_dirs:
+                    if dn == x or dn.startswith("%s/" % x):
+                        match = True
+                        break
+                if match:
+                    continue
+
+            if dirn in result:
+                continue
+
+            result[dirn] = tree_a[dirn]
+        return result
+
+    @staticmethod
+    def sorting(k):
+        """
+        Sorting using the is_rpm and is_image funcs. Images are first, extras
+        next, rpm's last.
+        """
+        rolling = (0 if Idents.is_image(k) else 2 if Idents.is_rpm(k) else 1, k)
+        return rolling
+
+    @staticmethod
+    def is_rpm(k):
+        """
+        Is this an RPM? :o
+        """
+        result = k.endswith(".rpm")
+        return result
+
+    @staticmethod
+    def is_image(k):
+        """
+        Is this an image? :o
+        """
+        if (
+                k.startswith("images/") or
+                k.startswith("isolinux/") or
+                k.startswith("EFI/") or
+                k.startswith("etc/") or
+                k.startswith("ppc/")
+           ):
+            return True
+
+        if (
+                k.endswith(".img") or
+                k.endswith(".ins")
+           ):
+            return True
+
+        return False
+
+    @staticmethod
+    def get_vol_id(opts):
+        """
+        Gets a volume ID
+        """

iso/empanadas/poetry.lock

@@ -1,6 +1,6 @@
 [[package]]
 name = "atomicwrites"
-version = "1.4.0"
+version = "1.4.1"
 description = "Atomic file writes."
 category = "dev"
 optional = false
@@ -22,14 +22,14 @@ tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>
 
 [[package]]
 name = "boto3"
-version = "1.24.22"
+version = "1.26.89"
 description = "The AWS SDK for Python"
 category = "main"
 optional = false
 python-versions = ">= 3.7"
 
 [package.dependencies]
-botocore = ">=1.27.22,<1.28.0"
+botocore = ">=1.29.89,<1.30.0"
 jmespath = ">=0.7.1,<2.0.0"
 s3transfer = ">=0.6.0,<0.7.0"
 
@@ -38,7 +38,7 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
 
 [[package]]
 name = "botocore"
-version = "1.27.22"
+version = "1.29.89"
 description = "Low-level, data-driven core of boto 3."
 category = "main"
 optional = false
@@ -50,11 +50,11 @@ python-dateutil = ">=2.1,<3.0.0"
 urllib3 = ">=1.25.4,<1.27"
 
 [package.extras]
-crt = ["awscrt (==0.13.8)"]
+crt = ["awscrt (==0.16.9)"]
 
 [[package]]
 name = "certifi"
-version = "2022.6.15"
+version = "2022.12.7"
 description = "Python package for providing Mozilla's CA Bundle."
 category = "main"
 optional = false
@@ -62,26 +62,23 @@ python-versions = ">=3.6"
 
 [[package]]
 name = "charset-normalizer"
-version = "2.1.0"
+version = "3.1.0"
 description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
 category = "main"
 optional = false
-python-versions = ">=3.6.0"
+python-versions = ">=3.7.0"
-
-[package.extras]
-unicode_backport = ["unicodedata2"]
 
 [[package]]
 name = "colorama"
-version = "0.4.5"
+version = "0.4.6"
 description = "Cross-platform colored terminal text."
 category = "dev"
 optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
 
 [[package]]
 name = "idna"
-version = "3.3"
+version = "3.4"
 description = "Internationalized Domain Names in Applications (IDNA)"
 category = "main"
 optional = false
@@ -89,7 +86,7 @@ python-versions = ">=3.5"
 
 [[package]]
 name = "importlib-metadata"
-version = "4.12.0"
+version = "6.0.0"
 description = "Read metadata from Python packages"
 category = "dev"
 optional = false
@@ -100,13 +97,13 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""}
 zipp = ">=0.5"
 
 [package.extras]
-docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"]
+docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"]
 perf = ["ipython"]
-testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"]
+testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8", "importlib-resources (>=1.3)"]
 
 [[package]]
 name = "importlib-resources"
-version = "5.8.0"
+version = "5.12.0"
 description = "Read resources from Python packages"
 category = "main"
 optional = false
@@ -116,8 +113,8 @@ python-versions = ">=3.7"
 zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""}
 
 [package.extras]
-docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"]
+docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"]
-testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"]
+testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"]
 
 [[package]]
 name = "jinja2"
@@ -162,22 +159,19 @@ python-versions = ">=3.6"
 
 [[package]]
 name = "more-itertools"
-version = "8.13.0"
+version = "9.1.0"
 description = "More routines for operating on iterables, beyond itertools"
 category = "dev"
 optional = false
-python-versions = ">=3.5"
+python-versions = ">=3.7"
 
 [[package]]
 name = "packaging"
-version = "21.3"
+version = "23.0"
 description = "Core utilities for Python packages"
 category = "dev"
 optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.7"
-
-[package.dependencies]
-pyparsing = ">=2.0.2,<3.0.5 || >3.0.5"
 
 [[package]]
 name = "pluggy"
@@ -212,17 +206,6 @@ category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 
-[[package]]
-name = "pyparsing"
-version = "3.0.9"
-description = "pyparsing module - Classes and methods to define and execute parsing grammars"
-category = "dev"
-optional = false
-python-versions = ">=3.6.8"
-
-[package.extras]
-diagrams = ["railroad-diagrams", "jinja2"]
-
 [[package]]
 name = "pytest"
 version = "5.4.3"
@@ -267,7 +250,7 @@ python-versions = ">=3.6"
 
 [[package]]
 name = "requests"
-version = "2.28.1"
+version = "2.28.2"
 description = "Python HTTP for Humans."
 category = "main"
 optional = false
@@ -275,7 +258,7 @@ python-versions = ">=3.7, <4"
 
 [package.dependencies]
 certifi = ">=2017.4.17"
-charset-normalizer = ">=2,<3"
+charset-normalizer = ">=2,<4"
 idna = ">=2.5,<4"
 urllib3 = ">=1.21.1,<1.27"
 
@@ -315,7 +298,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
 
 [[package]]
 name = "typing-extensions"
-version = "4.3.0"
+version = "4.5.0"
 description = "Backported and Experimental Type Hints for Python 3.7+"
 category = "dev"
 optional = false
@@ -323,20 +306,20 @@ python-versions = ">=3.7"
 
 [[package]]
 name = "urllib3"
-version = "1.26.9"
+version = "1.26.15"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
 category = "main"
 optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
 
 [package.extras]
 brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"]
-secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"]
+secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"]
 socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
 
 [[package]]
 name = "wcwidth"
-version = "0.2.5"
+version = "0.2.6"
 description = "Measures the displayed width of unicode strings in a terminal"
 category = "dev"
 optional = false
@@ -352,15 +335,15 @@ python-versions = ">=3.4"
 
 [[package]]
 name = "zipp"
-version = "3.8.0"
+version = "3.15.0"
 description = "Backport of pathlib-compatible object wrapper for zip files"
 category = "main"
 optional = false
 python-versions = ">=3.7"
 
 [package.extras]
-docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"]
+docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"]
-testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"]
+testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "jaraco.functools", "more-itertools", "big-o", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"]
 
 [metadata]
 lock-version = "1.1"
@@ -368,46 +351,19 @@ python-versions = ">=3.7,<4"
 content-hash = "42676fd0ceb350c8cd90246dc688cfcd404e14d22229052d0527fe342c135b95"
 
 [metadata.files]
-atomicwrites = [
+atomicwrites = []
-    {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"},
-    {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"},
-]
 attrs = [
     {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"},
     {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"},
 ]
-boto3 = [
+boto3 = []
-    {file = "boto3-1.24.22-py3-none-any.whl", hash = "sha256:c9a9f893561f64f5b81de197714ac4951251a328672a8dba28ad4c4a589c3adf"},
+botocore = []
-    {file = "boto3-1.24.22.tar.gz", hash = "sha256:67d404c643091d4aa37fc485193289ad859f1f65f94d0fa544e13bdd1d4187c1"},
+certifi = []
-]
+charset-normalizer = []
-botocore = [
+colorama = []
-    {file = "botocore-1.27.22-py3-none-any.whl", hash = "sha256:7145d9b7cae87999a9f074de700d02a1b3222ee7d1863aa631ff56c5fc868035"},
+idna = []
-    {file = "botocore-1.27.22.tar.gz", hash = "sha256:f57cb33446deef92e552b0be0e430d475c73cf64bc9e46cdb4783cdfe39cb6bb"},
+importlib-metadata = []
-]
+importlib-resources = []
-certifi = [
-    {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"},
-    {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"},
-]
-charset-normalizer = [
-    {file = "charset-normalizer-2.1.0.tar.gz", hash = "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413"},
-    {file = "charset_normalizer-2.1.0-py3-none-any.whl", hash = "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5"},
-]
-colorama = [
-    {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"},
-    {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"},
-]
-idna = [
-    {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"},
-    {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"},
-]
-importlib-metadata = [
-    {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"},
-    {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"},
-]
-importlib-resources = [
-    {file = "importlib_resources-5.8.0-py3-none-any.whl", hash = "sha256:7952325ffd516c05a8ad0858c74dff2c3343f136fe66a6002b2623dd1d43f223"},
-    {file = "importlib_resources-5.8.0.tar.gz", hash = "sha256:568c9f16cb204f9decc8d6d24a572eeea27dacbb4cee9e6b03a8025736769751"},
-]
 jinja2 = [
     {file = "Jinja2-2.11.3-py2.py3-none-any.whl", hash = "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419"},
     {file = "Jinja2-2.11.3.tar.gz", hash = "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6"},
@@ -416,9 +372,7 @@ jmespath = [
     {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"},
     {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"},
 ]
-kobo = [
+kobo = []
-    {file = "kobo-0.24.2.tar.gz", hash = "sha256:1b3c17260a93d933d2238884373fbf3485ecd417d930acf984285dc012410e2b"},
-]
 markupsafe = [
     {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53"},
     {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38"},
@@ -490,14 +444,8 @@ markupsafe = [
     {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"},
     {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"},
 ]
-more-itertools = [
+more-itertools = []
-    {file = "more-itertools-8.13.0.tar.gz", hash = "sha256:a42901a0a5b169d925f6f217cd5a190e32ef54360905b9c39ee7db5313bfec0f"},
+packaging = []
-    {file = "more_itertools-8.13.0-py3-none-any.whl", hash = "sha256:c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb"},
-]
-packaging = [
-    {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"},
-    {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"},
-]
 pluggy = [
     {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"},
     {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"},
@@ -510,10 +458,6 @@ py = [
     {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"},
     {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"},
 ]
-pyparsing = [
-    {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"},
-    {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"},
-]
 pytest = [
     {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"},
     {file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"},
@@ -557,10 +501,7 @@ pyyaml = [
     {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"},
     {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"},
 ]
-requests = [
+requests = []
-    {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"},
-    {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"},
-]
 rpm-py-installer = [
     {file = "rpm-py-installer-1.1.0.tar.gz", hash = "sha256:66e5f4f9247752ed386345642683103afaee50fb16928878a204bc12504b9bbe"},
 ]
@@ -572,23 +513,11 @@ six = [
     {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
     {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
 ]
-typing-extensions = [
+typing-extensions = []
-    {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"},
+urllib3 = []
-    {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"},
+wcwidth = []
-]
-urllib3 = [
-    {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"},
-    {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"},
-]
-wcwidth = [
-    {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"},
-    {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"},
-]
 xmltodict = [
     {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"},
     {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"},
 ]
-zipp = [
+zipp = []
-    {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"},
-    {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"},
-]

mangle/fix-aws-quotas.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# Source common variables
+# shellcheck disable=SC2046,1091,1090
+source "$(dirname "${BASH_SOURCE[0]}")/common"
+
+usage() {
+	echo "usage: $0"
+}
+
+aws() {
+	# shellcheck disable=SC2068
+	command aws --profile resf-ami --output json $@
+}
+
+# Get the quota code for Public AMIs once
+quota_code=$(aws service-quotas list-service-quotas --service-code ec2 --region us-east-1 --query "Quotas[*].{QuotaCode:QuotaCode,QuotaName:QuotaName}" | jq '.[] | select(.QuotaName == "Public AMIs") | .QuotaCode' | tr -d '"')
+
+function get_current_quota() {
+	region=$1
+	# Get the current value of the quota
+	current_value=$(aws service-quotas get-service-quota --service-code ec2 --quota-code "$quota_code" --region "$region" 2>/dev/null | jq .Quota.Value 2>/dev/null)
+	# shellcheck disable=SC2181
+	if [[ $? -gt 0 ]]; then
+		echo "ERR"
+	fi
+	echo "$current_value"
+}
+
+function request_quota_increase() {
+	mapfile -t regions <<<"$@"
+	for region in "${regions[@]}"; do
+		# Get the current value of the quota
+		current_value=$(get_current_quota "$region")
+		if ((current_value >= QUOTA)); then
+			echo "Quota for Public AMIs in region $region is already set to $current_value, skipping request."
+		else
+			# Request the quota increase
+			request_output=$(aws service-quotas request-service-quota-increase --service-code ec2 --quota-code "$quota_code" --region "$region" --desired-value "$QUOTA")
+			request_id=$(echo "$request_output" | jq .RequestedQuota.Id | tr -d '"')
+			echo "Successfully submitted request with ID: $request_id"
+		fi
+	done
+}
+
+function list_quotas() {
+	mapfile -t regions <<<"$@"
+	output="Region\tQuota\tDesired\tStatus"
+	for region in "${regions[@]}"; do
+		current_quota=$(get_current_quota "$region")
+		request_info=$(aws service-quotas list-requested-service-quota-change-history-by-quota --service-code ec2 --quota-code "$quota_code" --region "$region" --query "RequestedQuotas[-1:].{DesiredValue:DesiredValue,Status:Status}" 2>/dev/null)
+		requested_value=$(echo "$request_info" | jq .[].DesiredValue)
+		case_status=$(echo "$request_info" | jq .[].Status | tr -d '"')
+		output="$output\n$region $current_quota $requested_value $case_status"
+	done
+	echo -e "$output" | column -t
+}
+
+REGIONS=$(aws ec2 describe-regions \
+	--all-regions \
+	--query "Regions[].{Name:RegionName}" \
+	--output text)
+
+QUOTA=50
+
+list_quotas "$REGIONS"
+
+# request_quota_increase "$REGIONS"

mangle/generators/.gitignore

@@ -1,3 +1,5 @@
-__pycache__/                                                                                                                                                                                                         *.py[cod]
+__pycache__/
+*.py[cod]
 *$py.class
 *.so
+generate_prepopulate_from_self

mangle/generators/common

@@ -17,4 +17,6 @@ if [ "$?" -ne 0 ]; then
 fi
 
 STREAM_COMPOSE_BASEURL="https://composes.stream.centos.org/production"
+STREAM_KOJI_REPO="https://kojihub.stream.centos.org/kojifiles/repos"
+PERIDOT_REPO="https://yumrepofs.build.resf.org/v1/projects"
 COMPOSE_BASEDIR="/mnt/compose"

mangle/generators/common.py

@@ -1,6 +1,18 @@
+import sys
+
 class common:
-    def rlver(self, rlver):
+    def rlver(self, rlver, stream=False, all_repo=False):
         default = "Not Supported"
+        if stream and all_repo:
+            print("incompatible options used")
+            sys.exit(1)
+
+        if stream:
+            return getattr(self, 'c' + str(rlver) + 's', lambda: default)()
+
+        if all_repo:
+            return getattr(self, 'rl' + str(rlver) + 'all', lambda: default)()
+
         return getattr(self, 'rl' + str(rlver), lambda: default)()
 
     def rl8(self):
@@ -28,4 +40,23 @@ class common:
         }
         return REPOS
 
+    def rl9all(self):
+        REPOS = {
+                'all': ['aarch64', 'ppc64le', 's390x', 'x86_64'],
+        }
+        return REPOS
+
+    # Parse tags of koji
+    def c8s(self):
+        REPOS = {
+                'c8s-build': ['aarch64', 'ppc64le', 'x86_64'],
+        }
+        return REPOS
+
+    def c9s(self):
+        REPOS = {
+                'c9s-build': ['aarch64', 'ppc64le', 's390x', 'x86_64'],
+        }
+        return REPOS
+
 switcher = common()

mangle/generators/common_9

@@ -2,3 +2,16 @@
 
 REPO=("BaseOS" "AppStream" "CRB" "HighAvailability" "ResilientStorage" "NFV" "RT" "SAP" "SAPHANA")
 ARCH=("aarch64" "ppc64le" "s390x" "x86_64")
+
+VER="r${MAJOR}${LH}"
+
+case "$VER" in
+  r9)
+    export PERIDOT_PROJECT_ID=0048077b-1573-4cb7-8ba7-cce823857ba5 ;;
+  r9lh)
+    export PERIDOT_PROJECT_ID=6794b5a8-290b-4d0d-ad5a-47164329cbb0 ;;
+  *)
+    echo "Not supported"
+    exit 1
+    ;;
+esac

mangle/generators/generate_catalog_branched

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Creates a prepopulate from KPR and then creates a catalog
+set -e
+
+if [ -n "$1" ]; then
+  NEWMAJOR=$1
+else
+  echo "Major version not specified"
+  exit 1
+fi
+
+tmpdir=$(mktemp -d)
+bash generate_prepopulate_from_kpr ${NEWMAJOR}
+git clone https://github.com/rocky-linux/peridot-releng.git "${tmpdir}/peridot-releng"
+cp prepopcatalog.py "${tmpdir}/peridot-releng/pungicatalog/"
+pushd "${tmpdir}/peridot-releng" || { echo "Could not change directory"; exit 1; }
+python3 pungicatalog/prepopcatalog.py --major ${NEWMAJOR} --minor 0 --prepopulate-path /tmp/prepopulate.json --output-path "${tmpdir}/branched.cfg"
+popd || { echo "Could not change back..."; exit 1; }
+
+echo "File located at: ${tmpdir}/branched.cfg"

mangle/generators/generate_prepopulate_from_kpr

@@ -0,0 +1,42 @@
+#!/bin/bash
+# Parses a koji tag for a branch out
+#set -x
+
+if [ -n "$1" ]; then
+  MAJOR=$1
+else
+  echo "Major version not specified"
+  exit 1
+fi
+
+export RLVER=$MAJOR
+source common
+
+drop="${PREPOPDROP}"
+current=$(pwd)
+tmpdir=$(mktemp -d)
+tag_template="c${MAJOR}s-build"
+stream_repo_url="${STREAM_KOJI_REPO}/${tag_template}/latest"
+
+pushd "${tmpdir}" || { echo "Could not change directory"; exit 1; }
+for y in "${ARCH[@]}"; do
+  repodatas=( $(dnf reposync --repofrompath ${tag_template},${stream_repo_url}/${y} --download-metadata --repoid=${tag_template} -p ${tag_template}/${y} --forcearch ${y} --norepopath --remote-time --assumeyes -u | grep repodata) )
+  mkdir -p "${tag_template}/${y}/repodata"
+  pushd "${tag_template}/${y}/repodata" || { echo "Could not change directory"; exit 1; }
+  for z in "${repodatas[@]}"; do
+    wget -q -nc "${z}"
+  done
+  wget -q -nc "${stream_repo_url}/${y}/repodata/repomd.xml"
+  popd || { echo "Could not change back..."; exit 1; }
+done
+/usr/bin/python3 "${current}/prepopulate_parser.py" --version $RLVER --stream
+ret_val=$?
+popd || { echo "Could not change back..."; exit 1; }
+
+if [ "$ret_val" -ne "0" ]; then
+  echo "There was an error running through the parser."
+  exit 1
+fi
+
+sed -i "s|${tag_template}|branched|g" $drop
+echo "File located at: $drop"

mangle/generators/generate_prepopulate_from_pungi

@@ -27,7 +27,7 @@ current=$(pwd)
 compose_dir="${COMPOSE_BASEDIR}/${MAJOR}/Rocky-${MAJOR}-${DATE}/compose"
 
 pushd "${compose_dir}" || { echo "Could not change directory"; exit 1; }
-/usr/bin/python3 "${current}/prepopulate_parser.py"
+/usr/bin/python3 "${current}/prepopulate_parser.py" --version ${RLVER}
 ret_val=$?
 popd || { echo "Could not change back..."; exit 1; }
 

mangle/generators/generate_prepopulate_from_pungi_latest

@@ -17,7 +17,7 @@ current=$(pwd)
 compose_dir="${COMPOSE_BASEDIR}/${MAJOR}/latest-Rocky-${MAJOR}/compose"
 
 pushd "${compose_dir}" || { echo "Could not change directory"; exit 1; }
-/usr/bin/python3 "${current}/prepopulate_parser.py" --pungi
+/usr/bin/python3 "${current}/prepopulate_parser.py" --pungi --version ${RLVER}
 ret_val=$?
 popd || { echo "Could not change back..."; exit 1; }
 

mangle/generators/generate_prepopulate_from_stream

@@ -41,7 +41,7 @@ for x in "${REPO[@]}"; do
     popd || { echo "Could not change back..."; exit 1; }
   done
 done
-/usr/bin/python3 "${current}/prepopulate_parser.py"
+/usr/bin/python3 "${current}/prepopulate_parser.py" --version ${MAJOR}
 ret_val=$?
 popd || { echo "Could not change back..."; exit 1; }
 

mangle/generators/generate_versions_from_kpr

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Parses a CentOS Stream Koji Tag
+#set -x
+
+if [ -n "$1" ]; then
+  MAJOR="$1"
+else
+  echo "Major version not specified"
+  exit 1
+fi
+
+export RLVER="${MAJOR}"
+source common
+
+drop="${VERSDROP}"
+current=$(pwd)
+tmpdir=$(mktemp -d)
+tag_template="c${MAJOR}s-build"
+stream_repo_url="${STREAM_KOJI_REPO}/${tag_template}/latest"
+
+pushd "${tmpdir}" || { echo "Could not change directory"; exit 1; }
+for y in "${ARCH[@]}"; do
+  repodatas=( $(dnf reposync --repofrompath ${tag_template},${stream_repo_url}/${y} --download-metadata --repoid=${tag_template} -p ${tag_template}/${y} --forcearch ${y} --norepopath --remote-time --assumeyes -u | grep repodata) )
+  mkdir -p "${tag_template}/${y}/repodata"
+  pushd "${tag_template}/${y}/repodata" || { echo "Could not change directory"; exit 1; }
+  for z in "${repodatas[@]}"; do
+    wget -q -nc "${z}"
+  done
+  wget -q -nc "${stream_repo_url}/${y}/repodata/repomd.xml"
+  popd || { echo "Could not change back..."; exit 1; }
+done
+/usr/bin/python3 "${current}/version_parser.py" --version "${MAJOR}" --stream
+ret_val=$?
+popd || { echo "Could not change back..."; exit 1; }
+
+if [ "$ret_val" -ne "0" ]; then
+  echo "There was an error running through the parser."
+  exit 1
+fi
+
+echo "File located at: $drop"

mangle/generators/generate_versions_from_ppr

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Parses a peridot project
+#set -x
+
+if [ -n "$1" ]; then
+  MAJOR="$1"
+else
+  echo "Major version not specified"
+  exit 1
+fi
+
+if [ -n "$2" ] && [[ "$2" == "lh" ]]; then
+  export LH="lh"
+fi
+
+export RLVER="${MAJOR}"
+source common
+
+drop="${VERSDROP}"
+current=$(pwd)
+tmpdir=$(mktemp -d)
+tag_template="all"
+peridot_repo_url="${PERIDOT_REPO}/${PERIDOT_PROJECT_ID}/repo/${tag_template}"
+
+pushd "${tmpdir}" || { echo "Could not change directory"; exit 1; }
+for y in "${ARCH[@]}"; do
+  repodatas=( $(dnf reposync --repofrompath ${tag_template},${peridot_repo_url}/${y} --download-metadata --repoid=${tag_template} -p ${tag_template}/${y} --forcearch ${y} --norepopath --remote-time --assumeyes -u | grep repodata) )
+  mkdir -p "${tag_template}/${y}/repodata"
+  pushd "${tag_template}/${y}/repodata" || { echo "Could not change directory"; exit 1; }
+  for z in "${repodatas[@]}"; do
+    wget -q -nc "${z}"
+  done
+  wget -q -nc "${peridot_repo_url}/${y}/repodata/repomd.xml"
+  popd || { echo "Could not change back..."; exit 1; }
+done
+/usr/bin/python3 "${current}/version_parser.py" --version "${MAJOR}" --all-repo
+ret_val=$?
+popd || { echo "Could not change back..."; exit 1; }
+
+if [ "$ret_val" -ne "0" ]; then
+  echo "There was an error running through the parser."
+  exit 1
+fi
+
+echo "File located at: $drop"

mangle/generators/generate_versions_from_stream

@@ -3,8 +3,8 @@
 #set -x
 
 if [ -n "$1" ] && [ -n "$2" ]; then
-  MAJOR=$1
+  MAJOR="$1"
-  DATE=$2
+  DATE="$2"
 else
   echo "Major version not specified"
   exit 1
@@ -18,7 +18,7 @@ if [ "$grep_val" -ne 0 ]; then
   echo "Date format incorrect. You must use: YYYYMMDD.X"
 fi
 
-export RLVER=$MAJOR
+export RLVER="${MAJOR}"
 source common
 
 drop="${VERSDROP}"
@@ -40,7 +40,7 @@ for x in "${REPO[@]}"; do
     popd || { echo "Could not change back..."; exit 1; }
   done
 done
-/usr/bin/python3 "${current}/version_parser.py"
+/usr/bin/python3 "${current}/version_parser.py" --version "${MAJOR}"
 ret_val=$?
 popd || { echo "Could not change back..."; exit 1; }
 

mangle/generators/prepopcatalog.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+# This is copied into peridot-releng and ran. This file is likely temporary
+# until we decide if this goes into that git repo.
+import argparse
+import os
+import json
+
+from catalog import (
+    PeridotCatalogSync,
+    PeridotCatalogSyncPackage,
+    PeridotCatalogSyncPackageType,
+    PeridotCatalogSyncRepository,
+)
+
+def main(prepopulate: str, output_path: str, major: int, minor: int):
+    print(f"Using prepopulate file: {prepopulate}")
+
+    with open(prepopulate) as json_file:
+        prepop = json.load(json_file)
+        json_file.close()
+
+    # Create a catalog
+    catalog = PeridotCatalogSync()
+    catalog.major = major
+    catalog.minor = minor
+
+    # Create indexes
+    package_index = {}
+    repo_module_index = {}
+    module_name_index = {}
+    module_defaults = []
+
+    # Read prepopulate json and create package objects
+    all_arches = []
+    for repo in prepop.keys():
+        for arch in prepop[repo].keys():
+            if arch not in all_arches:
+                all_arches.append(arch)
+            for package in prepop[repo][arch].keys():
+                if package not in package_index:
+                    package_index[package] = {}
+                if repo not in package_index[package]:
+                    package_index[package][repo] = {
+                        "include_filter": [],
+                        "multilib": [],
+                    }
+                na_list = prepop[repo][arch][package]
+                for na in na_list:
+                    splitted = na.split(".")
+                    arch_package = splitted[len(splitted) - 1]
+                    if arch != arch_package and arch_package != "noarch":
+                        if arch not in package_index[package][repo]["multilib"]:
+                            package_index[package][repo]["multilib"].append(arch)
+                    if na not in package_index[package][repo]["include_filter"]:
+                        package_index[package][repo]["include_filter"].append(na)
+
+    arch_specific_excludes = {}
+    na_index = {}
+    for pkg in package_index.keys():
+        for repo in package_index[pkg].keys():
+            na_list = list(
+                filter(
+                    lambda x: x.endswith(".noarch"),
+                    package_index[pkg][repo]["include_filter"],
+                )
+            )
+            if not na_list:
+                continue
+            exclude_arches = {}
+            for na in na_list:
+                for arch in all_arches:
+                    if (
+                        arch not in prepop[repo]
+                        or pkg not in prepop[repo][arch]
+                        or na not in prepop[repo][arch][pkg]
+                    ):
+                        if na not in exclude_arches:
+                            exclude_arches[na] = []
+                        exclude_arches[na].append(arch)
+                na_index[na] = na
+            if not exclude_arches:
+                continue
+            if pkg not in arch_specific_excludes:
+                arch_specific_excludes[pkg] = {}
+            if repo not in arch_specific_excludes[pkg]:
+                arch_specific_excludes[pkg][repo] = []
+            arch_specific_excludes[pkg][repo].append(exclude_arches)
+
+    # Index arch specific excludes by repo and arch
+    repo_arch_index = {}
+    for pkg in arch_specific_excludes.keys():
+        for repo in arch_specific_excludes[pkg].keys():
+            if repo not in repo_arch_index:
+                repo_arch_index[repo] = {}
+            for arches2 in arch_specific_excludes[pkg][repo]:
+                for na in arches2.keys():
+                    for arch in arches2[na]:
+                        if arch not in repo_arch_index[repo]:
+                            repo_arch_index[repo][arch] = []
+                        if na not in repo_arch_index[repo][arch]:
+                            repo_arch_index[repo][arch].append(na)
+
+    # Add noarch packages not in a specific arch to exclude filter
+    for repo in repo_arch_index.keys():
+        repo_key = f"^{repo}$"
+        filter_tuple = {}
+        for arch in repo_arch_index[repo].keys():
+            if arch not in filter_tuple:
+                filter_tuple[arch] = []
+            for na in repo_arch_index[repo][arch]:
+                na = na.removesuffix(".noarch")
+                if na not in filter_tuple[arch]:
+                    filter_tuple[arch].append(na)
+        catalog.exclude_filter.append((repo_key, filter_tuple))
+
+    for package in package_index.keys():
+        package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_FORK
+        if package in module_name_index:
+            package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_FORK_MODULE
+        elif package.startswith("rocky-"):
+            package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_SRC
+
+        catalog.add_package(
+            PeridotCatalogSyncPackage(
+                package,
+                package_type,
+                [
+                    PeridotCatalogSyncRepository(
+                        x,
+                        package_index[package][x]["include_filter"],
+                        package_index[package][x]["multilib"],
+                        (get_modules_for_repo(package, x, repo_module_index) if x in repo_module_index else None) if package in module_name_index else None,
+                    )
+                    for x in package_index[package].keys()
+                ],
+            )
+        )
+
+    print(f"Found {len(catalog.packages)} packages")
+
+    f = open(output_path, "w")
+    f.write(catalog.to_prototxt())
+    f.close()
+
+    print(f"Catalog written to {output_path}")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Create a catalog from a standalone prepopulate.json"
+    )
+    parser.add_argument("--prepopulate-path", type=str, required=True)
+    parser.add_argument("--major", type=int, required=True)
+    parser.add_argument("--minor", type=int, required=True)
+    parser.add_argument("--output-path", type=str, default="hidden.cfg")
+    args = parser.parse_args()
+    main(args.prepopulate_path, args.output_path, args.major, args.minor)

mangle/generators/prepopulate_parser.py

@@ -7,8 +7,6 @@ import dnf
 import createrepo_c as cr
 from common import *
 
-REPOS = switcher.rlver((os.environ['RLVER']))
-
 # Source packages we do not ship or are rocky branded
 IGNORES = [
         'insights-client',
@@ -22,10 +20,16 @@ IGNORES = [
 ]
 
 parser = argparse.ArgumentParser()
+parser.add_argument('--version', type=str, required=True)
 parser.add_argument("--pungi", help="local pungi is here", action='store_true')
-args = parser.parse_args()
+parser.add_argument('--stream', action='store_true', help="Stream koji only")
 
-if args.pungi:
+results = parser.parse_args()
+
+REPOS = switcher.rlver(results.version,
+                       stream=results.stream)
+
+if results.pungi:
     APPEND_TO_PATH = '/os'
 else:
     APPEND_TO_PATH = ''

mangle/generators/version_metadata_parser.py

@@ -10,7 +10,7 @@ conf = base.conf
 base.read_all_repos()
 all_repos = base.repos.all()
 all_repos.disable()
-base.repos.add_new_repo('all-source', conf, baseurl=['https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/all/src'])
+base.repos.add_new_repo('all-source', conf, baseurl=['https://yumrepofs.build.resf.org/v1/projects/0048077b-1573-4cb7-8ba7-cce823857ba5/repo/all/src'])
 base.fill_sack()
 
 q = base.sack.query()

mangle/generators/version_parser.py

@@ -2,11 +2,20 @@
 import os
 import os.path
 import json
+import argparse
 import dnf
 import createrepo_c as cr
 from common import *
 
-REPOS = switcher.rlver((os.environ['RLVER']))
+parser = argparse.ArgumentParser(description="Version Parser")
+parser.add_argument('--version', type=str, required=True)
+parser.add_argument('--stream', action='store_true', help="Stream koji only")
+parser.add_argument('--all-repo', action='store_true', help="Looks at the all repo for peridot")
+results = parser.parse_args()
+
+REPOS = switcher.rlver(results.version,
+                       stream=results.stream,
+                       all_repo=results.all_repo)
 
 # Source packages we do not ship or are rocky branded
 IGNORES = [

mangle/quotas.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"sync"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/servicequotas"
+)
+
+func getQuotaCode(sqSvc *servicequotas.ServiceQuotas) string {
+	input := &servicequotas.ListServiceQuotasInput{
+		ServiceCode: aws.String("ec2"),
+	}
+
+	for {
+		output, err := sqSvc.ListServiceQuotas(input)
+		if err != nil {
+			log.Println("Error getting quota code:", err)
+			os.Exit(1)
+		}
+
+		for _, quota := range output.Quotas {
+			if *quota.QuotaName == "Public AMIs" {
+				return *quota.QuotaCode
+			}
+		}
+
+		if output.NextToken == nil {
+			break
+		}
+		input.NextToken = output.NextToken
+	}
+	log.Println("Quota code not found")
+	os.Exit(1)
+	return ""
+}
+
+func getRegions(ec2Svc *ec2.EC2) ([]*string, error) {
+	input := &ec2.DescribeRegionsInput{}
+
+	output, err := ec2Svc.DescribeRegions(input)
+	if err != nil {
+		return nil, err
+	}
+
+	var regions []*string
+	for _, region := range output.Regions {
+		regions = append(regions, region.RegionName)
+	}
+
+	return regions, nil
+}
+
+type QuotaInfo struct {
+	CurrentQuota float64
+	DesiredValue float64
+	Status       string
+	CaseId       string
+}
+
+func getQuotaInfo(sqSvc *servicequotas.ServiceQuotas, quotaCode string, region string) *QuotaInfo {
+	input := &servicequotas.GetServiceQuotaInput{
+		ServiceCode: aws.String("ec2"),
+		QuotaCode:   aws.String(quotaCode),
+	}
+
+	output, err := sqSvc.GetServiceQuota(input)
+	if err != nil {
+		log.Printf("Error getting quota info for %s: %s\n", region, err)
+		return nil
+		// os.Exit(1)
+	}
+
+	currentValue := *output.Quota.Value
+	requestOutput, err := sqSvc.ListRequestedServiceQuotaChangeHistoryByQuota(&servicequotas.ListRequestedServiceQuotaChangeHistoryByQuotaInput{
+		ServiceCode: aws.String("ec2"),
+		QuotaCode:   aws.String(quotaCode),
+	})
+
+	if err != nil {
+		log.Println("Error getting request info:", err)
+		os.Exit(1)
+	}
+	var desiredValue float64
+	var status string
+	var caseId string
+	if len(requestOutput.RequestedQuotas) > 0 {
+		desiredValue = *requestOutput.RequestedQuotas[len(requestOutput.RequestedQuotas)-1].DesiredValue
+		status = *requestOutput.RequestedQuotas[len(requestOutput.RequestedQuotas)-1].Status
+		caseId = *requestOutput.RequestedQuotas[len(requestOutput.RequestedQuotas)-1].CaseId
+	}
+	return &QuotaInfo{currentValue, desiredValue, status, caseId}
+}
+
+func listQuotas(sess *session.Session, quotaCode string, regions []*string) {
+	fmt.Println("Region\tQuota\tDesired\tStatus\tCaseId")
+
+	var wg sync.WaitGroup
+	wg.Add(len(regions))
+
+	for _, region := range regions {
+		go func(region string) {
+			defer wg.Done()
+			regionSqSvc := servicequotas.New(sess, &aws.Config{Region: aws.String(region)})
+			quotaInfo := getQuotaInfo(regionSqSvc, quotaCode, region)
+			if quotaInfo != nil {
+				fmt.Printf("%s\t%.0f\t%.0f\t%s\t%s\n", region, quotaInfo.CurrentQuota, quotaInfo.DesiredValue, quotaInfo.Status, quotaInfo.CaseId)
+			}
+		}(aws.StringValue(region))
+	}
+	wg.Wait()
+}
+
+func main() {
+	// Create session
+	sess := session.Must(session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+	}))
+
+	// Create EC2 client
+	ec2Svc := ec2.New(sess, &aws.Config{Region: aws.String("us-east-1")})
+
+	// Create Service Quotas client
+	sqSvc := servicequotas.New(sess)
+
+	// Get the quota code for Public AMIs once
+	quotaCode := getQuotaCode(sqSvc)
+
+	// Get all regions
+	regions, err := getRegions(ec2Svc)
+	if err != nil {
+		log.Println("Error getting regions:", err)
+		os.Exit(1)
+	}
+
+	// List quotas for all regions
+	listQuotas(sess, quotaCode, regions)
+
+	// Request quota increase for all regions
+	// requestQuotaIncrease(sqSvc, quotaCode, regions)
+}

sync/common

@@ -15,9 +15,14 @@ if [ -z "$RLVER" ]; then
   export RLVER=8
 fi
 
+if [ -z "$RLREL" ]; then
+  export RLREL=stable
+fi
+
 # Override: Not Allowed
 STAGING_ROOT="/mnt/repos-staging"
 PRODUCTION_ROOT="/mnt/repos-production"
+COMPOSE_ROOT="/mnt/compose"
 
 # Override: Not Allowed
 # relative to ${ENV}_ROOT

sync/common_8

@@ -1,20 +1,35 @@
 # To be sourced by scripts to use
 
 # Revision must always start with a major number
-REVISION=8.7
+case "${RLREL}" in
-
+  stable)
-MAJOR="${REVISION:0:1}"
+    REVISION=8.7
-MINOR="${REVISION:2:1}"
+    APPEND_TO_DIR="-RC1"
-
+    ;;
-# comment or blank if needed
+  beta)
-APPEND_TO_DIR="-RC1"
+    REVISION=8.8
+    APPEND_TO_DIR="-BETA"
+    COMPOSE_APPEND="${APPEND_TO_DIR}"
+    ;;
+  lh)
+    REVISION=8.8
+    APPEND_TO_DIR="-LookAhead"
+    COMPOSE_APPEND="${APPEND_TO_DIR}"
+    ;;
+  *)
+    echo "option no recognized"
+    ;;
+esac
 
+MAJOR="${REVISION%.*}"
+MINOR="${REVISION#*.}"
 STAGING_ROOT="/mnt/repos-staging"
 PRODUCTION_ROOT="/mnt/repos-production"
 
 # relative to ${ENV}_ROOT
 CATEGORY_STUB="mirror/pub/rocky"
 RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}"
+RELEASE_COMPOSE_ROOT="${COMPOSE_ROOT}/${MAJOR}${COMPOSE_APPEND}/latest-${SHORT}-${MAJOR}"
 
 # Set all repos that have no comps/groups associated with them. This is even in
 # cases where repos will not be available by normal means. It's just for
@@ -23,7 +38,6 @@ NONMODS_REPOS=(
   extras
   Devel
   plus
-  rockyrpi
 )
 
 # These repos were originally separate from the main compose and need symlinks

sync/common_9

@@ -1,22 +1,34 @@
 # To be sourced by scripts to use
 
 # Revision must always start with a major number
-REVISION=9.1
+case "${RLREL}" in
+  stable)
+    REVISION=9.1
+    APPEND_TO_DIR="-RC4"
+    ;;
+  beta)
+    REVISION=9.2
+    APPEND_TO_DIR="-beta"
+    ;;
+  lh)
+    REVISION=9.2
+    APPEND_TO_DIR="-lookahead"
+    ;;
+  *)
+    echo "option no recognized"
+    ;;
+esac
 
+MAJOR="${REVISION%.*}"
+MINOR="${REVISION#*.}"
 ARCHES=(x86_64 aarch64 ppc64le s390x)
-
-MAJOR="${REVISION:0:1}"
-MINOR="${REVISION:2:1}"
-
-# comment or blank if needed
-APPEND_TO_DIR="-RC4"
-
 STAGING_ROOT="/mnt/repos-staging"
 PRODUCTION_ROOT="/mnt/repos-production"
 
 # relative to ${ENV}_ROOT
 CATEGORY_STUB="mirror/pub/rocky"
 RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}"
+RELEASE_COMPOSE_ROOT="${COMPOSE_ROOT}/${MAJOR}/latest-${SHORT}-${MAJOR}"
 
 # Set all repos that have no comps/groups associated with them. This is even in
 # cases where repos will not be available by normal means. It's just for
@@ -25,7 +37,6 @@ NONMODS_REPOS=(
   extras
   Devel
   plus
-  rockyrpi
 )
 
 # These repos were originally separate from the main compose and need symlinks

sync/lh-release-sync-to-staging.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+# Performs a full on sync of a minor release, directories and all. It calls the
+# other scripts in this directory to assist where necessary.
+# Note that this is EL8 specific
+#
+# Source common variables
+# shellcheck disable=SC2046,1091,1090
+source $(dirname "$0")/common
+
+# Major Version (eg, 8)
+MAJ=${RLVER}
+
+# sync all pieces of a release, including extras, nfv, etc
+for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
+  echo "${COMPOSE}: Syncing"
+  pushd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; }
+
+  if [[ "${COMPOSE}" == "Rocky" ]]; then
+    # ISO Work before syncing
+    mkdir -p isos/{x86_64,aarch64}
+
+    # Sort the ISO's
+    for ARCH in "${ARCHES[@]}"; do
+      for x in BaseOS Minimal; do
+        echo "${x} ${ARCH}: Moving ISO images"
+        mv "${x}/${ARCH}/iso/"* "isos/${ARCH}/"
+      done
+      pushd "isos/${ARCH}" || { echo "${ARCH}: Failed to change directory"; break; }
+      ln -s "Rocky-${REVISION}-${ARCH}-boot.iso" "Rocky-${ARCH}-boot.iso"
+      ln -s "Rocky-${REVISION}-${ARCH}-dvd1.iso" "Rocky-${ARCH}-dvd1.iso"
+      ln -s "Rocky-${REVISION}-${ARCH}-dvd1.iso" "Rocky-${ARCH}-dvd.iso"
+      ln -s "Rocky-${REVISION}-${ARCH}-minimal.iso" "Rocky-${ARCH}-minimal.iso"
+      for file in *.iso; do
+        printf "# %s: %s bytes\n%s\n" \
+          "${file}" \
+          "$(stat -c %s ${file})" \
+          "$(sha256sum --tag ${file})" \
+        | sudo tee -a CHECKSUM;
+      done
+      popd || { echo "Could not change directory"; break; }
+    done
+  fi
+
+  TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}"
+  mkdir -p "${TARGET}"
+  # disabling because none of our files should be starting with dashes. If they
+  # are something is *seriously* wrong here.
+  # shellcheck disable=SC2035
+  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
+      {} "${TARGET}"
+
+  if [[ "${COMPOSE}" == "Rocky" ]]; then
+    cp COMPOSE_ID "${TARGET}"
+    chown 10004:10005 "${TARGET}/COMPOSE_ID"
+    rsync -av --chown=10004:10005 --progress --relative --human-readable metadata "${TARGET}"
+  fi
+
+  # Return back to where we started
+  popd || { echo "${COMPOSE}: Failed to change back"; break; }
+
+  # Create extra stuff
+  pushd "${TARGET}" || { echo "${COMPOSE}: Failed to change directory"; break;  }
+  mkdir -p Live/x86_64
+  ln -s Live live
+  popd || { echo "${COMPOSE}: Failed to change back"; break;  }
+done
+
+
+# Create symlinks for repos that were once separate from the main compose
+for LINK in "${LINK_REPOS[@]}"; do
+  ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${LINK}" \
+    "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${LINK_REPOS[$LINK]}"
+done
+
+# make a kickstart directory
+for ARCH in "${ARCHES[@]}"; do
+  for REPO in "${MODS_REPOS[@]}"; do
+    SOURCE="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/os"
+    TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/kickstart"
+    echo "Making golden kickstart directory"
+    cp -na "${SOURCE}" "${TARGET}"
+  done
+done
+
+# fix treeinfo
+for ARCH in "${ARCHES[@]}"; do
+  echo "Ensuring treeinfo is correct"
+  treeinfoModder "${ARCH}"
+  treeinfoModderKickstart "${ARCH}"
+  treeinfoSaver "${ARCH}"
+done
+
+# sign all repos
+echo "Signing all repositories"
+test -f $(dirname "$0")/sign-repos-only.sh
+ret_val=$?
+
+if [ "$ret_val" -eq 0 ]; then
+  $(dirname "$0")/sign-repos-only.sh
+fi
+
+# Change Symlink if required
+echo "Setting symlink to ${REV}"
+pushd "${STAGING_ROOT}/${CATEGORY_STUB}" || exit
+/bin/rm "${STAGING_ROOT}/${CATEGORY_STUB}/latest-8"
+ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" latest-8
+popd || exit

sync/minor-release-sync-to-staging.sh

@@ -13,7 +13,7 @@ MAJ=${RLVER}
 # sync all pieces of a release, including extras, nfv, etc
 for COMPOSE in "${NONSIG_COMPOSE[@]}"; do
   echo "${COMPOSE}: Syncing"
-  pushd "/mnt/compose/${MAJ}/latest-${COMPOSE}-${MAJ}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; }
+  pushd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; }
 
   if [[ "${COMPOSE}" == "Rocky" ]]; then
     # ISO Work before syncing

sync/prep-staging-8.sh

@@ -23,7 +23,7 @@ for y in "${ALL_REPOS[@]}"; do
   if [ "$ret_val" -eq 0 ]; then
     createrepo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree" \
       "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \
-      --workers 8
+      --workers 12
     sed -i '/<open-size><\/open-size>/d' \
       "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree/repodata/repomd.xml"
     test -f /root/bin/sign-repo.sh && /root/bin/sign-repo.sh \
@@ -46,7 +46,7 @@ for x in "${ARCHES[@]}"; do
       if [ "$ret_val" -eq 0 ]; then
         createrepo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \
           "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \
-          --workers 8
+          --workers 12
         sed -i '/<open-size><\/open-size>/d' \
           "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}/repodata/repomd.xml"
         test -f /root/bin/sign-repo.sh && /root/bin/sign-repo.sh \
@@ -63,7 +63,7 @@ for x in "${ARCHES[@]}"; do
     if [ "$ret_val" -eq 0 ]; then
       createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree" \
         "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \
-        --workers 8
+        --workers 12 --retain-old-md-by-age 1d
       sed -i '/<open-size><\/open-size>/d' \
         "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree/repodata/repomd.xml"
       test -f /root/bin/sign-repo.sh && /root/bin/sign-repo.sh \
@@ -83,7 +83,7 @@ for x in "${ARCHES[@]}"; do
         --groupfile="/mnt/compose/8/latest-Rocky-8/work/${x}/comps/comps-${y}.${x}.xml" \
         --xz --revision="${REVISION}" \
         "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \
-        --workers=8 --checksum=sha256
+        --workers=12 --checksum=sha256 --retain-old-md-by-age 1d
       sed -i '/<open-size><\/open-size>/d' \
         "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os/repodata/repomd.xml"
       sed -i '/<open-size><\/open-size>/d' \

sync/propagate-image.sh

@@ -33,6 +33,8 @@ function copy(){
   for region in $REGIONS; do 
     if find_image_by_name $region; then
       echo "Found copy of $source_ami in $region - $found_image_id - Skipping"
+      unset ami_ids[$region]
+      ami_ids[$region]=$(echo $found_image_id | tr -d "'")
       continue
     fi
     echo -n "Creating copy job for $region..."
@@ -68,16 +70,33 @@ function change_privacy(){
   local finished=false
   while ! $finished; do
     for region in "${!ami_ids[@]}"; do 
-      echo -n "Making ${ami_ids[$region]} in $region $status..."
+      image_id=${ami_ids[$region]}
-      aws --profile resf-ami ec2 modify-image-attribute \
+      echo -n "Making ${image_id} in $region $status..."
+      if aws --profile resf-ami ec2 modify-image-attribute \
         --region $region \
-        --image-id "${ami_ids[$region]}" \
+        --image-id "$image_id" \
-        --launch-permission "${launch_permission}" 2>/dev/null
+        --launch-permission "${launch_permission}" 2>/dev/null; then
-      if [[ $? -eq 0 ]]; then
+
+        snapshot_id=$(aws --profile resf-ami ec2 describe-images \
+          --region $region \
+          --image-ids "${image_id}" \
+          --query 'Images[*].BlockDeviceMappings[0].Ebs.SnapshotId' \
+          --output text 2>&1)
+        permissions=$(aws --profile resf-ami ec2 describe-snapshot-attribute \
+            --region $region \
+            --snapshot-id "${snapshot_id}" \
+            --attribute createVolumePermission \
+            --query 'CreateVolumePermissions[0].Group' \
+            --output text 2>&1)
+        if [[ "$permissions" == "all" ]] || aws --profile resf-ami ec2 modify-snapshot-attribute \
+            --region $region \
+            --snapshot-id "${snapshot_id}" \
+            --create-volume-permission "${launch_permission}" 2>/dev/null; then
             unset ami_ids[$region] 
             echo ". Done"
             continue
         fi
+      fi
       echo ". Still pending"
     done
     if [[ ${#ami_ids[@]} -gt 0 ]]; then

sync/sync-to-prod.delete.sh

@@ -1,6 +1,10 @@
 #!/bin/bash
 # Syncs everything from staging to production
 
+if [[ "$RLREL" == "lh" ]] || [[ "$RLREL" == "beta" ]]; then
+  exit 1
+fi
+
 # Source common variables
 # shellcheck disable=SC2046,1091,1090
 source "$(dirname "$0")/common"

sync/sync-to-prod.sh

@@ -1,6 +1,10 @@
 #!/bin/bash
 # Syncs everything from staging to production
 
+if [[ "$RLREL" == "lh" ]] || [[ "$RLREL" == "beta" ]]; then
+  exit 1
+fi
+
 # Source common variables
 # shellcheck disable=SC2046,1091,1090
 source "$(dirname "$0")/common"

sync/sync-to-staging.delete.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Short name (eg, NFV, extras, Rocky, gluster9)
+SHORT=${1}
+
 # Source common variables
 # shellcheck disable=SC2046,1091,1090
 source "$(dirname "$0")/common"
@@ -11,10 +14,8 @@ fi
 
 # Major Version (eg, 8)
 MAJ=${RLVER}
-# Short name (eg, NFV, extras, Rocky, gluster9)
-SHORT=${1}
 
-cd "/mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}/compose" || { echo "Failed to change directory"; ret_val=1; exit 1; }
+cd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "Failed to change directory"; ret_val=1; exit 1; }
 ret_val=$?
 
 if [ $ret_val -eq "0" ]; then

sync/sync-to-staging.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Short name (eg, NFV, extras, Rocky, gluster9)
+SHORT=${1}
+
 # Source common variables
 # shellcheck disable=SC2046,1091,1090
 source "$(dirname "$0")/common"
@@ -11,10 +14,8 @@ fi
 
 # Major Version (eg, 8)
 MAJ=${RLVER}
-# Short name (eg, NFV, extras, Rocky, gluster9)
-SHORT=${1}
 
-cd "/mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}/compose" || { echo "Failed to change directory"; ret_val=1; exit 1; }
+cd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "Failed to change directory"; ret_val=1; exit 1; }
 ret_val=$?
 
 if [ $ret_val -eq "0" ]; then