diff --git a/iso/empanadas/empanadas/__init__.py b/iso/empanadas/empanadas/__init__.py index 2b8877c..ef7eb44 100644 --- a/iso/empanadas/empanadas/__init__.py +++ b/iso/empanadas/empanadas/__init__.py @@ -1 +1 @@ -__version__ = '0.5.0' +__version__ = '0.6.0' diff --git a/iso/empanadas/empanadas/configs/el9.yaml b/iso/empanadas/empanadas/configs/el9.yaml index a56b346..e347588 100644 --- a/iso/empanadas/empanadas/configs/el9.yaml +++ b/iso/empanadas/empanadas/configs/el9.yaml @@ -109,6 +109,7 @@ XFCE: rocky-live-xfce.ks KDE: rocky-live-kde.ks MATE: rocky-live-mate.ks + Cinnamon: rocky-live-cinnamon.ks allowed_arches: - x86_64 - aarch64 diff --git a/iso/empanadas/empanadas/util/__init__.py b/iso/empanadas/empanadas/util/__init__.py index 828e595..ee2ac51 100644 --- a/iso/empanadas/empanadas/util/__init__.py +++ b/iso/empanadas/empanadas/util/__init__.py @@ -9,6 +9,7 @@ from empanadas.util.check import ( from empanadas.util.shared import ( Shared, ArchCheck, + Idents, ) from empanadas.util.dnf_utils import ( diff --git a/iso/empanadas/empanadas/util/dnf_utils.py b/iso/empanadas/empanadas/util/dnf_utils.py index 055fc0f..da02757 100644 --- a/iso/empanadas/empanadas/util/dnf_utils.py +++ b/iso/empanadas/empanadas/util/dnf_utils.py @@ -289,7 +289,9 @@ class RepoSync: Shared.deploy_extra_files(self.extra_files, sync_root, global_work_root, self.log) self.deploy_treeinfo(self.repo, sync_root, self.arch) self.tweak_treeinfo(self.repo, sync_root, self.arch) - self.symlink_to_latest(generated_dir) + #self.symlink_to_latest(generated_dir) + Shared.symlink_to_latest(self.shortname, self.major_version, + generated_dir, self.compose_latest_dir, self.log) if self.repoclosure: self.repoclosure_work(sync_root, work_root, log_root) @@ -323,7 +325,7 @@ class RepoSync: if self.parallel: self.podman_sync(repo, sync_root, work_root, log_root, global_work_root, arch) else: - Shared.dnf_sync(repo, sync_root, work_root, arch, self.log) + Shared.norm_dnf_sync(self, repo, sync_root, work_root, arch, self.log) def podman_sync( self, @@ -709,22 +711,6 @@ class RepoSync: 'No issues detected.' ) - def symlink_to_latest(self, generated_dir): - """ - Emulates pungi and symlinks latest-Rocky-X - - This link will be what is updated in full runs. Whatever is in this - 'latest' directory is what is rsynced on to staging after completion. - This link should not change often. - """ - try: - os.remove(self.compose_latest_dir) - except: - pass - - self.log.info('Symlinking to latest-{}-{}...'.format(self.shortname, self.major_version)) - os.symlink(generated_dir, self.compose_latest_dir) - def repoclosure_work(self, sync_root, work_root, log_root): """ This is where we run repoclosures, based on the configuration of each diff --git a/iso/empanadas/empanadas/util/iso_utils.py b/iso/empanadas/empanadas/util/iso_utils.py index b9cefd8..912d9e0 100644 --- a/iso/empanadas/empanadas/util/iso_utils.py +++ b/iso/empanadas/empanadas/util/iso_utils.py @@ -35,7 +35,7 @@ import productmd.treeinfo from jinja2 import Environment, FileSystemLoader from empanadas.common import Color, _rootdir -from empanadas.util import Shared, ArchCheck +from empanadas.util import Shared, ArchCheck, Idents class IsoBuild: """ @@ -138,10 +138,7 @@ class IsoBuild: self.compose_latest_dir = os.path.join( config['compose_root'], major, - "latest-{}-{}".format( - self.shortname, - self.profile - ) + f"latest-{self.shortname}-{self.profile}" ) self.compose_latest_sync = os.path.join( @@ -371,11 +368,7 @@ class IsoBuild: source_path = latest_artifacts[arch] - full_drop = '{}/lorax-{}-{}.tar.gz'.format( - lorax_arch_dir, - self.release, - arch - ) + full_drop = f'{lorax_arch_dir}/lorax-{self.release}-{arch}.tar.gz' if not os.path.exists(lorax_arch_dir): os.makedirs(lorax_arch_dir, exist_ok=True) @@ -403,10 +396,7 @@ class IsoBuild: self.log.info(Color.INFO + 'Beginning unpack phase...') for arch in arches_to_unpack: - tarname = 'lorax-{}-{}.tar.gz'.format( - self.release, - arch - ) + tarname = f'lorax-{self.release}-{arch}.tar.gz' tarball = os.path.join( self.lorax_work_dir, @@ -523,22 +513,13 @@ class IsoBuild: if self.release_candidate: rclevel = '-' + self.rclvl - discname = '{}-{}.{}{}-{}-{}.iso'.format( - self.shortname, - self.major_version, - self.minor_version, - rclevel, - arch, - 'boot' - ) + discname = f'{self.shortname}-{self.major_version}.{self.minor_version}{rclevel}-{arch}-boot.iso' isobootpath = os.path.join(iso_to_go, discname) - manifest = '{}.manifest'.format(isobootpath) - link_name = '{}-{}-boot.iso'.format(self.shortname, arch) + manifest = f'{isobootpath}.manifest' + link_name = f'{self.shortname}-{arch}-boot.iso' link_manifest = link_name + '.manifest' - latest_link_name = '{}-{}-latest-{}-boot.iso'.format(self.shortname, - self.major_version, - arch) + latest_link_name = f'{self.shortname}-{self.major_version}-latest-{arch}-boot.iso' latest_link_manifest = latest_link_name + '.manifest' isobootpath = os.path.join(iso_to_go, discname) linkbootpath = os.path.join(iso_to_go, link_name) @@ -813,11 +794,11 @@ class IsoBuild: xorriso_template = self.tmplenv.get_template('xorriso.tmpl.txt') iso_readme_template = self.tmplenv.get_template('ISOREADME.tmpl') - mock_iso_path = '/var/tmp/lorax-{}.cfg'.format(self.major_version) - mock_sh_path = '{}/extraisobuild-{}-{}.sh'.format(entries_dir, arch, image) - iso_template_path = '{}/buildExtraImage-{}-{}.sh'.format(entries_dir, arch, image) - xorriso_template_path = '{}/xorriso-{}-{}.txt'.format(entries_dir, arch, image) - iso_readme_path = '{}/{}/README'.format(self.iso_work_dir, arch) + mock_iso_path = f'/var/tmp/lorax-{self.major_version}.cfg' + mock_sh_path = f'{entries_dir}/extraisobuild-{arch}-{image}.sh' + iso_template_path = f'{entries_dir}/buildExtraImage-{arch}-{image}.sh' + xorriso_template_path = f'{entries_dir}/xorriso-{arch}-{image}.txt' + iso_readme_path = f'{self.iso_work_dir}/{arch}/README' #print(iso_readme_path) log_root = os.path.join( @@ -829,7 +810,7 @@ class IsoBuild: if not os.path.exists(log_root): os.makedirs(log_root, exist_ok=True) - log_path_command = '| tee -a {}/{}-{}.log'.format(log_root, arch, image) + log_path_command = f'| tee -a {log_root}/{arch}-{image}.log' # This is kind of a hack. Installing xorrisofs sets the alternatives to # it, so backwards compatibility is sort of guaranteed. But we want to @@ -850,31 +831,10 @@ class IsoBuild: if self.updated_image: datestamp = '-' + self.updated_image_date - volid = '{}-{}-{}{}-{}-{}'.format( - self.shortname, - self.major_version, - self.minor_version, - rclevel, - arch, - volname - ) - - isoname = '{}-{}{}{}-{}-{}.iso'.format( - self.shortname, - self.revision, - rclevel, - datestamp, - arch, - image - ) - - generic_isoname = '{}-{}-{}.iso'.format(self.shortname, arch, image) - latest_isoname = '{}-{}-latest-{}-{}.iso'.format( - self.shortname, - self.major_version, - arch, - image - ) + volid = f'{self.shortname}-{self.major_version}-{self.minor_version}{rclevel}-{arch}-{volname}' + isoname = f'{self.shortname}-{self.revision}{rclevel}{datestamp}-{arch}-{image}.iso' + generic_isoname = f'{self.shortname}-{arch}-{image}.iso' + latest_isoname = f'{self.shortname}-{self.major_version}-latest-{arch}-{image}.iso' lorax_pkg_cmd = '/usr/bin/dnf install {} -y {}'.format( ' '.join(required_pkgs), @@ -986,7 +946,7 @@ class IsoBuild: have mock available. """ entries_dir = os.path.join(work_root, "entries") - extra_iso_cmd = '/bin/bash {}/extraisobuild-{}-{}.sh'.format(entries_dir, arch, image) + extra_iso_cmd = f'/bin/bash {entries_dir}/extraisobuild-{arch}-{image}.sh' self.log.info('Starting mock build...') p = subprocess.call(shlex.split(extra_iso_cmd)) if p != 0: @@ -1022,7 +982,7 @@ class IsoBuild: arch_sync = arches.copy() for a in arch_sync: - entry_name = 'buildExtraImage-{}-{}.sh'.format(a, i) + entry_name = f'buildExtraImage-{a}-{i}.sh' entry_name_list.append(entry_name) rclevel = '' @@ -1080,10 +1040,7 @@ class IsoBuild: join_all_pods = ' '.join(entry_name_list) time.sleep(3) self.log.info(Color.INFO + 'Building ' + i + ' ...') - pod_watcher = '{} wait {}'.format( - cmd, - join_all_pods - ) + pod_watcher = f'{cmd} wait {join_all_pods}' watch_man = subprocess.call( shlex.split(pod_watcher), @@ -1095,10 +1052,7 @@ class IsoBuild: # code. pattern = "Exited (0)" for pod in entry_name_list: - checkcmd = '{} ps -f status=exited -f name={}'.format( - cmd, - pod - ) + checkcmd = f'{cmd} ps -f status=exited -f name={pod}' podcheck = subprocess.Popen( checkcmd, stdout=subprocess.PIPE, @@ -1111,10 +1065,7 @@ class IsoBuild: self.log.error(Color.FAIL + pod) bad_exit_list.append(pod) - rmcmd = '{} rm {}'.format( - cmd, - join_all_pods - ) + rmcmd = f'{cmd} rm {join_all_pods}' rmpod = subprocess.Popen( rmcmd, @@ -1202,32 +1153,41 @@ class IsoBuild: for k, v in self._get_grafts([rd_for_var]).items(): files[os.path.join(repo, "repodata", k)] = v - grafts = '{}/{}-{}-grafts'.format( - lorax_base_dir, - iso, - arch - ) + grafts = f'{lorax_base_dir}/{iso}-{arch}-grafts' - xorrs = '{}/xorriso-{}.txt'.format( - lorax_base_dir, - arch - ) + xorrs = f'{lorax_base_dir}/xorriso-{iso}-{arch}.txt' + + # Generate exclusion list/dict from boot.iso manifest + boot_manifest = f'{lorax_base_dir}/lorax/images/boot.iso.manifest' + # Boot configs and images that may change + # It's unlikely these will be changed in empanadas, they're used as is + # and it works fine. This is a carry over from a recent pungi commit, + # based on an issue I had filed. The above was the original part, the + # below is a pungi "buildinstall" thing that we don't do, but may + # include as a feature if it ever happens. + updatable_files = set(ArchCheck.boot_configs + ArchCheck.boot_images + ['.discinfo']) + ignores = set() + updatables = set() + + try: + with open(boot_manifest) as i: + # ignores = set(line.lstrip("/").rstrip("\n") for line in i) + for line in i: + path = line.lstrip("/").rstrip("\n") + if path in updatable_files: + updatables.add(path) + else: + ignores.add(path) + except Exception as e: + self.log.error(Color.FAIL + 'File was likely not found.') + raise SystemExit(e) self._write_grafts( grafts, xorrs, files, - exclude=[ - "*/lost+found", - "*/boot.iso", - "*/boot.iso.manifest", - "EFI/*", - "images/*", - "isolinux/*", - "boot/*", - "ppc/*", - "generic.ins" - ] + exclude=ignores, + update=updatables ) if self.iso_map['xorrisofs']: @@ -1249,12 +1209,12 @@ class IsoBuild: if isinstance(p, dict): tree = p else: - tree = self._scanning(p) - result = self._merging(result, tree) + tree = Idents.scanning(p) + result = Idents.merging(result, tree) for p in exclusive_paths: - tree = self._scanning(p) - result = self._merging(result, tree, exclusive=True) + tree = Idents.scanning(p) + result = Idents.merging(result, tree, exclusive=True) # Resolves possible symlinks for key in result.keys(): @@ -1267,12 +1227,17 @@ class IsoBuild: return result - def _write_grafts(self, filepath, xorrspath, u, exclude=None): + def _write_grafts(self, filepath, xorrspath, u, exclude=None, update=None): """ Write out the graft points """ seen = set() + # There are files that are on the exclude list typically. exclude = exclude or [] + # There is a chance files may get updated before being placed in a + # variant ISO - it's rare though. most that will be different is + # .discinfo + update = update or [] result = {} for zl in sorted(u, reverse=True): dirn = os.path.dirname(zl) @@ -1291,119 +1256,43 @@ class IsoBuild: result[zl] = u[zl] seen.add(dirn) + # We check first if a file needs to be updated first before relying on + # the boot.iso manifest to exclude a file if self.iso_map['xorrisofs']: fx = open(xorrspath, "w") - for zm in sorted(result, key=self._sorting): + for zm in sorted(result, key=Idents.sorting): found = False + replace = False + for upda in update: + if fnmatch(zm, upda): + #print(f'updating: {zm} {upda}') + replace = True + break for excl in exclude: if fnmatch(zm, excl): + #print(f'ignoring: {zm} {excl}') found = True break if found: continue - fx.write("-map %s %s\n" % (u[zm], zm)) + mcmd = "-update" if replace else "-map" + fx.write("%s %s %s\n" % (mcmd, u[zm], zm)) fx.close() else: fh = open(filepath, "w") - for zl in sorted(result, key=self._sorting): - found = False - for excl in exclude: - if fnmatch(zl, excl): - found = True - break - if found: - continue + self.log.info(Color.WARN + 'Nothing should be excluded in legacy ' + + 'genisoimage. Ignoring exclude list.') + for zl in sorted(result, key=Idents.sorting): + #found = False + #for excl in exclude: + # if fnmatch(zl, excl): + # found = True + # break + #if found: + # continue fh.write("%s=%s\n" % (zl, u[zl])) fh.close() - def _scanning(self, p): - """ - Scan tree - """ - path = os.path.abspath(p) - result = {} - for root, dirs, files in os.walk(path): - for file in files: - abspath = os.path.join(root, file) - relpath = kobo.shortcuts.relative_path(abspath, path.rstrip("/") + "/") - result[relpath] = abspath - - # Include empty directories too - if root != path: - abspath = os.path.join(root, "") - relpath = kobo.shortcuts.relative_path(abspath, path.rstrip("/") + "/") - result[relpath] = abspath - - return result - - - def _merging(self, tree_a, tree_b, exclusive=False): - """ - Merge tree - """ - result = tree_b.copy() - all_dirs = set( - [os.path.dirname(dirn).rstrip("/") for dirn in result if os.path.dirname(dirn) != ""] - ) - - for dirn in tree_a: - dn = os.path.dirname(dirn) - if exclusive: - match = False - for x in all_dirs: - if dn == x or dn.startswith("%s/" % x): - match = True - break - if match: - continue - - if dirn in result: - continue - - result[dirn] = tree_a[dirn] - return result - - def _sorting(self, k): - """ - Sorting using the is_rpm and is_image funcs. Images are first, extras - next, rpm's last. - """ - rolling = (0 if self._is_image(k) else 2 if self._is_rpm(k) else 1, k) - return rolling - - def _is_rpm(self, k): - """ - Is this an RPM? :o - """ - result = k.endswith(".rpm") - return result - - def _is_image(self, k): - """ - Is this an image? :o - """ - if ( - k.startswith("images/") or - k.startswith("isolinux/") or - k.startswith("EFI/") or - k.startswith("etc/") or - k.startswith("ppc/") - ): - return True - - if ( - k.endswith(".img") or - k.endswith(".ins") - ): - return True - - return False - - def _get_vol_id(self): - """ - Gets a volume ID - """ - def run_pull_generic_images(self): """ Pulls generic images built in peridot and places them where they need @@ -1500,15 +1389,9 @@ class IsoBuild: drop_name = source_path.split('/')[-3] + fsuffix checksum_name = drop_name + '.CHECKSUM' - full_drop = '{}/{}'.format( - image_arch_dir, - drop_name - ) + full_drop = f'{image_arch_dir}/{drop_name}' - checksum_drop = '{}/{}.CHECKSUM'.format( - image_arch_dir, - drop_name - ) + checksum_drop = f'{image_arch_dir}/{drop_name}.CHECKSUM' if not os.path.exists(image_arch_dir): os.makedirs(image_arch_dir, exist_ok=True) @@ -1694,10 +1577,7 @@ class LiveBuild: self.compose_latest_dir = os.path.join( config['compose_root'], major, - "latest-{}-{}".format( - self.shortname, - self.profile - ) + f"latest-{self.shortname}-{self.profile}" ) self.compose_latest_sync = os.path.join( @@ -1820,17 +1700,9 @@ class LiveBuild: if self.peridot: kloc = 'peridot' - mock_iso_path = '/var/tmp/live-{}.cfg'.format(self.major_version) - mock_sh_path = '{}/liveisobuild-{}-{}.sh'.format( - entries_dir, - self.current_arch, - image - ) - iso_template_path = '{}/buildLiveImage-{}-{}.sh'.format( - entries_dir, - self.current_arch, - image - ) + mock_iso_path = f'/var/tmp/live-{self.major_version}.cfg' + mock_sh_path = f'{entries_dir}/liveisobuild-{self.current_arch}-{image}.sh' + iso_template_path = f'{entries_dir}/buildLiveImage-{self.current_arch}-{image}.sh' log_root = os.path.join( work_root, @@ -1843,27 +1715,12 @@ class LiveBuild: if not os.path.exists(log_root): os.makedirs(log_root, exist_ok=True) - log_path_command = '| tee -a {}/{}-{}.log'.format( - log_root, - self.current_arch, - image - ) + log_path_command = f'| tee -a {log_root}/{self.current_arch}-{image}.log' required_pkgs = self.livemap['required_pkgs'] - volid = '{}-{}-{}-{}'.format( - self.shortname, - self.major_version, - self.minor_version, - image - ) + volid = f'{self.shortname}-{self.major_version}-{self.minor_version}-{image}' - isoname = '{}-{}-{}-{}-{}.iso'.format( - self.shortname, - self.release, - image, - self.current_arch, - self.date - ) + isoname = f'{self.shortname}-{self.release}-{image}-{self.current_arch}-{self.date}.iso' live_pkg_cmd = '/usr/bin/dnf install {} -y {}'.format( ' '.join(required_pkgs), @@ -1960,17 +1817,10 @@ class LiveBuild: self.log.warn(Color.WARN + 'This mode does not work properly. It will fail.') for i in images: image_name = i - entry_name = 'buildLiveImage-{}-{}.sh'.format(arch, i) + entry_name = f'buildLiveImage-{arch}-{i}.sh' entry_name_list.append(entry_name) - isoname = '{}/{}-{}-{}-{}-{}.iso'.format( - arch, - self.shortname, - i, - self.major_version, - arch, - self.date - ) + isoname = f'{arch}/{self.shortname}-{i}-{self.major_version}-{arch}-{self.date}.iso' checksum_list.append(isoname) @@ -1998,10 +1848,7 @@ class LiveBuild: time.sleep(3) self.log.info(Color.INFO + 'Building requested live images ...') - pod_watcher = '{} wait {}'.format( - cmd, - join_all_pods - ) + pod_watcher = f'{cmd} wait {join_all_pods}' watch_man = subprocess.call( shlex.split(pod_watcher), @@ -2013,10 +1860,7 @@ class LiveBuild: # code. pattern = "Exited (0)" for pod in entry_name_list: - checkcmd = '{} ps -f status=exited -f name={}'.format( - cmd, - pod - ) + checkcmd = f'{cmd} ps -f status=exited -f name={pod}' podcheck = subprocess.Popen( checkcmd, stdout=subprocess.PIPE, @@ -2029,10 +1873,7 @@ class LiveBuild: self.log.error(Color.FAIL + pod) bad_exit_list.append(pod) - rmcmd = '{} rm {}'.format( - cmd, - join_all_pods - ) + rmcmd = f'{cmd} rm {join_all_pods}' rmpod = subprocess.Popen( rmcmd, @@ -2072,25 +1913,9 @@ class LiveBuild: """ entries_dir = os.path.join(work_root, "entries") live_dir_arch = os.path.join(self.live_work_dir, arch) - isoname = '{}-{}-{}-{}-{}.iso'.format( - self.shortname, - self.release, - image, - arch, - self.date - ) - isolink = '{}-{}-{}-{}-{}.iso'.format( - self.shortname, - self.major_version, - image, - arch, - 'latest' - ) - live_res_dir = '/var/lib/mock/{}-{}-{}/result'.format( - self.shortname.lower(), - self.major_version, - arch - ) + isoname = f'{self.shortname}-{self.release}-{image}-{arch}-{self.date}.iso' + isolink = f'{self.shortname}-{self.major_version}-{image}-{arch}-latest.iso' + live_res_dir = f'/var/lib/mock/{self.shortname.lower()}-{self.major_version}-{arch}/result' if self.justcopyit: if os.path.exists(os.path.join(live_dir_arch, isoname)): @@ -2101,7 +1926,7 @@ class LiveBuild: self.log.warn(Color.WARN + 'Skipping.') return - live_iso_cmd = '/bin/bash {}/liveisobuild-{}-{}.sh'.format(entries_dir, arch, image) + live_iso_cmd = f'/bin/bash {entries_dir}/liveisobuild-{arch}-{image}.sh' self.log.info('Starting mock build...') p = subprocess.call(shlex.split(live_iso_cmd)) if p != 0: diff --git a/iso/empanadas/empanadas/util/shared.py b/iso/empanadas/empanadas/util/shared.py index a2761e5..4245c9f 100644 --- a/iso/empanadas/empanadas/util/shared.py +++ b/iso/empanadas/empanadas/util/shared.py @@ -40,6 +40,18 @@ class ArchCheck: ] } + # These are files that can potentially change on an image. + boot_configs = [ + "isolinux/isolinux.cfg", + "etc/yaboot.conf", + "ppc/ppc64/yaboot.conf", + "EFI/BOOT/BOOTX64.conf", + "EFI/BOOT/grub.cfg" + ] + boot_images = [ + "images/efiboot.img" + ] + class Shared: """ Quick utilities that may be commonly used @@ -73,7 +85,7 @@ class Shared: base = os.path.basename(path) # This emulates our current syncing scripts that runs stat and # sha256sum and what not with a very specific output. - return "%s: %s bytes\n%s (%s) = %s\n" % ( + return "# %s: %s bytes\n%s (%s) = %s\n" % ( base, stat.st_size, hashtype.upper(), @@ -1141,3 +1153,208 @@ class Shared: logger.error('DNF syncing has been removed.') logger.error('Please install podman and enable parallel') raise SystemExit() + + @staticmethod + def norm_dnf_sync(data, repo, sync_root, work_root, arch, logger): + """ + This is for normal dnf syncs. This is very slow. + """ + cmd = Shared.reposync_cmd(logger) + sync_single_arch = False + arches_to_sync = data.arches + if arch: + sync_single_arch = True + arches_to_sync = [arch] + + logger.info( + Color.BOLD + '!! WARNING !! ' + Color.END + 'You are performing a ' + 'local reposync, which will incur delays in your compose.' + ) + + if data.fullrun: + logger.info( + Color.BOLD + '!! WARNING !! ' + Color.END + 'This is a full ' + 'sync. Expect a few days for it to complete.' + ) + + for r in repos_to_sync: + for a in arches_to_sync: + repo_name = r + if r in data.repo_renames: + repo_name = data.repo_renames[r] + + os_sync_path = os.path.join( + sync_root, + repo_name, + a, + 'os' + ) + + debug_sync_path = os.path.join( + sync_root, + repo_name, + a, + 'debug/tree' + ) + + sync_cmd = "{} -c {} --download-metadata --repoid={} -p {} --forcearch {} --norepopath".format( + cmd, + data.dnf_config, + r, + os_sync_path, + a + ) + + debug_sync_cmd = "{} -c {} --download-metadata --repoid={}-debug -p {} --forcearch {} --norepopath".format( + cmd, + data.dnf_config, + r, + debug_sync_path, + a + ) + + logger.info('Syncing {} {}'.format(r, a)) + #logger.info(sync_cmd) + # Try to figure out where to send the actual output of this... + # Also consider on running a try/except here? Basically if + # something happens (like a repo doesn't exist for some arch, + # eg RT for aarch64), make a note of it somehow (but don't + # break the entire sync). As it stands with this + # implementation, if something fails, it just continues on. + process = subprocess.call( + shlex.split(sync_cmd), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + if not data.ignore_debug: + logger.info('Syncing {} {} (debug)'.format(r, a)) + process_debug = subprocess.call( + shlex.split(debug_sync_cmd), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + # There should be a check here that if it's "all" and multilib + # is on, i686 should get synced too. + + if not data.ignore_source: + source_sync_path = os.path.join( + sync_root, + repo_name, + 'source/tree' + ) + + source_sync_cmd = "{} -c {} --download-metadata --repoid={}-source -p {} --norepopath".format( + cmd, + data.dnf_config, + r, + source_sync_path + ) + + + logger.info('Syncing {} source'.format(r)) + process_source = subprocess.call( + shlex.split(source_sync_cmd), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + logger.info('Syncing complete') + +class Idents: + """ + Identifiers or locators + """ + @staticmethod + def scanning(p): + """ + Scan tree + """ + path = os.path.abspath(p) + result = {} + for root, dirs, files in os.walk(path): + for file in files: + abspath = os.path.join(root, file) + relpath = kobo.shortcuts.relative_path(abspath, path.rstrip("/") + "/") + result[relpath] = abspath + + # Include empty directories too + if root != path: + abspath = os.path.join(root, "") + relpath = kobo.shortcuts.relative_path(abspath, path.rstrip("/") + "/") + result[relpath] = abspath + + return result + + @staticmethod + def merging(tree_a, tree_b, exclusive=False): + """ + Merge tree + """ + result = tree_b.copy() + all_dirs = set( + [os.path.dirname(dirn).rstrip("/") for dirn in result if os.path.dirname(dirn) != ""] + ) + + for dirn in tree_a: + dn = os.path.dirname(dirn) + if exclusive: + match = False + for x in all_dirs: + if dn == x or dn.startswith("%s/" % x): + match = True + break + if match: + continue + + if dirn in result: + continue + + result[dirn] = tree_a[dirn] + return result + + @staticmethod + def sorting(k): + """ + Sorting using the is_rpm and is_image funcs. Images are first, extras + next, rpm's last. + """ + rolling = (0 if Idents.is_image(k) else 2 if Idents.is_rpm(k) else 1, k) + return rolling + + @staticmethod + def is_rpm(k): + """ + Is this an RPM? :o + """ + result = k.endswith(".rpm") + return result + + @staticmethod + def is_image(k): + """ + Is this an image? :o + """ + if ( + k.startswith("images/") or + k.startswith("isolinux/") or + k.startswith("EFI/") or + k.startswith("etc/") or + k.startswith("ppc/") + ): + return True + + if ( + k.endswith(".img") or + k.endswith(".ins") + ): + return True + + return False + + @staticmethod + def get_vol_id(opts): + """ + Gets a volume ID + """ diff --git a/iso/empanadas/poetry.lock b/iso/empanadas/poetry.lock index 7a23d84..f56cd86 100644 --- a/iso/empanadas/poetry.lock +++ b/iso/empanadas/poetry.lock @@ -1,6 +1,6 @@ [[package]] name = "atomicwrites" -version = "1.4.0" +version = "1.4.1" description = "Atomic file writes." category = "dev" optional = false @@ -22,14 +22,14 @@ tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (> [[package]] name = "boto3" -version = "1.24.22" +version = "1.26.89" description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" [package.dependencies] -botocore = ">=1.27.22,<1.28.0" +botocore = ">=1.29.89,<1.30.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -38,7 +38,7 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.27.22" +version = "1.29.89" description = "Low-level, data-driven core of boto 3." category = "main" optional = false @@ -50,11 +50,11 @@ python-dateutil = ">=2.1,<3.0.0" urllib3 = ">=1.25.4,<1.27" [package.extras] -crt = ["awscrt (==0.13.8)"] +crt = ["awscrt (==0.16.9)"] [[package]] name = "certifi" -version = "2022.6.15" +version = "2022.12.7" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -62,26 +62,23 @@ python-versions = ">=3.6" [[package]] name = "charset-normalizer" -version = "2.1.0" +version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false -python-versions = ">=3.6.0" - -[package.extras] -unicode_backport = ["unicodedata2"] +python-versions = ">=3.7.0" [[package]] name = "colorama" -version = "0.4.5" +version = "0.4.6" description = "Cross-platform colored terminal text." category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" [[package]] name = "idna" -version = "3.3" +version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false @@ -89,7 +86,7 @@ python-versions = ">=3.5" [[package]] name = "importlib-metadata" -version = "4.12.0" +version = "6.0.0" description = "Read metadata from Python packages" category = "dev" optional = false @@ -100,13 +97,13 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8", "importlib-resources (>=1.3)"] [[package]] name = "importlib-resources" -version = "5.8.0" +version = "5.12.0" description = "Read resources from Python packages" category = "main" optional = false @@ -116,8 +113,8 @@ python-versions = ">=3.7" zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"] [[package]] name = "jinja2" @@ -162,22 +159,19 @@ python-versions = ">=3.6" [[package]] name = "more-itertools" -version = "8.13.0" +version = "9.1.0" description = "More routines for operating on iterables, beyond itertools" category = "dev" optional = false -python-versions = ">=3.5" +python-versions = ">=3.7" [[package]] name = "packaging" -version = "21.3" +version = "23.0" description = "Core utilities for Python packages" category = "dev" optional = false -python-versions = ">=3.6" - -[package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" +python-versions = ">=3.7" [[package]] name = "pluggy" @@ -212,17 +206,6 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -[[package]] -name = "pyparsing" -version = "3.0.9" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "dev" -optional = false -python-versions = ">=3.6.8" - -[package.extras] -diagrams = ["railroad-diagrams", "jinja2"] - [[package]] name = "pytest" version = "5.4.3" @@ -267,7 +250,7 @@ python-versions = ">=3.6" [[package]] name = "requests" -version = "2.28.1" +version = "2.28.2" description = "Python HTTP for Humans." category = "main" optional = false @@ -275,7 +258,7 @@ python-versions = ">=3.7, <4" [package.dependencies] certifi = ">=2017.4.17" -charset-normalizer = ">=2,<3" +charset-normalizer = ">=2,<4" idna = ">=2.5,<4" urllib3 = ">=1.21.1,<1.27" @@ -315,7 +298,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "typing-extensions" -version = "4.3.0" +version = "4.5.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "dev" optional = false @@ -323,20 +306,20 @@ python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.9" +version = "1.26.15" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.extras] brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "wcwidth" -version = "0.2.5" +version = "0.2.6" description = "Measures the displayed width of unicode strings in a terminal" category = "dev" optional = false @@ -352,15 +335,15 @@ python-versions = ">=3.4" [[package]] name = "zipp" -version = "3.8.0" +version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "main" optional = false python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "jaraco.functools", "more-itertools", "big-o", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"] [metadata] lock-version = "1.1" @@ -368,46 +351,19 @@ python-versions = ">=3.7,<4" content-hash = "42676fd0ceb350c8cd90246dc688cfcd404e14d22229052d0527fe342c135b95" [metadata.files] -atomicwrites = [ - {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, - {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, -] +atomicwrites = [] attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, ] -boto3 = [ - {file = "boto3-1.24.22-py3-none-any.whl", hash = "sha256:c9a9f893561f64f5b81de197714ac4951251a328672a8dba28ad4c4a589c3adf"}, - {file = "boto3-1.24.22.tar.gz", hash = "sha256:67d404c643091d4aa37fc485193289ad859f1f65f94d0fa544e13bdd1d4187c1"}, -] -botocore = [ - {file = "botocore-1.27.22-py3-none-any.whl", hash = "sha256:7145d9b7cae87999a9f074de700d02a1b3222ee7d1863aa631ff56c5fc868035"}, - {file = "botocore-1.27.22.tar.gz", hash = "sha256:f57cb33446deef92e552b0be0e430d475c73cf64bc9e46cdb4783cdfe39cb6bb"}, -] -certifi = [ - {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, - {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, -] -charset-normalizer = [ - {file = "charset-normalizer-2.1.0.tar.gz", hash = "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413"}, - {file = "charset_normalizer-2.1.0-py3-none-any.whl", hash = "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5"}, -] -colorama = [ - {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, - {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, -] -idna = [ - {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, - {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, -] -importlib-metadata = [ - {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, - {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, -] -importlib-resources = [ - {file = "importlib_resources-5.8.0-py3-none-any.whl", hash = "sha256:7952325ffd516c05a8ad0858c74dff2c3343f136fe66a6002b2623dd1d43f223"}, - {file = "importlib_resources-5.8.0.tar.gz", hash = "sha256:568c9f16cb204f9decc8d6d24a572eeea27dacbb4cee9e6b03a8025736769751"}, -] +boto3 = [] +botocore = [] +certifi = [] +charset-normalizer = [] +colorama = [] +idna = [] +importlib-metadata = [] +importlib-resources = [] jinja2 = [ {file = "Jinja2-2.11.3-py2.py3-none-any.whl", hash = "sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419"}, {file = "Jinja2-2.11.3.tar.gz", hash = "sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6"}, @@ -416,9 +372,7 @@ jmespath = [ {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] -kobo = [ - {file = "kobo-0.24.2.tar.gz", hash = "sha256:1b3c17260a93d933d2238884373fbf3485ecd417d930acf984285dc012410e2b"}, -] +kobo = [] markupsafe = [ {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53"}, {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38"}, @@ -490,14 +444,8 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"}, {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"}, ] -more-itertools = [ - {file = "more-itertools-8.13.0.tar.gz", hash = "sha256:a42901a0a5b169d925f6f217cd5a190e32ef54360905b9c39ee7db5313bfec0f"}, - {file = "more_itertools-8.13.0-py3-none-any.whl", hash = "sha256:c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb"}, -] -packaging = [ - {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, - {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, -] +more-itertools = [] +packaging = [] pluggy = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, @@ -510,10 +458,6 @@ py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] -pyparsing = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, -] pytest = [ {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"}, {file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"}, @@ -557,10 +501,7 @@ pyyaml = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] -requests = [ - {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, - {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, -] +requests = [] rpm-py-installer = [ {file = "rpm-py-installer-1.1.0.tar.gz", hash = "sha256:66e5f4f9247752ed386345642683103afaee50fb16928878a204bc12504b9bbe"}, ] @@ -572,23 +513,11 @@ six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -typing-extensions = [ - {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, - {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, -] -urllib3 = [ - {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"}, - {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, -] -wcwidth = [ - {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, - {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, -] +typing-extensions = [] +urllib3 = [] +wcwidth = [] xmltodict = [ {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, ] -zipp = [ - {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, - {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"}, -] +zipp = [] diff --git a/mangle/fix-aws-quotas.sh b/mangle/fix-aws-quotas.sh new file mode 100644 index 0000000..78842da --- /dev/null +++ b/mangle/fix-aws-quotas.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Source common variables +# shellcheck disable=SC2046,1091,1090 +source "$(dirname "${BASH_SOURCE[0]}")/common" + +usage() { + echo "usage: $0" +} + +aws() { + # shellcheck disable=SC2068 + command aws --profile resf-ami --output json $@ +} + +# Get the quota code for Public AMIs once +quota_code=$(aws service-quotas list-service-quotas --service-code ec2 --region us-east-1 --query "Quotas[*].{QuotaCode:QuotaCode,QuotaName:QuotaName}" | jq '.[] | select(.QuotaName == "Public AMIs") | .QuotaCode' | tr -d '"') + +function get_current_quota() { + region=$1 + # Get the current value of the quota + current_value=$(aws service-quotas get-service-quota --service-code ec2 --quota-code "$quota_code" --region "$region" 2>/dev/null | jq .Quota.Value 2>/dev/null) + # shellcheck disable=SC2181 + if [[ $? -gt 0 ]]; then + echo "ERR" + fi + echo "$current_value" +} + +function request_quota_increase() { + mapfile -t regions <<<"$@" + for region in "${regions[@]}"; do + # Get the current value of the quota + current_value=$(get_current_quota "$region") + if ((current_value >= QUOTA)); then + echo "Quota for Public AMIs in region $region is already set to $current_value, skipping request." + else + # Request the quota increase + request_output=$(aws service-quotas request-service-quota-increase --service-code ec2 --quota-code "$quota_code" --region "$region" --desired-value "$QUOTA") + request_id=$(echo "$request_output" | jq .RequestedQuota.Id | tr -d '"') + echo "Successfully submitted request with ID: $request_id" + fi + done +} + +function list_quotas() { + mapfile -t regions <<<"$@" + output="Region\tQuota\tDesired\tStatus" + for region in "${regions[@]}"; do + current_quota=$(get_current_quota "$region") + request_info=$(aws service-quotas list-requested-service-quota-change-history-by-quota --service-code ec2 --quota-code "$quota_code" --region "$region" --query "RequestedQuotas[-1:].{DesiredValue:DesiredValue,Status:Status}" 2>/dev/null) + requested_value=$(echo "$request_info" | jq .[].DesiredValue) + case_status=$(echo "$request_info" | jq .[].Status | tr -d '"') + output="$output\n$region $current_quota $requested_value $case_status" + done + echo -e "$output" | column -t +} + +REGIONS=$(aws ec2 describe-regions \ + --all-regions \ + --query "Regions[].{Name:RegionName}" \ + --output text) + +QUOTA=50 + +list_quotas "$REGIONS" + +# request_quota_increase "$REGIONS" diff --git a/mangle/generators/.gitignore b/mangle/generators/.gitignore index 67e33fd..f8f5cf6 100644 --- a/mangle/generators/.gitignore +++ b/mangle/generators/.gitignore @@ -1,3 +1,5 @@ -__pycache__/ *.py[cod] +__pycache__/ +*.py[cod] *$py.class *.so +generate_prepopulate_from_self diff --git a/mangle/generators/common b/mangle/generators/common index 9d102cc..e6be9bd 100644 --- a/mangle/generators/common +++ b/mangle/generators/common @@ -17,4 +17,6 @@ if [ "$?" -ne 0 ]; then fi STREAM_COMPOSE_BASEURL="https://composes.stream.centos.org/production" +STREAM_KOJI_REPO="https://kojihub.stream.centos.org/kojifiles/repos" +PERIDOT_REPO="https://yumrepofs.build.resf.org/v1/projects" COMPOSE_BASEDIR="/mnt/compose" diff --git a/mangle/generators/common.py b/mangle/generators/common.py index 602e1f4..f0a3d7f 100644 --- a/mangle/generators/common.py +++ b/mangle/generators/common.py @@ -1,6 +1,18 @@ +import sys + class common: - def rlver(self, rlver): + def rlver(self, rlver, stream=False, all_repo=False): default = "Not Supported" + if stream and all_repo: + print("incompatible options used") + sys.exit(1) + + if stream: + return getattr(self, 'c' + str(rlver) + 's', lambda: default)() + + if all_repo: + return getattr(self, 'rl' + str(rlver) + 'all', lambda: default)() + return getattr(self, 'rl' + str(rlver), lambda: default)() def rl8(self): @@ -28,4 +40,23 @@ class common: } return REPOS + def rl9all(self): + REPOS = { + 'all': ['aarch64', 'ppc64le', 's390x', 'x86_64'], + } + return REPOS + + # Parse tags of koji + def c8s(self): + REPOS = { + 'c8s-build': ['aarch64', 'ppc64le', 'x86_64'], + } + return REPOS + + def c9s(self): + REPOS = { + 'c9s-build': ['aarch64', 'ppc64le', 's390x', 'x86_64'], + } + return REPOS + switcher = common() diff --git a/mangle/generators/common_9 b/mangle/generators/common_9 index d48ac58..2a27006 100644 --- a/mangle/generators/common_9 +++ b/mangle/generators/common_9 @@ -2,3 +2,16 @@ REPO=("BaseOS" "AppStream" "CRB" "HighAvailability" "ResilientStorage" "NFV" "RT" "SAP" "SAPHANA") ARCH=("aarch64" "ppc64le" "s390x" "x86_64") + +VER="r${MAJOR}${LH}" + +case "$VER" in + r9) + export PERIDOT_PROJECT_ID=0048077b-1573-4cb7-8ba7-cce823857ba5 ;; + r9lh) + export PERIDOT_PROJECT_ID=6794b5a8-290b-4d0d-ad5a-47164329cbb0 ;; + *) + echo "Not supported" + exit 1 + ;; +esac diff --git a/mangle/generators/generate_catalog_branched b/mangle/generators/generate_catalog_branched new file mode 100644 index 0000000..0769e7c --- /dev/null +++ b/mangle/generators/generate_catalog_branched @@ -0,0 +1,20 @@ +#!/bin/bash +# Creates a prepopulate from KPR and then creates a catalog +set -e + +if [ -n "$1" ]; then + NEWMAJOR=$1 +else + echo "Major version not specified" + exit 1 +fi + +tmpdir=$(mktemp -d) +bash generate_prepopulate_from_kpr ${NEWMAJOR} +git clone https://github.com/rocky-linux/peridot-releng.git "${tmpdir}/peridot-releng" +cp prepopcatalog.py "${tmpdir}/peridot-releng/pungicatalog/" +pushd "${tmpdir}/peridot-releng" || { echo "Could not change directory"; exit 1; } +python3 pungicatalog/prepopcatalog.py --major ${NEWMAJOR} --minor 0 --prepopulate-path /tmp/prepopulate.json --output-path "${tmpdir}/branched.cfg" +popd || { echo "Could not change back..."; exit 1; } + +echo "File located at: ${tmpdir}/branched.cfg" diff --git a/mangle/generators/generate_prepopulate_from_kpr b/mangle/generators/generate_prepopulate_from_kpr new file mode 100755 index 0000000..c1632c7 --- /dev/null +++ b/mangle/generators/generate_prepopulate_from_kpr @@ -0,0 +1,42 @@ +#!/bin/bash +# Parses a koji tag for a branch out +#set -x + +if [ -n "$1" ]; then + MAJOR=$1 +else + echo "Major version not specified" + exit 1 +fi + +export RLVER=$MAJOR +source common + +drop="${PREPOPDROP}" +current=$(pwd) +tmpdir=$(mktemp -d) +tag_template="c${MAJOR}s-build" +stream_repo_url="${STREAM_KOJI_REPO}/${tag_template}/latest" + +pushd "${tmpdir}" || { echo "Could not change directory"; exit 1; } +for y in "${ARCH[@]}"; do + repodatas=( $(dnf reposync --repofrompath ${tag_template},${stream_repo_url}/${y} --download-metadata --repoid=${tag_template} -p ${tag_template}/${y} --forcearch ${y} --norepopath --remote-time --assumeyes -u | grep repodata) ) + mkdir -p "${tag_template}/${y}/repodata" + pushd "${tag_template}/${y}/repodata" || { echo "Could not change directory"; exit 1; } + for z in "${repodatas[@]}"; do + wget -q -nc "${z}" + done + wget -q -nc "${stream_repo_url}/${y}/repodata/repomd.xml" + popd || { echo "Could not change back..."; exit 1; } +done +/usr/bin/python3 "${current}/prepopulate_parser.py" --version $RLVER --stream +ret_val=$? +popd || { echo "Could not change back..."; exit 1; } + +if [ "$ret_val" -ne "0" ]; then + echo "There was an error running through the parser." + exit 1 +fi + +sed -i "s|${tag_template}|branched|g" $drop +echo "File located at: $drop" diff --git a/mangle/generators/generate_prepopulate_from_pungi b/mangle/generators/generate_prepopulate_from_pungi index 96cff93..fa3771d 100755 --- a/mangle/generators/generate_prepopulate_from_pungi +++ b/mangle/generators/generate_prepopulate_from_pungi @@ -27,7 +27,7 @@ current=$(pwd) compose_dir="${COMPOSE_BASEDIR}/${MAJOR}/Rocky-${MAJOR}-${DATE}/compose" pushd "${compose_dir}" || { echo "Could not change directory"; exit 1; } -/usr/bin/python3 "${current}/prepopulate_parser.py" +/usr/bin/python3 "${current}/prepopulate_parser.py" --version ${RLVER} ret_val=$? popd || { echo "Could not change back..."; exit 1; } diff --git a/mangle/generators/generate_prepopulate_from_pungi_latest b/mangle/generators/generate_prepopulate_from_pungi_latest index 5566fc2..9f84f65 100755 --- a/mangle/generators/generate_prepopulate_from_pungi_latest +++ b/mangle/generators/generate_prepopulate_from_pungi_latest @@ -17,7 +17,7 @@ current=$(pwd) compose_dir="${COMPOSE_BASEDIR}/${MAJOR}/latest-Rocky-${MAJOR}/compose" pushd "${compose_dir}" || { echo "Could not change directory"; exit 1; } -/usr/bin/python3 "${current}/prepopulate_parser.py" --pungi +/usr/bin/python3 "${current}/prepopulate_parser.py" --pungi --version ${RLVER} ret_val=$? popd || { echo "Could not change back..."; exit 1; } diff --git a/mangle/generators/generate_prepopulate_from_stream b/mangle/generators/generate_prepopulate_from_stream index 3d51a5c..f734912 100755 --- a/mangle/generators/generate_prepopulate_from_stream +++ b/mangle/generators/generate_prepopulate_from_stream @@ -41,7 +41,7 @@ for x in "${REPO[@]}"; do popd || { echo "Could not change back..."; exit 1; } done done -/usr/bin/python3 "${current}/prepopulate_parser.py" +/usr/bin/python3 "${current}/prepopulate_parser.py" --version ${MAJOR} ret_val=$? popd || { echo "Could not change back..."; exit 1; } diff --git a/mangle/generators/generate_versions_from_kpr b/mangle/generators/generate_versions_from_kpr new file mode 100755 index 0000000..25d45bc --- /dev/null +++ b/mangle/generators/generate_versions_from_kpr @@ -0,0 +1,41 @@ +#!/bin/bash +# Parses a CentOS Stream Koji Tag +#set -x + +if [ -n "$1" ]; then + MAJOR="$1" +else + echo "Major version not specified" + exit 1 +fi + +export RLVER="${MAJOR}" +source common + +drop="${VERSDROP}" +current=$(pwd) +tmpdir=$(mktemp -d) +tag_template="c${MAJOR}s-build" +stream_repo_url="${STREAM_KOJI_REPO}/${tag_template}/latest" + +pushd "${tmpdir}" || { echo "Could not change directory"; exit 1; } +for y in "${ARCH[@]}"; do + repodatas=( $(dnf reposync --repofrompath ${tag_template},${stream_repo_url}/${y} --download-metadata --repoid=${tag_template} -p ${tag_template}/${y} --forcearch ${y} --norepopath --remote-time --assumeyes -u | grep repodata) ) + mkdir -p "${tag_template}/${y}/repodata" + pushd "${tag_template}/${y}/repodata" || { echo "Could not change directory"; exit 1; } + for z in "${repodatas[@]}"; do + wget -q -nc "${z}" + done + wget -q -nc "${stream_repo_url}/${y}/repodata/repomd.xml" + popd || { echo "Could not change back..."; exit 1; } +done +/usr/bin/python3 "${current}/version_parser.py" --version "${MAJOR}" --stream +ret_val=$? +popd || { echo "Could not change back..."; exit 1; } + +if [ "$ret_val" -ne "0" ]; then + echo "There was an error running through the parser." + exit 1 +fi + +echo "File located at: $drop" diff --git a/mangle/generators/generate_versions_from_ppr b/mangle/generators/generate_versions_from_ppr new file mode 100755 index 0000000..2394940 --- /dev/null +++ b/mangle/generators/generate_versions_from_ppr @@ -0,0 +1,45 @@ +#!/bin/bash +# Parses a peridot project +#set -x + +if [ -n "$1" ]; then + MAJOR="$1" +else + echo "Major version not specified" + exit 1 +fi + +if [ -n "$2" ] && [[ "$2" == "lh" ]]; then + export LH="lh" +fi + +export RLVER="${MAJOR}" +source common + +drop="${VERSDROP}" +current=$(pwd) +tmpdir=$(mktemp -d) +tag_template="all" +peridot_repo_url="${PERIDOT_REPO}/${PERIDOT_PROJECT_ID}/repo/${tag_template}" + +pushd "${tmpdir}" || { echo "Could not change directory"; exit 1; } +for y in "${ARCH[@]}"; do + repodatas=( $(dnf reposync --repofrompath ${tag_template},${peridot_repo_url}/${y} --download-metadata --repoid=${tag_template} -p ${tag_template}/${y} --forcearch ${y} --norepopath --remote-time --assumeyes -u | grep repodata) ) + mkdir -p "${tag_template}/${y}/repodata" + pushd "${tag_template}/${y}/repodata" || { echo "Could not change directory"; exit 1; } + for z in "${repodatas[@]}"; do + wget -q -nc "${z}" + done + wget -q -nc "${peridot_repo_url}/${y}/repodata/repomd.xml" + popd || { echo "Could not change back..."; exit 1; } +done +/usr/bin/python3 "${current}/version_parser.py" --version "${MAJOR}" --all-repo +ret_val=$? +popd || { echo "Could not change back..."; exit 1; } + +if [ "$ret_val" -ne "0" ]; then + echo "There was an error running through the parser." + exit 1 +fi + +echo "File located at: $drop" diff --git a/mangle/generators/generate_versions_from_stream b/mangle/generators/generate_versions_from_stream index bca3846..318237a 100755 --- a/mangle/generators/generate_versions_from_stream +++ b/mangle/generators/generate_versions_from_stream @@ -3,8 +3,8 @@ #set -x if [ -n "$1" ] && [ -n "$2" ]; then - MAJOR=$1 - DATE=$2 + MAJOR="$1" + DATE="$2" else echo "Major version not specified" exit 1 @@ -18,7 +18,7 @@ if [ "$grep_val" -ne 0 ]; then echo "Date format incorrect. You must use: YYYYMMDD.X" fi -export RLVER=$MAJOR +export RLVER="${MAJOR}" source common drop="${VERSDROP}" @@ -40,7 +40,7 @@ for x in "${REPO[@]}"; do popd || { echo "Could not change back..."; exit 1; } done done -/usr/bin/python3 "${current}/version_parser.py" +/usr/bin/python3 "${current}/version_parser.py" --version "${MAJOR}" ret_val=$? popd || { echo "Could not change back..."; exit 1; } diff --git a/mangle/generators/prepopcatalog.py b/mangle/generators/prepopcatalog.py new file mode 100644 index 0000000..9a00c71 --- /dev/null +++ b/mangle/generators/prepopcatalog.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +# This is copied into peridot-releng and ran. This file is likely temporary +# until we decide if this goes into that git repo. +import argparse +import os +import json + +from catalog import ( + PeridotCatalogSync, + PeridotCatalogSyncPackage, + PeridotCatalogSyncPackageType, + PeridotCatalogSyncRepository, +) + +def main(prepopulate: str, output_path: str, major: int, minor: int): + print(f"Using prepopulate file: {prepopulate}") + + with open(prepopulate) as json_file: + prepop = json.load(json_file) + json_file.close() + + # Create a catalog + catalog = PeridotCatalogSync() + catalog.major = major + catalog.minor = minor + + # Create indexes + package_index = {} + repo_module_index = {} + module_name_index = {} + module_defaults = [] + + # Read prepopulate json and create package objects + all_arches = [] + for repo in prepop.keys(): + for arch in prepop[repo].keys(): + if arch not in all_arches: + all_arches.append(arch) + for package in prepop[repo][arch].keys(): + if package not in package_index: + package_index[package] = {} + if repo not in package_index[package]: + package_index[package][repo] = { + "include_filter": [], + "multilib": [], + } + na_list = prepop[repo][arch][package] + for na in na_list: + splitted = na.split(".") + arch_package = splitted[len(splitted) - 1] + if arch != arch_package and arch_package != "noarch": + if arch not in package_index[package][repo]["multilib"]: + package_index[package][repo]["multilib"].append(arch) + if na not in package_index[package][repo]["include_filter"]: + package_index[package][repo]["include_filter"].append(na) + + arch_specific_excludes = {} + na_index = {} + for pkg in package_index.keys(): + for repo in package_index[pkg].keys(): + na_list = list( + filter( + lambda x: x.endswith(".noarch"), + package_index[pkg][repo]["include_filter"], + ) + ) + if not na_list: + continue + exclude_arches = {} + for na in na_list: + for arch in all_arches: + if ( + arch not in prepop[repo] + or pkg not in prepop[repo][arch] + or na not in prepop[repo][arch][pkg] + ): + if na not in exclude_arches: + exclude_arches[na] = [] + exclude_arches[na].append(arch) + na_index[na] = na + if not exclude_arches: + continue + if pkg not in arch_specific_excludes: + arch_specific_excludes[pkg] = {} + if repo not in arch_specific_excludes[pkg]: + arch_specific_excludes[pkg][repo] = [] + arch_specific_excludes[pkg][repo].append(exclude_arches) + + # Index arch specific excludes by repo and arch + repo_arch_index = {} + for pkg in arch_specific_excludes.keys(): + for repo in arch_specific_excludes[pkg].keys(): + if repo not in repo_arch_index: + repo_arch_index[repo] = {} + for arches2 in arch_specific_excludes[pkg][repo]: + for na in arches2.keys(): + for arch in arches2[na]: + if arch not in repo_arch_index[repo]: + repo_arch_index[repo][arch] = [] + if na not in repo_arch_index[repo][arch]: + repo_arch_index[repo][arch].append(na) + + # Add noarch packages not in a specific arch to exclude filter + for repo in repo_arch_index.keys(): + repo_key = f"^{repo}$" + filter_tuple = {} + for arch in repo_arch_index[repo].keys(): + if arch not in filter_tuple: + filter_tuple[arch] = [] + for na in repo_arch_index[repo][arch]: + na = na.removesuffix(".noarch") + if na not in filter_tuple[arch]: + filter_tuple[arch].append(na) + catalog.exclude_filter.append((repo_key, filter_tuple)) + + for package in package_index.keys(): + package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_FORK + if package in module_name_index: + package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_FORK_MODULE + elif package.startswith("rocky-"): + package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_SRC + + catalog.add_package( + PeridotCatalogSyncPackage( + package, + package_type, + [ + PeridotCatalogSyncRepository( + x, + package_index[package][x]["include_filter"], + package_index[package][x]["multilib"], + (get_modules_for_repo(package, x, repo_module_index) if x in repo_module_index else None) if package in module_name_index else None, + ) + for x in package_index[package].keys() + ], + ) + ) + + print(f"Found {len(catalog.packages)} packages") + + f = open(output_path, "w") + f.write(catalog.to_prototxt()) + f.close() + + print(f"Catalog written to {output_path}") + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Create a catalog from a standalone prepopulate.json" + ) + parser.add_argument("--prepopulate-path", type=str, required=True) + parser.add_argument("--major", type=int, required=True) + parser.add_argument("--minor", type=int, required=True) + parser.add_argument("--output-path", type=str, default="hidden.cfg") + args = parser.parse_args() + main(args.prepopulate_path, args.output_path, args.major, args.minor) diff --git a/mangle/generators/prepopulate_parser.py b/mangle/generators/prepopulate_parser.py index bf21ea1..cb9046b 100755 --- a/mangle/generators/prepopulate_parser.py +++ b/mangle/generators/prepopulate_parser.py @@ -7,8 +7,6 @@ import dnf import createrepo_c as cr from common import * -REPOS = switcher.rlver((os.environ['RLVER'])) - # Source packages we do not ship or are rocky branded IGNORES = [ 'insights-client', @@ -22,10 +20,16 @@ IGNORES = [ ] parser = argparse.ArgumentParser() +parser.add_argument('--version', type=str, required=True) parser.add_argument("--pungi", help="local pungi is here", action='store_true') -args = parser.parse_args() +parser.add_argument('--stream', action='store_true', help="Stream koji only") -if args.pungi: +results = parser.parse_args() + +REPOS = switcher.rlver(results.version, + stream=results.stream) + +if results.pungi: APPEND_TO_PATH = '/os' else: APPEND_TO_PATH = '' diff --git a/mangle/generators/version_metadata_parser.py b/mangle/generators/version_metadata_parser.py index 62777d4..9c166c1 100644 --- a/mangle/generators/version_metadata_parser.py +++ b/mangle/generators/version_metadata_parser.py @@ -10,7 +10,7 @@ conf = base.conf base.read_all_repos() all_repos = base.repos.all() all_repos.disable() -base.repos.add_new_repo('all-source', conf, baseurl=['https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/all/src']) +base.repos.add_new_repo('all-source', conf, baseurl=['https://yumrepofs.build.resf.org/v1/projects/0048077b-1573-4cb7-8ba7-cce823857ba5/repo/all/src']) base.fill_sack() q = base.sack.query() diff --git a/mangle/generators/version_parser.py b/mangle/generators/version_parser.py index 9491c52..2d21889 100755 --- a/mangle/generators/version_parser.py +++ b/mangle/generators/version_parser.py @@ -2,11 +2,20 @@ import os import os.path import json +import argparse import dnf import createrepo_c as cr from common import * -REPOS = switcher.rlver((os.environ['RLVER'])) +parser = argparse.ArgumentParser(description="Version Parser") +parser.add_argument('--version', type=str, required=True) +parser.add_argument('--stream', action='store_true', help="Stream koji only") +parser.add_argument('--all-repo', action='store_true', help="Looks at the all repo for peridot") +results = parser.parse_args() + +REPOS = switcher.rlver(results.version, + stream=results.stream, + all_repo=results.all_repo) # Source packages we do not ship or are rocky branded IGNORES = [ diff --git a/mangle/quotas.go b/mangle/quotas.go new file mode 100644 index 0000000..a088058 --- /dev/null +++ b/mangle/quotas.go @@ -0,0 +1,146 @@ +package main + +import ( + "fmt" + "log" + "os" + "sync" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/servicequotas" +) + +func getQuotaCode(sqSvc *servicequotas.ServiceQuotas) string { + input := &servicequotas.ListServiceQuotasInput{ + ServiceCode: aws.String("ec2"), + } + + for { + output, err := sqSvc.ListServiceQuotas(input) + if err != nil { + log.Println("Error getting quota code:", err) + os.Exit(1) + } + + for _, quota := range output.Quotas { + if *quota.QuotaName == "Public AMIs" { + return *quota.QuotaCode + } + } + + if output.NextToken == nil { + break + } + input.NextToken = output.NextToken + } + log.Println("Quota code not found") + os.Exit(1) + return "" +} + +func getRegions(ec2Svc *ec2.EC2) ([]*string, error) { + input := &ec2.DescribeRegionsInput{} + + output, err := ec2Svc.DescribeRegions(input) + if err != nil { + return nil, err + } + + var regions []*string + for _, region := range output.Regions { + regions = append(regions, region.RegionName) + } + + return regions, nil +} + +type QuotaInfo struct { + CurrentQuota float64 + DesiredValue float64 + Status string + CaseId string +} + +func getQuotaInfo(sqSvc *servicequotas.ServiceQuotas, quotaCode string, region string) *QuotaInfo { + input := &servicequotas.GetServiceQuotaInput{ + ServiceCode: aws.String("ec2"), + QuotaCode: aws.String(quotaCode), + } + + output, err := sqSvc.GetServiceQuota(input) + if err != nil { + log.Printf("Error getting quota info for %s: %s\n", region, err) + return nil + // os.Exit(1) + } + + currentValue := *output.Quota.Value + requestOutput, err := sqSvc.ListRequestedServiceQuotaChangeHistoryByQuota(&servicequotas.ListRequestedServiceQuotaChangeHistoryByQuotaInput{ + ServiceCode: aws.String("ec2"), + QuotaCode: aws.String(quotaCode), + }) + + if err != nil { + log.Println("Error getting request info:", err) + os.Exit(1) + } + var desiredValue float64 + var status string + var caseId string + if len(requestOutput.RequestedQuotas) > 0 { + desiredValue = *requestOutput.RequestedQuotas[len(requestOutput.RequestedQuotas)-1].DesiredValue + status = *requestOutput.RequestedQuotas[len(requestOutput.RequestedQuotas)-1].Status + caseId = *requestOutput.RequestedQuotas[len(requestOutput.RequestedQuotas)-1].CaseId + } + return &QuotaInfo{currentValue, desiredValue, status, caseId} +} + +func listQuotas(sess *session.Session, quotaCode string, regions []*string) { + fmt.Println("Region\tQuota\tDesired\tStatus\tCaseId") + + var wg sync.WaitGroup + wg.Add(len(regions)) + + for _, region := range regions { + go func(region string) { + defer wg.Done() + regionSqSvc := servicequotas.New(sess, &aws.Config{Region: aws.String(region)}) + quotaInfo := getQuotaInfo(regionSqSvc, quotaCode, region) + if quotaInfo != nil { + fmt.Printf("%s\t%.0f\t%.0f\t%s\t%s\n", region, quotaInfo.CurrentQuota, quotaInfo.DesiredValue, quotaInfo.Status, quotaInfo.CaseId) + } + }(aws.StringValue(region)) + } + wg.Wait() +} + +func main() { + // Create session + sess := session.Must(session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + })) + + // Create EC2 client + ec2Svc := ec2.New(sess, &aws.Config{Region: aws.String("us-east-1")}) + + // Create Service Quotas client + sqSvc := servicequotas.New(sess) + + // Get the quota code for Public AMIs once + quotaCode := getQuotaCode(sqSvc) + + // Get all regions + regions, err := getRegions(ec2Svc) + if err != nil { + log.Println("Error getting regions:", err) + os.Exit(1) + } + + // List quotas for all regions + listQuotas(sess, quotaCode, regions) + + // Request quota increase for all regions + // requestQuotaIncrease(sqSvc, quotaCode, regions) +} diff --git a/sync/common b/sync/common index 33c0852..0c50eac 100644 --- a/sync/common +++ b/sync/common @@ -15,9 +15,14 @@ if [ -z "$RLVER" ]; then export RLVER=8 fi +if [ -z "$RLREL" ]; then + export RLREL=stable +fi + # Override: Not Allowed STAGING_ROOT="/mnt/repos-staging" PRODUCTION_ROOT="/mnt/repos-production" +COMPOSE_ROOT="/mnt/compose" # Override: Not Allowed # relative to ${ENV}_ROOT diff --git a/sync/common_8 b/sync/common_8 index 735c1a6..08810d0 100644 --- a/sync/common_8 +++ b/sync/common_8 @@ -1,20 +1,35 @@ # To be sourced by scripts to use # Revision must always start with a major number -REVISION=8.7 - -MAJOR="${REVISION:0:1}" -MINOR="${REVISION:2:1}" - -# comment or blank if needed -APPEND_TO_DIR="-RC1" +case "${RLREL}" in + stable) + REVISION=8.7 + APPEND_TO_DIR="-RC1" + ;; + beta) + REVISION=8.8 + APPEND_TO_DIR="-BETA" + COMPOSE_APPEND="${APPEND_TO_DIR}" + ;; + lh) + REVISION=8.8 + APPEND_TO_DIR="-LookAhead" + COMPOSE_APPEND="${APPEND_TO_DIR}" + ;; + *) + echo "option no recognized" + ;; +esac +MAJOR="${REVISION%.*}" +MINOR="${REVISION#*.}" STAGING_ROOT="/mnt/repos-staging" PRODUCTION_ROOT="/mnt/repos-production" # relative to ${ENV}_ROOT CATEGORY_STUB="mirror/pub/rocky" RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}" +RELEASE_COMPOSE_ROOT="${COMPOSE_ROOT}/${MAJOR}${COMPOSE_APPEND}/latest-${SHORT}-${MAJOR}" # Set all repos that have no comps/groups associated with them. This is even in # cases where repos will not be available by normal means. It's just for @@ -23,7 +38,6 @@ NONMODS_REPOS=( extras Devel plus - rockyrpi ) # These repos were originally separate from the main compose and need symlinks diff --git a/sync/common_9 b/sync/common_9 index e3dbcad..8062626 100644 --- a/sync/common_9 +++ b/sync/common_9 @@ -1,22 +1,34 @@ # To be sourced by scripts to use # Revision must always start with a major number -REVISION=9.1 +case "${RLREL}" in + stable) + REVISION=9.1 + APPEND_TO_DIR="-RC4" + ;; + beta) + REVISION=9.2 + APPEND_TO_DIR="-beta" + ;; + lh) + REVISION=9.2 + APPEND_TO_DIR="-lookahead" + ;; + *) + echo "option no recognized" + ;; +esac +MAJOR="${REVISION%.*}" +MINOR="${REVISION#*.}" ARCHES=(x86_64 aarch64 ppc64le s390x) - -MAJOR="${REVISION:0:1}" -MINOR="${REVISION:2:1}" - -# comment or blank if needed -APPEND_TO_DIR="-RC4" - STAGING_ROOT="/mnt/repos-staging" PRODUCTION_ROOT="/mnt/repos-production" # relative to ${ENV}_ROOT CATEGORY_STUB="mirror/pub/rocky" RELEASE_DIR="${CATEGORY_STUB}/${REVISION}${APPEND_TO_DIR}" +RELEASE_COMPOSE_ROOT="${COMPOSE_ROOT}/${MAJOR}/latest-${SHORT}-${MAJOR}" # Set all repos that have no comps/groups associated with them. This is even in # cases where repos will not be available by normal means. It's just for @@ -25,7 +37,6 @@ NONMODS_REPOS=( extras Devel plus - rockyrpi ) # These repos were originally separate from the main compose and need symlinks diff --git a/sync/lh-release-sync-to-staging.sh b/sync/lh-release-sync-to-staging.sh new file mode 100644 index 0000000..846d457 --- /dev/null +++ b/sync/lh-release-sync-to-staging.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# Performs a full on sync of a minor release, directories and all. It calls the +# other scripts in this directory to assist where necessary. +# Note that this is EL8 specific +# +# Source common variables +# shellcheck disable=SC2046,1091,1090 +source $(dirname "$0")/common + +# Major Version (eg, 8) +MAJ=${RLVER} + +# sync all pieces of a release, including extras, nfv, etc +for COMPOSE in "${NONSIG_COMPOSE[@]}"; do + echo "${COMPOSE}: Syncing" + pushd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; } + + if [[ "${COMPOSE}" == "Rocky" ]]; then + # ISO Work before syncing + mkdir -p isos/{x86_64,aarch64} + + # Sort the ISO's + for ARCH in "${ARCHES[@]}"; do + for x in BaseOS Minimal; do + echo "${x} ${ARCH}: Moving ISO images" + mv "${x}/${ARCH}/iso/"* "isos/${ARCH}/" + done + pushd "isos/${ARCH}" || { echo "${ARCH}: Failed to change directory"; break; } + ln -s "Rocky-${REVISION}-${ARCH}-boot.iso" "Rocky-${ARCH}-boot.iso" + ln -s "Rocky-${REVISION}-${ARCH}-dvd1.iso" "Rocky-${ARCH}-dvd1.iso" + ln -s "Rocky-${REVISION}-${ARCH}-dvd1.iso" "Rocky-${ARCH}-dvd.iso" + ln -s "Rocky-${REVISION}-${ARCH}-minimal.iso" "Rocky-${ARCH}-minimal.iso" + for file in *.iso; do + printf "# %s: %s bytes\n%s\n" \ + "${file}" \ + "$(stat -c %s ${file})" \ + "$(sha256sum --tag ${file})" \ + | sudo tee -a CHECKSUM; + done + popd || { echo "Could not change directory"; break; } + done + fi + + TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" + mkdir -p "${TARGET}" + # disabling because none of our files should be starting with dashes. If they + # are something is *seriously* wrong here. + # shellcheck disable=SC2035 + sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \ + {} "${TARGET}" + + if [[ "${COMPOSE}" == "Rocky" ]]; then + cp COMPOSE_ID "${TARGET}" + chown 10004:10005 "${TARGET}/COMPOSE_ID" + rsync -av --chown=10004:10005 --progress --relative --human-readable metadata "${TARGET}" + fi + + # Return back to where we started + popd || { echo "${COMPOSE}: Failed to change back"; break; } + + # Create extra stuff + pushd "${TARGET}" || { echo "${COMPOSE}: Failed to change directory"; break; } + mkdir -p Live/x86_64 + ln -s Live live + popd || { echo "${COMPOSE}: Failed to change back"; break; } +done + + +# Create symlinks for repos that were once separate from the main compose +for LINK in "${LINK_REPOS[@]}"; do + ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${LINK}" \ + "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${LINK_REPOS[$LINK]}" +done + +# make a kickstart directory +for ARCH in "${ARCHES[@]}"; do + for REPO in "${MODS_REPOS[@]}"; do + SOURCE="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/os" + TARGET="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${REPO}/${ARCH}/kickstart" + echo "Making golden kickstart directory" + cp -na "${SOURCE}" "${TARGET}" + done +done + +# fix treeinfo +for ARCH in "${ARCHES[@]}"; do + echo "Ensuring treeinfo is correct" + treeinfoModder "${ARCH}" + treeinfoModderKickstart "${ARCH}" + treeinfoSaver "${ARCH}" +done + +# sign all repos +echo "Signing all repositories" +test -f $(dirname "$0")/sign-repos-only.sh +ret_val=$? + +if [ "$ret_val" -eq 0 ]; then + $(dirname "$0")/sign-repos-only.sh +fi + +# Change Symlink if required +echo "Setting symlink to ${REV}" +pushd "${STAGING_ROOT}/${CATEGORY_STUB}" || exit +/bin/rm "${STAGING_ROOT}/${CATEGORY_STUB}/latest-8" +ln -sr "${STAGING_ROOT}/${CATEGORY_STUB}/${REV}" latest-8 +popd || exit diff --git a/sync/minor-release-sync-to-staging.sh b/sync/minor-release-sync-to-staging.sh index f671528..846d457 100644 --- a/sync/minor-release-sync-to-staging.sh +++ b/sync/minor-release-sync-to-staging.sh @@ -13,7 +13,7 @@ MAJ=${RLVER} # sync all pieces of a release, including extras, nfv, etc for COMPOSE in "${NONSIG_COMPOSE[@]}"; do echo "${COMPOSE}: Syncing" - pushd "/mnt/compose/${MAJ}/latest-${COMPOSE}-${MAJ}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; } + pushd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "${COMPOSE}: Failed to change directory"; break; } if [[ "${COMPOSE}" == "Rocky" ]]; then # ISO Work before syncing diff --git a/sync/prep-staging-8.sh b/sync/prep-staging-8.sh index c5c2c22..fd9269c 100644 --- a/sync/prep-staging-8.sh +++ b/sync/prep-staging-8.sh @@ -23,7 +23,7 @@ for y in "${ALL_REPOS[@]}"; do if [ "$ret_val" -eq 0 ]; then createrepo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree" \ "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \ - --workers 8 + --workers 12 sed -i '/<\/open-size>/d' \ "${STAGING_ROOT}/${RELEASE_DIR}/${y}/source/tree/repodata/repomd.xml" test -f /root/bin/sign-repo.sh && /root/bin/sign-repo.sh \ @@ -46,7 +46,7 @@ for x in "${ARCHES[@]}"; do if [ "$ret_val" -eq 0 ]; then createrepo "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}" \ "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \ - --workers 8 + --workers 12 sed -i '/<\/open-size>/d' \ "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/${z}/repodata/repomd.xml" test -f /root/bin/sign-repo.sh && /root/bin/sign-repo.sh \ @@ -63,7 +63,7 @@ for x in "${ARCHES[@]}"; do if [ "$ret_val" -eq 0 ]; then createrepo --update "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree" \ "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \ - --workers 8 + --workers 12 --retain-old-md-by-age 1d sed -i '/<\/open-size>/d' \ "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/debug/tree/repodata/repomd.xml" test -f /root/bin/sign-repo.sh && /root/bin/sign-repo.sh \ @@ -83,7 +83,7 @@ for x in "${ARCHES[@]}"; do --groupfile="/mnt/compose/8/latest-Rocky-8/work/${x}/comps/comps-${y}.${x}.xml" \ --xz --revision="${REVISION}" \ "--distro=cpe:/o:rocky:rocky:${REVISION:0:1},Rocky Linux ${REVISION:0:1}" \ - --workers=8 --checksum=sha256 + --workers=12 --checksum=sha256 --retain-old-md-by-age 1d sed -i '/<\/open-size>/d' \ "${STAGING_ROOT}/${RELEASE_DIR}/${y}/${x}/os/repodata/repomd.xml" sed -i '/<\/open-size>/d' \ diff --git a/sync/propagate-image.sh b/sync/propagate-image.sh index a5e8fa7..68524dd 100644 --- a/sync/propagate-image.sh +++ b/sync/propagate-image.sh @@ -33,6 +33,8 @@ function copy(){ for region in $REGIONS; do if find_image_by_name $region; then echo "Found copy of $source_ami in $region - $found_image_id - Skipping" + unset ami_ids[$region] + ami_ids[$region]=$(echo $found_image_id | tr -d "'") continue fi echo -n "Creating copy job for $region..." @@ -68,15 +70,32 @@ function change_privacy(){ local finished=false while ! $finished; do for region in "${!ami_ids[@]}"; do - echo -n "Making ${ami_ids[$region]} in $region $status..." - aws --profile resf-ami ec2 modify-image-attribute \ + image_id=${ami_ids[$region]} + echo -n "Making ${image_id} in $region $status..." + if aws --profile resf-ami ec2 modify-image-attribute \ --region $region \ - --image-id "${ami_ids[$region]}" \ - --launch-permission "${launch_permission}" 2>/dev/null - if [[ $? -eq 0 ]]; then - unset ami_ids[$region] - echo ". Done" - continue + --image-id "$image_id" \ + --launch-permission "${launch_permission}" 2>/dev/null; then + + snapshot_id=$(aws --profile resf-ami ec2 describe-images \ + --region $region \ + --image-ids "${image_id}" \ + --query 'Images[*].BlockDeviceMappings[0].Ebs.SnapshotId' \ + --output text 2>&1) + permissions=$(aws --profile resf-ami ec2 describe-snapshot-attribute \ + --region $region \ + --snapshot-id "${snapshot_id}" \ + --attribute createVolumePermission \ + --query 'CreateVolumePermissions[0].Group' \ + --output text 2>&1) + if [[ "$permissions" == "all" ]] || aws --profile resf-ami ec2 modify-snapshot-attribute \ + --region $region \ + --snapshot-id "${snapshot_id}" \ + --create-volume-permission "${launch_permission}" 2>/dev/null; then + unset ami_ids[$region] + echo ". Done" + continue + fi fi echo ". Still pending" done diff --git a/sync/sync-to-prod.delete.sh b/sync/sync-to-prod.delete.sh index b1446b2..9058255 100644 --- a/sync/sync-to-prod.delete.sh +++ b/sync/sync-to-prod.delete.sh @@ -1,6 +1,10 @@ #!/bin/bash # Syncs everything from staging to production +if [[ "$RLREL" == "lh" ]] || [[ "$RLREL" == "beta" ]]; then + exit 1 +fi + # Source common variables # shellcheck disable=SC2046,1091,1090 source "$(dirname "$0")/common" diff --git a/sync/sync-to-prod.sh b/sync/sync-to-prod.sh index d62db36..da92f66 100644 --- a/sync/sync-to-prod.sh +++ b/sync/sync-to-prod.sh @@ -1,6 +1,10 @@ #!/bin/bash # Syncs everything from staging to production +if [[ "$RLREL" == "lh" ]] || [[ "$RLREL" == "beta" ]]; then + exit 1 +fi + # Source common variables # shellcheck disable=SC2046,1091,1090 source "$(dirname "$0")/common" diff --git a/sync/sync-to-staging.delete.sh b/sync/sync-to-staging.delete.sh index de1be80..dd8be5b 100644 --- a/sync/sync-to-staging.delete.sh +++ b/sync/sync-to-staging.delete.sh @@ -1,5 +1,8 @@ #!/bin/bash +# Short name (eg, NFV, extras, Rocky, gluster9) +SHORT=${1} + # Source common variables # shellcheck disable=SC2046,1091,1090 source "$(dirname "$0")/common" @@ -11,10 +14,8 @@ fi # Major Version (eg, 8) MAJ=${RLVER} -# Short name (eg, NFV, extras, Rocky, gluster9) -SHORT=${1} -cd "/mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}/compose" || { echo "Failed to change directory"; ret_val=1; exit 1; } +cd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "Failed to change directory"; ret_val=1; exit 1; } ret_val=$? if [ $ret_val -eq "0" ]; then diff --git a/sync/sync-to-staging.sh b/sync/sync-to-staging.sh index dc14ba2..156e4d3 100644 --- a/sync/sync-to-staging.sh +++ b/sync/sync-to-staging.sh @@ -1,5 +1,8 @@ #!/bin/bash +# Short name (eg, NFV, extras, Rocky, gluster9) +SHORT=${1} + # Source common variables # shellcheck disable=SC2046,1091,1090 source "$(dirname "$0")/common" @@ -11,10 +14,8 @@ fi # Major Version (eg, 8) MAJ=${RLVER} -# Short name (eg, NFV, extras, Rocky, gluster9) -SHORT=${1} -cd "/mnt/compose/${MAJ}/latest-${SHORT}-${MAJ}/compose" || { echo "Failed to change directory"; ret_val=1; exit 1; } +cd "${RELEASE_COMPOSE_ROOT}/compose" || { echo "Failed to change directory"; ret_val=1; exit 1; } ret_val=$? if [ $ret_val -eq "0" ]; then