From 9135214802fbf661e0ef3a323964bc0bd158ce36 Mon Sep 17 00:00:00 2001 From: Louis Abel Date: Sat, 4 Mar 2023 00:31:30 -0700 Subject: [PATCH] Add catalog branching --- mangle/generators/generate_catalog_branched | 20 +++ mangle/generators/prepopcatalog.py | 156 ++++++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 mangle/generators/generate_catalog_branched create mode 100644 mangle/generators/prepopcatalog.py diff --git a/mangle/generators/generate_catalog_branched b/mangle/generators/generate_catalog_branched new file mode 100644 index 0000000..0769e7c --- /dev/null +++ b/mangle/generators/generate_catalog_branched @@ -0,0 +1,20 @@ +#!/bin/bash +# Creates a prepopulate from KPR and then creates a catalog +set -e + +if [ -n "$1" ]; then + NEWMAJOR=$1 +else + echo "Major version not specified" + exit 1 +fi + +tmpdir=$(mktemp -d) +bash generate_prepopulate_from_kpr ${NEWMAJOR} +git clone https://github.com/rocky-linux/peridot-releng.git "${tmpdir}/peridot-releng" +cp prepopcatalog.py "${tmpdir}/peridot-releng/pungicatalog/" +pushd "${tmpdir}/peridot-releng" || { echo "Could not change directory"; exit 1; } +python3 pungicatalog/prepopcatalog.py --major ${NEWMAJOR} --minor 0 --prepopulate-path /tmp/prepopulate.json --output-path "${tmpdir}/branched.cfg" +popd || { echo "Could not change back..."; exit 1; } + +echo "File located at: ${tmpdir}/branched.cfg" diff --git a/mangle/generators/prepopcatalog.py b/mangle/generators/prepopcatalog.py new file mode 100644 index 0000000..9a00c71 --- /dev/null +++ b/mangle/generators/prepopcatalog.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +# This is copied into peridot-releng and ran. This file is likely temporary +# until we decide if this goes into that git repo. +import argparse +import os +import json + +from catalog import ( + PeridotCatalogSync, + PeridotCatalogSyncPackage, + PeridotCatalogSyncPackageType, + PeridotCatalogSyncRepository, +) + +def main(prepopulate: str, output_path: str, major: int, minor: int): + print(f"Using prepopulate file: {prepopulate}") + + with open(prepopulate) as json_file: + prepop = json.load(json_file) + json_file.close() + + # Create a catalog + catalog = PeridotCatalogSync() + catalog.major = major + catalog.minor = minor + + # Create indexes + package_index = {} + repo_module_index = {} + module_name_index = {} + module_defaults = [] + + # Read prepopulate json and create package objects + all_arches = [] + for repo in prepop.keys(): + for arch in prepop[repo].keys(): + if arch not in all_arches: + all_arches.append(arch) + for package in prepop[repo][arch].keys(): + if package not in package_index: + package_index[package] = {} + if repo not in package_index[package]: + package_index[package][repo] = { + "include_filter": [], + "multilib": [], + } + na_list = prepop[repo][arch][package] + for na in na_list: + splitted = na.split(".") + arch_package = splitted[len(splitted) - 1] + if arch != arch_package and arch_package != "noarch": + if arch not in package_index[package][repo]["multilib"]: + package_index[package][repo]["multilib"].append(arch) + if na not in package_index[package][repo]["include_filter"]: + package_index[package][repo]["include_filter"].append(na) + + arch_specific_excludes = {} + na_index = {} + for pkg in package_index.keys(): + for repo in package_index[pkg].keys(): + na_list = list( + filter( + lambda x: x.endswith(".noarch"), + package_index[pkg][repo]["include_filter"], + ) + ) + if not na_list: + continue + exclude_arches = {} + for na in na_list: + for arch in all_arches: + if ( + arch not in prepop[repo] + or pkg not in prepop[repo][arch] + or na not in prepop[repo][arch][pkg] + ): + if na not in exclude_arches: + exclude_arches[na] = [] + exclude_arches[na].append(arch) + na_index[na] = na + if not exclude_arches: + continue + if pkg not in arch_specific_excludes: + arch_specific_excludes[pkg] = {} + if repo not in arch_specific_excludes[pkg]: + arch_specific_excludes[pkg][repo] = [] + arch_specific_excludes[pkg][repo].append(exclude_arches) + + # Index arch specific excludes by repo and arch + repo_arch_index = {} + for pkg in arch_specific_excludes.keys(): + for repo in arch_specific_excludes[pkg].keys(): + if repo not in repo_arch_index: + repo_arch_index[repo] = {} + for arches2 in arch_specific_excludes[pkg][repo]: + for na in arches2.keys(): + for arch in arches2[na]: + if arch not in repo_arch_index[repo]: + repo_arch_index[repo][arch] = [] + if na not in repo_arch_index[repo][arch]: + repo_arch_index[repo][arch].append(na) + + # Add noarch packages not in a specific arch to exclude filter + for repo in repo_arch_index.keys(): + repo_key = f"^{repo}$" + filter_tuple = {} + for arch in repo_arch_index[repo].keys(): + if arch not in filter_tuple: + filter_tuple[arch] = [] + for na in repo_arch_index[repo][arch]: + na = na.removesuffix(".noarch") + if na not in filter_tuple[arch]: + filter_tuple[arch].append(na) + catalog.exclude_filter.append((repo_key, filter_tuple)) + + for package in package_index.keys(): + package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_FORK + if package in module_name_index: + package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_FORK_MODULE + elif package.startswith("rocky-"): + package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_SRC + + catalog.add_package( + PeridotCatalogSyncPackage( + package, + package_type, + [ + PeridotCatalogSyncRepository( + x, + package_index[package][x]["include_filter"], + package_index[package][x]["multilib"], + (get_modules_for_repo(package, x, repo_module_index) if x in repo_module_index else None) if package in module_name_index else None, + ) + for x in package_index[package].keys() + ], + ) + ) + + print(f"Found {len(catalog.packages)} packages") + + f = open(output_path, "w") + f.write(catalog.to_prototxt()) + f.close() + + print(f"Catalog written to {output_path}") + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Create a catalog from a standalone prepopulate.json" + ) + parser.add_argument("--prepopulate-path", type=str, required=True) + parser.add_argument("--major", type=int, required=True) + parser.add_argument("--minor", type=int, required=True) + parser.add_argument("--output-path", type=str, default="hidden.cfg") + args = parser.parse_args() + main(args.prepopulate_path, args.output_path, args.major, args.minor)