diff --git a/pungicatalog/catalog.py b/pungicatalog/catalog.py index 22b8db1..0848a71 100644 --- a/pungicatalog/catalog.py +++ b/pungicatalog/catalog.py @@ -51,6 +51,12 @@ class PeridotCatalogSyncRepository: name: str include_filter: list[str] multilib: list[str] + module_streams: list[str] + + def module_streams_to_prototxt(self): + return "\n" + "\n".join( + [f' module_stream: "{f}"' for f in self.module_streams] + ) def include_filter_to_prototxt(self): return "\n" + "\n".join( @@ -65,22 +71,15 @@ class PeridotCatalogSyncRepository: class PeridotCatalogSyncPackage: name: str type: PeridotCatalogSyncPackageType - module_components: list[str] repositories: list[PeridotCatalogSyncRepository] - def mc_to_prototxt(self): - return "\n" + "\n".join( - [ - f' module_component: "{component}"' - for component in self.module_components - ] - ) - def repos_to_prototxt(self): return "\n".join( [ f""" repository {{ name: \"{repo.name}\"{ + repo.module_streams_to_prototxt() if repo.module_streams else "" + }{ repo.include_filter_to_prototxt() if repo.include_filter else "" }{ repo.multilib_to_prototxt() if repo.multilib else "" @@ -97,12 +96,50 @@ class PeridotCatalogSync: exclude_filter: list[tuple[str, dict]] = [] include_filter: list[tuple[str, dict]] = [] packages: list[PeridotCatalogSyncPackage] = [] + module_defaults = None + major = 0 + minor = 0 def add_package(self, package: PeridotCatalogSyncPackage): self.packages.append(package) - def additional_multilib_to_prototxt(self): + def module_profile_to_prototxt(self, profile): + return "\n".join([f" name: \"{p}\"" for p in profile]) + + def module_defaults_profiles_to_prototxt(self, profiles): + if not profiles: + return "" + return "\n" + "\n".join( + [f""" profile {{ + stream: \"{f}\" +{self.module_profile_to_prototxt(profiles[f])} + }} +""" for f in profiles.keys()] + ) + + def module_defaults_to_prototxt(self): return "\n".join( + [f""" default {{ + name: \"{f["data"]["module"]}\" + stream: \"{f["data"].get("stream", "")}\"{ + self.module_defaults_profiles_to_prototxt(f["data"].get("profiles", [])) +} }}""" for f in self.module_defaults] + ) if self.module_defaults else "" + + def module_configuration_to_prototxt(self): + if not self.module_defaults: + return "" + return f"""module_configuration {{ + platform {{ + major: {self.major} + minor: {self.minor} + patch: 0 + }} +{self.module_defaults_to_prototxt()} +}}""" + + def additional_multilib_to_prototxt(self): + return "\n" + "\n".join( [f'additional_multilib: "{f}"' for f in self.additional_multilib] ) @@ -152,7 +189,9 @@ class PeridotCatalogSync: def to_prototxt(self): ret = f"""# kind: resf.peridot.v1.CatalogSync -{self.additional_multilib_to_prototxt()}{ +{self.module_configuration_to_prototxt()}{ + self.additional_multilib_to_prototxt() + }{ self.exclude_multilib_filter_to_prototxt() }{ self.exclude_filter_to_prototxt() @@ -163,9 +202,7 @@ class PeridotCatalogSync: for pkg in self.packages: ret += f"""package {{ name: "{pkg.name}" - type: {pkg.type}{ - pkg.mc_to_prototxt() if pkg.module_components else "" - } + type: {pkg.type} {pkg.repos_to_prototxt()} }} """ diff --git a/pungicatalog/pungicatalog.py b/pungicatalog/pungicatalog.py index 37387a6..05db0d0 100644 --- a/pungicatalog/pungicatalog.py +++ b/pungicatalog/pungicatalog.py @@ -29,6 +29,7 @@ import argparse import os +import yaml import kobo.conf @@ -40,19 +41,49 @@ from catalog import ( ) from scm import SCM +def get_modules_for_repo(package, repo, module_index): + if not repo in module_index: + return None -def main(pungi_conf_path: str, output_path: str): + modules = [] + for module in module_index[repo]: + if module.startswith(f"{package}:"): + modules.append(module.split(":")[1]) + + if len(modules) == 0: + return None + + return modules + +def main(pungi_conf_path: str, output_path: str, major: int, minor: int): pungi_base = os.path.dirname(pungi_conf_path) + print(f"Using pungi base: {pungi_base}") conf = kobo.conf.PyConfigParser() conf.load_from_file(pungi_conf_path) + print(f"Loaded pungi config: {pungi_conf_path}") + print("Loading prepopulate...") gather_prepopulate_scm_dict = conf.get("gather_prepopulate") gpscm = SCM(pungi_base, gather_prepopulate_scm_dict) gpjson = gpscm.json() + # Get variants + print("Loading variants...") + variants_file_scm_dict = conf.get("variants_file") + vscm = SCM(pungi_base, variants_file_scm_dict) + vxml = vscm.xml() + + # Get module defaults + print("Loading module defaults...") + module_defaults_file_scm_dict = conf.get("module_defaults_dir") + mdscm = SCM(pungi_base, module_defaults_file_scm_dict, ext_filters=[".yaml"]) + mdtexts = mdscm.texts() + # Create a catalog catalog = PeridotCatalogSync() + catalog.major = major + catalog.minor = minor # Set multilib filters catalog.additional_multilib.extend(list(conf.get("multilib_whitelist").values())[0]) @@ -66,6 +97,32 @@ def main(pungi_conf_path: str, output_path: str): # Create indexes package_index = {} + repo_module_index = {} + module_name_index = {} + module_defaults = [] + + # Add modules + for repo in gpjson.keys(): + xml_path = f".//variant[@id='{repo}']/modules/module" + modules = vxml.findall(xml_path) + # No modules in repo, continue + if len(modules) == 0: + continue + for module in modules: + module_name = module.text.split(":")[0] + if not repo in repo_module_index: + repo_module_index[repo] = [] + repo_module_index[repo].append(module.text) + module_name_index[module_name] = True + print(f"Found module: {module.text}") + + # Add module defaults + for mdtext in mdtexts: + md = yaml.safe_load(mdtext) + module_defaults.append(md) + + if len(module_defaults) > 0: + catalog.module_defaults = module_defaults # Read prepopulate json and create package objects all_arches = [] @@ -151,36 +208,43 @@ def main(pungi_conf_path: str, output_path: str): catalog.exclude_filter.append((repo_key, filter_tuple)) for package in package_index.keys(): + package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_FORK + if package in module_name_index: + package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_FORK_MODULE + elif package.startswith("rocky-"): + package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_SRC + catalog.add_package( PeridotCatalogSyncPackage( package, - PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_FORK - if not package.startswith("rocky-") - else PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_SRC, - [], + package_type, [ PeridotCatalogSyncRepository( x, package_index[package][x]["include_filter"], package_index[package][x]["multilib"], + (get_modules_for_repo(package, x, repo_module_index) if x in repo_module_index else None) if package in module_name_index else None, ) for x in package_index[package].keys() ], ) ) + print(f"Found {len(catalog.packages)} packages") + f = open(output_path, "w") f.write(catalog.to_prototxt()) f.close() - pass - + print(f"Catalog written to {output_path}") if __name__ == "__main__": parser = argparse.ArgumentParser( description="Convert Pungi configuration to Peridot compatible " "catalogs." ) parser.add_argument("--pungi-conf-path", type=str, required=True) + parser.add_argument("--major", type=int, required=True) + parser.add_argument("--minor", type=int, required=True) parser.add_argument("--output-path", type=str, default="catalog.cfg") args = parser.parse_args() - main(args.pungi_conf_path, args.output_path) + main(args.pungi_conf_path, args.output_path, args.major, args.minor) diff --git a/pungicatalog/scm.py b/pungicatalog/scm.py index 0185c97..901304f 100644 --- a/pungicatalog/scm.py +++ b/pungicatalog/scm.py @@ -27,31 +27,84 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +import xml.etree.ElementTree as ET import json import os +import tempfile +from git import Repo class SCM: - def __init__(self, pungi_base, scm_dict): - if isinstance(scm_dict, str) or scm_dict["scm"] == "file": - file_path = "" - if not isinstance(scm_dict, str): - file_path = os.path.join(pungi_base, scm_dict["file"]) + def __init__(self, pungi_base, scm_dict, ext_filters=None): + # Temporary hack since pungi-rocky usually has everything in one repo anyways + # todo(mustafa): remove this hack + base_file_path = "" + base_file_dir = "" + if isinstance(scm_dict, str): + base_file_path = scm_dict + else: + if scm_dict["scm"] == "file": + base_file_path = scm_dict["file"] + elif scm_dict["scm"] == "git": + if "file" in scm_dict: + base_file_path = scm_dict["file"] + elif "dir" in scm_dict: + base_file_dir = scm_dict["dir"] else: - file_path = os.path.join(pungi_base, scm_dict) + raise Exception("Unsupported SCM type") + + file_contents = None + file_list_contents = [] + + if isinstance(scm_dict, str) or scm_dict["scm"] == "file": + file_path = os.path.join(pungi_base, base_file_path) f = open(file_path, "r") file_contents = f.read() + f.close() + elif scm_dict["scm"] == "git": + with tempfile.TemporaryDirectory() as d: + print(f"Cloning {scm_dict['repo']}") + Repo.clone_from(scm_dict["repo"], d, branch=scm_dict["branch"], depth=1) - if file_path.endswith(".json"): + if base_file_path: + print(f"Found file {base_file_path}") + file_path = os.path.join(d, base_file_path) + f = open(file_path, "r") + file_contents = f.read() + f.close() + elif base_file_dir: + print(f"Reading files from {base_file_dir}") + file_dir = os.path.join(d, base_file_dir) + for file in os.listdir(file_dir): + if file in [".git"]: + continue + if ext_filters: + if not any(file.endswith(ext) for ext in ext_filters): + continue + file_path = os.path.join(file_dir, file) + f = open(file_path, "r") + file_list_contents.append(f.read()) + f.close() + + if file_contents: + if base_file_path.endswith(".json"): self.json_value = json.loads(file_contents) + elif base_file_path.endswith(".xml"): + self.xml_value = ET.fromstring(file_contents) else: self.text_value = file_contents - - f.close() + elif file_list_contents: + self.text_values = file_list_contents def json(self): return self.json_value def text(self): return self.text_value + + def xml(self): + return self.xml_value + + def texts(self): + return self.text_values