Add module support to pungicatalog

pungicatalog/catalog.py

@@ -51,6 +51,12 @@ class PeridotCatalogSyncRepository:
     name: str
     include_filter: list[str]
     multilib: list[str]
+    module_streams: list[str]
+
+    def module_streams_to_prototxt(self):
+        return "\n" + "\n".join(
+            [f'    module_stream: "{f}"' for f in self.module_streams]
+        )
 
     def include_filter_to_prototxt(self):
         return "\n" + "\n".join(
@@ -65,22 +71,15 @@ class PeridotCatalogSyncRepository:
 class PeridotCatalogSyncPackage:
     name: str
     type: PeridotCatalogSyncPackageType
-    module_components: list[str]
     repositories: list[PeridotCatalogSyncRepository]
 
-    def mc_to_prototxt(self):
-        return "\n" + "\n".join(
-            [
-                f'  module_component: "{component}"'
-                for component in self.module_components
-            ]
-        )
-
     def repos_to_prototxt(self):
         return "\n".join(
             [
                 f"""  repository {{
     name: \"{repo.name}\"{
+            repo.module_streams_to_prototxt() if repo.module_streams else ""
+            }{
             repo.include_filter_to_prototxt() if repo.include_filter else ""
             }{
             repo.multilib_to_prototxt() if repo.multilib else ""
@@ -97,12 +96,50 @@ class PeridotCatalogSync:
     exclude_filter: list[tuple[str, dict]] = []
     include_filter: list[tuple[str, dict]] = []
     packages: list[PeridotCatalogSyncPackage] = []
+    module_defaults = None
+    major = 0
+    minor = 0
 
     def add_package(self, package: PeridotCatalogSyncPackage):
         self.packages.append(package)
 
-    def additional_multilib_to_prototxt(self):
+    def module_profile_to_prototxt(self, profile):
+        return "\n".join([f"            name: \"{p}\"" for p in profile])
+
+    def module_defaults_profiles_to_prototxt(self, profiles):
+        if not profiles:
+            return ""
+        return "\n" + "\n".join(
+            [f"""        profile {{
+            stream: \"{f}\"
+{self.module_profile_to_prototxt(profiles[f])}
+        }}
+""" for f in profiles.keys()]
+        )
+
+    def module_defaults_to_prototxt(self):
         return "\n".join(
+            [f"""    default {{
+        name: \"{f["data"]["module"]}\"
+        stream: \"{f["data"].get("stream", "")}\"{
+        self.module_defaults_profiles_to_prototxt(f["data"].get("profiles", []))
+}    }}""" for f in self.module_defaults]
+        ) if self.module_defaults else ""
+
+    def module_configuration_to_prototxt(self):
+        if not self.module_defaults:
+            return ""
+        return f"""module_configuration {{
+    platform {{
+        major: {self.major}
+        minor: {self.minor}
+        patch: 0
+    }}
+{self.module_defaults_to_prototxt()}
+}}"""
+
+    def additional_multilib_to_prototxt(self):
+        return "\n" + "\n".join(
             [f'additional_multilib: "{f}"' for f in self.additional_multilib]
         )
 
@@ -152,7 +189,9 @@ class PeridotCatalogSync:
 
     def to_prototxt(self):
         ret = f"""# kind: resf.peridot.v1.CatalogSync
-{self.additional_multilib_to_prototxt()}{
+{self.module_configuration_to_prototxt()}{
+        self.additional_multilib_to_prototxt()
+        }{
         self.exclude_multilib_filter_to_prototxt()
         }{
         self.exclude_filter_to_prototxt()
@@ -163,9 +202,7 @@ class PeridotCatalogSync:
         for pkg in self.packages:
             ret += f"""package {{
   name: "{pkg.name}"
-  type: {pkg.type}{
+  type: {pkg.type}
-            pkg.mc_to_prototxt() if pkg.module_components else ""
-            }
 {pkg.repos_to_prototxt()}
 }}
 """

pungicatalog/pungicatalog.py

@@ -29,6 +29,7 @@
 
 import argparse
 import os
+import yaml
 
 import kobo.conf
 
@@ -40,19 +41,49 @@ from catalog import (
 )
 from scm import SCM
 
+def get_modules_for_repo(package, repo, module_index):
+    if not repo in module_index:
+        return None
 
-def main(pungi_conf_path: str, output_path: str):
+    modules = []
+    for module in module_index[repo]:
+        if module.startswith(f"{package}:"):
+            modules.append(module.split(":")[1])
+
+    if len(modules) == 0:
+        return None
+
+    return modules
+
+def main(pungi_conf_path: str, output_path: str, major: int, minor: int):
     pungi_base = os.path.dirname(pungi_conf_path)
+    print(f"Using pungi base: {pungi_base}")
 
     conf = kobo.conf.PyConfigParser()
     conf.load_from_file(pungi_conf_path)
+    print(f"Loaded pungi config: {pungi_conf_path}")
 
+    print("Loading prepopulate...")
     gather_prepopulate_scm_dict = conf.get("gather_prepopulate")
     gpscm = SCM(pungi_base, gather_prepopulate_scm_dict)
     gpjson = gpscm.json()
 
+    # Get variants
+    print("Loading variants...")
+    variants_file_scm_dict = conf.get("variants_file")
+    vscm = SCM(pungi_base, variants_file_scm_dict)
+    vxml = vscm.xml()
+
+    # Get module defaults
+    print("Loading module defaults...")
+    module_defaults_file_scm_dict = conf.get("module_defaults_dir")
+    mdscm = SCM(pungi_base, module_defaults_file_scm_dict, ext_filters=[".yaml"])
+    mdtexts = mdscm.texts()
+
     # Create a catalog
     catalog = PeridotCatalogSync()
+    catalog.major = major
+    catalog.minor = minor
 
     # Set multilib filters
     catalog.additional_multilib.extend(list(conf.get("multilib_whitelist").values())[0])
@@ -66,6 +97,32 @@ def main(pungi_conf_path: str, output_path: str):
 
     # Create indexes
     package_index = {}
+    repo_module_index = {}
+    module_name_index = {}
+    module_defaults = []
+
+    # Add modules
+    for repo in gpjson.keys():
+        xml_path = f".//variant[@id='{repo}']/modules/module"
+        modules = vxml.findall(xml_path)
+        # No modules in repo, continue
+        if len(modules) == 0:
+            continue
+        for module in modules:
+            module_name = module.text.split(":")[0]
+            if not repo in repo_module_index:
+                repo_module_index[repo] = []
+            repo_module_index[repo].append(module.text)
+            module_name_index[module_name] = True
+            print(f"Found module: {module.text}")
+
+    # Add module defaults
+    for mdtext in mdtexts:
+        md = yaml.safe_load(mdtext)
+        module_defaults.append(md)
+
+    if len(module_defaults) > 0:
+        catalog.module_defaults = module_defaults
 
     # Read prepopulate json and create package objects
     all_arches = []
@@ -151,36 +208,43 @@ def main(pungi_conf_path: str, output_path: str):
         catalog.exclude_filter.append((repo_key, filter_tuple))
 
     for package in package_index.keys():
+        package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_FORK
+        if package in module_name_index:
+            package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_FORK_MODULE
+        elif package.startswith("rocky-"):
+            package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_SRC
+
         catalog.add_package(
             PeridotCatalogSyncPackage(
                 package,
-                PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_FORK
+                package_type,
-                if not package.startswith("rocky-")
-                else PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_SRC,
-                [],
                 [
                     PeridotCatalogSyncRepository(
                         x,
                         package_index[package][x]["include_filter"],
                         package_index[package][x]["multilib"],
+                        (get_modules_for_repo(package, x, repo_module_index) if x in repo_module_index else None) if package in module_name_index else None,
                     )
                     for x in package_index[package].keys()
                 ],
             )
         )
 
+    print(f"Found {len(catalog.packages)} packages")
+
     f = open(output_path, "w")
     f.write(catalog.to_prototxt())
     f.close()
 
-    pass
+    print(f"Catalog written to {output_path}")
-
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Convert Pungi configuration to Peridot compatible " "catalogs."
     )
     parser.add_argument("--pungi-conf-path", type=str, required=True)
+    parser.add_argument("--major", type=int, required=True)
+    parser.add_argument("--minor", type=int, required=True)
     parser.add_argument("--output-path", type=str, default="catalog.cfg")
     args = parser.parse_args()
-    main(args.pungi_conf_path, args.output_path)
+    main(args.pungi_conf_path, args.output_path, args.major, args.minor)

pungicatalog/scm.py

@@ -27,31 +27,84 @@
 #  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 #  POSSIBILITY OF SUCH DAMAGE.
 
+import xml.etree.ElementTree as ET
 import json
 import os
+import tempfile
 
+from git import Repo
 
 class SCM:
-    def __init__(self, pungi_base, scm_dict):
+    def __init__(self, pungi_base, scm_dict, ext_filters=None):
-        if  isinstance(scm_dict, str) or scm_dict["scm"] == "file":
+        # Temporary hack since pungi-rocky usually has everything in one repo anyways
-            file_path = ""
+        # todo(mustafa): remove this hack
-            if not isinstance(scm_dict, str):
+        base_file_path = ""
-                file_path = os.path.join(pungi_base, scm_dict["file"])
+        base_file_dir = ""
+        if isinstance(scm_dict, str):
+            base_file_path = scm_dict
         else:
-                file_path = os.path.join(pungi_base, scm_dict)
+            if scm_dict["scm"] == "file":
+                base_file_path = scm_dict["file"]
+            elif scm_dict["scm"] == "git":
+                if "file" in scm_dict:
+                    base_file_path = scm_dict["file"]
+                elif "dir" in scm_dict:
+                    base_file_dir = scm_dict["dir"]
+            else:
+                raise Exception("Unsupported SCM type")
+
+        file_contents = None
+        file_list_contents = []
+
+        if  isinstance(scm_dict, str) or scm_dict["scm"] == "file":
+            file_path = os.path.join(pungi_base, base_file_path)
 
             f = open(file_path, "r")
             file_contents = f.read()
+            f.close()
+        elif scm_dict["scm"] == "git":
+            with tempfile.TemporaryDirectory() as d:
+                print(f"Cloning {scm_dict['repo']}")
+                Repo.clone_from(scm_dict["repo"], d, branch=scm_dict["branch"], depth=1)
 
-            if file_path.endswith(".json"):
+                if base_file_path:
+                    print(f"Found file {base_file_path}")
+                    file_path = os.path.join(d, base_file_path)
+                    f = open(file_path, "r")
+                    file_contents = f.read()
+                    f.close()
+                elif base_file_dir:
+                    print(f"Reading files from {base_file_dir}")
+                    file_dir = os.path.join(d, base_file_dir)
+                    for file in os.listdir(file_dir):
+                        if file in [".git"]:
+                            continue
+                        if ext_filters:
+                            if not any(file.endswith(ext) for ext in ext_filters):
+                                continue
+                        file_path = os.path.join(file_dir, file)
+                        f = open(file_path, "r")
+                        file_list_contents.append(f.read())
+                        f.close()
+
+        if file_contents:
+            if base_file_path.endswith(".json"):
                 self.json_value = json.loads(file_contents)
+            elif base_file_path.endswith(".xml"):
+                self.xml_value = ET.fromstring(file_contents)
             else:
                 self.text_value = file_contents
-
+        elif file_list_contents:
-            f.close()
+            self.text_values = file_list_contents
 
     def json(self):
         return self.json_value
 
     def text(self):
         return self.text_value
+
+    def xml(self):
+        return self.xml_value
+
+    def texts(self):
+        return self.text_values