sig_core
/
peridot-releng
mirror of https://github.com/rocky-linux/peridot-releng.git
7
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add module support to pungicatalog

This commit is contained in:
Mustafa Gezen 2022-11-03 07:50:30 +01:00
parent 9a4d612f9b
commit 080ab0a19b
Signed by: mustafa
GPG Key ID: DCDF010D946438C1
3 changed files with 185 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
pungicatalog/catalog.py
View File

@ -51,6 +51,12 @@ class PeridotCatalogSyncRepository:
name: str
include_filter: list[str]
multilib: list[str]
module_streams: list[str]
def module_streams_to_prototxt(self):
return "\n" + "\n".join(
[f' module_stream: "{f}"' for f in self.module_streams]
)
def include_filter_to_prototxt(self):
return "\n" + "\n".join(
@ -65,22 +71,15 @@ class PeridotCatalogSyncRepository:
class PeridotCatalogSyncPackage:
name: str
type: PeridotCatalogSyncPackageType
module_components: list[str]
repositories: list[PeridotCatalogSyncRepository]
def mc_to_prototxt(self):
return "\n" + "\n".join(
[
f' module_component: "{component}"'
for component in self.module_components
]
)
def repos_to_prototxt(self):
return "\n".join(
[
f""" repository {{
name: \"{repo.name}\"{
repo.module_streams_to_prototxt() if repo.module_streams else ""
}{
repo.include_filter_to_prototxt() if repo.include_filter else ""
}{
repo.multilib_to_prototxt() if repo.multilib else ""
@ -97,12 +96,50 @@ class PeridotCatalogSync:
exclude_filter: list[tuple[str, dict]] = []
include_filter: list[tuple[str, dict]] = []
packages: list[PeridotCatalogSyncPackage] = []
module_defaults = None
major = 0
minor = 0
def add_package(self, package: PeridotCatalogSyncPackage):
self.packages.append(package)
def additional_multilib_to_prototxt(self):
def module_profile_to_prototxt(self, profile):
return "\n".join([f" name: \"{p}\"" for p in profile])
def module_defaults_profiles_to_prototxt(self, profiles):
if not profiles:
return ""
return "\n" + "\n".join(
[f""" profile {{
stream: \"{f}\"
{self.module_profile_to_prototxt(profiles[f])}
}}
""" for f in profiles.keys()]
)
def module_defaults_to_prototxt(self):
return "\n".join(
[f""" default {{
name: \"{f["data"]["module"]}\"
stream: \"{f["data"].get("stream", "")}\"{
self.module_defaults_profiles_to_prototxt(f["data"].get("profiles", []))
} }}""" for f in self.module_defaults]
) if self.module_defaults else ""
def module_configuration_to_prototxt(self):
if not self.module_defaults:
return ""
return f"""module_configuration {{
platform {{
major: {self.major}
minor: {self.minor}
patch: 0
}}
{self.module_defaults_to_prototxt()}
}}"""
def additional_multilib_to_prototxt(self):
return "\n" + "\n".join(
[f'additional_multilib: "{f}"' for f in self.additional_multilib]
)
@ -152,7 +189,9 @@ class PeridotCatalogSync:
def to_prototxt(self):
ret = f"""# kind: resf.peridot.v1.CatalogSync
{self.additional_multilib_to_prototxt()}{
{self.module_configuration_to_prototxt()}{
self.additional_multilib_to_prototxt()
}{
self.exclude_multilib_filter_to_prototxt()
}{
self.exclude_filter_to_prototxt()
@ -163,9 +202,7 @@ class PeridotCatalogSync:
for pkg in self.packages:
ret += f"""package {{
name: "{pkg.name}"
type: {pkg.type}{
pkg.mc_to_prototxt() if pkg.module_components else ""
}
type: {pkg.type}
{pkg.repos_to_prototxt()}
}}
"""

80
pungicatalog/pungicatalog.py
View File

@ -29,6 +29,7 @@
import argparse
import os
import yaml
import kobo.conf
@ -40,19 +41,49 @@ from catalog import (
)
from scm import SCM
def get_modules_for_repo(package, repo, module_index):
if not repo in module_index:
return None
def main(pungi_conf_path: str, output_path: str):
modules = []
for module in module_index[repo]:
if module.startswith(f"{package}:"):
modules.append(module.split(":")[1])
if len(modules) == 0:
return None
return modules
def main(pungi_conf_path: str, output_path: str, major: int, minor: int):
pungi_base = os.path.dirname(pungi_conf_path)
print(f"Using pungi base: {pungi_base}")
conf = kobo.conf.PyConfigParser()
conf.load_from_file(pungi_conf_path)
print(f"Loaded pungi config: {pungi_conf_path}")
print("Loading prepopulate...")
gather_prepopulate_scm_dict = conf.get("gather_prepopulate")
gpscm = SCM(pungi_base, gather_prepopulate_scm_dict)
gpjson = gpscm.json()
# Get variants
print("Loading variants...")
variants_file_scm_dict = conf.get("variants_file")
vscm = SCM(pungi_base, variants_file_scm_dict)
vxml = vscm.xml()
# Get module defaults
print("Loading module defaults...")
module_defaults_file_scm_dict = conf.get("module_defaults_dir")
mdscm = SCM(pungi_base, module_defaults_file_scm_dict, ext_filters=[".yaml"])
mdtexts = mdscm.texts()
# Create a catalog
catalog = PeridotCatalogSync()
catalog.major = major
catalog.minor = minor
# Set multilib filters
catalog.additional_multilib.extend(list(conf.get("multilib_whitelist").values())[0])
@ -66,6 +97,32 @@ def main(pungi_conf_path: str, output_path: str):
# Create indexes
package_index = {}
repo_module_index = {}
module_name_index = {}
module_defaults = []
# Add modules
for repo in gpjson.keys():
xml_path = f".//variant[@id='{repo}']/modules/module"
modules = vxml.findall(xml_path)
# No modules in repo, continue
if len(modules) == 0:
continue
for module in modules:
module_name = module.text.split(":")[0]
if not repo in repo_module_index:
repo_module_index[repo] = []
repo_module_index[repo].append(module.text)
module_name_index[module_name] = True
print(f"Found module: {module.text}")
# Add module defaults
for mdtext in mdtexts:
md = yaml.safe_load(mdtext)
module_defaults.append(md)
if len(module_defaults) > 0:
catalog.module_defaults = module_defaults
# Read prepopulate json and create package objects
all_arches = []
@ -151,36 +208,43 @@ def main(pungi_conf_path: str, output_path: str):
catalog.exclude_filter.append((repo_key, filter_tuple))
for package in package_index.keys():
package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_FORK
if package in module_name_index:
package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_FORK_MODULE
elif package.startswith("rocky-"):
package_type = PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_SRC
catalog.add_package(
PeridotCatalogSyncPackage(
package,
PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_FORK
if not package.startswith("rocky-")
else PeridotCatalogSyncPackageType.PACKAGE_TYPE_NORMAL_SRC,
[],
package_type,
[
PeridotCatalogSyncRepository(
x,
package_index[package][x]["include_filter"],
package_index[package][x]["multilib"],
(get_modules_for_repo(package, x, repo_module_index) if x in repo_module_index else None) if package in module_name_index else None,
)
for x in package_index[package].keys()
],
)
)
print(f"Found {len(catalog.packages)} packages")
f = open(output_path, "w")
f.write(catalog.to_prototxt())
f.close()
pass
print(f"Catalog written to {output_path}")
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Convert Pungi configuration to Peridot compatible " "catalogs."
)
parser.add_argument("--pungi-conf-path", type=str, required=True)
parser.add_argument("--major", type=int, required=True)
parser.add_argument("--minor", type=int, required=True)
parser.add_argument("--output-path", type=str, default="catalog.cfg")
args = parser.parse_args()
main(args.pungi_conf_path, args.output_path)
main(args.pungi_conf_path, args.output_path, args.major, args.minor)

71
pungicatalog/scm.py
View File

@ -27,31 +27,84 @@
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
import xml.etree.ElementTree as ET
import json
import os
import tempfile
from git import Repo
class SCM:
def __init__(self, pungi_base, scm_dict):
if isinstance(scm_dict, str) or scm_dict["scm"] == "file":
file_path = ""
if not isinstance(scm_dict, str):
file_path = os.path.join(pungi_base, scm_dict["file"])
def __init__(self, pungi_base, scm_dict, ext_filters=None):
# Temporary hack since pungi-rocky usually has everything in one repo anyways
# todo(mustafa): remove this hack
base_file_path = ""
base_file_dir = ""
if isinstance(scm_dict, str):
base_file_path = scm_dict
else:
if scm_dict["scm"] == "file":
base_file_path = scm_dict["file"]
elif scm_dict["scm"] == "git":
if "file" in scm_dict:
base_file_path = scm_dict["file"]
elif "dir" in scm_dict:
base_file_dir = scm_dict["dir"]
else:
file_path = os.path.join(pungi_base, scm_dict)
raise Exception("Unsupported SCM type")
file_contents = None
file_list_contents = []
if isinstance(scm_dict, str) or scm_dict["scm"] == "file":
file_path = os.path.join(pungi_base, base_file_path)
f = open(file_path, "r")
file_contents = f.read()
f.close()
elif scm_dict["scm"] == "git":
with tempfile.TemporaryDirectory() as d:
print(f"Cloning {scm_dict['repo']}")
Repo.clone_from(scm_dict["repo"], d, branch=scm_dict["branch"], depth=1)
if file_path.endswith(".json"):
if base_file_path:
print(f"Found file {base_file_path}")
file_path = os.path.join(d, base_file_path)
f = open(file_path, "r")
file_contents = f.read()
f.close()
elif base_file_dir:
print(f"Reading files from {base_file_dir}")
file_dir = os.path.join(d, base_file_dir)
for file in os.listdir(file_dir):
if file in [".git"]:
continue
if ext_filters:
if not any(file.endswith(ext) for ext in ext_filters):
continue
file_path = os.path.join(file_dir, file)
f = open(file_path, "r")
file_list_contents.append(f.read())
f.close()
if file_contents:
if base_file_path.endswith(".json"):
self.json_value = json.loads(file_contents)
elif base_file_path.endswith(".xml"):
self.xml_value = ET.fromstring(file_contents)
else:
self.text_value = file_contents
f.close()
elif file_list_contents:
self.text_values = file_list_contents
def json(self):
return self.json_value
def text(self):
return self.text_value
def xml(self):
return self.xml_value
def texts(self):
return self.text_values