diff --git a/.idea/peridot-releng.iml b/.idea/peridot-releng.iml
index 586a970..351e709 100644
--- a/.idea/peridot-releng.iml
+++ b/.idea/peridot-releng.iml
@@ -5,6 +5,7 @@
+
diff --git a/comps2peridot/README.md b/comps2peridot/README.md
new file mode 100644
index 0000000..7870d70
--- /dev/null
+++ b/comps2peridot/README.md
@@ -0,0 +1,7 @@
+# comps2peridot
+Pre-render comps for Peridot consumption
+
+### Usage
+```
+python3 comps2peridot/comps2peridot.py --comps-path /tmp/pungi-rocky/comps.xml --variants-path /tmp/pungi-rocky/variants.xml --output-path /tmp/comps.cfg
+```
diff --git a/comps2peridot/comps2peridot.py b/comps2peridot/comps2peridot.py
new file mode 100644
index 0000000..7871b50
--- /dev/null
+++ b/comps2peridot/comps2peridot.py
@@ -0,0 +1,330 @@
+# -- peridot-releng-header-v0.1 --
+# Copyright (c) Peridot-Releng Authors. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+# noinspection PyPep8Naming
+import xml.etree.ElementTree as ET
+from xml.dom import minidom
+
+from group import Group, PackageReq, Environment, EnvGroup
+
+
+def write_variant(groups, environments, categories, out):
+ root = ET.Element('comps')
+ for group in groups:
+ group_elem = ET.SubElement(root, 'group')
+ ET.SubElement(group_elem, 'id').text = group.id
+ for lang in group.name:
+ name = ET.SubElement(group_elem, 'name')
+ if lang != "":
+ name.set('xml:lang', lang)
+ name.text = group.name[lang]
+ for lang in group.description:
+ description = ET.SubElement(group_elem, 'description')
+ if lang != "":
+ description.set('xml:lang', lang)
+ description.text = group.description[lang]
+ ET.SubElement(group_elem, 'default').text = str(group.default).lower()
+ ET.SubElement(group_elem, 'uservisible').text = str(
+ group.user_visible).lower()
+ package_list = ET.SubElement(group_elem, 'packagelist')
+ for package in group.packages:
+ package_elem = ET.SubElement(package_list, 'packagereq')
+ package_elem.set('type', package.type)
+ package_elem.text = package.name
+ for environment in environments:
+ env_elem = ET.SubElement(root, 'environment')
+ ET.SubElement(env_elem, 'id').text = environment.id
+ for lang in environment.name:
+ name = ET.SubElement(env_elem, 'name')
+ if lang != "":
+ name.set('xml:lang', lang)
+ name.text = environment.name[lang]
+ for lang in environment.description:
+ description = ET.SubElement(env_elem, 'description')
+ if lang != "":
+ description.set('xml:lang', lang)
+ description.text = environment.description[lang]
+ ET.SubElement(env_elem, 'display_order').text = str(
+ environment.display_order)
+ group_list = ET.SubElement(env_elem, 'grouplist')
+ for group in environment.group_list:
+ ET.SubElement(group_list, 'groupid').text = group.name
+ option_list = ET.SubElement(env_elem, 'optionlist')
+ for option in environment.option_list:
+ ET.SubElement(option_list, 'optionid').text = option.name
+ for category_name in categories.keys():
+ category = categories[category_name]
+ new_group_list = []
+ for group in category.group_list:
+ for ggroup in groups:
+ if ggroup.id == group.name:
+ new_group_list.append(group)
+ break
+ if len(new_group_list) == 0:
+ continue
+ category_elem = ET.SubElement(root, 'category')
+ ET.SubElement(category_elem, 'id').text = category_name
+ for lang in category.name:
+ name = ET.SubElement(category_elem, 'name')
+ if lang != "":
+ name.set('xml:lang', lang)
+ name.text = category.name[lang]
+ for lang in category.description:
+ description = ET.SubElement(category_elem, 'description')
+ if lang != "":
+ description.set('xml:lang', lang)
+ description.text = category.description[lang]
+ ET.SubElement(category_elem, 'display_order').text = str(
+ category.display_order)
+ group_list = ET.SubElement(category_elem, 'grouplist')
+ for group in new_group_list:
+ ET.SubElement(group_list, 'groupid').text = group.name
+ ET.ElementTree(root).write(out, encoding='utf-8', xml_declaration=False)
+
+ with open(out, 'r') as f:
+ data = f.read()
+ with open(out, 'w') as f:
+ f.writelines("""
+
+""" + minidom.parseString(data).toprettyxml(indent=" ").replace('\n', ''))
+
+
+def main(comps_path: str, variants_path: str, output_path: str):
+ default_arches = ['x86_64', 'aarch64', 'ppc64le', 's390x']
+ variants = {}
+ environments = {}
+ categories = {}
+
+ tree = ET.parse(comps_path)
+ root = tree.getroot()
+ for gchild in root:
+ if gchild.tag == 'group':
+ group_name = {}
+ group_desc = {}
+ group_id = ''
+ is_default = False
+ is_visible = False
+ variant = ''
+ package_list_xml = None
+ if 'variant' in gchild.attrib:
+ variant = gchild.attrib['variant']
+ if 'arch' in gchild.attrib:
+ arches = gchild.attrib['arch'].split(',')
+ else:
+ arches = default_arches
+ for gattr in gchild:
+ if gattr.tag == 'id':
+ group_id = gattr.text
+ elif gattr.tag == 'name':
+ if '{http://www.w3.org/XML/1998/namespace}lang' in gattr.attrib:
+ group_name[gattr.attrib[
+ '{http://www.w3.org/XML/1998/namespace}lang']] = gattr.text
+ else:
+ group_name[""] = gattr.text
+ elif gattr.tag == 'description':
+ if '{http://www.w3.org/XML/1998/namespace}lang' in gattr.attrib:
+ group_desc[gattr.attrib[
+ '{http://www.w3.org/XML/1998/namespace}lang']] = gattr.text
+ else:
+ group_desc[""] = gattr.text
+ elif gattr.tag == 'default':
+ is_default = gattr.text == 'true'
+ elif gattr.tag == 'uservisible':
+ is_visible = gattr.text == 'true'
+ elif gattr.tag == 'packagelist':
+ package_list_xml = gattr
+ package_list = {}
+ if variant != '':
+ package_list[variant] = {}
+ for reqxml in package_list_xml:
+ req_variant = variant
+ req_type = 'default'
+ if 'variant' in reqxml.attrib:
+ req_variant = reqxml.attrib['variant']
+ if 'type' in reqxml.attrib:
+ req_type = reqxml.attrib['type']
+ if 'arch' in reqxml.attrib:
+ req_arches = reqxml.attrib['arch'].split(',')
+ else:
+ req_arches = arches
+ if req_variant not in package_list:
+ package_list[req_variant] = {}
+ for arch in req_arches:
+ if arch not in package_list[req_variant]:
+ package_list[req_variant][arch] = []
+ package_list[req_variant][arch].append(
+ PackageReq(reqxml.text, req_type, req_arches))
+ for variant in package_list:
+ if variant not in variants:
+ variants[variant] = {}
+ for arch in arches:
+ if arch not in package_list[variant]:
+ package_list[variant][arch] = []
+ if group_id not in variants[variant]:
+ variants[variant][group_id] = {}
+ variants[variant][group_id][arch] = Group(group_id,
+ group_name,
+ group_desc,
+ is_default,
+ is_visible,
+ package_list[
+ variant][
+ arch])
+ elif gchild.tag == 'environment' or gchild.tag == 'category':
+ env_name = {}
+ env_desc = {}
+ env_id = ''
+ display_order = 0
+ group_list = []
+ option_list = []
+ for gattr in gchild:
+ if gattr.tag == 'id':
+ env_id = gattr.text
+ elif gattr.tag == 'name':
+ if '{http://www.w3.org/XML/1998/namespace}lang' in gattr.attrib:
+ env_name[gattr.attrib[
+ '{http://www.w3.org/XML/1998/namespace}lang']] = gattr.text
+ else:
+ env_name[""] = gattr.text
+ elif gattr.tag == 'description':
+ if '{http://www.w3.org/XML/1998/namespace}lang' in gattr.attrib:
+ env_desc[gattr.attrib[
+ '{http://www.w3.org/XML/1998/namespace}lang']] = gattr.text
+ else:
+ env_desc[""] = gattr.text
+ elif gattr.tag == 'display_order':
+ display_order = gattr.text
+ elif gattr.tag == 'grouplist':
+ for group in gattr:
+ if 'arch' in group.attrib:
+ arches = group.attrib['arch'].split(',')
+ else:
+ arches = default_arches
+ group_list.append(EnvGroup(group.text, arches))
+ elif gattr.tag == 'optionlist':
+ for group in gattr:
+ if 'arch' in group.attrib:
+ arches = group.attrib['arch'].split(',')
+ else:
+ arches = default_arches
+ option_list.append(EnvGroup(group.text, arches))
+ new_env = Environment(env_id, env_name, env_desc, display_order,
+ group_list, option_list)
+ dictmap = categories
+ if gchild.tag == 'environment':
+ dictmap = environments
+ if 'arch' in gchild.attrib:
+ arches = gchild.attrib['arch'].split(',')
+ else:
+ arches = default_arches
+ for arch in arches:
+ if arch not in dictmap:
+ dictmap[arch] = {}
+ dictmap[arch][env_id] = new_env
+
+ environment_id_index = {}
+ for arch in environments.keys():
+ for env in environments[arch].values():
+ if env.id not in environment_id_index:
+ environment_id_index[env.id] = {}
+ environment_id_index[env.id][arch] = env
+
+ variant_arch_index = {}
+ environment_arch_index = {}
+ pungi_variants_tree = ET.parse(variants_path).getroot()
+ for pungi_variant in pungi_variants_tree:
+ if pungi_variant.tag == 'variant':
+ if pungi_variant.attrib['type'] != 'variant':
+ continue
+ arches = []
+ groups = {}
+ n_environments = {}
+ variant_id = pungi_variant.attrib['id']
+ for child in pungi_variant:
+ if child.tag == 'arches':
+ for arch in child:
+ arches.append(arch.text)
+ elif child.tag == 'groups':
+ for group in child:
+ groupbase = variants[""]
+ if variant_id in variants:
+ groupbase = variants[variant_id]
+ if group.text not in groupbase:
+ continue
+ groupind = groupbase[group.text]
+ for arch_group in groupind.keys():
+ if arch_group not in groups:
+ groups[arch_group] = []
+ groups[arch_group].append(groupind[arch_group])
+ elif child.tag == 'environments':
+ for environment in child:
+ envind = environment_id_index[environment.text]
+ for arch_environment in envind.keys():
+ if arch_environment not in n_environments:
+ n_environments[arch_environment] = []
+ n_environments[arch_environment].append(
+ envind[arch_environment])
+ for arch in arches:
+ if arch in groups:
+ if arch not in variant_arch_index:
+ variant_arch_index[arch] = {}
+ if variant_id not in variant_arch_index[arch]:
+ variant_arch_index[arch][variant_id] = []
+ variant_arch_index[arch][variant_id].extend(groups[arch])
+ if arch in n_environments:
+ if arch not in environment_arch_index:
+ environment_arch_index[arch] = {}
+ if variant_id not in environment_arch_index[arch]:
+ environment_arch_index[arch][variant_id] = []
+ environment_arch_index[arch][variant_id].extend(
+ n_environments[arch])
+
+ for arch in variant_arch_index.keys():
+ for variant in variant_arch_index[arch].keys():
+ write_variant(variant_arch_index[arch][variant] if variant in
+ variant_arch_index[
+ arch] else [],
+ environment_arch_index[arch][variant] if variant in
+ environment_arch_index[
+ arch] else [],
+ categories[arch].copy(),
+ f'{output_path}/{variant}-{arch}.xml')
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(
+ description='Convert comps to Peridot compatible configuration.')
+ parser.add_argument('--comps-path', type=str, required=True)
+ parser.add_argument('--variants-path', type=str, required=True)
+ parser.add_argument('--output-path', type=str, default=".")
+ args = parser.parse_args()
+ main(args.comps_path, args.variants_path, args.output_path)
diff --git a/comps2peridot/group.py b/comps2peridot/group.py
new file mode 100644
index 0000000..ed66372
--- /dev/null
+++ b/comps2peridot/group.py
@@ -0,0 +1,63 @@
+# -- peridot-releng-header-v0.1 --
+# Copyright (c) Peridot-Releng Authors. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from dataclasses import dataclass
+
+
+@dataclass
+class PackageReq:
+ name: str
+ type: str
+ arches: list[str]
+
+
+@dataclass
+class Group:
+ id: str
+ name: dict[str, str]
+ description: dict[str, str]
+ default: bool
+ user_visible: bool
+ packages: list[PackageReq]
+
+
+@dataclass
+class EnvGroup:
+ name: str
+ arch: list[str]
+
+
+@dataclass
+class Environment:
+ id: str
+ name: dict[str, str]
+ description: dict[str, str]
+ display_order: int
+ group_list: list[EnvGroup]
+ option_list: list[EnvGroup]
diff --git a/scripts/common.py b/scripts/common.py
new file mode 100644
index 0000000..00fb196
--- /dev/null
+++ b/scripts/common.py
@@ -0,0 +1,42 @@
+# -- peridot-releng-header-v0.1 --
+# Copyright (c) Peridot-Releng Authors. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+BASE_URL = 'https://peridot-api.build.resf.org/v1'
+PROJECT_ID_PROD = '55b17281-bc54-4929-8aca-a8a11d628738'
+
+
+def construct_url(path, project_id=PROJECT_ID_PROD):
+ return f'{BASE_URL}/projects/{project_id}{path}'
+
+
+def build_batches_url(batch_type, task_id, page, status,
+ project_id=PROJECT_ID_PROD):
+ return construct_url(
+ f'/{batch_type}_batches/{task_id}?page={page}&limit=100&filter.status={status}',
+ project_id)
diff --git a/scripts/create-batch-task-list.py b/scripts/create-batch-task-list.py
index 39cb5dc..c14ae3d 100644
--- a/scripts/create-batch-task-list.py
+++ b/scripts/create-batch-task-list.py
@@ -33,10 +33,11 @@ import sys
import requests
import json
+from common import build_batches_url
+
def get_batch(batch_type, task_id, status, page):
- r = requests.get(
- f'https://peridot.pdot-dev.rockylinux.org/api/v1/projects/c4fa14a2-5af6-4634-bfea-847a9fd639c7/{batch_type}_batches/{task_id}?page={page}&limit=100&filter.status={status}')
+ r = requests.get(build_batches_url(batch_type, task_id, page, status))
return r.json()[f'{batch_type}s']
@@ -56,7 +57,6 @@ if __name__ == '__main__':
task_id = sys.argv[2]
batch_items = process_batch(batch_type, task_id, 4)
- # batch_items.extend(process_batch(batch_type, task_id, 5))
req = {}
key = f'{batch_type}s'
diff --git a/scripts/create-no-build-batch.py b/scripts/create-no-build-batch.py
new file mode 100644
index 0000000..76ce169
--- /dev/null
+++ b/scripts/create-no-build-batch.py
@@ -0,0 +1,70 @@
+# -- peridot-releng-header-v0.1 --
+# Copyright (c) Peridot-Releng Authors. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import sys
+import requests
+import json
+
+from itertools import islice
+
+from common import construct_url
+
+
+def chunks(lst, n):
+ for i in range(0, len(lst), n):
+ yield lst[i:i + n]
+
+
+def get_packages(page):
+ r = requests.get(
+ construct_url(f'/packages?limit=100&page={page}&filters.no_builds=1'))
+ return r.json()['packages']
+
+
+def process_packages():
+ ret = []
+ page = 0
+ while True:
+ res = get_packages(page)
+ if len(res) == 0:
+ return ret
+ ret.extend(res)
+ page = page + 1
+
+
+if __name__ == '__main__':
+ batch_items = process_packages()
+
+ builds = []
+ for item in batch_items:
+ builds.append({
+ 'package_name': item['name']
+ })
+ for chunk in chunks(builds, 400):
+ print(json.dumps({"builds": chunk}))