diff --git a/.idea/peridot-releng.iml b/.idea/peridot-releng.iml index 586a970..351e709 100644 --- a/.idea/peridot-releng.iml +++ b/.idea/peridot-releng.iml @@ -5,6 +5,7 @@ + diff --git a/comps2peridot/README.md b/comps2peridot/README.md new file mode 100644 index 0000000..7870d70 --- /dev/null +++ b/comps2peridot/README.md @@ -0,0 +1,7 @@ +# comps2peridot +Pre-render comps for Peridot consumption + +### Usage +``` +python3 comps2peridot/comps2peridot.py --comps-path /tmp/pungi-rocky/comps.xml --variants-path /tmp/pungi-rocky/variants.xml --output-path /tmp/comps.cfg +``` diff --git a/comps2peridot/comps2peridot.py b/comps2peridot/comps2peridot.py new file mode 100644 index 0000000..7871b50 --- /dev/null +++ b/comps2peridot/comps2peridot.py @@ -0,0 +1,330 @@ +# -- peridot-releng-header-v0.1 -- +# Copyright (c) Peridot-Releng Authors. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import argparse +# noinspection PyPep8Naming +import xml.etree.ElementTree as ET +from xml.dom import minidom + +from group import Group, PackageReq, Environment, EnvGroup + + +def write_variant(groups, environments, categories, out): + root = ET.Element('comps') + for group in groups: + group_elem = ET.SubElement(root, 'group') + ET.SubElement(group_elem, 'id').text = group.id + for lang in group.name: + name = ET.SubElement(group_elem, 'name') + if lang != "": + name.set('xml:lang', lang) + name.text = group.name[lang] + for lang in group.description: + description = ET.SubElement(group_elem, 'description') + if lang != "": + description.set('xml:lang', lang) + description.text = group.description[lang] + ET.SubElement(group_elem, 'default').text = str(group.default).lower() + ET.SubElement(group_elem, 'uservisible').text = str( + group.user_visible).lower() + package_list = ET.SubElement(group_elem, 'packagelist') + for package in group.packages: + package_elem = ET.SubElement(package_list, 'packagereq') + package_elem.set('type', package.type) + package_elem.text = package.name + for environment in environments: + env_elem = ET.SubElement(root, 'environment') + ET.SubElement(env_elem, 'id').text = environment.id + for lang in environment.name: + name = ET.SubElement(env_elem, 'name') + if lang != "": + name.set('xml:lang', lang) + name.text = environment.name[lang] + for lang in environment.description: + description = ET.SubElement(env_elem, 'description') + if lang != "": + description.set('xml:lang', lang) + description.text = environment.description[lang] + ET.SubElement(env_elem, 'display_order').text = str( + environment.display_order) + group_list = ET.SubElement(env_elem, 'grouplist') + for group in environment.group_list: + ET.SubElement(group_list, 'groupid').text = group.name + option_list = ET.SubElement(env_elem, 'optionlist') + for option in environment.option_list: + ET.SubElement(option_list, 'optionid').text = option.name + for category_name in categories.keys(): + category = categories[category_name] + new_group_list = [] + for group in category.group_list: + for ggroup in groups: + if ggroup.id == group.name: + new_group_list.append(group) + break + if len(new_group_list) == 0: + continue + category_elem = ET.SubElement(root, 'category') + ET.SubElement(category_elem, 'id').text = category_name + for lang in category.name: + name = ET.SubElement(category_elem, 'name') + if lang != "": + name.set('xml:lang', lang) + name.text = category.name[lang] + for lang in category.description: + description = ET.SubElement(category_elem, 'description') + if lang != "": + description.set('xml:lang', lang) + description.text = category.description[lang] + ET.SubElement(category_elem, 'display_order').text = str( + category.display_order) + group_list = ET.SubElement(category_elem, 'grouplist') + for group in new_group_list: + ET.SubElement(group_list, 'groupid').text = group.name + ET.ElementTree(root).write(out, encoding='utf-8', xml_declaration=False) + + with open(out, 'r') as f: + data = f.read() + with open(out, 'w') as f: + f.writelines(""" + +""" + minidom.parseString(data).toprettyxml(indent=" ").replace('\n', '')) + + +def main(comps_path: str, variants_path: str, output_path: str): + default_arches = ['x86_64', 'aarch64', 'ppc64le', 's390x'] + variants = {} + environments = {} + categories = {} + + tree = ET.parse(comps_path) + root = tree.getroot() + for gchild in root: + if gchild.tag == 'group': + group_name = {} + group_desc = {} + group_id = '' + is_default = False + is_visible = False + variant = '' + package_list_xml = None + if 'variant' in gchild.attrib: + variant = gchild.attrib['variant'] + if 'arch' in gchild.attrib: + arches = gchild.attrib['arch'].split(',') + else: + arches = default_arches + for gattr in gchild: + if gattr.tag == 'id': + group_id = gattr.text + elif gattr.tag == 'name': + if '{http://www.w3.org/XML/1998/namespace}lang' in gattr.attrib: + group_name[gattr.attrib[ + '{http://www.w3.org/XML/1998/namespace}lang']] = gattr.text + else: + group_name[""] = gattr.text + elif gattr.tag == 'description': + if '{http://www.w3.org/XML/1998/namespace}lang' in gattr.attrib: + group_desc[gattr.attrib[ + '{http://www.w3.org/XML/1998/namespace}lang']] = gattr.text + else: + group_desc[""] = gattr.text + elif gattr.tag == 'default': + is_default = gattr.text == 'true' + elif gattr.tag == 'uservisible': + is_visible = gattr.text == 'true' + elif gattr.tag == 'packagelist': + package_list_xml = gattr + package_list = {} + if variant != '': + package_list[variant] = {} + for reqxml in package_list_xml: + req_variant = variant + req_type = 'default' + if 'variant' in reqxml.attrib: + req_variant = reqxml.attrib['variant'] + if 'type' in reqxml.attrib: + req_type = reqxml.attrib['type'] + if 'arch' in reqxml.attrib: + req_arches = reqxml.attrib['arch'].split(',') + else: + req_arches = arches + if req_variant not in package_list: + package_list[req_variant] = {} + for arch in req_arches: + if arch not in package_list[req_variant]: + package_list[req_variant][arch] = [] + package_list[req_variant][arch].append( + PackageReq(reqxml.text, req_type, req_arches)) + for variant in package_list: + if variant not in variants: + variants[variant] = {} + for arch in arches: + if arch not in package_list[variant]: + package_list[variant][arch] = [] + if group_id not in variants[variant]: + variants[variant][group_id] = {} + variants[variant][group_id][arch] = Group(group_id, + group_name, + group_desc, + is_default, + is_visible, + package_list[ + variant][ + arch]) + elif gchild.tag == 'environment' or gchild.tag == 'category': + env_name = {} + env_desc = {} + env_id = '' + display_order = 0 + group_list = [] + option_list = [] + for gattr in gchild: + if gattr.tag == 'id': + env_id = gattr.text + elif gattr.tag == 'name': + if '{http://www.w3.org/XML/1998/namespace}lang' in gattr.attrib: + env_name[gattr.attrib[ + '{http://www.w3.org/XML/1998/namespace}lang']] = gattr.text + else: + env_name[""] = gattr.text + elif gattr.tag == 'description': + if '{http://www.w3.org/XML/1998/namespace}lang' in gattr.attrib: + env_desc[gattr.attrib[ + '{http://www.w3.org/XML/1998/namespace}lang']] = gattr.text + else: + env_desc[""] = gattr.text + elif gattr.tag == 'display_order': + display_order = gattr.text + elif gattr.tag == 'grouplist': + for group in gattr: + if 'arch' in group.attrib: + arches = group.attrib['arch'].split(',') + else: + arches = default_arches + group_list.append(EnvGroup(group.text, arches)) + elif gattr.tag == 'optionlist': + for group in gattr: + if 'arch' in group.attrib: + arches = group.attrib['arch'].split(',') + else: + arches = default_arches + option_list.append(EnvGroup(group.text, arches)) + new_env = Environment(env_id, env_name, env_desc, display_order, + group_list, option_list) + dictmap = categories + if gchild.tag == 'environment': + dictmap = environments + if 'arch' in gchild.attrib: + arches = gchild.attrib['arch'].split(',') + else: + arches = default_arches + for arch in arches: + if arch not in dictmap: + dictmap[arch] = {} + dictmap[arch][env_id] = new_env + + environment_id_index = {} + for arch in environments.keys(): + for env in environments[arch].values(): + if env.id not in environment_id_index: + environment_id_index[env.id] = {} + environment_id_index[env.id][arch] = env + + variant_arch_index = {} + environment_arch_index = {} + pungi_variants_tree = ET.parse(variants_path).getroot() + for pungi_variant in pungi_variants_tree: + if pungi_variant.tag == 'variant': + if pungi_variant.attrib['type'] != 'variant': + continue + arches = [] + groups = {} + n_environments = {} + variant_id = pungi_variant.attrib['id'] + for child in pungi_variant: + if child.tag == 'arches': + for arch in child: + arches.append(arch.text) + elif child.tag == 'groups': + for group in child: + groupbase = variants[""] + if variant_id in variants: + groupbase = variants[variant_id] + if group.text not in groupbase: + continue + groupind = groupbase[group.text] + for arch_group in groupind.keys(): + if arch_group not in groups: + groups[arch_group] = [] + groups[arch_group].append(groupind[arch_group]) + elif child.tag == 'environments': + for environment in child: + envind = environment_id_index[environment.text] + for arch_environment in envind.keys(): + if arch_environment not in n_environments: + n_environments[arch_environment] = [] + n_environments[arch_environment].append( + envind[arch_environment]) + for arch in arches: + if arch in groups: + if arch not in variant_arch_index: + variant_arch_index[arch] = {} + if variant_id not in variant_arch_index[arch]: + variant_arch_index[arch][variant_id] = [] + variant_arch_index[arch][variant_id].extend(groups[arch]) + if arch in n_environments: + if arch not in environment_arch_index: + environment_arch_index[arch] = {} + if variant_id not in environment_arch_index[arch]: + environment_arch_index[arch][variant_id] = [] + environment_arch_index[arch][variant_id].extend( + n_environments[arch]) + + for arch in variant_arch_index.keys(): + for variant in variant_arch_index[arch].keys(): + write_variant(variant_arch_index[arch][variant] if variant in + variant_arch_index[ + arch] else [], + environment_arch_index[arch][variant] if variant in + environment_arch_index[ + arch] else [], + categories[arch].copy(), + f'{output_path}/{variant}-{arch}.xml') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Convert comps to Peridot compatible configuration.') + parser.add_argument('--comps-path', type=str, required=True) + parser.add_argument('--variants-path', type=str, required=True) + parser.add_argument('--output-path', type=str, default=".") + args = parser.parse_args() + main(args.comps_path, args.variants_path, args.output_path) diff --git a/comps2peridot/group.py b/comps2peridot/group.py new file mode 100644 index 0000000..ed66372 --- /dev/null +++ b/comps2peridot/group.py @@ -0,0 +1,63 @@ +# -- peridot-releng-header-v0.1 -- +# Copyright (c) Peridot-Releng Authors. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from dataclasses import dataclass + + +@dataclass +class PackageReq: + name: str + type: str + arches: list[str] + + +@dataclass +class Group: + id: str + name: dict[str, str] + description: dict[str, str] + default: bool + user_visible: bool + packages: list[PackageReq] + + +@dataclass +class EnvGroup: + name: str + arch: list[str] + + +@dataclass +class Environment: + id: str + name: dict[str, str] + description: dict[str, str] + display_order: int + group_list: list[EnvGroup] + option_list: list[EnvGroup] diff --git a/scripts/common.py b/scripts/common.py new file mode 100644 index 0000000..00fb196 --- /dev/null +++ b/scripts/common.py @@ -0,0 +1,42 @@ +# -- peridot-releng-header-v0.1 -- +# Copyright (c) Peridot-Releng Authors. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +BASE_URL = 'https://peridot-api.build.resf.org/v1' +PROJECT_ID_PROD = '55b17281-bc54-4929-8aca-a8a11d628738' + + +def construct_url(path, project_id=PROJECT_ID_PROD): + return f'{BASE_URL}/projects/{project_id}{path}' + + +def build_batches_url(batch_type, task_id, page, status, + project_id=PROJECT_ID_PROD): + return construct_url( + f'/{batch_type}_batches/{task_id}?page={page}&limit=100&filter.status={status}', + project_id) diff --git a/scripts/create-batch-task-list.py b/scripts/create-batch-task-list.py index 39cb5dc..c14ae3d 100644 --- a/scripts/create-batch-task-list.py +++ b/scripts/create-batch-task-list.py @@ -33,10 +33,11 @@ import sys import requests import json +from common import build_batches_url + def get_batch(batch_type, task_id, status, page): - r = requests.get( - f'https://peridot.pdot-dev.rockylinux.org/api/v1/projects/c4fa14a2-5af6-4634-bfea-847a9fd639c7/{batch_type}_batches/{task_id}?page={page}&limit=100&filter.status={status}') + r = requests.get(build_batches_url(batch_type, task_id, page, status)) return r.json()[f'{batch_type}s'] @@ -56,7 +57,6 @@ if __name__ == '__main__': task_id = sys.argv[2] batch_items = process_batch(batch_type, task_id, 4) - # batch_items.extend(process_batch(batch_type, task_id, 5)) req = {} key = f'{batch_type}s' diff --git a/scripts/create-no-build-batch.py b/scripts/create-no-build-batch.py new file mode 100644 index 0000000..76ce169 --- /dev/null +++ b/scripts/create-no-build-batch.py @@ -0,0 +1,70 @@ +# -- peridot-releng-header-v0.1 -- +# Copyright (c) Peridot-Releng Authors. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import sys +import requests +import json + +from itertools import islice + +from common import construct_url + + +def chunks(lst, n): + for i in range(0, len(lst), n): + yield lst[i:i + n] + + +def get_packages(page): + r = requests.get( + construct_url(f'/packages?limit=100&page={page}&filters.no_builds=1')) + return r.json()['packages'] + + +def process_packages(): + ret = [] + page = 0 + while True: + res = get_packages(page) + if len(res) == 0: + return ret + ret.extend(res) + page = page + 1 + + +if __name__ == '__main__': + batch_items = process_packages() + + builds = [] + for item in batch_items: + builds.append({ + 'package_name': item['name'] + }) + for chunk in chunks(builds, 400): + print(json.dumps({"builds": chunk}))