diff --git a/fifloader-rocky.py b/fifloader-rocky.py
new file mode 100755
index 00000000..9194f17d
--- /dev/null
+++ b/fifloader-rocky.py
@@ -0,0 +1,357 @@
+#!/usr/bin/python3
+
+# Copyright Red Hat
+#
+# This file is part of os-autoinst-distri-fedora.
+#
+# os-autoinst-distri-fedora is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation, either version 2 of
+# the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+# Author: Adam Williamson
+# Modified: Trevor Cooper
+
+"""This is an openQA template loader/converter for FIF, the Fedora Intermediate Format. It reads
+from one or more files expected to contain FIF JSON-formatted template data; read on for details
+on this format as it compares to the upstream format. It produces data in the upstream format; it
+can write this data to a JSON file and/or call the upstream loader on it directly, depending on
+the command-line arguments specified.
+
+The input data must contain definitions of Machines, Products, TestSuites, and Profiles. The input
+data *may* contain JobTemplates, but does not have to and is expected to contain none or only a few
+oddballs.
+
+The format for Machines, Products and TestSuites is based on the upstream format but with various
+quality-of-life improvements. Upstream, each of these is a list-of-dicts, each dict containing a
+'name' key. This loader expects each to be a dict-of-dicts, with the names as keys (this is both
+easier to read and easier to access). In the upstream format, each Machine, Product and TestSuite
+dict can contain an entry with the key 'settings' which defines variables. The value (for some
+reason...) is a list of dicts, each dict of the format {"key": keyname, "value": value}. This
+loader expects a more obvious and simple format where the value of the 'settings' key is simply a
+dict of keys and values.
+
+The expected format of the Profiles dict is a dict-of-dicts. For each entry, the key is a unique
+name, and the value is a dict with keys 'machine' and 'product', each value being a valid name from
+the Machines or Products dict respectively. The name of each profile can be anything as long as
+it's unique.
+
+For TestSuites, this loader then expects an additional 'profiles' key in each dict, whose value is
+a dict indicating the Profiles from which we should generate one or more job templates for that
+test suite. For each entry in the dict, the key is a profile name from the Profiles dict, and the
+value is the priority to give the generated job template.
+
+This loader will generate JobTemplates from the combination of TestSuites and Profiles. It means
+that, for instance, if you want to add a new test suite and run it on the same set of images and
+arches as several other tests are already run, you do not need to do a large amount of copying and
+pasting to create a bunch of JobTemplates that look a lot like other existing JobTemplates but with
+a different test_suite value; you can just specify an appropriate profiles dict, which is much
+shorter and easier and less error-prone. Thus specifying JobTemplates directly is not usually
+needed and is expected to be used only for some oddball case which the generation system does not
+handle.
+
+The loader will automatically set the group_name for each job template based on Fedora-specific
+logic which we previously followed manually when creating job templates (e.g. it is set to 'Fedora
+PowerPC' for compose tests run on the PowerPC arch); thus this loader is not really generic but
+specific to Fedora conventions. This could possibly be changed (e.g. by allowing the logic for
+deciding group names to be configurable) if anyone else wants to use it.
+
+Multiple input files will be combined. Mostly this involves simply updating dicts, but there is
+special handling for TestSuites to allow multiple input files to each include entries for 'the
+same' test suite, but with different profile dicts. So for instance one input file may contain a
+complete TestSuite definition, with the value of its `profiles` key as `{'foo': 10}`. Another input
+file may contain a TestSuite entry with the same key (name) as the complete definition in the other
+file, and the value as a dict with only a `profiles` key (with the value `{'bar': 20}`). This
+loader will combine those into a single complete TestSuite entry with the `profiles` value
+`{'foo': 10, 'bar': 20}`.
+"""
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+
+import jsonschema
+
+SCHEMAPATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'schemas')
+
+def schema_validate(instance, fif=True, complete=True, schemapath=SCHEMAPATH):
+ """Validate some input against one of our JSON schemas. We have
+ 'complete' and 'incomplete' schemas for FIF and the upstream
+ template format. The 'complete' schemas expect the validated
+ input to contain a complete set of data (everything needed for
+ an openQA deployment to actually run tests). The 'incomplete'
+ schemas expect the validated input to contain at least *some*
+ valid data - they are intended for validating input files which
+ will be combined into 'complete' data, or which will be loaded
+ without --clean, to add to an existing configuration.
+ """
+ filename = 'openqa-'
+ if fif:
+ filename = 'fif-'
+ if complete:
+ filename += 'complete.json'
+ else:
+ filename += 'incomplete.json'
+ base_uri = "file://{0}/".format(schemapath)
+ resolver = jsonschema.RefResolver(base_uri, None)
+ schemafile = os.path.join(schemapath, filename)
+ with open(schemafile, 'r') as schemafh:
+ schema = json.load(schemafh)
+ # raises an exception if it fails
+ jsonschema.validate(instance=instance, schema=schema, resolver=resolver)
+ return True
+
+# you could refactor this just using a couple of dicts, but I don't
+# think that would really make it *better*
+# pylint:disable=too-many-locals, too-many-branches
+def merge_inputs(inputs, validate=False, clean=False):
+ """Merge multiple input files. Expects JSON file names. Optionally
+ validates the input files before merging, and the merged output.
+ Returns a 5-tuple of machines, products, profiles, testsuites and
+ jobtemplates (the first four as dicts, the fifth as a list).
+ """
+ machines = {}
+ products = {}
+ profiles = {}
+ testsuites = {}
+ jobtemplates = []
+
+ for _input in inputs:
+ try:
+ with open(_input, 'r') as inputfh:
+ data = json.load(inputfh)
+ # we're just wrapping the exception a bit, so this is fine
+ # pylint:disable=broad-except
+ except Exception as err:
+ print("Reading input file {} failed!".format(_input))
+ sys.exit(str(err))
+ # validate against incomplete schema
+ if validate:
+ schema_validate(data, fif=True, complete=False)
+
+ # simple merges for all these
+ for (datatype, tgt) in (
+ ('Machines', machines),
+ ('Products', products),
+ ('Profiles', profiles),
+ ('JobTemplates', jobtemplates),
+ ):
+ if datatype in data:
+ if datatype == 'JobTemplates':
+ tgt.extend(data[datatype])
+ else:
+ tgt.update(data[datatype])
+ # special testsuite merging as described in the docstring
+ if 'TestSuites' in data:
+ for (name, newsuite) in data['TestSuites'].items():
+ try:
+ existing = testsuites[name]
+ # combine and stash the profiles
+ existing['profiles'].update(newsuite['profiles'])
+ combinedprofiles = existing['profiles']
+ # now update the existing suite with the new one, this
+ # will overwrite the profiles
+ existing.update(newsuite)
+ # now restore the combined profiles
+ existing['profiles'] = combinedprofiles
+ except KeyError:
+ testsuites[name] = newsuite
+
+ # validate combined data, against complete schema if clean is True
+ # (as we'd expect data to be loaded with --clean to be complete),
+ # incomplete schema otherwise
+ if validate:
+ merged = {}
+ if machines:
+ merged['Machines'] = machines
+ if products:
+ merged['Products'] = products
+ if profiles:
+ merged['Profiles'] = profiles
+ if testsuites:
+ merged['TestSuites'] = testsuites
+ if jobtemplates:
+ merged['JobTemplates'] = jobtemplates
+ schema_validate(merged, fif=True, complete=clean)
+ print("Input template data is valid")
+
+ return (machines, products, profiles, testsuites, jobtemplates)
+
+def generate_job_templates(products, profiles, testsuites):
+ """Given machines, products, profiles and testsuites (after
+ merging, but still in intermediate format), generates job
+ templates and returns them as a list.
+ """
+ jobtemplates = []
+ for (name, suite) in testsuites.items():
+ if 'profiles' not in suite:
+ print("Warning: no profiles for test suite {}".format(name))
+ continue
+ for (profile, prio) in suite['profiles'].items():
+ jobtemplate = {'test_suite_name': name, 'prio': prio}
+ # x86_64 compose
+ jobtemplate['group_name'] = 'Rocky'
+ jobtemplate['machine_name'] = profiles[profile]['machine']
+ product = products[profiles[profile]['product']]
+ jobtemplate['arch'] = product['arch']
+ jobtemplate['flavor'] = product['flavor']
+ jobtemplate['distri'] = product['distri']
+ jobtemplate['version'] = product['version']
+ if jobtemplate['machine_name'] == 'ppc64le':
+ if 'updates' in product['flavor']:
+ jobtemplate['group_name'] = "Rocky PowerPC Updates"
+ else:
+ jobtemplate['group_name'] = "Rocky PowerPC"
+ elif jobtemplate['machine_name'] in ('aarch64', 'ARM'):
+ if 'updates' in product['flavor']:
+ jobtemplate['group_name'] = "Rocky AArch64 Updates"
+ else:
+ jobtemplate['group_name'] = "Rocky AArch64"
+ elif 'updates' in product['flavor']:
+ # x86_64 updates
+ jobtemplate['group_name'] = "Rocky Updates"
+ jobtemplates.append(jobtemplate)
+ return jobtemplates
+
+def reverse_qol(machines, products, testsuites):
+ """Reverse all our quality-of-life improvements in Machines,
+ Products and TestSuites. We don't do profiles as only this loader
+ uses them, upstream loader does not. We don't do jobtemplates as
+ we don't do any QOL stuff for that. Returns the same tuple it's
+ passed.
+ """
+ # first, some nested convenience functions
+ def to_list_of_dicts(datadict):
+ """Convert our nice dicts to upstream's stupid list-of-dicts-with
+ -name-keys.
+ """
+ converted = []
+ for (name, item) in datadict.items():
+ item['name'] = name
+ converted.append(item)
+ return converted
+
+ def dumb_settings(settdict):
+ """Convert our sensible settings dicts to upstream's weird-ass
+ list-of-dicts format.
+ """
+ converted = []
+ for (key, value) in settdict.items():
+ converted.append({'key': key, 'value': value})
+ return converted
+
+ # drop profiles from test suites - these are only used for job
+ # template generation and should not be in final output. if suite
+ # *only* contained profiles, drop it
+ for suite in testsuites.values():
+ del suite['profiles']
+ testsuites = {name: suite for (name, suite) in testsuites.items() if suite}
+
+ machines = to_list_of_dicts(machines)
+ products = to_list_of_dicts(products)
+ testsuites = to_list_of_dicts(testsuites)
+ for datatype in (machines, products, testsuites):
+ for item in datatype:
+ if 'settings' in item:
+ item['settings'] = dumb_settings(item['settings'])
+
+ return (machines, products, testsuites)
+
+def parse_args(args):
+ """Parse arguments with argparse."""
+ parser = argparse.ArgumentParser(description=(
+ "Alternative openQA template loader/generator, using a more "
+ "convenient input format. See docstring for details. "))
+ parser.add_argument(
+ '-l', '--load', help="Load the generated templates into openQA.",
+ action='store_true')
+ parser.add_argument(
+ '--loader', help="Loader to use with --load",
+ default="/usr/share/openqa/script/load_templates")
+ parser.add_argument(
+ '-w', '--write', help="Write the generated templates in JSON "
+ "format.", action='store_true')
+ parser.add_argument(
+ '--filename', help="Filename to write with --write",
+ default="generated.json")
+ parser.add_argument(
+ '--host', help="If specified with --load, gives a host "
+ "to load the templates to. Is passed unmodified to upstream "
+ "loader.")
+ parser.add_argument(
+ '-c', '--clean', help="If specified with --load, passed to "
+ "upstream loader and behaves as documented there.",
+ action='store_true')
+ parser.add_argument(
+ '-u', '--update', help="If specified with --load, passed to "
+ "upstream loader and behaves as documented there.",
+ action='store_true')
+ parser.add_argument(
+ '--no-validate', help="Do not do schema validation on input "
+ "or output data", action='store_false', dest='validate')
+ parser.add_argument(
+ 'files', help="Input JSON files", nargs='+')
+ return parser.parse_args(args)
+
+def run(args):
+ """Read in arguments and run the appropriate steps."""
+ args = parse_args(args)
+ if not args.validate and not args.write and not args.load:
+ sys.exit("--no-validate specified and neither --write nor --load specified! Doing nothing.")
+ (machines, products, profiles, testsuites, jobtemplates) = merge_inputs(
+ args.files, validate=args.validate, clean=args.clean)
+ jobtemplates.extend(generate_job_templates(products, profiles, testsuites))
+ (machines, products, testsuites) = reverse_qol(machines, products, testsuites)
+ # now produce the output in upstream-compatible format
+ out = {}
+ if jobtemplates:
+ out['JobTemplates'] = jobtemplates
+ if machines:
+ out['Machines'] = machines
+ if products:
+ out['Products'] = products
+ if testsuites:
+ out['TestSuites'] = testsuites
+ if args.validate:
+ # validate generated data against upstream schema
+ schema_validate(out, fif=False, complete=args.clean)
+ print("Generated template data is valid")
+ if args.write:
+ # write generated output to given filename
+ with open(args.filename, 'w') as outfh:
+ json.dump(out, outfh, indent=4)
+ if args.load:
+ # load generated output with given loader (defaults to
+ # /usr/share/openqa/script/load_templates)
+ loadargs = [args.loader]
+ if args.host:
+ loadargs.extend(['--host', args.host])
+ if args.clean:
+ loadargs.append('--clean')
+ if args.update:
+ loadargs.append('--update')
+ loadargs.append('-')
+ subprocess.run(loadargs, input=json.dumps(out), text=True, check=True)
+
+def main():
+ """Main loop."""
+ try:
+ run(args=sys.argv[1:])
+ except KeyboardInterrupt:
+ sys.stderr.write("Interrupted, exiting...\n")
+ sys.exit(1)
+
+if __name__ == '__main__':
+ main()
+
+# vim: set textwidth=100 ts=8 et sw=4: