report results using wikitcms directly, not via relval

Also refactor the result submission a bit to share the code
between CLI and module modes. This requires python-wikitcms
1.11+.

This should be substantially more efficient - it should make
only two wiki roundtrips per result page, and only init and
login to the wiki once. Going via relval requires a wiki init
and then two roundtrips for *each result*.

https://phab.qadevel.cloud.fedoraproject.org/D316

openqa_trigger/report_job_results.py

@@ -1,9 +1,11 @@
 import requests
 import argparse
-import os
+import sys
 import time
 import conf_test_suites
 
+from operator import attrgetter
+from wikitcms.wiki import Wiki, ResTuple
 
 API_ROOT = "http://localhost/api/v1"
 SLEEPTIME = 60
@@ -27,47 +29,53 @@ def get_passed_testcases(job_ids):
         if running_jobs:
            time.sleep(SLEEPTIME)
 
-    passed_testcases = {} # key = VERSION_BUILD_ARCH
+    passed_testcases = set()
     for job_id in job_ids:
         job = finished_jobs[job_id]
         if job['result'] =='passed':
-            key = (job['settings']['VERSION'], job['settings']['FLAVOR'], job['settings'].get('BUILD', None), job['settings']['ARCH'])
+            (release, milestone, compose) = job['settings']['BUILD'].split('_')
-            passed_testcases.setdefault(key, [])
+            testsuite = job['settings']['TEST']
-            passed_testcases[key].extend(conf_test_suites.TESTSUITES[job['settings']['TEST']])
+            arch = job['settings']['ARCH']
+            flavor = job['settings']['FLAVOR']
 
-    for key, value in passed_testcases.iteritems():
+            for testcase in conf_test_suites.TESTSUITES[testsuite]:
-        passed_testcases[key] = sorted(list(set(value)))
+                # each 'testsuite' is a list using testcase names to indicate which Wikitcms tests have
-    return passed_testcases
+                # passed if this job passes. Each testcase name is the name of a dict in the TESTCASES
+                # dict-of-dicts which more precisely identifies the 'test instance' (when there is more
+                # than one for a testcase) and environment for which the result should be filed.
+                uniqueres = conf_test_suites.TESTCASES[testcase]
+                testname = ''
+                if 'name_cb' in uniqueres:
+                    testname = uniqueres['name_cb'](flavor)
+                env = arch if uniqueres['env'] == '$RUNARCH$' else uniqueres['env']
+                result = ResTuple(
+                    testtype=uniqueres['type'], release=release, milestone=milestone, compose=compose,
+                    testcase=testcase, section=uniqueres['section'], testname=testname, env=env, status='pass')
+                passed_testcases.add(result)
 
+    return sorted(list(passed_testcases), key=attrgetter('testcase'))
 
-def get_relval_commands(passed_testcases):
+def report_results(job_ids, printcases=False, report=True):
-    relval_template = "relval report-auto"
+    passed_testcases = get_passed_testcases(job_ids)
-    commands = []
+    if printcases:
-    for key in passed_testcases:
+        for restup in passed_testcases:
-        cmd_ = relval_template
+            print(restup)
-        version, flavor, build, arch = key
-        cmd_ += ' --release "%s" --milestone "%s" --compose "%s"' % tuple(build.split('_'))
 
-        for tc_name in passed_testcases[key]:
+    if report:
-            testcase = conf_test_suites.TESTCASES[tc_name]
+        print "Reporting test passes:"
-            tc_env = arch if testcase['env'] == '$RUNARCH$' else testcase['env']
+        wiki = Wiki()
-            tc_type = testcase['type']
+        wiki.login()
-            tc_section = testcase['section']
+        if not wiki.logged_in:
-            if 'name_cb' in testcase:
+            sys.exit("Could not log in to wiki!")
-                tc_name = testcase['name_cb'](flavor)
 
-            commands.append('%s --environment "%s" --testtype "%s" --section "%s" --testcase "%s" pass' % (cmd_, tc_env, tc_type,  tc_section, tc_name))
+        # Submit the results
-
+        (insuffs, dupes) = wiki.report_validation_results(passed_testcases)
-    return commands
+        for dupe in dupes:
-
+            tmpl = "Already reported result for test {0}, env {1}! Will not report dupe."
-
+            print(tmpl.format(dupe.testcase, dupe.env))
-def report_results(job_ids):
-    commands = get_relval_commands(get_passed_testcases(job_ids))
-    print "Running relval commands:"
-    for command in commands:
-        print command
-        os.system(command)
 
+    else:
+        print "\n\n### No reporting is done! ###\n\n"
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Evaluate per-testcase results from OpenQA job runs")
@@ -75,17 +83,4 @@ if __name__ == "__main__":
     parser.add_argument('--report', default=False, action='store_true')
 
     args = parser.parse_args()
-
+    report_results(args.jobs, printcases=True, report=args.report)
-    passed_testcases = get_passed_testcases(args.jobs)
-    commands = get_relval_commands(passed_testcases)
-
-    import pprint
-    pprint.pprint(passed_testcases)
-    if not args.report:
-        print "\n\n### No reporting is done! ###\n\n"
-        pprint.pprint(commands)
-    else:
-        for command in commands:
-            print command
-            os.system(command)
-