From cbe776974887fb3cce350b80561ab4f07856fca2 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Wed, 18 Mar 2015 14:51:01 -0700 Subject: [PATCH] handling scheduling of jobs for multiple images This handles scheduling of jobs for more than one type of image; currently we'll run tests for Workstation live as well. It requires some cleverness to run some tests for *all* images (currently just default_boot_and_install) but run all the tests that can be run with any non-live installer image with the best image available for the compose. We introduce a special (openQA, not fedfind) 'flavor' called 'universal'; we run a couple of checks to find the best image in the compose for running the universal tests, and schedule tests for the 'universal' flavor with that image. The 'best' image is a server or 'generic' DVD if possible, and if not, a server or 'generic' boot.iso. ISO files have the compose's version identifier prepended to their names. Otherwise they retain their original names, which should usually be unique within a given compose, except for boot.iso files, which have their payload and arch added into their names to ensure they don't overwrite each other. This also adds a mechanism for TESTCASES (in conf_test_suites) to define a callback which will be called with the flavor of the image being tested; the result of the callback will be used as the 'test name' for relval result reporting purposes. This allows us to report results against the correct 'test instance' for the image being tested, for tests like Boot_default_install which have 'test instances' for each image. We can extend this general approach in future for other cases where we have multiple 'test instances' for a single test case. --- openqa_trigger/conf_test_suites.py | 40 ++++++------- openqa_trigger/openqa_trigger.py | 85 +++++++++++++++++++++------- openqa_trigger/report_job_results.py | 4 +- 3 files changed, 88 insertions(+), 41 deletions(-) diff --git a/openqa_trigger/conf_test_suites.py b/openqa_trigger/conf_test_suites.py index 851a13d..5ad64c9 100644 --- a/openqa_trigger/conf_test_suites.py +++ b/openqa_trigger/conf_test_suites.py @@ -1,10 +1,15 @@ +def default_install_cb(flavor): + """Figure out the correct test case name for a default_boot_and_ + install pass for a given flavor. + """ + (payload, imagetype) = flavor.split('_') + imagetype = imagetype.replace('boot', 'netinst') + imagetype = imagetype.replace('dvd', 'offline') + return "{0} {1}".format(payload, imagetype) + TESTCASES = { - "Server offline": { - "section": 'Default boot and install', - "env": "$RUNARCH$", - "type": "Installation", - }, - "Server netinst": { + "QA:Testcase_Boot_default_install": { + "name_cb": default_install_cb, "section": 'Default boot and install', "env": "$RUNARCH$", "type": "Installation", @@ -120,6 +125,7 @@ TESTCASES = { "type": "Installation", }, # "": { +# "name_cb": callbackfunc # optional, called with 'flavor' # "section": "", # "env": "x86", # "type": "Installation", @@ -128,16 +134,21 @@ TESTCASES = { TESTSUITES = { - "server_simple":[ - "Server netinst", + "default_install":[ + "QA:Testcase_Boot_default_install", "QA:Testcase_install_to_VirtIO", "QA:Testcase_partitioning_guided_empty", "QA:Testcase_Anaconda_User_Interface_Graphical", "QA:Testcase_Anaconda_user_creation", + ], + "package_set_minimal":[ + "QA:Testcase_partitioning_guided_empty", + "QA:Testcase_install_to_VirtIO", + "QA:Testcase_Anaconda_User_Interface_Graphical", + "QA:Testcase_Anaconda_user_creation", "QA:Testcase_Package_Sets_Minimal_Package_Install", ], "server_delete_pata":[ - "Server netinst", "QA:Testcase_install_to_PATA", "QA:Testcase_partitioning_guided_delete_all", "QA:Testcase_Anaconda_User_Interface_Graphical", @@ -145,7 +156,6 @@ TESTSUITES = { "QA:Testcase_Package_Sets_Minimal_Package_Install", ], "server_sata_multi":[ - "Server netinst", "QA:Testcase_install_to_SATA", "QA:Testcase_partitioning_guided_multi_select", "QA:Testcase_Anaconda_User_Interface_Graphical", @@ -153,7 +163,6 @@ TESTSUITES = { "QA:Testcase_Package_Sets_Minimal_Package_Install", ], "server_scsi_updates_img":[ - "Server netinst", "QA:Testcase_install_to_SCSI", "QA:Testcase_partitioning_guided_empty", "QA:Testcase_Anaconda_updates.img_via_URL", @@ -168,7 +177,6 @@ TESTSUITES = { "QA:Testcase_Kickstart_Http_Server_Ks_Cfg", ], "server_mirrorlist_graphical":[ - "Server netinst", "QA:Testcase_install_to_VirtIO", "QA:Testcase_partitioning_guided_empty", "QA:Testcase_Anaconda_User_Interface_Graphical", @@ -177,7 +185,6 @@ TESTSUITES = { "QA:Testcase_Package_Sets_Minimal_Package_Install", ], "server_repository_http_graphical":[ - "Server netinst", "QA:Testcase_install_to_VirtIO", "QA:Testcase_partitioning_guided_empty", "QA:Testcase_Anaconda_User_Interface_Graphical", @@ -186,7 +193,6 @@ TESTSUITES = { "QA:Testcase_Package_Sets_Minimal_Package_Install", ], "server_repository_http_variation":[ - "Server netinst", "QA:Testcase_install_to_VirtIO", "QA:Testcase_partitioning_guided_empty", "QA:Testcase_Anaconda_User_Interface_Graphical", @@ -195,7 +201,6 @@ TESTSUITES = { "QA:Testcase_Package_Sets_Minimal_Package_Install", ], "server_mirrorlist_http_variation":[ - "Server netinst", "QA:Testcase_install_to_VirtIO", "QA:Testcase_partitioning_guided_empty", "QA:Testcase_Anaconda_User_Interface_Graphical", @@ -204,7 +209,6 @@ TESTSUITES = { "QA:Testcase_Package_Sets_Minimal_Package_Install", ], "server_simple_encrypted": [ - "Server netinst", "QA:Testcase_install_to_VirtIO", "QA:Testcase_partitioning_guided_empty", "QA:Testcase_Anaconda_User_Interface_Graphical", @@ -213,7 +217,6 @@ TESTSUITES = { "QA:Testcase_partitioning_guided_encrypted", ], "server_delete_partial": [ - "Server netinst", "QA:Testcase_install_to_VirtIO", "QA:Testcase_partitioning_guided_delete_partial", "QA:Testcase_Anaconda_User_Interface_Graphical", @@ -221,7 +224,6 @@ TESTSUITES = { "QA:Testcase_Package_Sets_Minimal_Package_Install", ], "server_simple_free_space": [ - "Server netinst", "QA:Testcase_install_to_VirtIO", "QA:Testcase_partitioning_guided_free_space", "QA:Testcase_Anaconda_User_Interface_Graphical", @@ -229,7 +231,6 @@ TESTSUITES = { "QA:Testcase_Package_Sets_Minimal_Package_Install", ], "server_multi_empty": [ - "Server netinst", "QA:Testcase_install_to_VirtIO", "QA:Testcase_partitioning_guided_multi_empty_all", "QA:Testcase_Anaconda_User_Interface_Graphical", @@ -237,7 +238,6 @@ TESTSUITES = { "QA:Testcase_Package_Sets_Minimal_Package_Install", ], "server_software_raid": [ - "Server netinst", "QA:Testcase_install_to_VirtIO", "QA:Testcase_Partitioning_On_Software_RAID", "QA:Testcase_Anaconda_User_Interface_Graphical", diff --git a/openqa_trigger/openqa_trigger.py b/openqa_trigger/openqa_trigger.py index cce5dad..6f18f8c 100755 --- a/openqa_trigger/openqa_trigger.py +++ b/openqa_trigger/openqa_trigger.py @@ -20,7 +20,7 @@ from report_job_results import report_results PERSISTENT = "/var/tmp/openqa_watcher.json" ISO_PATH = "/var/lib/openqa/factory/iso/" -RUN_COMMAND = "/var/lib/openqa/script/client isos post ISO=%s DISTRI=fedora VERSION=rawhide FLAVOR=server ARCH=%s BUILD=%s" +RUN_COMMAND = "/var/lib/openqa/script/client isos post ISO=%s DISTRI=fedora VERSION=rawhide FLAVOR=%s ARCH=%s BUILD=%s" VERSIONS = ['i386', 'x86_64'] # read last tested version from file @@ -39,20 +39,24 @@ def read_last(): return result, json_parsed def download_image(image): - """Download a given image with a name that should be unique for - this event and arch (until we start testing different images - for the same event and arch). Returns the filename of the image - (not the path). + """Download a given image with a name that should be unique. + Returns the filename of the image (not the path). """ - isoname = "{0}_{1}.iso".format(image.version.replace(' ', '_'), image.arch) + ver = image.version.replace(' ', '_') + if image.imagetype == 'boot': + isoname = "{0}_{1}_{2}_boot.iso".format(ver, image.payload, image.arch) + else: + isoname = "{0}_{1}".format(ver, image.filename) filename = os.path.join(ISO_PATH, isoname) if not os.path.isfile(filename): + print("Downloading {0} ({1}) to {2}...".format( + image.url, image.desc, filename)) # Icky hack around a urlgrabber bug: # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=715416 urlgrabber.urlgrab(image.url.replace('https', 'http'), filename) return isoname -def run_openqa_jobs(isoname, arch, image_version): +def run_openqa_jobs(isoname, flavor, arch, build): """# run OpenQA 'isos' job on selected isoname, with given arch and a version string. **NOTE**: the version passed to OpenQA as BUILD and is parsed back into the 'relval report-auto' arguments @@ -61,7 +65,7 @@ def run_openqa_jobs(isoname, arch, image_version): will be passed as --release, --compose and --milestone. Returns list of job IDs. """ - command = RUN_COMMAND % (isoname, arch, image_version) + command = RUN_COMMAND % (isoname, flavor, arch, build) # starts OpenQA jobs output = subprocess.check_output(command.split()) @@ -113,23 +117,64 @@ def jobs_from_fedfind(ff_release, arches=VERSIONS): jobs on them. arches is an iterable of arches to run on, if not specified, we'll use our constant. """ - # Find boot.iso images for our arches; third query is a bit of a - # bodge till I know what 22 TCs/RCs will actually look like, - # ideally we want a query that will reliably return one image per - # arch without us having to filter further, but we can always just - # take the first image for each arch if necessary + # Find currently-testable images for our arches. jobs = [] queries = ( - fedfind.release.Query('imagetype', ('boot',)), + fedfind.release.Query('imagetype', ('boot', 'live')), fedfind.release.Query('arch', arches), - fedfind.release.Query('payload', ('server', 'generic'))) + fedfind.release.Query('payload', ('server', 'generic', 'workstation'))) + images = ff_release.find_images(queries) - for image in ff_release.find_images(queries): - print("{0} {1}".format(image.url, image.desc)) + # Now schedule jobs. First, let's get the BUILD value for openQA. + build = '_'.join( + (ff_release.release, ff_release.milestone, ff_release.compose)) + + # Next let's schedule the 'universal' tests. + # We have different images in different composes: nightlies only + # have a generic boot.iso, TC/RC builds have Server netinst/boot + # and DVD. We always want to run *some* tests - + # default_boot_and_install at least - for all images we find, then + # we want to run all the tests that are not image-dependent on + # just one image. So we have a special 'universal' flavor and + # product in openQA; all the image-independent test suites run for + # that product. Here, we find the 'best' image we can for the + # compose we're running on (a DVD if possible, a boot.iso or + # netinst if not), and schedule the 'universal' jobs on that + # image. + for arch in arches: + okimgs = (img for img in images if img.arch == arch and + any(img.imagetype == okt + for okt in ('dvd', 'boot', 'netinst'))) + bestscore = 0 + bestimg = None + for img in okimgs: + if img.imagetype == 'dvd': + score = 10 + else: + score = 1 + if img.payload == 'generic': + score += 5 + elif img.payload == 'server': + score += 3 + elif img.payload == 'workstation': + score += 1 + if score > bestscore: + bestimg = img + bestscore = score + if not bestimg: + print("No universal tests image found for {0)!".format(arch)) + continue + print("Running universal tests for {0} with {1}!".format( + arch, bestimg.desc)) + isoname = download_image(bestimg) + job_ids = run_openqa_jobs(isoname, 'universal', arch, build) + jobs.extend(job_ids) + + # Now schedule per-image jobs. + for image in images: isoname = download_image(image) - version = '_'.join( - (ff_release.release, ff_release.milestone, ff_release.compose)) - job_ids = run_openqa_jobs(isoname, image.arch, version) + flavor = '_'.join((image.payload, image.imagetype)) + job_ids = run_openqa_jobs(isoname, flavor, image.arch, build) jobs.extend(job_ids) return jobs diff --git a/openqa_trigger/report_job_results.py b/openqa_trigger/report_job_results.py index edce03b..ef58fd7 100644 --- a/openqa_trigger/report_job_results.py +++ b/openqa_trigger/report_job_results.py @@ -45,7 +45,7 @@ def get_relval_commands(passed_testcases): commands = [] for key in passed_testcases: cmd_ = relval_template - version, _, build, arch = key + version, flavor, build, arch = key cmd_ += ' --release "%s" --milestone "%s" --compose "%s"' % tuple(build.split('_')) for tc_name in passed_testcases[key]: @@ -53,6 +53,8 @@ def get_relval_commands(passed_testcases): tc_env = arch if testcase['env'] == '$RUNARCH$' else testcase['env'] tc_type = testcase['type'] tc_section = testcase['section'] + if 'name_cb' in testcase: + tc_name = testcase['name_cb'](flavor) commands.append('%s --environment "%s" --testtype "%s" --section "%s" --testcase "%s" pass' % (cmd_, tc_env, tc_type, tc_section, tc_name))