Handle running update tests on Koji tasks

We quite often want to run the update tests on a Koji task (not
a Bodhi update) for some reason - usually to test a potential
fix for an issue, or at a maintainer's request to test a change
before it is merged upstream and officially sent out as an
update. Up till now I've always hacked up utils.pm on the
staging server by hand to do this, which is horrible. Together
with a commit to fedora_openqa, this should allow us to do it in
a nice, sane way via the CLI. It's mostly just tweaking the
"updates" repo setup in utils.pm as you'd expect, but there's a
bit of subtlety to it because of the installer tests that use
%ADVISORY% as a variable substitution in the disk image name;
you can't do something like `%ADVISORY or KOJITASK%`, sadly, so
I had to have almost-redundant variables ADVISORY, KOJITASK and
ADVISORY_OR_TASK (we could kinda just live with ADVISORY_OR_TASK
except I didn't want to drop ADVISORY as it's an unnecessary
change from previous behavior).

Signed-off-by: Adam Williamson <awilliam@redhat.com>

VARIABLES.md

@@ -87,6 +87,7 @@ These variables should be set when tests are scheduled (when running `isos post`
 | `ISO` | contains filename of ISO that is used for booting. If `ISO_URL` is not set, file must exist in /var/lib/openqa/share/factory/iso. If `ISO_URL` is set, it will be downloaded to this name instead of its original name |
 | `ISO_URL` | contains URL for ISO to boot, openQA will download it |
 | `ADVISORY` | A Bodhi update ID. If set, the 'update testing' flow will be used: post-install tests will be run with the packages from the update, starting from the stable release base disk images |
+| `KOJITASK` | A Koji task ID. If set, the modified 'update testing' flow for testing scratch builds will be used: post-install tests will be run with the packages from the update, starting from the stable release base disk images |
 | `DISTRI` | contains distribution name (should be same as in WebUI, probably `fedora`) |
 | `VERSION` | contains version of distribution |
 | `DEVELOPMENT` | set for update tests if the update is for a development release (not a stable release) |

lib/utils.pm

@@ -375,17 +375,24 @@ sub _repo_setup_updates {
         }
     }
 
-    # Set up an additional repo containing the update packages. We do
-    # this rather than simply running a one-time update because it may
-    # be the case that a package from the update isn't installed *now*
-    # but will be installed by one of the tests; by setting up a repo
-    # containing the update and enabling it here, we ensure all later
-    # 'dnf install' calls will get the packages from the update.
+    # Set up an additional repo containing the update or task packages. We do
+    # this rather than simply running a one-time update because it may be the
+    # case that a package from the update isn't installed *now* but will be
+    # installed by one of the tests; by setting up a repo containing the
+    # update and enabling it here, we ensure all later 'dnf install' calls
+    # will get the packages from the update.
     assert_script_run "mkdir -p /opt/update_repo";
     assert_script_run "cd /opt/update_repo";
     assert_script_run "dnf -y install bodhi-client git createrepo koji", 300;
     # download the packages
-    assert_script_run "bodhi updates download --updateid " . get_var("ADVISORY"), 600;
+    if (get_var("ADVISORY")) {
+        # regular update case
+        assert_script_run "bodhi updates download --updateid " . get_var("ADVISORY"), 600;
+    }
+    else {
+        # Koji task case (KOJITASK will be set)
+        assert_script_run "koji download-task --arch=" . get_var("ARCH") . " --arch=noarch " . get_var("KOJITASK"), 600;
+    }
     # for upgrade tests, we want to do the 'development' changes *after* we
     # set up the update repo. We don't do the f28 fixups as we don't have
     # f28 fedora-repos.
@@ -408,7 +415,7 @@ sub _repo_setup_updates {
     assert_script_run "createrepo .";
     # write a repo config file
     assert_script_run 'printf "[advisory]\nname=Advisory repo\nbaseurl=file:///opt/update_repo\nenabled=1\nmetadata_expire=3600\ngpgcheck=0" > /etc/yum.repos.d/advisory.repo';
-    # mark via a variable that we've set up the update repo and done
+    # mark via a variable that we've set up the update/task repo and done
     # all the logging stuff above
     set_var('_ADVISORY_REPO_DONE', '1');
     # run an update now (except for upgrade tests)
@@ -417,7 +424,7 @@ sub _repo_setup_updates {
 
 sub repo_setup {
     # Run the appropriate sub-function for the job
-    get_var("ADVISORY") ? _repo_setup_updates : _repo_setup_compose;
+    get_var("ADVISORY_OR_TASK") ? _repo_setup_updates : _repo_setup_compose;
     # This repo does not always exist for Rawhide or Branched, and
     # some things (at least realmd) try to update the repodata for
     # it even though it is disabled, and fail. At present none of the

main.pm

@@ -268,14 +268,14 @@ sub load_postinstall_tests() {
         autotest::loadtest "tests/_post_network_static.pm";
     }
 
-    # if scheduler passed an advisory, update packages from that advisory
-    # (intended for the updates testing workflow, so we install the updates
-    # to be tested). Don't do this for UPGRADE tests, as the update gets
-    # installed as part of the upgrade in that case and we don't need the
-    # extra reboot. Don't do this for INSTALL test(s); these are checking
-    # that an installer image built from the update works and do not install
-    # the update themselves.
-    if (get_var("ADVISORY") && !get_var("UPGRADE") && !get_var("INSTALL")) {
+    # if scheduler passed an advisory or task ID, update packages from that
+    # advisory or task ID (intended for the updates testing workflow, so we
+    # install the updates to be tested). Don't do this for UPGRADE tests, as
+    # the update gets installed as part of the upgrade in that case and we
+    # don't need the extra reboot. Don't do this for INSTALL test(s); these
+    # are checking that an installer image built from the update works and do
+    # not install the update themselves in this manner
+    if (get_var("ADVISORY_OR_TASK") && !get_var("UPGRADE") && !get_var("INSTALL")) {
         autotest::loadtest "tests/_advisory_update.pm";
         # now load the early boot tests again, as _advisory_update reboots
         _load_early_postinstall_tests(2);
@@ -316,11 +316,11 @@ sub load_postinstall_tests() {
         }
     }
 
-    # load the ADVISORY post-install test - this records which update
-    # packages were actually installed during the test. Don't do this
-    # for INSTALL test(s); these are checking that an installer image
+    # load the ADVISORY / KOJITASK post-install test - this records which
+    # update or task packages were actually installed during the test. Don't
+    # do this for INSTALL test(s); these are checking that an installer image
     # built from the update works and do not install the update themselves.
-    if (get_var("ADVISORY") && !get_var("INSTALL")) {
+    if (get_var("ADVISORY_OR_TASK") && !get_var("INSTALL")) {
         autotest::loadtest "tests/_advisory_post.pm";
     }
 

templates-updates

@@ -997,7 +997,7 @@
                       settings => [
                         { key => "INSTALL", value => "1" },
                         { key => "INSTALL_UNLOCK", value => "support_ready" },
-                        { key => "ISO", value => "%ADVISORY%-netinst-%ARCH%.iso" },
+                        { key => "ISO", value => "%ADVISORY_OR_TASK%-netinst-%ARCH%.iso" },
                         { key => "NICTYPE", value => "tap" },
                         { key => "PACKAGE_SET", value => "default" },
                         { key => "PARALLEL_WITH", value => "support_server" },

tests/_graphical_wait_login.pm

@@ -59,10 +59,10 @@ sub run {
 
     # Handle initial-setup, for GNOME, unless START_AFTER_TEST
     # is set in which case it will have been done already. Always
-    # do it if ADVISORY is set, as for the update testing flow,
+    # do it if ADVISORY_OR_TASK is set, as for the update testing flow,
     # START_AFTER_TEST is set but a no-op and this hasn't happened
-    if (get_var("DESKTOP") eq 'gnome' && (get_var("ADVISORY") || !get_var("START_AFTER_TEST"))) {
-        # as this test gets loaded twice on the ADVISORY flow, and
+    if (get_var("DESKTOP") eq 'gnome' && (get_var("ADVISORY_OR_TASK") || !get_var("START_AFTER_TEST"))) {
+        # as this test gets loaded twice on the ADVISORY_OR_TASK flow, and
         # we might be on the INSTALL_NO_USER flow, check whether
         # this happened already
         unless (get_var("_setup_done")) {

tests/_installer_build.pm

@@ -6,7 +6,7 @@ use utils;
 sub run {
     my $self = shift;
     my $version = get_var("VERSION");
-    my $advisory = get_var("ADVISORY");
+    my $advortask = get_var("ADVISORY_OR_TASK");
     my $arch = get_var("ARCH");
     # we need lorax from u-t for f28 atm it seems
     my $loraxcmd = "dnf -y ";
@@ -30,8 +30,8 @@ sub run {
     }
     $cmd .= " --repo=/etc/yum.repos.d/advisory.repo ./results";
     assert_script_run $cmd, 1500;
-    assert_script_run "mv results/images/boot.iso ./${advisory}-netinst-${arch}.iso";
-    upload_asset "./${advisory}-netinst-x86_64.iso";
+    assert_script_run "mv results/images/boot.iso ./${advortask}-netinst-${arch}.iso";
+    upload_asset "./${advortask}-netinst-x86_64.iso";
 }
 
 sub test_flags {

tests/_support_server.pm

@@ -34,7 +34,7 @@ sub run {
     # get the kickstart
     assert_script_run "curl -o /export/root-user-crypted-net.ks https://jskladan.fedorapeople.org/kickstarts/root-user-crypted-net.ks";
     # for update tests, set up the update repository and export it
-    if (get_var("ADVISORY")) {
+    if (get_var("ADVISORY_OR_TASK")) {
         assert_script_run "echo '/opt/update_repo 10.0.2.0/24(ro)' >> /etc/exports";
     }
     # for compose tests, we do all this stuff