add desktop_terminal test, refactor test loading a bit

Summary:
I really just want to add the desktop_terminal test, but I think
this refactor is in order now. It splits up loading of the
various test phases (much as SUSE do it) and allows us to run
the post-install tests without the install tests, for e.g. I
tweaked things to allow the upgrade tests to use the existing
_wait_login tests for final login and combine the two upgrade
postinstall tests into one simple one.

This comes with a bit of a behaviour change to make graphical
wait login behave the same as console wait login: it will log
in unless USER_LOGIN is set to 'false'. Previously it only
logged in if both USER_LOGIN and USER_PASSWORD were set, which
I don't think ever happened in a graphical test, so we never
actually did a graphical login. The intent here is we should do
a login on the default_install tests. That's going a bit beyond
the test case, but it seems like a reasonable thing to test. We
can set USER_LOGIN to false if we don't want to do it.

Test Plan:
Do a full test run, make sure the new tests work and
no old tests break.

Reviewers: jskladan, garretraziel

Reviewed By: garretraziel

Subscribers: tflink

Differential Revision: https://phab.qadevel.cloud.fedoraproject.org/D839

lib/installedtest.pm

@@ -44,6 +44,18 @@ sub check_release {
     validate_script_output $check_command, sub { $_ =~ m/REDHAT_SUPPORT_PRODUCT_VERSION=$release/ };
 }
 
+sub menu_launch_type {
+    my $self = shift;
+    my $app = shift;
+    # super does not work on KDE, because fml
+    send_key 'alt-f1';
+    # srsly KDE y u so slo
+    wait_still_screen 3;
+    type_string "$app";
+    wait_still_screen 3;
+    send_key 'ret';
+}
+
 1;
 
 # vim: set sw=4 et:

main.pm

@@ -24,6 +24,9 @@ my $distri = testapi::get_var("CASEDIR") . '/lib/fedoradistribution.pm';
 require $distri;
 testapi::set_distribution(fedoradistribution->new());
 
+## UTILITY SUBROUTINES
+
+
 # Stolen from openSUSE.
 sub unregister_needle_tags($) {
     my $tag = shift;
@@ -90,29 +93,20 @@ sub cleanup_needles() {
 }
 $needle::cleanuphandler = \&cleanup_needles;
 
-if (get_var('LIVE')) {
+## TEST LOADING SUBROUTINES
-    # No package set selection for lives.
-    set_var('PACKAGE_SET', "default");
-}
 
-# if user set ENTRYPOINT, run required test directly
+
-# (good for tests where it doesn't make sense to use _boot_to_anaconda, _software_selection etc.)
+sub load_upgrade_tests() {
-if (get_var("ENTRYPOINT"))
-{
-    autotest::loadtest "tests/".get_var("ENTRYPOINT").".pm";
-}
-elsif (get_var("UPGRADE"))
-{
     # all upgrade tests consist of: preinstall phase (where packages are upgraded and
     # dnf-plugin-system-upgrade is installed), run phase (where upgrade is run) and postinstall
     # phase (where is checked if fedora was upgraded successfully)
     autotest::loadtest "tests/upgrade_preinstall.pm";
     autotest::loadtest "tests/upgrade_run.pm";
-    # UPGRADE can be set to "minimal", "encrypted", "desktop"...
+    # set postinstall test
-    autotest::loadtest "tests/upgrade_postinstall_".get_var("UPGRADE").".pm";
+    set_var('POSTINSTALL', "upgrade" );
 }
-else
+
-{
+sub load_install_tests() {
     # normal installation test consists of several phases, from which some of them are
     # loaded automatically and others are loaded based on what env variables are set
 
@@ -122,10 +116,8 @@ else
     # boot phase is loaded automatically every time
     autotest::loadtest "tests/_boot_to_anaconda.pm";
 
-    # with kickstart tests, booting to anaconda is the only thing required (kickstart file handles
+    # if this is a kickstart install, that's all folks
-    # everything else)
+    return if (get_var("KICKSTART"));
-    unless (get_var("KICKSTART"))
-    {
 
     ## Installation source
     if (get_var('MIRRORLIST_GRAPHICAL') || get_var("REPOSITORY_GRAPHICAL")){
@@ -135,6 +127,11 @@ else
         autotest::loadtest "tests/install_source_variation.pm";
     }
 
+    if (get_var('LIVE')) {
+        # No package set selection for lives.
+        set_var('PACKAGE_SET', "default");
+    }
+
     ## Select package set. Minimal is the default, if 'default' is specified, skip selection.
     autotest::loadtest "tests/_software_selection.pm";
 
@@ -163,6 +160,7 @@ else
     autotest::loadtest "tests/_do_install_and_reboot.pm";
 }
 
+sub load_postinstall_tests() {
     # Unlock encrypted storage volumes, if necessary. The test name here
     # follows the 'storage post-install' convention, but must be run earlier.
     if (get_var("ENCRYPT_PASSWORD")) {
@@ -204,8 +202,27 @@ else
     }
 }
 
+## LOADING STARTS HERE
 
 
+# if user set ENTRYPOINT, run required test directly
+# (good for tests where it doesn't make sense to use _boot_to_anaconda, _software_selection etc.)
+if (get_var("ENTRYPOINT")) {
+    autotest::loadtest "tests/".get_var("ENTRYPOINT").".pm";
+}
+elsif (get_var("UPGRADE")) {
+    load_upgrade_tests;
+}
+elsif (!get_var("START_AFTER_TEST")) {
+    # for now we can assume START_AFTER_TEST means this test picks up
+    # after an install, so we skip to post-install
+    load_install_tests;
+}
+
+if (!get_var("ENTRYPOINT")) {
+    load_postinstall_tests;
+}
+
 1;
 
 # vim: set sw=4 et:

needles/getting_started.json

@@ -0,0 +1,15 @@
+{
+  "area": [
+    {
+      "xpos": 454,
+      "ypos": 34,
+      "width": 115,
+      "height": 17,
+      "type": "match"
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "getting_started"
+  ]
+}

needles/next_button-gtk.json

@@ -0,0 +1,15 @@
+{
+  "area": [
+    {
+      "xpos": 958,
+      "ypos": 38,
+      "width": 55,
+      "height": 25,
+      "type": "match"
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "next_button"
+  ]
+}

needles/skip_button-gtk.json

@@ -0,0 +1,15 @@
+{
+  "area": [
+    {
+      "xpos": 965,
+      "ypos": 41,
+      "width": 44,
+      "height": 21,
+      "type": "match"
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "skip_button"
+  ]
+}

templates

@@ -248,6 +248,28 @@
                                     },
                       test_suite => { name => "base_service_manipulation" },
                     },
+                    {
+                      machine    => { name => "64bit" },
+                      prio       => 30,
+                      product    => {
+                                      arch    => "x86_64",
+                                      distri  => "fedora",
+                                      flavor  => "Workstation-live-iso",
+                                      version => "*",
+                                    },
+                      test_suite => { name => "desktop_terminal" },
+                    },
+                    {
+                      machine    => { name => "64bit" },
+                      prio       => 32,
+                      product    => {
+                                      arch    => "x86_64",
+                                      distri  => "fedora",
+                                      flavor  => "KDE-live-iso",
+                                      version => "*",
+                                    },
+                      test_suite => { name => "desktop_terminal" },
+                    },
                     {
                       machine    => { name => "64bit" },
                       prio       => 30,
@@ -1293,18 +1315,21 @@
                       name => "upgrade_minimal_64bit",
                       settings => [
                         { key => "ROOT_PASSWORD", value => "weakpassword" },
+                        { key => "USER_LOGIN", value => "false" },
                         { key => "BOOTFROM", value => "c" },
                         { key => "HDD_1", value => "disk_f%CURRREL%_minimal_x86_64.img" },
-                        { key => "UPGRADE", value => "minimal" },
+                        { key => "UPGRADE", value => "1" },
                       ],
                     },
                     {
                       name => "upgrade_desktop_64bit",
                       settings => [
                         { key => "ROOT_PASSWORD", value => "weakpassword" },
+                        { key => "USER_LOGIN", value => "test" },
+                        { key => "USER_PASSWORD", value => "weakpassword" },
                         { key => "BOOTFROM", value => "c" },
-                        { key => "HDD_1", value => "disk_f%CURRREL%_desktop_x86_64.img" },
+                        { key => "HDD_1", value => "disk_f%CURRREL%_desktop_2_x86_64.img" },
-                        { key => "UPGRADE", value => "desktop" },
+                        { key => "UPGRADE", value => "1" },
                         { key => "DESKTOP", value => "gnome" },
                       ],
                     },
@@ -1312,18 +1337,22 @@
                       name => "upgrade_server_64bit",
                       settings => [
                         { key => "ROOT_PASSWORD", value => "weakpassword" },
+                        { key => "USER_LOGIN", value => "test" },
+                        { key => "USER_PASSWORD", value => "weakpassword" },
                         { key => "BOOTFROM", value => "c" },
-                        { key => "HDD_1", value => "disk_f%CURRREL%_server_x86_64.img" },
+                        { key => "HDD_1", value => "disk_f%CURRREL%_server_2_x86_64.img" },
-                        { key => "UPGRADE", value => "minimal" },
+                        { key => "UPGRADE", value => "1" },
                       ],
                     },
                     {
                       name => "upgrade_kde_64bit",
                       settings => [
                         { key => "ROOT_PASSWORD", value => "weakpassword" },
+                        { key => "USER_LOGIN", value => "test" },
+                        { key => "USER_PASSWORD", value => "weakpassword" },
                         { key => "BOOTFROM", value => "c" },
-                        { key => "HDD_1", value => "disk_f%CURRREL%_kde_x86_64.img" },
+                        { key => "HDD_1", value => "disk_f%CURRREL%_kde_2_x86_64.img" },
-                        { key => "UPGRADE", value => "desktop" },
+                        { key => "UPGRADE", value => "1" },
                         { key => "DESKTOP", value => "kde" },
                       ],
                     },
@@ -1331,37 +1360,45 @@
                       name => "upgrade_2_minimal_64bit",
                       settings => [
                         { key => "ROOT_PASSWORD", value => "weakpassword" },
+                        { key => "USER_LOGIN", value => "false" },
                         { key => "BOOTFROM", value => "c" },
                         { key => "HDD_1", value => "disk_f%PREVREL%_minimal_x86_64.img" },
-                        { key => "UPGRADE", value => "minimal" },
+                        { key => "UPGRADE", value => "1" },
                       ],
                     },
                     {
                       name => "upgrade_2_desktop_64bit",
                       settings => [
                         { key => "ROOT_PASSWORD", value => "weakpassword" },
+                        { key => "USER_LOGIN", value => "test" },
+                        { key => "USER_PASSWORD", value => "weakpassword" },
                         { key => "BOOTFROM", value => "c" },
-                        { key => "HDD_1", value => "disk_f%PREVREL%_desktop_x86_64.img" },
+                        { key => "HDD_1", value => "disk_f%PREVREL%_desktop_2_x86_64.img" },
-                        { key => "UPGRADE", value => "desktop" },
+                        { key => "UPGRADE", value => "1" },
                         { key => "DESKTOP", value => "gnome" },
+
                       ],
                     },
                     {
                       name => "upgrade_2_server_64bit",
                       settings => [
                         { key => "ROOT_PASSWORD", value => "weakpassword" },
+                        { key => "USER_LOGIN", value => "test" },
+                        { key => "USER_PASSWORD", value => "weakpassword" },
                         { key => "BOOTFROM", value => "c" },
-                        { key => "HDD_1", value => "disk_f%PREVREL%_server_x86_64.img" },
+                        { key => "HDD_1", value => "disk_f%PREVREL%_server_2_x86_64.img" },
-                        { key => "UPGRADE", value => "minimal" },
+                        { key => "UPGRADE", value => "1" },
                       ],
                     },
                     {
                       name => "upgrade_2_kde_64bit",
                       settings => [
                         { key => "ROOT_PASSWORD", value => "weakpassword" },
+                        { key => "USER_LOGIN", value => "test" },
+                        { key => "USER_PASSWORD", value => "weakpassword" },
                         { key => "BOOTFROM", value => "c" },
-                        { key => "HDD_1", value => "disk_f%PREVREL%_kde_x86_64.img" },
+                        { key => "HDD_1", value => "disk_f%PREVREL%_kde_2_x86_64.img" },
-                        { key => "UPGRADE", value => "desktop" },
+                        { key => "UPGRADE", value => "1" },
                         { key => "DESKTOP", value => "kde" },
                       ],
                     },
@@ -1369,9 +1406,11 @@
                       name => "upgrade_desktop_32bit",
                       settings => [
                         { key => "ROOT_PASSWORD", value => "weakpassword" },
+                        { key => "USER_LOGIN", value => "test" },
+                        { key => "USER_PASSWORD", value => "weakpassword" },
                         { key => "BOOTFROM", value => "c" },
-                        { key => "HDD_1", value => "disk_f%CURRREL%_desktop_i686.img" },
+                        { key => "HDD_1", value => "disk_f%CURRREL%_desktop_2_i686.img" },
-                        { key => "UPGRADE", value => "desktop" },
+                        { key => "UPGRADE", value => "1" },
                         { key => "DESKTOP", value => "gnome" },
                       ],
                     },
@@ -1379,9 +1418,11 @@
                       name => "upgrade_2_desktop_32bit",
                       settings => [
                         { key => "ROOT_PASSWORD", value => "weakpassword" },
+                        { key => "USER_LOGIN", value => "test" },
+                        { key => "USER_PASSWORD", value => "weakpassword" },
                         { key => "BOOTFROM", value => "c" },
-                        { key => "HDD_1", value => "disk_f%PREVREL%_desktop_i686.img" },
+                        { key => "HDD_1", value => "disk_f%PREVREL%_desktop_2_i686.img" },
-                        { key => "UPGRADE", value => "desktop" },
+                        { key => "UPGRADE", value => "1" },
                         { key => "DESKTOP", value => "gnome" },
                       ],
                     },
@@ -1431,6 +1472,7 @@
                       settings => [
                         { key => "DESKTOP", value => "kde" },
                         { key => "PACKAGE_SET", value => "kde" },
+                        { key => "USER_LOGIN", value => "false" },
                       ],
                     },
                     {
@@ -1508,5 +1550,14 @@
                         { key => "WORKER_CLASS", value => "tap" },
                       ],
                     },
+                    {
+                      name => "desktop_terminal",
+                      settings => [
+                        { key => "POSTINSTALL", value => "desktop_terminal" },
+                        { key => "START_AFTER_TEST", value => "install_default_upload" },
+                        { key => "BOOTFROM", value => "c" },
+                        { key => "HDD_1", value => "disk_%FLAVOR%_%MACHINE%.qcow2" },
+                      ],
+                    },
                   ],
 }

tests/_console_wait_login.pm

@@ -4,12 +4,13 @@ use testapi;
 
 sub run {
     my $self = shift;
+    # If KICKSTART is set, then the wait_time needs to consider the
+    # install time. if UPGRADE, we have to wait for the entire upgrade
+    my $wait_time = 300;
+    $wait_time = 1800 if (get_var("KICKSTART"));
+    $wait_time = 6000 if (get_var("UPGRADE"));
 
-    # If KICKSTART is set, then the wait_time needs to
+    # Wait for the text login
-    #  consider the install time
-    my $wait_time = get_var("KICKSTART") ? 1800 : 300;
-
-    # Reboot and wait for the text login
     assert_screen "text_console_login", $wait_time;
 
     # do user login unless USER_LOGIN is set to string 'false'

tests/_graphical_wait_login.pm

@@ -3,22 +3,53 @@ use strict;
 use testapi;
 
 sub run {
-    # If KICKSTART is set, then the wait_time needs to
+    # If KICKSTART is set, then the wait_time needs to consider the
-    #  consider the install time
+    # install time. if UPGRADE, we have to wait for the entire upgrade
-    my $wait_time = get_var("KICKSTART") ? 1800 : 300;
+    my $wait_time = 300;
+    $wait_time = 1800 if (get_var("KICKSTART"));
+    $wait_time = 6000 if (get_var("UPGRADE"));
 
     # Wait for the login screen
     assert_screen "graphical_login", $wait_time;
+    # handle the qemu display buffer sometimes showing the DM from the
+    # *previous* boot - https://openqa.stg.fedoraproject.org/tests/17116
+    wait_still_screen;
+    assert_screen "graphical_login";
+    # do user login unless USER_LOGIN is set to string 'false'
+    unless (get_var("USER_LOGIN") eq "false") {
+        if (get_var("DESKTOP") eq 'gnome') {
+            # we have to hit enter to get the password dialog
+            send_key "ret";
+        }
+        assert_screen "graphical_login_input";
+        type_string get_var("USER_PASSWORD", "weakpassword");
+        send_key "ret";
 
-    if (get_var("USER_LOGIN") && get_var("USER_PASSWORD")) {
+        # Handle initial-setup, for GNOME, unless START_AFTER_TEST
+        # is set in which case it will have been done already
+        if (get_var("DESKTOP") eq 'gnome' && !get_var("START_AFTER_TEST")) {
+            for my $n (1..3) {
+                # click 'Next' three times, moving the mouse to avoid
+                # highlight problems, sleeping to give it time to get
+                # to the next screen between clicks
+                mouse_set(100, 100);
+                wait_screen_change { assert_and_click "next_button", 60; };
+            }
+            # click 'Skip' one time
+            mouse_set(100,100);
+            wait_screen_change { assert_and_click "skip_button"; };
             send_key "ret";
-        type_string get_var("USER_PASSWORD");
+            # wait for the stupid 'help' screen to show and kill it
-        send_key "ret";
+            assert_screen "getting_started";
-        # Move the mouse somewhere it won't highlight the match areas
+            send_key "alt-f4";
-        mouse_set(300, 200);
+            wait_still_screen 5;
-        assert_screen "graphical_desktop_clean", 30;
         }
 
+        # Move the mouse somewhere it won't highlight the match areas
+        mouse_set(300, 200);
+        # KDE can take ages to start up
+        assert_screen "graphical_desktop_clean", 120;
+    }
 }
 
 sub test_flags {

tests/desktop_terminal_postinstall.pm

@@ -0,0 +1,30 @@
+use base "installedtest";
+use strict;
+use testapi;
+
+sub run {
+    my $self=shift;
+    assert_screen 'graphical_desktop_clean';
+    $self->menu_launch_type('terminal');
+    wait_still_screen 5;
+    # need to be root
+    my $rootpass = get_var("ROOT_PASSWORD", "weakpassword");
+    type_string "su\n";
+    wait_still_screen 3;
+    type_string "$rootpass\n";
+    wait_still_screen 3;
+    # if we can do an assert_script_run, we're at a console
+    assert_script_run 'ls';
+}
+
+sub test_flags {
+    # without anything - rollback to 'lastgood' snapshot if failed
+    # 'fatal' - whole test suite is in danger if this fails
+    # 'milestone' - after this test succeeds, update 'lastgood'
+    # 'important' - if this fails, set the overall state to 'fail'
+    return { fatal => 1 };
+}
+
+1;
+
+# vim: set sw=4 et:

tests/upgrade_postinstall.pm

@@ -5,9 +5,7 @@ use testapi;
 
 sub run {
     my $self = shift;
-
     # try to login, check whether target release is installed
-    $self->boot_to_login_screen();
     $self->root_console(tty=>3);
     $self->check_release(lc(get_var('VERSION')));
 }

tests/upgrade_postinstall_desktop.pm

@@ -1,38 +0,0 @@
-use base "installedtest";
-use strict;
-use testapi;
-
-
-sub run {
-    my $self = shift;
-    my $password = get_var("USER_PASSWORD", "weakpassword");
-
-    # wait for DM to appear
-    $self->boot_to_login_screen("graphical_login", 20);
-
-    # login as normal user
-    if (get_var("DESKTOP") eq 'gnome') {
-        send_key "ret";
-    }
-    assert_screen "graphical_login_input";
-    type_string $password;
-    send_key "ret";
-    # wait until desktop appears
-    assert_screen "graphical_desktop_clean", 60;
-    # check an upgrade actually happened (and we can log into a console)
-    $self->root_console(tty=>3);
-    $self->check_release(lc(get_var('VERSION')));
-}
-
-
-sub test_flags {
-    # without anything - rollback to 'lastgood' snapshot if failed
-    # 'fatal' - whole test suite is in danger if this fails
-    # 'milestone' - after this test succeeds, update 'lastgood'
-    # 'important' - if this fails, set the overall state to 'fail'
-    return { fatal => 1 };
-}
-
-1;
-
-# vim: set sw=4 et:

tests/upgrade_preinstall.pm

@@ -5,9 +5,9 @@ use testapi;
 sub run {
     my $self = shift;
 
-    # wait for either GDM or text login
+    # wait for either graphical or text login
-    if (get_var('UPGRADE') eq "desktop") {
+    if (get_var('DESKTOP')) {
-        $self->boot_to_login_screen("graphical_login", 30); # GDM takes time to load
+        $self->boot_to_login_screen("graphical_login", 15, 90); # DM takes time to load
     } else {
         $self->boot_to_login_screen();
     }
@@ -22,15 +22,15 @@ sub run {
 
     script_run "reboot";
 
-    if (get_var('UPGRADE') eq "desktop") {
+    if (get_var('DESKTOP')) {
-        $self->boot_to_login_screen("graphical_login", 30); # GDM takes time to load
+        $self->boot_to_login_screen("graphical_login", 15, 90); # DM takes time to load
     } else {
         $self->boot_to_login_screen();
     }
     $self->root_console(tty=>3);
 
     my $update_command = 'dnf -y --enablerepo=updates-testing install dnf-plugin-system-upgrade';
-    assert_script_run $update_command, 1800;
+    assert_script_run $update_command, 300;
 }
 
 

tests/upgrade_run.pm

@@ -16,10 +16,7 @@ sub run {
     # disable screen blanking (download can take a long time)
     script_run "setterm -blank 0";
 
-    script_run "dnf -y system-upgrade download ${args}";
+    assert_script_run "dnf -y system-upgrade download ${args}", 6000;
-
-    # wait until dnf finishes its work (screen stops moving for 30 seconds)
-    wait_still_screen 30, 6000; # TODO: shorter timeout, longer stillscreen?
 
     upload_logs "/var/log/dnf.log";
     upload_logs "/var/log/dnf.rpm.log";
@@ -27,14 +24,6 @@ sub run {
     script_run "dnf system-upgrade reboot";
     # fail immediately if we see a DNF error message
     die "DNF reported failure" if (check_screen "upgrade_fail");
-
-    # now offline upgrading starts. user doesn't have to do anything, just wait untill
-    # system reboots and login screen is shown
-    if (get_var('UPGRADE') eq "desktop") {
-        assert_screen "graphical_login", 6000;
-    } else {
-        assert_screen "text_console_login", 6000;
-    }
 }