add comments and documentation

Differential Revision: https://phab.qadevel.cloud.fedoraproject.org/D481

README

@@ -1 +0,0 @@
-OpenQA tests for the Fedora distribution

README.md

@@ -0,0 +1,48 @@
+OpenQA tests for the Fedora distribution
+========================================
+
+This repository contains tests and images for testing [Fedora](https://getfedora.org/) with
+[OpenQA](http://os-autoinst.github.io/openQA/). For additional tools, Installation Guide and
+Docker images, see [this repository](https://bitbucket.org/rajcze/openqa_fedora_tools).
+
+Test development
+----------------
+See official documentation [on basic concept](https://github.com/os-autoinst/openQA/blob/master/docs/GettingStarted.asciidoc), [test development (including API specification)](https://github.com/os-autoinst/openQA/blob/master/docs/WritingTests.asciidoc), [needles specification](https://github.com/os-autoinst/os-autoinst/blob/master/doc/needles.txt) and [supported variables for backend](https://github.com/os-autoinst/os-autoinst/blob/master/doc/backend_vars.asciidoc). See
+[this example repo](https://github.com/os-autoinst/os-autoinst-distri-example) on how tests should be structured.
+
+In short, since OpenQA uses only one entrypoint for all tests (`main.pm`), we have decided to utilize
+this feature and make tests modular. It means that basic passing through `main.pm` (without any variables set)
+results in most basic installation test executed. Developer can then customize it with additional variables
+(for example by setting `PACKAGE_SET=minimal` to do installation only with minimal package set).
+
+Make your test modular, so that it utilizes `_boot_to_anaconda()` and `_do_install_and_reboot()`
+tests (that are loaded automatically). Break your test into smaller parts, each dealing with one
+specific feature (e. g. partitioning, user creation...) and add their loading into `main.pm` based
+on reasonable variable setting (so they can be used in other tests also).
+
+### Test inheritance
+Your test can inherit from `basetest`, `fedorabase`, `installedtest` or `anacondatest`.
+
+* `basetest` is basic class provided by os-autoinst - it has empty `post_fail_hook()` and doesn't set any flags.
+* `fedorabase` doesn't neither set flags nor does anything in `post_fail_hook()`, but it provides basic functions
+that will be useful during testing Fedora, like `console_login()` or `boot_to_login_screen()`. It should be used
+when no other, more specific class can be used.
+* `anacondatest` should be used in tests where Anaconda is running. It uploads Anaconda logs (for example
+`anaconda.log` or `packaging.log`) in `post_fail_hook()`. It also provides convenient methods for Anaconda
+like `select_disks()`.
+* `installedtest` should be used in tests that are running on installed system (either in postinstall phase
+or in upgrade tests). It uploads `/var/log` in `post_fail_hook()`.
+
+### Test development checklist
+
+1. Select test from [this document](https://bitbucket.org/rajcze/openqa_fedora_tools/src/develop/PhaseSeparation.md) or from [phabricator page](https://phab.qadevel.cloud.fedoraproject.org/maniphest/?statuses=open%28%29&projects=PHID-PROJ-epofbmazit3u2rndqccd#R)
+2. Put each part of your test as a separate file into `tests/` directory, reimplementing `run()` method
+and `test_flags()` method, inheriting from one of the classes mentioned above.
+3. Set correct variables (so that all test parts you have made are executed) in [WebUI -> Test suites](https://localhost:8080/admin/test_suites).
+4. Link your newly created Test suite to medium type in [WebUI -> Job groups](https://localhost:8080/admin/groups).
+5. Run test (see [openqa_fedora_tools repository](https://bitbucket.org/rajcze/openqa_fedora_tools)).
+6. Create needles (images) by using interactive mode and needles editor in WebUI.
+7. Add new Job template and Test suite into `templates` file.
+8. Add new Test suite and Test case into [`conf_test_suites.py`](https://bitbucket.org/rajcze/openqa_fedora_tools/src/develop/tools/openqa_trigger/conf_test_suites.py) file in openqa_fedora_tools repository.
+9. Mark your test in PhaseSeparation.md as done.
+10. Open differential request via phabricator.

lib/anacondatest.pm

@@ -1,11 +1,17 @@
 package anacondalog;
 use base 'fedorabase';
 
+# base class for all Anaconda (installation) tests
+
+# should be used in tests where Anaconda is running - when it makes sense
+# to upload Anaconda logs when something fails
+
 use testapi;
 
 sub post_fail_hook {
     my $self = shift;
 
+    # if error dialog is shown, click "report" - it then creates directory structure for ABRT
     my $has_traceback = 0;
     if (check_screen "anaconda_error", 10) {
         assert_and_click "anaconda_report_btn"; # Generage Anaconda ABRT logs
@@ -29,7 +35,7 @@ sub post_fail_hook {
             upload_logs "/var/tmp/var_tmp.tar.gz";
         }
 
-        # Upload Anaconda logs
+        # Upload Anaconda traceback logs
         type_string "tar czvf /tmp/anaconda_tb.tar.gz /tmp/anaconda-tb-*";
         send_key "ret";
         upload_logs "/tmp/anaconda_tb.tar.gz";
@@ -42,7 +48,7 @@ sub post_fail_hook {
 sub root_console {
     my $self = shift;
     my %args = (
-        check => 1,
+        check => 1, # whether to fail when console wasn't reached
         @_);
 
     if (get_var("LIVE")) {
@@ -107,4 +113,3 @@ sub custom_change_type {
 1;
 
 # vim: set sw=4 et:
-

lib/fedorabase.pm

@@ -1,8 +1,14 @@
 package fedorabase;
 use base 'basetest';
 
+# base class for all Fedora tests
+
+# use this class when using other base class doesn't make sense
+
 use testapi;
 
+# this subroutine handles logging in as a root/specified user into console
+# it requires TTY to be already displayed (handled by the root_console() method of subclasses)
 sub console_login {
     my $self = shift;
     my %args = (
@@ -63,7 +69,7 @@ sub console_login {
 
 sub boot_to_login_screen {
     my $self = shift;
-    my $boot_done_screen = shift;
+    my $boot_done_screen = shift; # what to expect when system is booted (e. g. GDM), can be ""
     my $stillscreen = shift || 10;
     my $timeout = shift || 60;
 

lib/fedoradistribution.pm

@@ -3,6 +3,17 @@ use base 'distribution';
 
 # Fedora distribution class
 
+# Distro-specific functions, that are actually part of the API
+# (and it's completely up to us to implement them) should be here
+
+# functions that can be reimplemented:
+# ensure_installed
+# x11_start_program
+# become_root
+# script_run
+# script_sudo
+# type_password
+
 use testapi qw(send_key type_string);
 
 sub init() {

lib/installedtest.pm

@@ -1,13 +1,18 @@
 package fedoralog;
 use base 'fedorabase';
 
+# base class for tests that run on installed system
+
+# should be used when with tests, where system is already installed, e. g all parts
+# of upgrade tests, postinstall phases...
+
 use testapi;
 
 sub root_console {
     my $self = shift;
     my %args = (
-        tty => 1,
+        tty => 1, # what TTY to login to
-        check => 1,
+        check => 1, # whether to fail when console wasn't reached
         @_);
 
     send_key "ctrl-alt-f$args{tty}";

main.pm

@@ -49,22 +49,35 @@ if (get_var('LIVE')) {
     set_var('PACKAGE_SET', "default");
 }
 
-# Boot to anaconda Hub in English
+# if user set ENTRYPOINT, run required test directly
-
+# (good for tests where it doesn't make sense to use _boot_to_anaconda, _software_selection etc.)
 if (get_var("ENTRYPOINT"))
 {
     autotest::loadtest get_var('CASEDIR')."/tests/".get_var("ENTRYPOINT").".pm";
 }
 elsif (get_var("UPGRADE"))
 {
+    # all upgrade tests consist of: preinstall phase (where packages are upgraded and fedup is
+    # installed), run phase (where fedup is run) and postinstall phase (where is checked if
+    # fedora was upgraded successfully)
     autotest::loadtest get_var('CASEDIR')."/tests/upgrade_preinstall.pm";
     autotest::loadtest get_var('CASEDIR')."/tests/upgrade_run.pm";
+    # UPGRADE can be set to "minimal", "encrypted", "desktop"...
     autotest::loadtest get_var('CASEDIR')."/tests/upgrade_postinstall_".get_var("UPGRADE").".pm";
 }
 else
 {
+    # normal installation test consists of several phases, from which some of them are
+    # loaded automatically and others are loaded based on what env variables are set
+
+    # generally speaking, install test consists of: boot phase, customization phase, installation
+    # and reboot phase, postinstall phase
+
+    # boot phase is loaded automatically every time
     autotest::loadtest get_var('CASEDIR')."/tests/_boot_to_anaconda.pm";
 
+    # with kickstart tests, booting to anaconda is the only thing required (kickstart file handles
+    # everything else)
     unless (get_var("KICKSTART"))
     {
 
@@ -105,6 +118,7 @@ else
         }
 
         # Start installation, set user & root passwords, reboot
+        # install and reboot phase is loaded automatically every time (except when KICKSTART is set)
         autotest::loadtest get_var('CASEDIR')."/tests/_do_install_and_reboot.pm";
     }
 
@@ -122,6 +136,8 @@ else
         autotest::loadtest get_var('CASEDIR')."/tests/_console_wait_login.pm";
     }
 
+    # from now on, we have fully installed and booted system with root/specified user logged in
+
     # If there is a post-install test to verify storage configuration worked
     # correctly, run it. Again we determine the test name based on the value
     # of DISK_CUSTOM or DISK_GUIDED.

tests/_boot_to_anaconda.pm

@@ -1,10 +1,9 @@
-use base "anacondalog";
+use base "anacondatest";
 use strict;
 use testapi;
 
 sub run {
     # Wait for bootloader to appear
-
     assert_screen "bootloader", 30;
 
     # Make sure we skip media check if it's selected by default. Standard
@@ -12,12 +11,14 @@ sub run {
     send_key "up";
     send_key "up";
 
+    # if variable GRUB is set, add its value into kernel line in grub
     if( get_var("GRUB")){
         send_key "tab";
         type_string " ".get_var("GRUB");
 
     }
 
+    # if variable REPOSITORY_VARIATION is set, construct inst.repo url and add it to kernel line
     if (get_var("REPOSITORY_VARIATION")){
         unless (get_var("GRUB")){
             send_key "tab";
@@ -27,10 +28,13 @@ sub run {
 
         $fedora_version = lc((split /_/, get_var("BUILD"))[0]);
 
+        # REPOSITORY_VARIATION should be set to repository URL without version and architecture
+        # appended (it will be appended automatically)
         $repourl = get_var("REPOSITORY_VARIATION")."/".$fedora_version."/".get_var("ARCH")."/os";
         type_string " inst.repo=".$repourl;
     }
 
+    # now we are on the correct "boot" menu item
     send_key "ret";
 
     unless (get_var("KICKSTART"))
@@ -52,7 +56,7 @@ sub run {
             assert_and_click "anaconda_rawhide_accept_fate";
         }
 
-        # Anaconda hub
+        # wait for Anaconda hub to appear
         assert_screen "anaconda_main_hub", 900; #
     }
 }

tests/_do_install_and_reboot.pm

@@ -1,4 +1,4 @@
-use base "anacondalog";
+use base "anacondatest";
 use strict;
 use testapi;
 
@@ -21,8 +21,6 @@ sub run {
     send_key "tab";
     type_string $root_password;
     assert_and_click "anaconda_spoke_done";
-    # weak password - click "done" once again"
-    #assert_and_click "anaconda_spoke_done";
 
     # Set user details
     sleep 1;
@@ -37,8 +35,6 @@ sub run {
     type_string $user_password;
     assert_and_click "anaconda_install_user_creation_make_admin";
     assert_and_click "anaconda_spoke_done";
-    # weak password - click "done" once again"
-    #assert_and_click "anaconda_spoke_done";
 
     # Wait for install to end
     assert_and_click "anaconda_install_done", '', 1800;

tests/_graphical_wait_login.pm

@@ -3,7 +3,6 @@ use strict;
 use testapi;
 
 sub run {
-
     # If KICKSTART is set, then the wait_time needs to
     #  consider the install time
     my $wait_time = get_var("KICKSTART") ? 1800 : 300;

tests/_software_selection.pm

@@ -1,4 +1,4 @@
-use base "anacondalog";
+use base "anacondatest";
 use strict;
 use testapi;
 
@@ -22,7 +22,7 @@ sub run {
     # select desired environment
     # go through the list 20 times at max (to prevent infinite loop when it's missing)
     for (my $i = 0; !check_screen("anaconda_".$packageset."_highlighted", 1) && $i < 20; $i++) {
-	send_key "down";
+	       send_key "down";
     }
 
     send_key "spc";

tests/disk_custom_software_raid_postinstall.pm

@@ -1,9 +1,10 @@
-use base "basetest";
+use base "installedtest";
 use strict;
 use testapi;
 
 sub run {
     assert_screen "root_console";
+    # check that RAID is used
     type_string "reset; cat /proc/mdstat";
     send_key "ret";
     assert_screen "console_raid_used";

tests/disk_guided_delete_all.pm

@@ -1,4 +1,4 @@
-use base "anacondalog";
+use base "anacondatest";
 use strict;
 use testapi;
 
@@ -9,7 +9,7 @@ sub run {
     $self->select_disks();
     assert_and_click "anaconda_spoke_done";
 
-    # the only provided disk should be full
+    # the only provided disk should be automatically selected and full
     assert_and_click "anaconda_install_destination_reclaim_space_btn";
 
     assert_and_click "anaconda_install_destination_delete_all_btn";

tests/disk_guided_delete_partial.pm

@@ -1,4 +1,4 @@
-use base "anacondalog";
+use base "anacondatest";
 use strict;
 use testapi;
 
@@ -9,9 +9,10 @@ sub run {
     $self->select_disks();
     assert_and_click "anaconda_spoke_done";
 
-    # Provided disk should be full
+    # the only provided disk should be automatically selected and full
     assert_and_click "anaconda_install_destination_reclaim_space_btn";
 
+    # Delete first partition, second should be untouched
     assert_and_click "anaconda_install_destination_reclaim_space_first_partition";
 
     assert_and_click "anaconda_install_destination_reclaim_space_delete_btn";

tests/disk_guided_delete_partial_postinstall.pm

@@ -1,10 +1,11 @@
-use base "basetest";
+use base "installedtest";
 use strict;
 use testapi;
 
 sub run {
     assert_screen "root_console";
-    type_string 'reset; mount /dev/vda2 /mnt; echo $?'; # if you use doublequotes, $? gets replaced by Perl with last error code
+    # mount second partition and check that it's intact
+    type_string 'reset; mount /dev/vda2 /mnt; echo $?';
     send_key "ret";
     assert_screen "console_command_success";
     type_string 'reset; cat /mnt/testfile';

tests/disk_guided_empty.pm

@@ -1,4 +1,4 @@
-use base "anacondalog";
+use base "anacondatest";
 use strict;
 use testapi;
 

tests/disk_guided_encrypted.pm

@@ -1,4 +1,4 @@
-use base "anacondalog";
+use base "anacondatest";
 use strict;
 use testapi;
 
@@ -7,9 +7,12 @@ sub run {
     # Anaconda hub
     # Go to INSTALLATION DESTINATION and ensure one disk is selected.
     $self->select_disks();
+
+    # check "encrypt data" checkbox
     assert_and_click "anaconda_install_destination_encrypt_data";
     assert_and_click "anaconda_spoke_done";
 
+    # type password for disk encryption
     wait_idle 5;
     type_string get_var("ENCRYPT_PASSWORD");
     send_key "tab";

tests/disk_guided_encrypted_postinstall.pm

@@ -1,8 +1,9 @@
-use base "basetest";
+use base "installedtest";
 use strict;
 use testapi;
 
 sub run {
+    # decrypt disks during boot
     assert_screen "boot_enter_passphrase", 300; #
     type_string get_var("ENCRYPT_PASSWORD");
     send_key "ret";

tests/disk_guided_free_space_postinstall.pm

@@ -1,10 +1,11 @@
-use base "basetest";
+use base "installedtest";
 use strict;
 use testapi;
 
 sub run {
     assert_screen "root_console";
-    type_string 'reset; mount /dev/vda1 /mnt; echo $?'; # if you use doublequotes, $? gets replaced by Perl with last error code
+    # check that first partition is intact
+    type_string 'reset; mount /dev/vda1 /mnt; echo $?';
     send_key "ret";
     assert_screen "console_command_success";
     type_string 'reset; cat /mnt/testfile';

tests/disk_guided_multi.pm

@@ -1,4 +1,4 @@
-use base "anacondalog";
+use base "anacondatest";
 use strict;
 use testapi;
 

tests/disk_guided_multi_empty_all.pm

@@ -1,4 +1,4 @@
-use base "anacondalog";
+use base "anacondatest";
 use strict;
 use testapi;
 

tests/disk_guided_multi_empty_all_postinstall.pm

@@ -1,10 +1,10 @@
-use base "basetest";
+use base "installedtest";
 use strict;
 use testapi;
 
 sub run {
     assert_screen "root_console";
-    
+
     # when two disks are selected in installation, LVM is used
     type_string "reset; pvdisplay";
     send_key "ret";

tests/disk_guided_multi_postinstall.pm

@@ -1,10 +1,11 @@
-use base "basetest";
+use base "installedtest";
 use strict;
 use testapi;
 
 sub run {
     assert_screen "root_console";
-    type_string 'reset; mount /dev/sdb1 /mnt; echo $?'; # if you use doublequotes, $? gets replaced by Perl with last error code
+    # check that second disk is intact
+    type_string 'reset; mount /dev/sdb1 /mnt; echo $?';
     send_key "ret";
     assert_screen "console_command_success";
     type_string 'reset; cat /mnt/testfile';

tests/install_source_graphical.pm

@@ -1,4 +1,4 @@
-use base "anacondalog";
+use base "anacondatest";
 use strict;
 use testapi;
 
@@ -10,8 +10,6 @@ sub run {
     # Go into the Install Source spoke
     assert_and_click "anaconda_main_hub_installation_source";
 
-
-
     # select "http" on the network
     assert_and_click "anaconda_install_source_on_the_network";
     send_key "tab";
@@ -31,6 +29,8 @@ sub run {
         }
     }
 
+    # if either MIRRORLIST_GRAPHICAL or REPOSITORY_GRAPHICAL is set, type this into
+    # the repository url input
     if (get_var("MIRRORLIST_GRAPHICAL")){
         $repourl = "mirrors.fedoraproject.org/mirrorlist?repo=".$fedora_version."&arch=".get_var('ARCH');
         type_string $repourl;

tests/install_source_variation.pm

@@ -1,4 +1,4 @@
-use base "anacondalog";
+use base "anacondatest";
 use strict;
 use testapi;
 

tests/upgrade_postinstall_desktop.pm

@@ -1,4 +1,4 @@
-use base "fedoralog";
+use base "installedtest";
 use strict;
 use testapi;
 
@@ -7,12 +7,15 @@ sub run {
     my $self = shift;
     my $password = get_var("PASSWORD", "weakpassword");
 
+    # wait for GDM to appear
     $self->boot_to_login_screen("graphical_login", 20);
 
+    # login as normal user
     send_key "ret";
     assert_screen "graphical_login_input";
     type_string $password;
     send_key "ret";
+    # wait until desktop appears
     assert_screen "graphical_desktop_clean", 30;
 }
 

tests/upgrade_postinstall_minimal.pm

@@ -1,4 +1,4 @@
-use base "fedoralog";
+use base "installedtest";
 use strict;
 use testapi;
 
@@ -6,6 +6,7 @@ use testapi;
 sub run {
     my $self = shift;
 
+    # try to login, check whether F22 is installed
     $self->boot_to_login_screen();
     $self->root_console(tty=>3);
 

tests/upgrade_preinstall.pm

@@ -1,18 +1,21 @@
-use base "fedoralog";
+use base "installedtest";
 use strict;
 use testapi;
 
 sub run {
     my $self = shift;
 
+    # wait for either GDM or text login
     if (get_var('UPGRADE') eq "desktop") {
         $self->boot_to_login_screen("graphical_login", 30); # GDM takes time to load
-    #} elsif (get_var('UPGRADE' eq "minimal")) {
     } else {
         $self->boot_to_login_screen();
     }
+    # switch to TTY3 for both, graphical and console tests
     $self->root_console(tty=>3);
 
+    # fedup should be installed on up-to-date system
+
     type_string 'yum -y update; echo $?';
     send_key "ret";
 
@@ -23,7 +26,6 @@ sub run {
 
     if (get_var('UPGRADE') eq "desktop") {
         $self->boot_to_login_screen("graphical_login", 30); # GDM takes time to load
-    #} elsif (get_var('UPGRADE' eq "minimal")) {
     } else {
         $self->boot_to_login_screen();
     }

tests/upgrade_run.pm

@@ -1,4 +1,4 @@
-use base "fedoralog";
+use base "installedtest";
 use strict;
 use testapi;
 
@@ -6,6 +6,7 @@ sub run {
     my $fedup_url;
     my $to_version;
     # FIXME: this is just a workaround, see https://phab.qadevel.cloud.fedoraproject.org/T478
+    # construct download URL
     if (get_var("BUILD") =~ /^(\d+)_Final_(.*)$/) {
         $fedup_url = "https://dl.fedoraproject.org/pub/alt/stage/".$1."_".$2."/Server/".get_var("ARCH")."/os";
         $to_version = $1;
@@ -18,6 +19,7 @@ sub run {
     type_string "fedup --network ".$to_version." --instrepo ".$fedup_url;
     send_key "ret";
 
+    # wait untill fedup finishes its work (screen stops moving for 30 seconds)
     wait_still_screen 30, 6000; # TODO: shorter timeout, longer stillscreen?
 
     upload_logs "/var/log/fedup.log";
@@ -25,12 +27,14 @@ sub run {
     type_string "reboot";
     send_key "ret";
 
+    # check that "upgrade" item is shown in GRUB
     assert_screen "grub_fedup", 30;
     send_key "ret";
 
+    # now offline upgrading starts. user doesn't have to do anything, just wait untill
+    # system reboots and login screen is shown
     if (get_var('UPGRADE') eq "desktop") {
         assert_screen "graphical_login", 6000;
-    #} elsif (get_var('UPGRADE' eq "minimal")) {
     } else {
         assert_screen "text_console_login", 6000;
     }