Enable tracking of executed applications in Application Start/Stop tests (FIXES #116)

This PR adds the following:
* moves out the presetting procedures, so that two long terminal tests do not have
to run twice
* add methods for application to register when successfully started
* adds a test that checks if all required applications have registered

lib/utils.pm

@@ -7,7 +7,11 @@ use Exporter;
 
 use lockapi;
 use testapi;
-our @EXPORT = qw/run_with_error_check type_safely type_very_safely desktop_vt boot_to_login_screen console_login console_switch_layout desktop_switch_layout console_loadkeys_us do_bootloader boot_decrypt check_release menu_launch_type repo_setup gnome_initial_setup anaconda_create_user check_desktop_clean download_modularity_tests quit_firefox advisory_get_installed_packages advisory_check_nonmatching_packages start_with_launcher quit_with_shortcut lo_dismiss_tip disable_firefox_studies select_rescue_mode copy_devcdrom_as_isofile bypass_1691487 get_release_number check_left_bar check_top_bar check_prerelease check_version spell_version_number _assert_and_click is_branched rec_log click_unwanted_notifications repos_mirrorlist/;
+our @EXPORT = qw/run_with_error_check type_safely type_very_safely desktop_vt boot_to_login_screen console_login console_switch_layout desktop_switch_layout console_loadkeys_us do_bootloader boot_decrypt check_release menu_launch_type start_cockpit repo_setup gnome_initial_setup anaconda_create_user check_desktop_clean download_modularity_tests quit_firefox advisory_get_installed_packages advisory_check_nonmatching_packages start_with_launcher quit_with_shortcut lo_dismiss_tip disable_firefox_studies select_rescue_mode copy_devcdrom_as_isofile bypass_1691487 get_release_number check_left_bar check_top_bar check_prerelease check_version spell_version_number _assert_and_click is_branched rec_log click_unwanted_notifications repos_mirrorlist register_application get_registered_applications/;
+
+# We introduce this global variable to hold the list of applications that have
+# registered during the apps_startstop_test when they have sucessfully run.
+our @application_list;
 
 sub run_with_error_check {
     my ($func, $error_screen) = @_;
@@ -1093,3 +1097,13 @@ sub click_unwanted_notifications {
     }
     return @closed;
 }
+
+# In each application test, when the application is started successfully, it
+# will register to the list of applications.
+sub register_application {
+    my $application = shift;
+    push(@application_list, $application);
+    print("APPLICATION REGISTERED: $application \n");
+}
+
+1;

main.pm

@@ -379,20 +379,23 @@ if (!get_var("ENTRYPOINT")) {
 if (get_var("STARTSTOP")) {
     my $desktop = get_var('DESKTOP');
     my $casedir = get_var('CASEDIR');
-    # Find all tests from a directory defined by the DESKTOP variable
-    my @apptests = glob "${casedir}/tests/apps_startstop/${desktop}/*.pm";
 
-    # Load the terminal test extra, because it must run first for settings
-    # when the desktop is Gnome.
     if ($desktop eq 'gnome') {
-        autotest::loadtest "tests/apps_startstop/${desktop}/terminal.pm";
+        # Run this test to preset the environment
+        autotest::loadtest "tests/apps_gnome_preset.pm";
     }   
 
-    # Load all desktop tests
+    # Find all tests from a directory defined by the DESKTOP variable
+    my @apptests = glob "${casedir}/tests/apps_startstop/${desktop}/*.pm";
+    # Now load them
     foreach my $filepath (@apptests) {
         my $file = basename($filepath);
         autotest::loadtest "tests/apps_startstop/${desktop}/${file}";
     }
+    if ($desktop eq 'gnome') {
+        # Run this test to check if required application have registered.
+        autotest::loadtest "tests/workstation_core_applications.pm";
+    }
 }
 1;
 

tests/apps_gnome_preset.pm

@@ -0,0 +1,41 @@
+use base "installedtest";
+use strict;
+use testapi;
+use utils;
+
+# This test tests if Terminal starts and uses it to change desktop settings for all the following tests.
+# Therefore, if you want to use all the tests from the APPS family, this should be the very first to do.
+
+sub run {
+    my $self = shift;
+    # open the application, let use the method that does not require any needles, 
+    # because this way, the terminal will always start even if some needles
+    # might fail because of changing background in various releases.
+    send_key 'alt-f1';
+    wait_still_screen 2;
+    type_very_safely 'terminal';
+    send_key 'ret';  
+    wait_still_screen 5;
+
+    # When the application opens, run command in it to set the background to black
+    type_very_safely "gsettings set org.gnome.desktop.background picture-uri ''";
+    send_key 'ret';
+    wait_still_screen 2;
+    type_very_safely "gsettings set org.gnome.desktop.background primary-color '#000000'";
+    send_key 'ret';
+    wait_still_screen 2;
+    quit_with_shortcut();
+    # check that is has changed color
+    assert_screen 'apps_settings_screen_black';
+}
+
+# If this test fails, the others will probably start failing too, 
+# so there is no need to continue.
+# Also, when subsequent tests fail, the suite will revert to this state for further testing.
+sub test_flags {
+    return { fatal => 1, milestone => 1 };
+}
+
+1;
+
+# vim: set sw=4 et:

tests/apps_startstop/gnome/abrt.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_abrt', 'apps_menu_utilities');
     # Check that it is started
     assert_screen 'apps_run_abrt';
+    # Register application
+    register_application('gnome-abrt');
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/archiver.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_archiver', 'apps_menu_utilities');
     # Check that is started
     assert_screen 'apps_run_archiver';
+    # Register application
+    register_application("menu-archiver");
     # Close the application
     quit_with_shortcut();
     

tests/apps_startstop/gnome/boxes.pm

@@ -21,6 +21,8 @@ sub run {
         send_key 'ret';
         assert_screen 'apps_run_boxes';
     }
+    # Register application
+    register_application("gnome-boxes");
     # Close the application
     quit_with_shortcut();
     

tests/apps_startstop/gnome/calculator.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_calculator', 'apps_menu_utilities');
     # Check that it is started
     assert_screen 'apps_run_calculator';
+    # Register application
+    register_application("gnome-calculator");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/calendar.pm

@@ -17,6 +17,8 @@ sub run {
     }
     wait_still_screen 2;
     assert_screen 'apps_run_calendar';
+    # Register application
+    register_application("gnome-calendar");
     # close the application
     quit_with_shortcut();
     

tests/apps_startstop/gnome/chars.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_chars', 'apps_menu_utilities');
     # Check that is started
     assert_screen 'apps_run_chars';
+    # Register application
+    register_application("gnome-characters");
     # Close the application
     quit_with_shortcut();
     

tests/apps_startstop/gnome/cheese.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_cheese');
     # Check that is started
     assert_screen 'apps_run_cheese';
+    # Register application
+    register_application("gnome-cheese");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/clocks.pm

@@ -14,6 +14,8 @@ sub run {
         assert_and_click 'apps_run_access';
     }
     assert_screen 'apps_run_clocks';
+    # Register application
+    register_application("gnome-clocks");
     # close the application
     quit_with_shortcut();
     

tests/apps_startstop/gnome/contacts.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_contacts');
     # Check that is started
     assert_screen 'apps_run_contacts';
+    # Register application
+    register_application("gnome-contacts");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/disks.pm

@@ -11,6 +11,8 @@ sub run {
     start_with_launcher('apps_menu_disks', 'apps_menu_utilities');
     # Check that is started
     assert_screen 'apps_run_disks';
+    # Register application
+    register_application("gnome-disks");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/diskusage.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_diskusage', 'apps_menu_utilities');
     # Check that is started
     assert_screen 'apps_run_diskusage';
+    # Register application
+    register_application("baobab");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/dviewer.pm

@@ -11,6 +11,8 @@ sub run {
     start_with_launcher('apps_menu_dviewer', 'apps_menu_utilities');
     # Check that is started
     assert_screen 'apps_run_dviewer';
+    # Register application
+    register_application("evince");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/files.pm

@@ -11,6 +11,8 @@ sub run {
     start_with_launcher('apps_menu_files');
     # Check that is started
     assert_screen 'apps_run_files';
+    # Register application
+    register_application("nautilus");
     # Close the application
     quit_with_shortcut();
     

tests/apps_startstop/gnome/firefox.pm

@@ -23,6 +23,8 @@ sub run {
         assert_and_click 'apps_run_firefox_stop';
     }
     wait_still_screen 2;
+    # Register application
+    register_application("firefox");
     # check that the application has stopped
     assert_screen 'workspace';
 }

tests/apps_startstop/gnome/fonts.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_fonts', 'apps_menu_utilities');
     # Check that is started
     assert_screen 'apps_run_fonts';
+    # Register application
+    register_application("gnome-font-viewer");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/help.pm

@@ -11,6 +11,8 @@ sub run {
     start_with_launcher('apps_menu_help', 'apps_menu_utilities');
     # Check that is started
     assert_screen 'apps_run_help';
+    # Register application
+    register_application("gnome-help");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/imageviewer.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_imageviewer', 'apps_menu_utilities');
     # Check that is started
     assert_screen 'apps_run_imageviewer';
+    # Register application
+    register_application("image-viewer");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/lcalc.pm

@@ -14,6 +14,8 @@ sub run {
     lo_dismiss_tip;
     # Check that is started
     assert_screen 'apps_run_lcalc';
+    # Register application
+    register_application("libreoffice-calc");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/ldraw.pm

@@ -14,6 +14,8 @@ sub run {
     lo_dismiss_tip;
     # Check that is started
     assert_screen 'apps_run_ldraw';
+    # Register application
+    register_application("libreoffice-draw");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/limpress.pm

@@ -15,6 +15,8 @@ sub run {
     # Check that is started
     assert_and_click 'apps_run_limpress_start';
     assert_screen 'apps_run_limpress';
+    # Register application
+    register_application("libreoffice-impress");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/logs.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_logs', 'apps_menu_utilities');
     # Check that is started
     assert_screen 'apps_run_logs';
+    # Register application
+    register_application("gnome-logs");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/lwriter.pm

@@ -14,6 +14,8 @@ sub run {
     lo_dismiss_tip;
     # Check that is started
     assert_screen 'apps_run_lwriter';
+    # Register application
+    register_application("libreoffice-writer");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/maps.pm

@@ -16,6 +16,8 @@ sub run {
         assert_and_click 'apps_run_access';
     }
     assert_screen 'apps_run_maps';
+    # Register application
+    register_application("gnome-maps");
     # Close the application
     quit_with_shortcut();
     

tests/apps_startstop/gnome/monitor.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_monitor', 'apps_menu_utilities');
     # Check that is started
     assert_screen 'apps_run_monitor';
+    # Register application
+    register_application("gnome-system-monitor");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/photos.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_photos');
     # Check that is started
     assert_screen 'apps_run_photos';
+    # Register application
+    register_application("gnome-photos");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/rhythmbox.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_rhythmbox');
     # Check that is started
     assert_screen 'apps_run_rhythmbox';
+    # Register application
+    register_application("rhythmbox");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/scan.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_scan');
     # Check that is started
     assert_screen 'apps_run_scan';
+    # Register application
+    register_application("simple-scan");
     # Close the application
     quit_with_shortcut();
     

tests/apps_startstop/gnome/screenshot.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_screenshot', 'apps_menu_utilities');
     # Check that is started
     assert_screen 'apps_run_screenshot';
+    # Register application
+    register_application("gnome-screenshot");
     # Close the application
     quit_with_shortcut();
     

tests/apps_startstop/gnome/settings.pm

@@ -22,6 +22,8 @@ sub run {
 
     # check that the screen really is black
     assert_screen 'workspace';
+    # Register application
+    register_application("gnome-control-center");
     
 }
 

tests/apps_startstop/gnome/software.pm

@@ -18,6 +18,8 @@ sub run {
         wait_still_screen 2;
     }
     assert_screen 'apps_run_software';
+    # Register application
+    register_application("gnome-software");
     # Close the application
     quit_with_shortcut();
     

tests/apps_startstop/gnome/terminal.pm

@@ -8,25 +8,18 @@ use utils;
 
 sub run {
     my $self = shift;
-    # open the application, let use the method that does not require any needles, 
+    # open the application
-    # because this way, the terminal will always start even if some needles
-    # might fail because of changing background in various releases.
     send_key 'alt-f1';
     wait_still_screen 2;
     type_very_safely 'terminal';
     send_key 'ret';  
     wait_still_screen 5;
 
-    # When the application opens, run command in it to set the background to black
+    # Register application
-    type_very_safely "gsettings set org.gnome.desktop.background picture-uri ''";
+    register_application("gnome-terminal");
-    send_key 'ret';
+
-    wait_still_screen 2;
+    # Close the application
-    type_very_safely "gsettings set org.gnome.desktop.background primary-color '#000000'";
-    send_key 'ret';
-    wait_still_screen 2;
     quit_with_shortcut();
-    # check that is has changed color
-    assert_screen 'apps_settings_screen_black';
 }
 
 # If this test fails, the others will probably start failing too, 

tests/apps_startstop/gnome/texteditor.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_texteditor');
     # Check that is started
     assert_screen 'apps_run_texteditor';
+    # Register application
+    register_application("gedit");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/videos.pm

@@ -12,6 +12,8 @@ sub run {
     start_with_launcher('apps_menu_videos');
     # Check that is started
     assert_screen 'apps_run_videos';
+    # Register application
+    register_application("totem");
     # Close the application
     quit_with_shortcut();
 }

tests/apps_startstop/gnome/weather.pm

@@ -15,6 +15,8 @@ sub run {
     }
     wait_still_screen 2;
     assert_screen 'apps_run_weather';
+    # Register application
+    register_application("gnome-weather");
     # close the application
     quit_with_shortcut();
     

tests/workstation_core_applications.pm

@@ -0,0 +1,39 @@
+use base "installedtest";
+use strict;
+use testapi;
+use utils;
+
+# This test collects the results of application registration (presence or absence).
+
+sub run {
+    my $self = shift;
+    $self->root_console(tty=>3);
+
+    # List of applications, that we want to track for their presence.
+    my @core_applications = ("gnome-software", "firefox", "gnome-terminal", "nautilus", "gedit", "gnome-boxes");
+
+    # Evaluate the results, make the log files and pass or fail the entire
+    # test suite.
+    my $failed;
+    foreach my $app (@core_applications) {
+        # @utils::application_list here is the list of registered apps
+        if (grep {$_ eq $app} @utils::application_list) {
+            assert_script_run "echo '$app=passed' >> registered.log";
+        }
+        else {
+            assert_script_run "echo '$app=failed' >> registered.log";
+            $failed = 1;
+        }
+    }
+    upload_logs "registered.log", failok=>1;
+    die "Some core applications could not be started. Check logs." if ($failed);
+}
+
+sub test_flags {
+    return {fatal => 1};
+}
+
+
+1;
+
+# vim: set sw=4 et: