Enable tracking of executed applications in Application Start/Stop tests (FIXES #116)

This PR adds the following:
* moves out the presetting procedures, so that two long terminal tests do not have
to run twice
* add methods for application to register when successfully started
* adds a test that checks if all required applications have registered
This commit is contained in:
Lukas Ruzicka 2019-09-19 16:03:50 +02:00 committed by Adam Williamson
parent b455e04188
commit 22b185abf6
38 changed files with 176 additions and 20 deletions

View File

@ -7,7 +7,11 @@ use Exporter;
use lockapi;
use testapi;
our @EXPORT = qw/run_with_error_check type_safely type_very_safely desktop_vt boot_to_login_screen console_login console_switch_layout desktop_switch_layout console_loadkeys_us do_bootloader boot_decrypt check_release menu_launch_type repo_setup gnome_initial_setup anaconda_create_user check_desktop_clean download_modularity_tests quit_firefox advisory_get_installed_packages advisory_check_nonmatching_packages start_with_launcher quit_with_shortcut lo_dismiss_tip disable_firefox_studies select_rescue_mode copy_devcdrom_as_isofile bypass_1691487 get_release_number check_left_bar check_top_bar check_prerelease check_version spell_version_number _assert_and_click is_branched rec_log click_unwanted_notifications repos_mirrorlist/;
our @EXPORT = qw/run_with_error_check type_safely type_very_safely desktop_vt boot_to_login_screen console_login console_switch_layout desktop_switch_layout console_loadkeys_us do_bootloader boot_decrypt check_release menu_launch_type start_cockpit repo_setup gnome_initial_setup anaconda_create_user check_desktop_clean download_modularity_tests quit_firefox advisory_get_installed_packages advisory_check_nonmatching_packages start_with_launcher quit_with_shortcut lo_dismiss_tip disable_firefox_studies select_rescue_mode copy_devcdrom_as_isofile bypass_1691487 get_release_number check_left_bar check_top_bar check_prerelease check_version spell_version_number _assert_and_click is_branched rec_log click_unwanted_notifications repos_mirrorlist register_application get_registered_applications/;
# We introduce this global variable to hold the list of applications that have
# registered during the apps_startstop_test when they have sucessfully run.
our @application_list;
sub run_with_error_check {
my ($func, $error_screen) = @_;
@ -1093,3 +1097,13 @@ sub click_unwanted_notifications {
}
return @closed;
}
# In each application test, when the application is started successfully, it
# will register to the list of applications.
sub register_application {
my $application = shift;
push(@application_list, $application);
print("APPLICATION REGISTERED: $application \n");
}
1;

15
main.pm
View File

@ -379,20 +379,23 @@ if (!get_var("ENTRYPOINT")) {
if (get_var("STARTSTOP")) {
my $desktop = get_var('DESKTOP');
my $casedir = get_var('CASEDIR');
# Find all tests from a directory defined by the DESKTOP variable
my @apptests = glob "${casedir}/tests/apps_startstop/${desktop}/*.pm";
# Load the terminal test extra, because it must run first for settings
# when the desktop is Gnome.
if ($desktop eq 'gnome') {
autotest::loadtest "tests/apps_startstop/${desktop}/terminal.pm";
# Run this test to preset the environment
autotest::loadtest "tests/apps_gnome_preset.pm";
}
# Load all desktop tests
# Find all tests from a directory defined by the DESKTOP variable
my @apptests = glob "${casedir}/tests/apps_startstop/${desktop}/*.pm";
# Now load them
foreach my $filepath (@apptests) {
my $file = basename($filepath);
autotest::loadtest "tests/apps_startstop/${desktop}/${file}";
}
if ($desktop eq 'gnome') {
# Run this test to check if required application have registered.
autotest::loadtest "tests/workstation_core_applications.pm";
}
}
1;

View File

@ -0,0 +1,41 @@
use base "installedtest";
use strict;
use testapi;
use utils;
# This test tests if Terminal starts and uses it to change desktop settings for all the following tests.
# Therefore, if you want to use all the tests from the APPS family, this should be the very first to do.
sub run {
my $self = shift;
# open the application, let use the method that does not require any needles,
# because this way, the terminal will always start even if some needles
# might fail because of changing background in various releases.
send_key 'alt-f1';
wait_still_screen 2;
type_very_safely 'terminal';
send_key 'ret';
wait_still_screen 5;
# When the application opens, run command in it to set the background to black
type_very_safely "gsettings set org.gnome.desktop.background picture-uri ''";
send_key 'ret';
wait_still_screen 2;
type_very_safely "gsettings set org.gnome.desktop.background primary-color '#000000'";
send_key 'ret';
wait_still_screen 2;
quit_with_shortcut();
# check that is has changed color
assert_screen 'apps_settings_screen_black';
}
# If this test fails, the others will probably start failing too,
# so there is no need to continue.
# Also, when subsequent tests fail, the suite will revert to this state for further testing.
sub test_flags {
return { fatal => 1, milestone => 1 };
}
1;
# vim: set sw=4 et:

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_abrt', 'apps_menu_utilities');
# Check that it is started
assert_screen 'apps_run_abrt';
# Register application
register_application('gnome-abrt');
# Close the application
quit_with_shortcut();
}

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_archiver', 'apps_menu_utilities');
# Check that is started
assert_screen 'apps_run_archiver';
# Register application
register_application("menu-archiver");
# Close the application
quit_with_shortcut();

View File

@ -21,6 +21,8 @@ sub run {
send_key 'ret';
assert_screen 'apps_run_boxes';
}
# Register application
register_application("gnome-boxes");
# Close the application
quit_with_shortcut();

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_calculator', 'apps_menu_utilities');
# Check that it is started
assert_screen 'apps_run_calculator';
# Register application
register_application("gnome-calculator");
# Close the application
quit_with_shortcut();
}

View File

@ -17,6 +17,8 @@ sub run {
}
wait_still_screen 2;
assert_screen 'apps_run_calendar';
# Register application
register_application("gnome-calendar");
# close the application
quit_with_shortcut();

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_chars', 'apps_menu_utilities');
# Check that is started
assert_screen 'apps_run_chars';
# Register application
register_application("gnome-characters");
# Close the application
quit_with_shortcut();

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_cheese');
# Check that is started
assert_screen 'apps_run_cheese';
# Register application
register_application("gnome-cheese");
# Close the application
quit_with_shortcut();
}

View File

@ -14,6 +14,8 @@ sub run {
assert_and_click 'apps_run_access';
}
assert_screen 'apps_run_clocks';
# Register application
register_application("gnome-clocks");
# close the application
quit_with_shortcut();

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_contacts');
# Check that is started
assert_screen 'apps_run_contacts';
# Register application
register_application("gnome-contacts");
# Close the application
quit_with_shortcut();
}

View File

@ -11,6 +11,8 @@ sub run {
start_with_launcher('apps_menu_disks', 'apps_menu_utilities');
# Check that is started
assert_screen 'apps_run_disks';
# Register application
register_application("gnome-disks");
# Close the application
quit_with_shortcut();
}

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_diskusage', 'apps_menu_utilities');
# Check that is started
assert_screen 'apps_run_diskusage';
# Register application
register_application("baobab");
# Close the application
quit_with_shortcut();
}

View File

@ -11,6 +11,8 @@ sub run {
start_with_launcher('apps_menu_dviewer', 'apps_menu_utilities');
# Check that is started
assert_screen 'apps_run_dviewer';
# Register application
register_application("evince");
# Close the application
quit_with_shortcut();
}

View File

@ -11,6 +11,8 @@ sub run {
start_with_launcher('apps_menu_files');
# Check that is started
assert_screen 'apps_run_files';
# Register application
register_application("nautilus");
# Close the application
quit_with_shortcut();

View File

@ -23,6 +23,8 @@ sub run {
assert_and_click 'apps_run_firefox_stop';
}
wait_still_screen 2;
# Register application
register_application("firefox");
# check that the application has stopped
assert_screen 'workspace';
}

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_fonts', 'apps_menu_utilities');
# Check that is started
assert_screen 'apps_run_fonts';
# Register application
register_application("gnome-font-viewer");
# Close the application
quit_with_shortcut();
}

View File

@ -11,6 +11,8 @@ sub run {
start_with_launcher('apps_menu_help', 'apps_menu_utilities');
# Check that is started
assert_screen 'apps_run_help';
# Register application
register_application("gnome-help");
# Close the application
quit_with_shortcut();
}

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_imageviewer', 'apps_menu_utilities');
# Check that is started
assert_screen 'apps_run_imageviewer';
# Register application
register_application("image-viewer");
# Close the application
quit_with_shortcut();
}

View File

@ -14,6 +14,8 @@ sub run {
lo_dismiss_tip;
# Check that is started
assert_screen 'apps_run_lcalc';
# Register application
register_application("libreoffice-calc");
# Close the application
quit_with_shortcut();
}

View File

@ -14,6 +14,8 @@ sub run {
lo_dismiss_tip;
# Check that is started
assert_screen 'apps_run_ldraw';
# Register application
register_application("libreoffice-draw");
# Close the application
quit_with_shortcut();
}

View File

@ -15,6 +15,8 @@ sub run {
# Check that is started
assert_and_click 'apps_run_limpress_start';
assert_screen 'apps_run_limpress';
# Register application
register_application("libreoffice-impress");
# Close the application
quit_with_shortcut();
}

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_logs', 'apps_menu_utilities');
# Check that is started
assert_screen 'apps_run_logs';
# Register application
register_application("gnome-logs");
# Close the application
quit_with_shortcut();
}

View File

@ -14,6 +14,8 @@ sub run {
lo_dismiss_tip;
# Check that is started
assert_screen 'apps_run_lwriter';
# Register application
register_application("libreoffice-writer");
# Close the application
quit_with_shortcut();
}

View File

@ -16,6 +16,8 @@ sub run {
assert_and_click 'apps_run_access';
}
assert_screen 'apps_run_maps';
# Register application
register_application("gnome-maps");
# Close the application
quit_with_shortcut();

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_monitor', 'apps_menu_utilities');
# Check that is started
assert_screen 'apps_run_monitor';
# Register application
register_application("gnome-system-monitor");
# Close the application
quit_with_shortcut();
}

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_photos');
# Check that is started
assert_screen 'apps_run_photos';
# Register application
register_application("gnome-photos");
# Close the application
quit_with_shortcut();
}

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_rhythmbox');
# Check that is started
assert_screen 'apps_run_rhythmbox';
# Register application
register_application("rhythmbox");
# Close the application
quit_with_shortcut();
}

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_scan');
# Check that is started
assert_screen 'apps_run_scan';
# Register application
register_application("simple-scan");
# Close the application
quit_with_shortcut();

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_screenshot', 'apps_menu_utilities');
# Check that is started
assert_screen 'apps_run_screenshot';
# Register application
register_application("gnome-screenshot");
# Close the application
quit_with_shortcut();

View File

@ -22,6 +22,8 @@ sub run {
# check that the screen really is black
assert_screen 'workspace';
# Register application
register_application("gnome-control-center");
}

View File

@ -18,6 +18,8 @@ sub run {
wait_still_screen 2;
}
assert_screen 'apps_run_software';
# Register application
register_application("gnome-software");
# Close the application
quit_with_shortcut();

View File

@ -8,25 +8,18 @@ use utils;
sub run {
my $self = shift;
# open the application, let use the method that does not require any needles,
# because this way, the terminal will always start even if some needles
# might fail because of changing background in various releases.
# open the application
send_key 'alt-f1';
wait_still_screen 2;
type_very_safely 'terminal';
send_key 'ret';
wait_still_screen 5;
# When the application opens, run command in it to set the background to black
type_very_safely "gsettings set org.gnome.desktop.background picture-uri ''";
send_key 'ret';
wait_still_screen 2;
type_very_safely "gsettings set org.gnome.desktop.background primary-color '#000000'";
send_key 'ret';
wait_still_screen 2;
# Register application
register_application("gnome-terminal");
# Close the application
quit_with_shortcut();
# check that is has changed color
assert_screen 'apps_settings_screen_black';
}
# If this test fails, the others will probably start failing too,

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_texteditor');
# Check that is started
assert_screen 'apps_run_texteditor';
# Register application
register_application("gedit");
# Close the application
quit_with_shortcut();
}

View File

@ -12,6 +12,8 @@ sub run {
start_with_launcher('apps_menu_videos');
# Check that is started
assert_screen 'apps_run_videos';
# Register application
register_application("totem");
# Close the application
quit_with_shortcut();
}

View File

@ -15,6 +15,8 @@ sub run {
}
wait_still_screen 2;
assert_screen 'apps_run_weather';
# Register application
register_application("gnome-weather");
# close the application
quit_with_shortcut();

View File

@ -0,0 +1,39 @@
use base "installedtest";
use strict;
use testapi;
use utils;
# This test collects the results of application registration (presence or absence).
sub run {
my $self = shift;
$self->root_console(tty=>3);
# List of applications, that we want to track for their presence.
my @core_applications = ("gnome-software", "firefox", "gnome-terminal", "nautilus", "gedit", "gnome-boxes");
# Evaluate the results, make the log files and pass or fail the entire
# test suite.
my $failed;
foreach my $app (@core_applications) {
# @utils::application_list here is the list of registered apps
if (grep {$_ eq $app} @utils::application_list) {
assert_script_run "echo '$app=passed' >> registered.log";
}
else {
assert_script_run "echo '$app=failed' >> registered.log";
$failed = 1;
}
}
upload_logs "registered.log", failok=>1;
die "Some core applications could not be started. Check logs." if ($failed);
}
sub test_flags {
return {fatal => 1};
}
1;
# vim: set sw=4 et: