Support for live image testing

This requires adding products, flavors and needles and test
cases, and tweaking some existing ones to handle the
slightly different behaviour of live images in shared tests.

To handle the different main hub screens in live and non-live,
a less stringent needle is added which is unregistered for
non-live tests, so they don't match on it before they've
finished updating repository metadata.

There are a few small bugfix tweaks in this too, like some
delays in user creation to try and avoid intermittent failures
there.

A new root_logged_in needle is also included, to handle a new
console font in Rawhide - that has nothing strictly to do with
live testing, it just happened to come up while working on
this.
This commit is contained in:
Adam Williamson 2015-03-18 14:28:03 -07:00
parent cab8bcc4de
commit 84a14cdb67
31 changed files with 477 additions and 221 deletions

View File

@ -6,7 +6,20 @@ use testapi;
sub post_fail_hook { sub post_fail_hook {
my $self = shift; my $self = shift;
send_key "ctrl-alt-f2"; send_key "ctrl-alt-f2";
if (check_screen "anaconda_console", 10) { my $logged_in = 0;
if (get_var("LIVE") && check_screen "text_console_login", 20) {
# On live installs, we need to log in
type_string "root";
send_key "ret";
if (check_screen "root_logged_in", 10) {
$logged_in = 1;
}
}
elsif (check_screen "anaconda_console", 10) {
$logged_in = 1;
}
if ($logged_in == 1) {
upload_logs "/tmp/X.log"; upload_logs "/tmp/X.log";
upload_logs "/tmp/anaconda.log"; upload_logs "/tmp/anaconda.log";
upload_logs "/tmp/packaging.log"; upload_logs "/tmp/packaging.log";
@ -25,6 +38,9 @@ sub post_fail_hook {
send_key "ret"; send_key "ret";
upload_logs "/tmp/anaconda_tb.tar.gz"; upload_logs "/tmp/anaconda_tb.tar.gz";
} }
else {
save_screenshot;
}
} }
1; 1;

26
lib/fedoradistribution.pm Normal file
View File

@ -0,0 +1,26 @@
package fedoradistribution;
use base 'distribution';
# Fedora distribution class
use testapi qw(send_key type_string);
sub init() {
my ($self) = @_;
$self->SUPER::init();
}
sub x11_start_program($$$) {
my ($self, $program, $timeout, $options) = @_;
# TODO: take screenshots of every goddamn 'run command' dialog in
# every goddamn desktop and keep them updated forever
send_key "alt-f2";
sleep 3;
type_string $program;
sleep 1;
send_key "ret", 1;
}
1;
# vim: set sw=4 et:

51
main.pm
View File

@ -17,6 +17,37 @@
use strict; use strict;
use testapi; use testapi;
use autotest; use autotest;
use needle;
# distribution-specific implementations of expected methods
my $distri = testapi::get_var("CASEDIR") . '/lib/fedoradistribution.pm';
require $distri;
testapi::set_distribution(fedoradistribution->new());
# Stolen from openSUSE.
sub unregister_needle_tags($) {
my $tag = shift;
my @a = @{ needle::tags($tag) };
for my $n (@a) { $n->unregister(); }
}
sub cleanup_needles() {
if (!get_var('LIVE')) {
## Unregister live-only installer needles. The main issue is the
## hub: on non-live we want to wait for repository setup to complete,
## but if we match that spoke's "ready" icon, it breaks live because
## it doesn't have that spoke. So we have a live needle which doesn't
## match on that icon, but we unregister it for non-live installs so
## they don't match on it too soon.
unregister_needle_tags("ENV-INSTALLER-live");
}
}
$needle::cleanuphandler = \&cleanup_needles;
if (get_var('LIVE')) {
# No package set selection for lives.
set_var('PACKAGE_SET', "default");
}
# Boot to anaconda Hub in English # Boot to anaconda Hub in English
@ -39,13 +70,12 @@ else
autotest::loadtest get_var('CASEDIR')."/tests/install_source_variation.pm"; autotest::loadtest get_var('CASEDIR')."/tests/install_source_variation.pm";
} }
## Select package set. Minimal is the default, if 'default' is specified, skip selection.
## Select minimal flavor my $packageset = get_var('PACKAGE_SET', 'minimal');
if (get_var("FLAVOR") eq "server") { unless ($packageset eq 'default') {
autotest::loadtest get_var('CASEDIR')."/tests/_select_minimal.pm"; autotest::loadtest get_var('CASEDIR')."/tests/_select_".$packageset.".pm";
} }
## Disk partitioning ## Disk partitioning
if (get_var('DISK_GUIDED_MULTI')) { if (get_var('DISK_GUIDED_MULTI')) {
autotest::loadtest get_var('CASEDIR')."/tests/disk_guided_multi.pm"; autotest::loadtest get_var('CASEDIR')."/tests/disk_guided_multi.pm";
@ -76,11 +106,18 @@ else
autotest::loadtest get_var('CASEDIR')."/tests/_do_install_and_reboot.pm"; autotest::loadtest get_var('CASEDIR')."/tests/_do_install_and_reboot.pm";
} }
# Wait for the login screen # Unlock encrypted storage volumes, if necessary
if (get_var("ENCRYPT_PASSWORD")){ if (get_var("ENCRYPT_PASSWORD")){
autotest::loadtest get_var('CASEDIR')."/tests/disk_guided_encrypted_postinstall.pm"; autotest::loadtest get_var('CASEDIR')."/tests/disk_guided_encrypted_postinstall.pm";
} }
autotest::loadtest get_var('CASEDIR')."/tests/_wait_for_login_screen.pm";
# Appropriate login method for install type
if (get_var("DESKTOP")) {
autotest::loadtest get_var('CASEDIR')."/tests/_graphical_wait_login.pm";
}
else {
autotest::loadtest get_var('CASEDIR')."/tests/_console_wait_login.pm";
}
if (get_var('DISK_GUIDED_MULTI')) { if (get_var('DISK_GUIDED_MULTI')) {
autotest::loadtest get_var('CASEDIR')."/tests/disk_guided_multi_postinstall.pm"; autotest::loadtest get_var('CASEDIR')."/tests/disk_guided_multi_postinstall.pm";

View File

@ -1,17 +1,24 @@
{ {
"area": [ "area": [
{ {
"xpos": 910, "xpos": 669,
"ypos": 680, "ypos": 639,
"width": 89, "height": 25,
"height": 46, "width": 330,
"type": "match" "type": "match"
} },
], {
"tags": [ "type": "match",
"anaconda_install_done", "width": 17,
"ENV-DESKTOP-default", "ypos": 688,
"ENV-DISTRI-fedora", "height": 28,
"ENV-INSTLANG-en_US" "xpos": 977
] }
} ],
"tags": [
"anaconda_install_done",
"ENV-DESKTOP-default",
"ENV-DISTRI-fedora",
"ENV-INSTLANG-en_US"
]
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 74 KiB

View File

@ -0,0 +1,25 @@
{
"tags": [
"anaconda_main_hub",
"ENV-DISTRI-fedora",
"ENV-INSTLANG-en_US",
"ENV-INSTALLER-live"
],
"area": [
{
"type": "match",
"width": 74,
"height": 57,
"ypos": 158,
"xpos": 184
},
{
"type": "match",
"width": 66,
"height": 66,
"ypos": 221,
"xpos": 186
}
],
"properties": []
}

View File

Before

Width:  |  Height:  |  Size: 100 KiB

After

Width:  |  Height:  |  Size: 100 KiB

View File

@ -0,0 +1,32 @@
{
"tags": [
"anaconda_main_hub",
"ENV-DISTRI-fedora",
"ENV-INSTLANG-en_US",
"ENV-FLAVOR-server"
],
"area": [
{
"type": "match",
"width": 74,
"height": 57,
"ypos": 158,
"xpos": 184
},
{
"type": "match",
"width": 66,
"ypos": 221,
"height": 66,
"xpos": 186
},
{
"width": 60,
"type": "match",
"xpos": 191,
"height": 60,
"ypos": 349
}
],
"properties": []
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 100 KiB

View File

@ -1,59 +0,0 @@
{
"tags": [
"anaconda_main_hub",
"ENV-DISTRI-fedora",
"ENV-INSTLANG-en_US",
"ENV-FLAVOR-server"
],
"area": [
{
"width": 227,
"height": 57,
"xpos": 168,
"ypos": 7,
"type": "match"
},
{
"height": 33,
"ypos": 153,
"xpos": 266,
"width": 101,
"type": "match"
},
{
"width": 180,
"height": 33,
"xpos": 699,
"ypos": 155,
"type": "match"
},
{
"width": 203,
"height": 38,
"ypos": 346,
"xpos": 267,
"type": "match"
},
{
"type": "match",
"height": 40,
"ypos": 461,
"xpos": 264,
"width": 252
},
{
"height": 18,
"ypos": 236,
"xpos": 272,
"width": 36,
"type": "match"
},
{
"type": "match",
"height": 13,
"ypos": 240,
"xpos": 330,
"width": 40
}
]
}

View File

@ -7,11 +7,11 @@
], ],
"area": [ "area": [
{ {
"xpos": 20, "xpos": 19,
"ypos": 37, "ypos": 35,
"width": 60, "width": 63,
"height": 27, "height": 30,
"type": "match" "type": "match"
} }
] ]
} }

View File

@ -1,17 +1,17 @@
{ {
"area": [ "area": [
{ {
"ypos": 326,
"height": 20,
"xpos": 91,
"type": "match", "type": "match",
"width": 76, "width": 449
"xpos": 941,
"ypos": 686,
"height": 33
} }
], ],
"tags": [ "tags": [
"anaconda_install_finish", "bootloader",
"ENV-DISTRI-fedora", "ENV-DISTRI-fedora",
"ENV-INSTLANG-en_US", "ENV-INSTLANG-en_US",
"ENV-FLAVOR-server" "ENV-FLAVOR-server"
] ]
} }

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.6 KiB

View File

@ -0,0 +1,17 @@
{
"area": [
{
"ypos": 326,
"height": 20,
"xpos": 91,
"type": "match",
"width": 449
}
],
"tags": [
"bootloader",
"ENV-DISTRI-fedora",
"ENV-INSTLANG-en_US",
"ENV-FLAVOR-server"
]
}

View File

Before

Width:  |  Height:  |  Size: 3.2 KiB

After

Width:  |  Height:  |  Size: 3.2 KiB

View File

@ -1,31 +0,0 @@
{
"area": [
{
"type": "match",
"width": 53,
"height": 24,
"xpos": 249,
"ypos": 147
},
{
"width": 114,
"height": 17,
"ypos": 216,
"xpos": 86,
"type": "match"
},
{
"xpos": 78,
"ypos": 232,
"height": 20,
"width": 264,
"type": "match"
}
],
"tags": [
"bootloader",
"ENV-DISTRI-fedora",
"ENV-INSTLANG-en_US",
"ENV-FLAVOR-server"
]
}

View File

@ -0,0 +1,24 @@
{
"tags": [
"ENV-DESKTOP-gnome",
"ENV-DISTRI-fedora",
"ENV-INSTLANG-en_US",
"graphical_desktop_clean"
],
"area": [
{
"xpos": 10,
"ypos": 7,
"width": 67,
"height": 15,
"type": "match"
},
{
"xpos": 968,
"ypos": 4,
"width": 43,
"height": 18,
"type": "match"
}
]
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 684 KiB

View File

@ -0,0 +1,17 @@
{
"tags": [
"ENV-DESKTOP-gnome",
"ENV-DISTRI-fedora",
"ENV-INSTLANG-en_US",
"graphical_login"
],
"area": [
{
"xpos": 335,
"ypos": 423,
"width": 67,
"height": 12,
"type": "match"
}
]
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 244 KiB

View File

@ -0,0 +1,17 @@
{
"tags": [
"root_logged_in",
"ENV-DISTRI-fedora",
"ENV-INSTLANG-en_US",
"ENV-FLAVOR-server"
],
"area": [
{
"xpos": 0,
"ypos": 80,
"width": 152,
"height": 14,
"type": "match"
}
]
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.6 KiB

View File

@ -3,7 +3,7 @@
"ENV-DESKTOP-default", "ENV-DESKTOP-default",
"ENV-DISTRI-fedora", "ENV-DISTRI-fedora",
"ENV-INSTLANG-en_US", "ENV-INSTLANG-en_US",
"clean_install_login" "text_console_login"
], ],
"area": [ "area": [
{ {

View File

@ -0,0 +1,17 @@
{
"tags": [
"live_initial_anaconda_launcher",
"ENV-DISTRI-fedora",
"ENV-DESKTOP-gnome"
],
"area": [
{
"xpos": 542,
"ypos": 205,
"width": 227,
"height": 180,
"type": "match"
}
],
"properties": []
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 531 KiB

122
templates
View File

@ -11,18 +11,48 @@
product => { product => {
arch => "x86_64", arch => "x86_64",
distri => "fedora", distri => "fedora",
flavor => "server", flavor => "workstation_live",
version => "rawhide", version => "*",
}, },
test_suite => { name => "server_simple" }, test_suite => { name => "default_install" },
}, },
{ {
machine => { name => "64bit" }, machine => { name => "64bit" },
product => { product => {
arch => "x86_64", arch => "x86_64",
distri => "fedora", distri => "fedora",
flavor => "server", flavor => "server_boot",
version => "rawhide", version => "*",
},
test_suite => { name => "default_install" },
},
{
machine => { name => "64bit" },
product => {
arch => "x86_64",
distri => "fedora",
flavor => "generic_boot",
version => "*",
},
test_suite => { name => "default_install" },
},
{
machine => { name => "64bit" },
product => {
arch => "x86_64",
distri => "fedora",
flavor => "universal",
version => "*",
},
test_suite => { name => "package_set_minimal" },
},
{
machine => { name => "64bit" },
product => {
arch => "x86_64",
distri => "fedora",
flavor => "universal",
version => "*",
}, },
test_suite => { name => "server_repository_http_variation" }, test_suite => { name => "server_repository_http_variation" },
}, },
@ -31,8 +61,8 @@
product => { product => {
arch => "x86_64", arch => "x86_64",
distri => "fedora", distri => "fedora",
flavor => "server", flavor => "universal",
version => "rawhide", version => "*",
}, },
test_suite => { name => "server_repository_http_graphical" }, test_suite => { name => "server_repository_http_graphical" },
}, },
@ -41,8 +71,8 @@
product => { product => {
arch => "x86_64", arch => "x86_64",
distri => "fedora", distri => "fedora",
flavor => "server", flavor => "universal",
version => "rawhide", version => "*",
}, },
test_suite => { name => "server_mirrorlist_graphical" }, test_suite => { name => "server_mirrorlist_graphical" },
}, },
@ -51,8 +81,8 @@
product => { product => {
arch => "x86_64", arch => "x86_64",
distri => "fedora", distri => "fedora",
flavor => "server", flavor => "universal",
version => "rawhide", version => "*",
}, },
test_suite => { name => "server_delete_pata" }, test_suite => { name => "server_delete_pata" },
}, },
@ -61,8 +91,8 @@
product => { product => {
arch => "x86_64", arch => "x86_64",
distri => "fedora", distri => "fedora",
flavor => "server", flavor => "universal",
version => "rawhide", version => "*",
}, },
test_suite => { name => "server_kickstart_user_creation" }, test_suite => { name => "server_kickstart_user_creation" },
}, },
@ -71,8 +101,8 @@
product => { product => {
arch => "x86_64", arch => "x86_64",
distri => "fedora", distri => "fedora",
flavor => "server", flavor => "universal",
version => "rawhide", version => "*",
}, },
test_suite => { name => "server_scsi_updates_img" }, test_suite => { name => "server_scsi_updates_img" },
}, },
@ -81,8 +111,8 @@
product => { product => {
arch => "x86_64", arch => "x86_64",
distri => "fedora", distri => "fedora",
flavor => "server", flavor => "universal",
version => "rawhide", version => "*",
}, },
test_suite => { name => "server_sata_multi" }, test_suite => { name => "server_sata_multi" },
}, },
@ -91,8 +121,8 @@
product => { product => {
arch => "x86_64", arch => "x86_64",
distri => "fedora", distri => "fedora",
flavor => "server", flavor => "universal",
version => "rawhide", version => "*",
}, },
test_suite => { name => "server_simple_encrypted" }, test_suite => { name => "server_simple_encrypted" },
}, },
@ -153,7 +183,9 @@
name => "64bit", name => "64bit",
settings => [ settings => [
{ key => "QEMUCPU", value => "qemu64" }, { key => "QEMUCPU", value => "qemu64" },
{ key => "QEMUCPUS", value => "2"} { key => "QEMUCPUS", value => "2"},
{ key => "QEMUVGA", value => "std"},
{ key => "QEMURAM", value => "2048"}
], ],
variables => "", variables => "",
}, },
@ -162,19 +194,61 @@
{ {
arch => "x86_64", arch => "x86_64",
distri => "fedora", distri => "fedora",
flavor => "server", flavor => "universal",
name => "", name => "",
settings => [ settings => [
], ],
variables => "", variables => "",
version => "rawhide", version => "*",
}, },
{
arch => "x86_64",
distri => "fedora",
flavor => "generic_boot",
name => "",
settings => [
],
variables => "",
version => "*",
},
{
arch => "x86_64",
distri => "fedora",
flavor => "server_boot",
name => "",
settings => [
],
variables => "",
version => "*",
},
{
arch => "x86_64",
distri => "fedora",
flavor => "workstation_live",
name => "",
settings => [
{ key => "LIVE", value => "1" },
{ key => "DESKTOP", value => "gnome" }
],
variables => "",
version => "*",
}
], ],
TestSuites => [ TestSuites => [
{ {
name => "server_simple", name => "default_install",
prio => 1, prio => 1,
settings => [], settings => [
{ key => "PACKAGE_SET", value => "default" },
],
variables => "",
},
{
name => "package_set_minimal",
prio => 1,
settings => [
{ key => "PACKAGE_SET", value => "minimal" },
],
variables => "", variables => "",
}, },
{ {

View File

@ -7,11 +7,10 @@ sub run {
assert_screen "bootloader", 30; assert_screen "bootloader", 30;
if ( get_var("FLAVOR") eq "server") # Make sure we skip media check if it's selected by default. Standard
{ # 'boot installer' menu entry is always first.
# Skip the media check on DVD send_key "up";
send_key "up"; send_key "up";
}
if( get_var("GRUB")){ if( get_var("GRUB")){
send_key "tab"; send_key "tab";
@ -36,8 +35,13 @@ sub run {
unless (get_var("KICKSTART")) unless (get_var("KICKSTART"))
{ {
# on lives, we have to explicitly launch anaconda
if (get_var('LIVE')) {
assert_and_click "live_initial_anaconda_launcher", '', 300;
} else {
assert_screen "anaconda_select_install_lang", 300;
}
# Select install language # Select install language
assert_screen "anaconda_select_install_lang", 300;
assert_and_click "anaconda_select_install_lang_input"; assert_and_click "anaconda_select_install_lang_input";
type_string "english"; type_string "english";
assert_and_click "anaconda_select_install_lang_english_filtered"; assert_and_click "anaconda_select_install_lang_english_filtered";

View File

@ -0,0 +1,57 @@
use base "basetest";
use strict;
use testapi;
sub run {
# If KICKSTART is set, then the wait_time needs to
# consider the install time
my $wait_time = get_var("KICKSTART") ? 1800 : 300;
# Reboot and wait for the text login
assert_screen "text_console_login", $wait_time;
if ((get_var("USER_LOGIN") && get_var("USER_PASSWORD")) || get_var("ROOT_PASSWORD"))
{
my $user_logged_in = 0;
if (get_var("USER_LOGIN"))
{
type_string get_var("USER_LOGIN");
send_key "ret";
type_string get_var("USER_PASSWORD");
send_key "ret";
assert_screen "user_logged_in", 10;
$user_logged_in = 1;
}
if (get_var("ROOT_PASSWORD"))
{
if ($user_logged_in == 1)
{
type_string "su -";
send_key "ret";
assert_screen "console_password_required", 10;
}
else
{
type_string "root";
send_key "ret";
}
type_string get_var("ROOT_PASSWORD");
send_key "ret";
assert_screen "root_logged_in", 10;
}
}
}
sub test_flags {
# without anything - rollback to 'lastgood' snapshot if failed
# 'fatal' - whole test suite is in danger if this fails
# 'milestone' - after this test succeeds, update 'lastgood'
# 'important' - if this fails, set the overall state to 'fail'
return { fatal => 1, milestone => 1 };
}
1;
# vim: set sw=4 et:

View File

@ -20,6 +20,7 @@ sub run {
#assert_and_click "anaconda_spoke_done"; #assert_and_click "anaconda_spoke_done";
# Set user details # Set user details
sleep 1;
my $user_login = get_var("USER_LOGIN") || "test"; my $user_login = get_var("USER_LOGIN") || "test";
my $user_password = get_var("USER_PASSWORD") || "weakpassword"; my $user_password = get_var("USER_PASSWORD") || "weakpassword";
assert_and_click "anaconda_install_user_creation"; assert_and_click "anaconda_install_user_creation";
@ -37,8 +38,10 @@ sub run {
#assert_and_click "anaconda_spoke_done"; #assert_and_click "anaconda_spoke_done";
# Wait for install to end # Wait for install to end
assert_screen "anaconda_install_done", 1800; assert_and_click "anaconda_install_done", '', 1800;
assert_and_click "anaconda_install_finish"; if (get_var('LIVE')) {
x11_start_program("reboot");
}
} }
sub test_flags { sub test_flags {

View File

@ -0,0 +1,35 @@
use base "basetest";
use strict;
use testapi;
sub run {
# If KICKSTART is set, then the wait_time needs to
# consider the install time
my $wait_time = get_var("KICKSTART") ? 1800 : 300;
# Wait for the login screen
assert_screen "graphical_login", $wait_time;
if (get_var("USER_LOGIN") && get_var("USER_PASSWORD")) {
send_key "ret";
type_string get_var("USER_PASSWORD");
send_key "ret";
# Move the mouse somewhere it won't highlight the match areas
mouse_set(300, 200);
assert_screen "graphical_desktop_clean", 30;
}
}
sub test_flags {
# without anything - rollback to 'lastgood' snapshot if failed
# 'fatal' - whole test suite is in danger if this fails
# 'milestone' - after this test succeeds, update 'lastgood'
# 'important' - if this fails, set the overall state to 'fail'
return { fatal => 1, milestone => 1 };
}
1;
# vim: set sw=4 et:

View File

@ -1,62 +0,0 @@
use base "basetest";
use strict;
use testapi;
sub run {
# If KICKSTART is set, then the wait_time needs to
# consider the install time
my $wait_time = get_var("KICKSTART") ? 1800 : 300;
# Reboot and wait for the text login
assert_screen "clean_install_login", $wait_time;
if ((get_var("USER_LOGIN") && get_var("USER_PASSWORD")) || get_var("ROOT_PASSWORD"))
{
if (get_var("FLAVOR") eq "server")
{
my $user_logged_in = 0;
if (get_var("USER_LOGIN"))
{
type_string get_var("USER_LOGIN");
send_key "ret";
assert_screen "password_required", 10;
type_string get_var("USER_PASSWORD");
send_key "ret";
assert_screen "user_logged_in", 10;
$user_logged_in = 1;
}
if (get_var("ROOT_PASSWORD"))
{
if ($user_logged_in == 1)
{
type_string "su -";
send_key "ret";
assert_screen "console_password_required", 10;
}
else
{
type_string "root";
send_key "ret";
}
assert_screen "password_required", 10;
type_string get_var("ROOT_PASSWORD");
send_key "ret";
assert_screen "root_logged_in", 10;
}
}
}
}
sub test_flags {
# without anything - rollback to 'lastgood' snapshot if failed
# 'fatal' - whole test suite is in danger if this fails
# 'milestone' - after this test succeeds, update 'lastgood'
# 'important' - if this fails, set the overall state to 'fail'
return { fatal => 1, milestone => 1 };
}
1;
# vim: set sw=4 et: