Automate the Anaconda_Save_Traceback_to_Bugzilla_Text test case.

This PR adds a new test that automates the above mentioned test case.
It starts the installation in text mode using the `install_text` test
case, which it interrupts using the Anaconda crash trigger.
When the crash happens, it goes through the process of reporting
the bug to Bugzilla, checks that Bugzilla sends a positive
confirmation of the action, but also performs some REST API
calls to do a proper check and then it closes the bug to clean up.
This commit is contained in:
Lukáš Růžička 2021-04-27 13:02:53 +02:00 committed by Adam Williamson
parent b87049217c
commit 395ed61a1a
16 changed files with 327 additions and 7 deletions

1
.gitignore vendored
View File

@ -7,4 +7,3 @@
.tox/
.coverage
coverage.xml

View File

@ -81,6 +81,10 @@ it also means that `B` conflicts `A` even if not shown in the table).
| `ANACONDA_STATIC` | string (IPv4 address) | not set | `ANACONDA_TEXT` | If set, will set up static networking using the chosen IP address during install |
| `POST_STATIC` | string (space-separated IPv4 address and hostname) | not set | nothing | If set, will set up static networking using the chosen IP address and hostname during early post-install |
| `NO_UEFI_POST` | boolean | `false`/not set | nothing | If set, `uefi_postinstall` test will not be loaded even if `UEFI` is set (can be useful for non-English tests to avoid `uefi_postinstall` running loadkeys) |
| `CRASH_REPORT` | boolean | not set | nothing | If set, Anaconda will perorm a simulated crash during installation, which will be used to create a report in the Bugzilla. Currently, this only affects Anaconda when it runs in text mode. |
| `BUGZILLA_LOGIN` | string | not set | used with `_SECRET_BUGZILLA_PASSWORD` | This is used to store a login string which does not get exposed in log files. |
| `_SECRET_BUGZILLA_PASSWORD` | string | not set | used with `BUGZILLA_LOGIN` | This is used to store a password string which does not get exposed in log files. |
| `_SECRET_BUGZILLA_APIKEY` | string | not set | used with other secrets | This is used to store an API key which does not get exposed in log files. |
Run variables
-------------

View File

@ -2,7 +2,7 @@
tasks:
- name: Install required packages
package:
name: ['os-autoinst', 'perl-Test-Strict', 'perl-Test-Harness']
name: ['os-autoinst', 'perl-Test-Strict', 'perl-Test-Harness', 'perl-JSON', 'perl-REST-Client']
state: present
become: yes
- name: Run perl tests

View File

@ -7,8 +7,9 @@ use Exporter;
use testapi;
use utils;
use bugzilla;
our @EXPORT = qw/select_disks custom_scheme_select custom_blivet_add_partition custom_blivet_format_partition custom_blivet_resize_partition custom_change_type custom_change_fs custom_change_device custom_delete_part get_full_repo get_mirrorlist_url check_help_on_pane/;
our @EXPORT = qw/select_disks custom_scheme_select custom_blivet_add_partition custom_blivet_format_partition custom_blivet_resize_partition custom_change_type custom_change_fs custom_change_device custom_delete_part get_full_repo get_mirrorlist_url crash_anaconda_text report_bug_text/;
sub select_disks {
# Handles disk selection. Has one optional argument - number of
@ -360,3 +361,111 @@ sub check_help_on_pane {
}
}
sub crash_anaconda_text {
# This routine uses the Anaconda crash trigger to break the ongoing Anaconda installation to simulate
# an Anaconda crash and runs a series of steps that results in creating a bug in Bugzilla.
# It is used in the `install_text.pm` test and can be switched on by using the CRASH_REPORT
# variable set to 1.
#
# First let us navigate to reach the shell window in Anaconda using the alt-f3 combo,
# this should take us to another terminal, where we can simulate the crash.
send_key "alt-f3";
assert_screen("anaconda_text_install_shell");
# We use the trigger command to do the simulated crash.
type_string "kill -USR1 `cat /var/run/anaconda.pid`\n";
# And navigate back to the main panel of Anaconda. This should require
send_key "alt-f1";
assert_screen("anaconda_text_install_main");
# We wait until the crash menu appears. This usually takes some time,
# so let's try for 300 seconds, this should be long enough.
my $trials = 1;
until (check_screen("anaconda_text_crash_menu_ready") || $trials > 30) {
sleep 10;
++$trials;
}
# If the crash menu never appears, let's assert it to fail.
if ($trials > 30) {
assert_screen("anaconda_text_crash_menu_ready");
}
}
sub report_bug_text {
# This routine handles the Bugzilla reporting after a simulated crash on
# a textual console.
# We will not create a needle for every menu item, and we will fail,
# if there will be no positive Bugzilla confirmation shown at the end
# of the process and then we will fail.
#
# Let us record the time of this test run. Later, we will use it to
# limit the Bugzilla search.
my $timestamp = time();
#
# First, collect the credentials.
my $login = get_var("BUGZILLA_LOGIN");
my $password = get_var("_SECRET_BUGZILLA_PASSWORD");
my $apikey = get_var("_SECRET_BUGZILLA_APIKEY");
# Choose item 1 - Report the bug.
type_string "1\n";
sleep 2;
# Choose item 1 - Report to Bugzilla
type_string "1\n";
sleep 5;
# Do login.
type_string $login;
type_string "\n";
sleep 5;
# Enter the name of the Zilla.
type_password $password;
type_string "\n";
sleep 10;
# Save the report without changing it.
# It would need some more tweaking to actually type into the report, but since
# it is reported even if unchanged, we leave it as such.
type_string ":wq\n";
# Wait until the Crash menu appears again.
# The same screen shows the result of the Bugzilla operation,
# so if the needle matches, the bug has been created in Bugzilla.
# Bugzilla connection is slow so we need to wait out some time,
# therefore let's use a cycle that will check each 10 seconds and
# ends if there is no correct answer from Bugzilla in 120 seconds.
my $counter = 0;
until (check_screen("anaconda_text_bug_reported") || $counter > 12) {
sleep 10;
++$counter;
}
# Sometimes, Bugzilla throws out a communication error although the bug has been
# created successfully. If this happens, we will softfail and leave the creation
# check to a later step.
if ($counter > 12) {
record_soft_failure "Warning: Bugzilla has reported an error which could mean that the bug has not been created correctly, but it probably is not a real problem, if the test has not failed completely. ";
}
# Now, let us check with Bugzilla directly, if the bug has been created.
# First, we shall get a Bugzilla format timestamp to use it in the query.
# The timestamp will limit the list of bugs to those that have been created since
# the then -> resulting with high probability in the one that this test run
# has just created.
$timestamp = convert_to_bz_timestamp($timestamp);
# Then we fetch the latest bug from Bugzilla.
my $lastbug = get_newest_bug($timestamp, $login);
unless ($lastbug) {
die "Bugzilla returned no newly created bug. It seems that the bug has not been created.";
}
else {
print("BUGZILLA: The last bug was found: $lastbug\n");
}
# We have found that the bug indeed is in the bugzilla (otherwise
# we would have died already) so now we close it to clean up after this test run.
my $result = close_notabug($lastbug, $apikey);
unless ($result) {
record_soft_failure "The bug has not been closed for some reason. Check manually.";
}
else {
print("BUGZILLA: The last bug $lastbug changed status to CLOSED.\n");
}
# Quit anaconda
type_string "4\n";
}

82
lib/bugzilla.pm Normal file
View File

@ -0,0 +1,82 @@
package bugzilla;
use strict;
use base 'Exporter';
use Exporter;
use lockapi;
use testapi;
use utils;
use POSIX qw(strftime);
use JSON;
use REST::Client;
our @EXPORT = qw(convert_to_bz_timestamp get_newest_bug check_bug_status_field close_notabug);
sub start_bugzilla_client {
# Start a Bugzilla REST client for setting up communication.
# This is a local subroutine, not intended for export.
my $bugzilla = REST::Client->new();
$bugzilla->setHost("https://bugzilla.redhat.com");
return $bugzilla;
}
sub convert_to_bz_timestamp {
# This subroutine takes the epoch time and converts it to
# the Bugzilla timestamp format (YYYY-MM-DDTHH:MM:SS)
# in the GMT time zone.
my $epochtime = shift;
my $bz_stamp = strftime("%FT%T", gmtime($epochtime));
return $bz_stamp;
}
sub get_newest_bug {
# This subroutine makes an API call to Bugzilla and
# fetches the newest bug that have been created.
# This will be the bug created by Anaconda in this
# test run.
my ($timestamp, $login) = @_;
$timestamp = convert_to_bz_timestamp($timestamp);
my $bugzilla = start_bugzilla_client();
my $api_call = $bugzilla->GET("/rest/bug?creator=$login&status=NEW&created_after=$timestamp");
my $rest_json = decode_json($api_call->responseContent());
my $last_id;
eval {
$last_id = $rest_json->{bugs}[-1]->{id};
1;
} or do {
record_soft_failure "Bugzilla returned an empty list of bugs which is unexpected!";
$last_id = 0;
};
return $last_id;
}
sub check_bug_status_field {
# This will check that the status field matches the one
# tested status. Arguments are bug_id and status.
my ($bug_id, $status) = @_;
my $bugzilla = start_bugzilla_client();
my $api_call = $bugzilla->GET("/rest/bug/$bug_id");
my $rest_json = decode_json($api_call->responseContent());
if ($rest_json->{bugs}[0]->{status} eq $status) {
return 1;
}
else {
return 0;
}
}
sub close_notabug {
# This will call Bugzilla and close the bug with the requested
# bug id as a NOTABUG.
my ($bug_id, $key) = @_;
my $bugzilla = start_bugzilla_client();
my $api_call = $bugzilla->PUT("/rest/bug/$bug_id?api_key=$key&status=CLOSED&resolution=NOTABUG");
my $rest_json = decode_json($api_call->responseContent());
if ($rest_json->{bugs}[0]->{changes}->{status}->{added} ne "CLOSED") {
return 0;
}
else {
return 1;
}
}

11
main.pm
View File

@ -251,7 +251,9 @@ sub _load_early_postinstall_tests {
if (get_var("SWITCHED_LAYOUT") || get_var("INPUT_METHOD")) {
_load_instance("tests/_graphical_input", $instance);
}
unless (get_var("DESKTOP")) {
# We do not want to run this on Desktop installations or when
# the installation is interrupted on purpose.
unless (get_var("DESKTOP") || get_var("CRASH_REPORT")) {
_load_instance("tests/_console_wait_login", $instance);
}
}
@ -311,12 +313,13 @@ sub load_postinstall_tests() {
# console avc / crash check
# it makes no sense to run this after logging in on most post-
# install tests (hence ! BOOTFROM) but we *do* want to run it on
# upgrade tests after upgrading (hence UPGRADE)
# install tests (hence ! BOOTFROM) and we do not want it
# on crashed installations (hence ! CRASH_REPORT) but we *do* want
# to run it on upgrade tests after upgrading (hence UPGRADE)
# desktops have specific tests for this (hence !DESKTOP). For
# desktop upgrades we should really upload a disk image at the end
# of upgrade and run all the desktop post-install tests on that
if (!get_var("DESKTOP") && (!get_var("BOOTFROM") || get_var("UPGRADE"))) {
if (!get_var("DESKTOP") && !get_var("CRASH_REPORT") && (!get_var("BOOTFROM") || get_var("UPGRADE"))) {
autotest::loadtest "tests/_console_avc_crash.pm";
}

View File

@ -0,0 +1,15 @@
{
"properties": [],
"tags": [
"anaconda_text_bug_reported"
],
"area": [
{
"xpos": 2,
"ypos": 304,
"width": 447,
"height": 17,
"type": "match"
}
]
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

View File

@ -0,0 +1,15 @@
{
"properties": [],
"tags": [
"anaconda_text_crash_menu_ready"
],
"area": [
{
"xpos": 2,
"ypos": 640,
"width": 107,
"height": 17,
"type": "match"
}
]
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

View File

@ -0,0 +1,15 @@
{
"properties": [],
"tags": [
"anaconda_text_install_main"
],
"area": [
{
"xpos": 94,
"ypos": 754,
"width": 47,
"height": 14,
"type": "match"
}
]
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

View File

@ -0,0 +1,15 @@
{
"properties": [],
"tags": [
"anaconda_text_install_shell"
],
"area": [
{
"xpos": 8,
"ypos": 81,
"width": 113,
"height": 15,
"type": "match"
}
]
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.9 KiB

View File

@ -369,6 +369,33 @@
"TEST_TARGET": "ISO"
},
"version": "*"
},
"fedora-seasonal-aarch64-*": {
"arch": "aarch64",
"distri": "fedora",
"flavor": "seasonal",
"settings": {
"TEST_TARGET": "ISO"
},
"version": "*"
},
"fedora-seasonal-ppc64le-*": {
"arch": "ppc64le",
"distri": "fedora",
"flavor": "seasonal",
"settings": {
"TEST_TARGET": "ISO"
},
"version": "*"
},
"fedora-seasonal-x86_64-*": {
"arch": "x86_64",
"distri": "fedora",
"flavor": "seasonal",
"settings": {
"TEST_TARGET": "ISO"
},
"version": "*"
}
},
"Profiles": {
@ -507,6 +534,22 @@
"fedora-universal-x86_64-*-uefi": {
"machine": "uefi",
"product": "fedora-universal-x86_64-*"
},
"fedora-seasonal-aarch64-*-aarch64": {
"machine": "aarch64",
"product": "fedora-seasonal-aarch64-*"
},
"fedora-seasonal-ppc64le-*-ppc64le": {
"machine": "ppc64le",
"product": "fedora-seasonal-ppc64le-*"
},
"fedora-seasonal-x86_64-*-64bit": {
"machine": "64bit",
"product": "fedora-seasonal-x86_64-*"
},
"fedora-seasonal-x86_64-*-uefi": {
"machine": "uefi",
"product": "fedora-seasonal-x86_64-*"
}
},
"TestSuites": {
@ -899,6 +942,17 @@
"ANACONDA_TEXT": "1"
}
},
"install_anaconda_text_save_traceback_bugzilla": {
"profiles": {
"fedora-seasonal-aarch64-*-aarch64": 20,
"fedora-seasonal-ppc64le-*-ppc64le": 20,
"fedora-seasonal-x86_64-*-64bit": 20
},
"settings": {
"ANACONDA_TEXT": "1",
"CRASH_REPORT": "1"
}
},
"install_arabic_language": {
"profiles": {
"fedora-universal-aarch64-*-aarch64": 40,

View File

@ -2,6 +2,7 @@ use base "anacondatest";
use strict;
use testapi;
use utils;
use anaconda;
# this enables you to send a command and some post-command wait time
@ -119,6 +120,14 @@ sub run {
# begin installation
console_type_wait("b\n");
# When simulated crash is planned, then proceed with the crash routines and finish,
# otherwise proceed normally and do
if (get_var("CRASH_REPORT")) {
crash_anaconda_text;
report_bug_text;
return;
}
# Wait for install to end. Give Rawhide a bit longer, in case
# we're on a debug kernel, debug kernel installs are really slow.
my $timeout = 1800;