Test upgrade of FreeIPA server and client deployment

Summary:
This adds an upgrade variant of the FreeIPA tests, with only
the simplest client enrolment (sssd) for now. The server test
starts from the N-1 release and deploys the domain controller
role. The client test similarly starts from the N-1 release
and, when the server is deployed, enrols as a domain client.
Then the server upgrades itself, while the client waits (as the
server is its name server). Then the client upgrades itself,
while the server does some self-checks. The server then waits
for the client to do its checks before decommissioning itself,
as usual. So, summary: *deployment* of both server and client
occurs on N-1, then both are upgraded, then the actual *checks*
occur on N.

In my testing, this all more or less works, except the role
decommission step fails. This failure seems to be a genuine one
so far as I can tell; I intend to file a bug for it soon.

Test Plan:
Run the new tests, check they work. Run the existing
FreeIPA tests (both the compose and the update variants), check
they both behave the same.

Reviewers: jsedlak, jskladan

Reviewed By: jsedlak

Subscribers: tflink

Differential Revision: https://phab.qa.fedoraproject.org/D1204

main.pm

@@ -97,13 +97,36 @@ $needle::cleanuphandler = \&cleanup_needles;
 
 
 sub load_upgrade_tests() {
-    # all upgrade tests consist of: preinstall phase (where packages are upgraded and
+    # all upgrade tests include: boot phase (where bootloader and
-    # dnf-plugin-system-upgrade is installed), run phase (where upgrade is run) and postinstall
+    # encryption are handled if necessary), preinstall phase (where
-    # phase (where is checked if fedora was upgraded successfully)
+    # packages are upgraded and dnf-plugin-system-upgrade installed),
+    # run phase (where upgrade is run) and postinstall phase (where
+    # is checked if fedora was upgraded successfully). The PREUPGRADE
+    # variable can be used to specify additional test modules to run
+    # after the preinstall phase but before the run phase, and the
+    # POSTINSTALL variable can be used to specify additional test
+    # modules to run after the upgrade postinstall phase.
+    autotest::loadtest "tests/upgrade_boot.pm";
+    # if static networking config is needed we must do it at this point
+    if (get_var("POST_STATIC")) {
+        autotest::loadtest "tests/_post_network_static.pm";
+    }
     autotest::loadtest "tests/upgrade_preinstall.pm";
+    # generic pre-upgrade test load
+    if (get_var("PREUPGRADE")) {
+        my @pus = split(/ /, get_var("PREUPGRADE"));
+        foreach my $pu (@pus) {
+            autotest::loadtest "tests/${pu}.pm";
+        }
+    }
     autotest::loadtest "tests/upgrade_run.pm";
-    # set postinstall test
+    # handle additional postinstall tests
-    set_var('POSTINSTALL', "upgrade_postinstall" );
+    if (get_var("POSTINSTALL")) {
+        set_var('POSTINSTALL', "upgrade_postinstall " . get_var("POSTINSTALL"));
+    }
+    else {
+        set_var('POSTINSTALL', "upgrade_postinstall");
+    }
 }
 
 sub load_install_tests() {
@@ -224,9 +247,11 @@ sub load_postinstall_tests() {
     _load_early_postinstall_tests();
 
     # do standard post-install static network config if the var is set
+    # and this is not an upgrade test (this is done elsewhere in the
+    # upgrade workflow)
     # this is here not in early_postinstall_tests as there's no need
     # to do it twice
-    if (get_var("POST_STATIC")) {
+    if (get_var("POST_STATIC") && !get_var("UPGRADE")) {
         autotest::loadtest "tests/_post_network_static.pm";
     }
 

templates

@@ -1227,6 +1227,28 @@
                                     },
                       test_suite => { name => "upgrade_server_64bit" },
                     },
+                    {
+                      machine    => { name => "64bit" },
+                      prio       => 30,
+                      product    => {
+                                      arch    => "x86_64",
+                                      distri  => "fedora",
+                                      flavor  => "universal",
+                                      version => "*",
+                                    },
+                      test_suite => { name => "upgrade_server_domain_controller" },
+                    },
+                    {
+                      machine    => { name => "64bit" },
+                      prio       => 30,
+                      product    => {
+                                      arch    => "x86_64",
+                                      distri  => "fedora",
+                                      flavor  => "universal",
+                                      version => "*",
+                                    },
+                      test_suite => { name => "upgrade_realmd_join_sssd" },
+                    },
                     {
                       machine    => { name => "64bit" },
                       prio       => 30,
@@ -2204,6 +2226,43 @@
                         { key => "TEST_TARGET", value => "COMPOSE" },
                       ],
                     },
+                    {
+                      name => "upgrade_server_domain_controller",
+                      settings => [
+                        { key => "ROOT_PASSWORD", value => "weakpassword" },
+                        { key => "HDD_1", value => "disk_f%CURRREL%_server_3_x86_64.img" },
+                        { key => "UPGRADE", value => "1" },
+                        { key => "TEST_TARGET", value => "COMPOSE" },
+                        { key => "PREUPGRADE", value => "role_deploy_domain_controller" },
+                        { key => "POSTINSTALL", value => "role_deploy_domain_controller_check" },
+                        { key => "USER_LOGIN", value => "false" },
+                        { key => "GRUB_POSTINSTALL", value => "net.ifnames=0 biosdevname=0" },
+                        { key => "BOOTFROM", value => "c" },
+                        { key => "GRUB", value => "net.ifnames=0 biosdevname=0" },
+                        { key => "POST_STATIC", value => "10.0.2.100 ipa001.domain.local" },
+                        { key => "NICTYPE", value => "tap" },
+                        { key => "WORKER_CLASS", value => "tap" },
+                      ],
+                    },
+                    {
+                      name => "upgrade_realmd_join_sssd",
+                      settings => [
+                        { key => "ROOT_PASSWORD", value => "weakpassword" },
+                        { key => "HDD_1", value => "disk_f%CURRREL%_server_3_x86_64.img" },
+                        { key => "UPGRADE", value => "1" },
+                        { key => "TEST_TARGET", value => "COMPOSE" },
+                        { key => "PREUPGRADE", value => "realmd_join_sssd" },
+                        { key => "POSTINSTALL", value => "freeipa_client" },
+                        { key => "PARALLEL_WITH", value => "upgrade_server_domain_controller" },
+                        { key => "USER_LOGIN", value => "false" },
+                        { key => "GRUB_POSTINSTALL", value => "net.ifnames=0 biosdevname=0" },
+                        { key => "BOOTFROM", value => "c" },
+                        { key => "GRUB", value => "net.ifnames=0 biosdevname=0" },
+                        { key => "POST_STATIC", value => "10.0.2.103 client003.domain.local" },
+                        { key => "NICTYPE", value => "tap" },
+                        { key => "WORKER_CLASS", value => "tap" },
+                      ],
+                    },
                     {
                       name => "upgrade_kde_64bit",
                       settings => [
@@ -2515,7 +2574,7 @@
                     {
                       name => "server_role_deploy_domain_controller",
                       settings => [
-                        { key => "POSTINSTALL", value => "role_deploy_domain_controller" },
+                        { key => "POSTINSTALL", value => "role_deploy_domain_controller role_deploy_domain_controller_check" },
                         { key => "USER_LOGIN", value => "false" },
                         { key => "ROOT_PASSWORD", value => "weakpassword" },
                         { key => "GRUB_POSTINSTALL", value => "net.ifnames=0 biosdevname=0" },

tests/realmd_join_sssd.pm

@@ -9,18 +9,30 @@ sub run {
     my $self=shift;
     # use FreeIPA server as DNS server
     assert_script_run "printf 'search domain.local\nnameserver 10.0.2.100' > /etc/resolv.conf";
+    assert_script_run "sed -i -e '/^DNS.*/d' /etc/sysconfig/network-scripts/ifcfg-eth0";
+    assert_script_run "printf '\nDNS1=10.0.2.100\n' >> /etc/sysconfig/network-scripts/ifcfg-eth0";
     # wait for the server to be ready (do it now just to make sure name
     # resolution is working before we proceed)
     mutex_lock "freeipa_ready";
     mutex_unlock "freeipa_ready";
-    # use compose repo, disable u-t, etc.
+    # use compose repo, disable u-t, etc. unless this is an upgrade
-    repo_setup();
+    # test (in which case we're on the 'old' release at this point;
+    # one of the upgrade test modules does repo_setup later)
+    repo_setup() unless get_var("UPGRADE");
     # do the enrolment
     assert_script_run "echo 'monkeys123' | realm join --user=admin ipa001.domain.local", 300;
     # set sssd debugging level higher (useful for debugging failures)
     # optional as it's not really part of the test
     script_run "dnf -y install sssd-tools", 180;
     script_run "sss_debuglevel 6";
+    # if upgrade test, report that we're enrolled
+    mutex_create('client_enrolled') if get_var("UPGRADE");
+    # if this is an upgrade test, wait for server to be upgraded before
+    # continuing, as we rely on it for name resolution
+    if (get_var("UPGRADE")) {
+        mutex_lock "server_upgraded";
+        mutex_unlock "server_upgraded";
+    }
 }
 
 sub test_flags {

tests/role_deploy_domain_controller.pm

@@ -10,8 +10,10 @@ sub run {
     my $self = shift;
     # login
     $self->root_console();
-    # use compose repo, disable u-t, etc.
+    # use compose repo, disable u-t, etc. unless this is an upgrade
-    repo_setup();
+    # test (in which case we're on the 'old' release at this point;
+    # one of the upgrade test modules does repo_setup later)
+    repo_setup() unless get_var("UPGRADE");
     # we need a lot of entropy for this, and we don't care how good
     # it is, so let's use haveged
     assert_script_run 'dnf -y install haveged', 300;
@@ -60,14 +62,6 @@ sub run {
     if ($release ne "rawhide" && $release < 25) {
         assert_script_run 'ipa-getcert resubmit -d /etc/httpd/alias -n Server-Cert -D $( uname -n )';
     }
-    # check the role status, should be 'running'
-    validate_script_output 'rolectl status domaincontroller/domain.local', sub { $_ =~ m/^running/ };
-    # check the admin password is listed in 'settings'
-    validate_script_output 'rolectl settings domaincontroller/domain.local', sub {$_ =~m/dm_password = \w{5,}/ };
-    # sanitize the settings
-    assert_script_run 'rolectl sanitize domaincontroller/domain.local';
-    # check the password now shows as 'None'
-    validate_script_output 'rolectl settings domaincontroller/domain.local', sub {$_ =~ m/dm_password = None/ };
     # kinit as admin
     assert_script_run 'echo "monkeys123" | kinit admin';
     # set up an OTP for client001 enrolment (it will enrol with a kickstart)
@@ -86,17 +80,15 @@ sub run {
     # kinit as each user and set a new password
     assert_script_run 'printf "correcthorse\nbatterystaple\nbatterystaple" | kinit test1@DOMAIN.LOCAL';
     assert_script_run 'printf "correcthorse\nbatterystaple\nbatterystaple" | kinit test2@DOMAIN.LOCAL';
-    # we're all ready for other jobs to run!
+    # we're ready for children to enrol, now
-    mutex_create('freeipa_ready');
+    mutex_create("freeipa_ready");
-    wait_for_children;
+    # if upgrade test, wait for children to enrol before upgrade
-    # once child jobs are done, stop the role
+    if (get_var("UPGRADE")) {
-    assert_script_run 'rolectl stop domaincontroller/domain.local';
+        my $children = get_children();
-    # check role is stopped
+        my $child_id = (keys %$children)[0];
-    validate_script_output 'rolectl status domaincontroller/domain.local', sub { $_ =~ m/^ready-to-start/ };
+        mutex_lock('client_enrolled', $child_id);
-    # decommission the role
+        mutex_unlock('client_enrolled');
-    assert_script_run 'rolectl decommission domaincontroller/domain.local', 300;
+    }
-    # check role is decommissioned
-    validate_script_output 'rolectl list instances', sub { $_ eq "" };
 }
 
 

tests/role_deploy_domain_controller_check.pm

@@ -0,0 +1,37 @@
+use base "installedtest";
+use strict;
+use testapi;
+use lockapi;
+use mmapi;
+
+sub run {
+    my $self = shift;
+    # if this is an update, notify clients that we're now up again
+    mutex_create('server_upgraded') if get_var("UPGRADE");
+    # check the role status, should be 'running'
+    validate_script_output 'rolectl status domaincontroller/domain.local', sub { $_ =~ m/^running/ };
+    # check the admin password is listed in 'settings'
+    validate_script_output 'rolectl settings domaincontroller/domain.local', sub {$_ =~m/dm_password = \w{5,}/ };
+    # sanitize the settings
+    assert_script_run 'rolectl sanitize domaincontroller/domain.local';
+    # check the password now shows as 'None'
+    validate_script_output 'rolectl settings domaincontroller/domain.local', sub {$_ =~ m/dm_password = None/ };
+    # once child jobs are done, stop the role
+    wait_for_children;
+    assert_script_run 'rolectl stop domaincontroller/domain.local';
+    # check role is stopped
+    validate_script_output 'rolectl status domaincontroller/domain.local', sub { $_ =~ m/^ready-to-start/ };
+    # decommission the role
+    assert_script_run 'rolectl decommission domaincontroller/domain.local', 300;
+    # check role is decommissioned
+    validate_script_output 'rolectl list instances', sub { $_ eq "" };
+}
+
+
+sub test_flags {
+    return { fatal => 1 };
+}
+
+1;
+
+# vim: set sw=4 et:

tests/upgrade_boot.pm

@@ -0,0 +1,32 @@
+use base "installedtest";
+use strict;
+use testapi;
+use utils;
+
+sub run {
+    my $self = shift;
+    # handle bootloader, if requested
+    if (get_var("GRUB_POSTINSTALL")) {
+        do_bootloader(postinstall=>1, params=>get_var("GRUB_POSTINSTALL"));
+    }
+
+    # decrypt disks during boot if necessary
+    if (get_var("ENCRYPT_PASSWORD")) {
+        boot_decrypt(60);
+    }
+
+    boot_to_login_screen;
+    # switch to TTY3 for both, graphical and console tests
+    $self->root_console(tty=>3);
+    # disable screen blanking (update can take a long time)
+    script_run "setterm -blank 0";
+}
+
+
+sub test_flags {
+    return { fatal => 1 };
+}
+
+1;
+
+# vim: set sw=4 et:

tests/upgrade_preinstall.pm

@@ -5,21 +5,15 @@ use utils;
 
 sub run {
     my $self = shift;
-    # decrypt disks during boot if necessary
-    if (get_var("ENCRYPT_PASSWORD")) {
-        boot_decrypt(60);
-    }
-
-    boot_to_login_screen;
-    # switch to TTY3 for both, graphical and console tests
-    $self->root_console(tty=>3);
-    # disable screen blanking (update can take a long time)
-    script_run "setterm -blank 0";
-
     # upgrader should be installed on up-to-date system
     assert_script_run 'dnf -y update', 1800;
     script_run "reboot", 0;
 
+    # handle bootloader, if requested
+    if (get_var("GRUB_POSTINSTALL")) {
+        do_bootloader(postinstall=>1, params=>get_var("GRUB_POSTINSTALL"));
+    }
+
     # decrypt if necessary
     if (get_var("ENCRYPT_PASSWORD")) {
         boot_decrypt(60);

tests/upgrade_run.pm

@@ -21,15 +21,22 @@ sub run {
     upload_logs "/var/log/dnf.rpm.log";
 
     script_run "dnf system-upgrade reboot", 0;
-    # fail immediately if we see a DNF error message
+    # fail immediately if we see a DNF error message, but keep an eye
-    die "DNF reported failure" if (check_screen "upgrade_fail", 15);
+    # out for the bootloader so we can handle it if requested
+    check_screen ["upgrade_fail", "bootloader"], 15;
+    die "DNF reported failure" if (match_has_tag "upgrade_fail");
+
+    # handle bootloader, if requested
+    if (get_var("GRUB_POSTINSTALL")) {
+        do_bootloader(postinstall=>1, params=>get_var("GRUB_POSTINSTALL"));
+    }
+
+    # decrypt, if encrypted
     if (get_var("ENCRYPT_PASSWORD")) {
         boot_decrypt(60);
-    }
         # in encrypted case we need to wait a bit so postinstall test
         # doesn't bogus match on the encryption prompt we just completed
         # before it disappears from view
-    if (get_var("ENCRYPT_PASSWORD")) {
         sleep 5;
     }
 }