add a cockpit realmd FreeIPA join test

Summary:
This requires a few other changes:

* turn clone_host_resolv into clone_host_file, letting you clone
  any given host file (cloning /etc/hosts seems to make both
  server deployment and client enrolment faster/more reliable)
* allow loading of multiple POSTINSTALL tests (so we can share
  the freeipa_client_postinstall test). Note this is compatible,
  existing uses will work fine
* move initial password change for the IPA test users into the
  server deployment test (so the client tests don't conflict over
  doing that)
* add GRUB_POSTINSTALL, for specifying boot parameters for boot of
  the installed system, and make it work by tweaking _console_wait
  _login (doesn't work for _graphical_wait_login yet, as I didn't
  need that)
* make the static networking config for tap tests into a library
  function so the tests can share it
* handle ABRT problem dirs showing up in /var/spool/abrt as well
  as /var/tmp/abrt (because the enrol attempt hits #1330766 and
  the crash report shows up in /var/spool/abrt, don't ask me why
  the difference, I just work here)
* specify the DNS servers from the worker host's resolv.conf as
  the forwarders for the FreeIPA server when deploying it; if we
  don't do this, rolekit defaults to using the root servers as
  forwarders(!) and thus we get the public, not phx2-appropriate,
  results for e.g. mirrors.fedoraproject.org, some of which the
  workers can't reach, so PackageKit package install always fails
  (boy, was it fun figuring THAT mess out)

Even after all that, the test still doesn't actually pass, but
I'm reasonably confident this is because it's hitting actual bugs,
not because it's broken. It runs into #1330766 nearly every time
(I think I saw *one* time the enrolment actually succeeded), and
seems to run into a subsequent bug I hadn't seen before when
trying to work around that by trying the join again (see
https://bugzilla.redhat.com/show_bug.cgi?id=1330766#c37 ).

Test Plan:
Run the test, see what happens. If you're really lucky,
it'll actually pass. But you'll probably run into #1330766#c37,
I'm mostly posting for comment. You'll need a tap-capable openQA
instance to test this.

Reviewers: jskladan, garretraziel

Reviewed By: garretraziel

Subscribers: tflink

Differential Revision: https://phab.qadevel.cloud.fedoraproject.org/D880

lib/fedorabase.pm

@@ -94,48 +94,53 @@ sub do_bootloader {
     # 'args' is a string of extra kernel args, if desired. 'mutex' is
     # a parallel test mutex lock to wait for before proceeding, if
     # desired. 'first' is whether to hit 'up' a couple of times to
-    # make sure we boot the first menu entry.
-    my ($self, $postinstall, $args, $mutex, $first, $bootloader, $uefi) = @_;
-    $uefi //= get_var("UEFI");
-    $postinstall //= 0;
+    # make sure we boot the first menu entry. 'timeout' is how long to
+    # wait for the bootloader screen.
+    my $self = shift;
+    my %args = (
+        postinstall => 0,
+        params => "",
+        mutex => "",
+        first => 1,
+        timeout => 30,
+        uefi => get_var("UEFI"),
+        @_
+    );
     # if not postinstall and not UEFI, syslinux
-    $bootloader //= ($uefi || $postinstall) ? "grub" : "syslinux";
-    $args //= "";
-    $mutex //= "";
-    $first //= 1;
-    if ($uefi) {
+    $args{bootloader} //= ($args{uefi} || $args{postinstall}) ? "grub" : "syslinux";
+    if ($args{uefi}) {
         # we don't just tag all screens with 'bootloader' because we
         # want to be sure we actually did a UEFI boot
-        assert_screen "bootloader_uefi", 30;
+        assert_screen "bootloader_uefi", $args{timeout};
     } else {
-        assert_screen "bootloader", 30;
+        assert_screen "bootloader", $args{timeout};
     }
-    if ($mutex) {
+    if ($args{mutex}) {
         # cancel countdown
         send_key "left";
-        mutex_lock $mutex;
-        mutex_unlock $mutex;
+        mutex_lock $args{mutex};
+        mutex_unlock $args{mutex};
     }
-    if ($first) {
+    if ($args{first}) {
         # press up a couple of times to make sure we're at first entry
         send_key "up";
         send_key "up";
     }
-    if ($args) {
-        if ($bootloader eq "syslinux") {
+    if ($args{params}) {
+        if ($args{bootloader} eq "syslinux") {
             send_key "tab";
         }
         else {
             send_key "e";
             # ternary: 13 'downs' to reach the kernel line for installed
             # system, 2 for UEFI installer
-            my $presses = $postinstall ? 13 : 2;
+            my $presses = $args{postinstall} ? 13 : 2;
             foreach my $i (1..$presses) {
                 send_key "down";
             }
             send_key "end";
         }
-        type_string " $args";
+        type_string " $args{params}";
     }
     # ctrl-X boots from grub editor mode
     send_key "ctrl-x";
@@ -162,20 +167,41 @@ sub get_milestone {
     return '';
 }
 
-sub clone_host_resolv {
-    # this is pretty crazy, but SUSE do almost the same thing...
-    # it's for openvswitch jobs to clone the host's resolv.conf, so
-    # we don't have to hard code 8.8.8.8 or have the templates pass
-    # in an address or something
+sub clone_host_file {
+    # copy a given file from the host into the guest. Mainly used
+    # for networking config on tap tests. this is pretty crazy, but
+    # SUSE do almost the same thing...
     my $self = shift;
-    my $resolv = '';
-    open(FH, '<', "/etc/resolv.conf");
-    while (<FH>) {
-        $resolv .= $_;
+    my $file = shift;
+    my $text = '';
+    open(my $fh, '<', $file);
+    while (<$fh>) {
+        $text .= $_;
     }
-    assert_script_run "printf '$resolv' > /etc/resolv.conf";
+    assert_script_run "printf '$text' > $file";
     # for debugging...
-    assert_script_run "cat /etc/resolv.conf";
+    assert_script_run "cat $file";
+}
+
+sub setup_tap_static {
+    # this is a common thing for tap tests, where we set up networking
+    # for the system with a static IP address and possibly a specific
+    # hostname
+    my $self = shift;
+    my $ip = shift;
+    my $hostname = shift || "";
+    if ($hostname) {
+        # assigning output of split to a single-item array gives us just
+        # the first split
+        my ($short) = split(/\./, $hostname);
+        # set hostname
+        assert_script_run "hostnamectl set-hostname $hostname";
+        # add entry to /etc/hosts
+        assert_script_run "echo '$ip $hostname $short' >> /etc/hosts";
+    }
+    # bring up network. DEFROUTE is *vital* here
+    assert_script_run "printf 'DEVICE=eth0\nBOOTPROTO=none\nIPADDR=$ip\nGATEWAY=10.0.2.2\nPREFIX=24\nDEFROUTE=yes' > /etc/sysconfig/network-scripts/ifcfg-eth0";
+    script_run "systemctl restart NetworkManager.service";
 }
 
 sub console_switch_layout {

lib/installedtest.pm

@@ -27,9 +27,15 @@ sub post_fail_hook {
     # If /var/tmp/abrt directory isn't empty (ls doesn't return empty string)
     my $vartmp = script_output "ls /var/tmp/abrt";
     if ($vartmp ne '') {
-        # Upload all ABRT logs
-        script_run "cd /var/tmp/abrt && tar czvf abrt.tar.gz *";
-        upload_logs "/var/tmp/abrt/abrt.tar.gz";
+        # Upload /var/tmp ABRT logs
+        script_run "cd /var/tmp/abrt && tar czvf tmpabrt.tar.gz *";
+        upload_logs "/var/tmp/abrt/tmpabrt.tar.gz";
+    }
+    my $varspool = script_output "ls /var/spool/abrt";
+    if ($varspool ne '') {
+        # Upload /var/spool ABRT logs
+        script_run "cd /var/spool/abrt && tar czvf spoolabrt.tar.gz *";
+        upload_logs "/var/spool/abrt/spoolabrt.tar.gz";
     }
 
     # Upload /var/log

main.pm

@@ -193,7 +193,10 @@ sub load_postinstall_tests() {
 
     # generic post-install test load
     if (get_var("POSTINSTALL")) {
-        autotest::loadtest "tests/".get_var('POSTINSTALL')."_postinstall.pm";
+        my @pis = split(/ /, get_var("POSTINSTALL"));
+        foreach my $pi (@pis) {
+            autotest::loadtest "tests/${pi}_postinstall.pm";
+        }
     }
 
     # we should shut down before uploading disk images

needles/cockpit/join_button.json

@@ -0,0 +1,15 @@
+{
+  "properties": [],
+  "tags": [
+    "cockpit_join_button"
+  ],
+  "area": [
+    {
+      "xpos": 840,
+      "ypos": 458,
+      "width": 31,
+      "height": 18,
+      "type": "match"
+    }
+  ]
+}

needles/cockpit/join_complete.json

@@ -0,0 +1,15 @@
+{
+  "properties": [],
+  "area": [
+    {
+      "xpos": 397,
+      "ypos": 290,
+      "width": 135,
+      "height": 17,
+      "type": "match"
+    }
+  ],
+  "tags": [
+    "cockpit_join_complete"
+  ]
+}

needles/cockpit/join_domain.json

@@ -0,0 +1,15 @@
+{
+  "tags": [
+    "cockpit_join_domain"
+  ],
+  "area": [
+    {
+      "xpos": 206,
+      "ypos": 168,
+      "width": 96,
+      "height": 16,
+      "type": "match"
+    }
+  ],
+  "properties": []
+}

needles/cockpit/join_domain_button.json

@@ -0,0 +1,15 @@
+{
+  "properties": [],
+  "tags": [
+    "cockpit_join_domain_button"
+  ],
+  "area": [
+    {
+      "xpos": 454,
+      "ypos": 291,
+      "width": 74,
+      "height": 15,
+      "type": "match"
+    }
+  ]
+}

needles/cockpit/join_progress.json

@@ -0,0 +1,15 @@
+{
+  "properties": [],
+  "area": [
+    {
+      "xpos": 237,
+      "ypos": 461,
+      "width": 125,
+      "height": 14,
+      "type": "match"
+    }
+  ],
+  "tags": [
+    "cockpit_join_progress"
+  ]
+}

templates

@@ -347,6 +347,17 @@
                                     },
                       test_suite => { name => "server_cockpit_basic" },
                     },
+                    {
+                      machine    => { name => "64bit" },
+                      prio       => 40,
+                      product    => {
+                                      arch    => "x86_64",
+                                      distri  => "fedora",
+                                      flavor  => "Server-dvd-iso",
+                                      version => "*",
+                                    },
+                      test_suite => { name => "realmd_join_cockpit" },
+                    },
                     {
                       machine    => { name => "64bit" },
                       prio       => 20,
@@ -1702,6 +1713,21 @@
                         { key => "HDD_1", value => "disk_%MACHINE%_cockpit.qcow2" },
                       ],
                     },
+                    {
+                      name => "realmd_join_cockpit",
+                      settings => [
+                        { key => "POSTINSTALL", value => "realmd_join_cockpit freeipa_client" },
+                        { key => "USER_LOGIN", value => "false" },
+                        { key => "ROOT_PASSWORD", value => "weakpassword" },
+                        { key => "START_AFTER_TEST", value => "server_cockpit_default" },
+                        { key => "PARALLEL_WITH", value => "server_role_deploy_domain_controller" },
+                        { key => "BOOTFROM", value => "c" },
+                        { key => "HDD_1", value => "disk_%MACHINE%_cockpit.qcow2" },
+                        { key => "GRUB_POSTINSTALL", value => "net.ifnames=0 biosdevname=0" },
+                        { key => "NICTYPE", value => "tap" },
+                        { key => "WORKER_CLASS", value => "tap" },
+                      ],
+                    },
                     {
                       name => "desktop_terminal",
                       settings => [

tests/_boot_to_anaconda.pm

@@ -4,29 +4,29 @@ use testapi;
 
 sub run {
     my $self = shift;
-    # construct the kernel args. the trick here is to wind up with
-    # spaced args if GRUB or GRUBADD is set, and just spaces if not,
+    # construct the kernel params. the trick here is to wind up with
+    # spaced params if GRUB or GRUBADD is set, and just spaces if not,
     # then check if we got all spaces. We wind up with a harmless
     # extra space if GRUBADD is set but GRUB is not.
-    my $args = "";
-    $args .= get_var("GRUB", "") . " ";
-    $args .= get_var("GRUBADD", "") . " ";
+    my $params = "";
+    $params .= get_var("GRUB", "") . " ";
+    $params .= get_var("GRUBADD", "") . " ";
     # Construct inst.repo arg for REPOSITORY_VARIATION
     my $repourl = get_var("REPOSITORY_VARIATION");
     if ($repourl) {
         my $version = lc(get_var("VERSION", ""));
         my $arch = get_var("ARCH", "");
         $repourl .= "/$version/Everything/$arch/os";
-        $args .= "inst.repo=$repourl";
+        $params .= "inst.repo=$repourl";
     }
-    # ternary: set $args to "" if it contains only spaces
-    $args = $args =~ /^\s+$/ ? "" : $args;
+    # ternary: set $params to "" if it contains only spaces
+    $params = $params =~ /^\s+$/ ? "" : $params;
 
     # set mutex wait if necessary
     my $mutex = get_var("INSTALL_UNLOCK");
 
-    # call do_bootloader with postinstall=0, the args, and the mutex
-    $self->do_bootloader(0, $args, $mutex);
+    # call do_bootloader with postinstall=0, the params, and the mutex
+    $self->do_bootloader(postinstall=>0, params=>$params, mutex=>$mutex);
 
     # proceed to installer
     unless (get_var("KICKSTART"))

tests/_console_wait_login.pm

@@ -10,6 +10,12 @@ sub run {
     $wait_time = 1800 if (get_var("KICKSTART"));
     $wait_time = 6000 if (get_var("UPGRADE"));
 
+    # handle bootloader, if requested
+    if (get_var("GRUB_POSTINSTALL")) {
+        $self->do_bootloader(postinstall=>1, params=>get_var("GRUB_POSTINSTALL"), timeout=>$wait_time);
+        $wait_time = 180;
+    }
+
     # Wait for the text login
     assert_screen "text_console_login", $wait_time;
 

tests/freeipa_client_postinstall.pm

@@ -9,12 +9,11 @@ sub run {
     # check we can see the admin user in getent
     assert_script_run 'getent passwd admin@DOMAIN.LOCAL';
     # check keytab entries
-    validate_script_output 'klist -k', sub { $_ =~ m/client001\.domain\.local\@DOMAIN.LOCAL/ };
+    my $hostname = script_output 'hostname';
+    my $qhost = quotemeta($hostname);
+    validate_script_output 'klist -k', sub { $_ =~ m/$qhost\@DOMAIN\.LOCAL/ };
     # check we can kinit with the host principal
-    assert_script_run 'kinit -k host/client001.domain.local@DOMAIN.LOCAL';
-    # kinit as each user and set a new password
-    assert_script_run 'printf "correcthorse\nbatterystaple\nbatterystaple" | kinit test1@DOMAIN.LOCAL';
-    assert_script_run 'printf "correcthorse\nbatterystaple\nbatterystaple" | kinit test2@DOMAIN.LOCAL';
+    assert_script_run "kinit -k host/$hostname\@DOMAIN.LOCAL";
     # switch to tty3
     send_key "ctrl-alt-f3";
     # try and login as test1, should work

tests/realmd_join_cockpit_postinstall.pm

@@ -0,0 +1,62 @@
+use base "installedtest";
+use strict;
+use testapi;
+use lockapi;
+
+sub run {
+    my $self=shift;
+    # clone host's /etc/hosts (for phx2 internal routing to work)
+    # must come *before* setup_tap_static or else it would overwrite
+    # its changes
+    $self->clone_host_file("/etc/hosts");
+    # set up networking
+    $self->setup_tap_static("10.0.2.102", "client002.domain.local");
+    # test
+    $self->clone_host_file("/etc/resolv.conf");
+    # use FreeIPA server as DNS server
+    assert_script_run "printf 'search domain.local\nnameserver 10.0.2.100' > /etc/resolv.conf";
+    # wait for the server to be ready (do it now just to make sure name
+    # resolution is working before we proceed)
+    mutex_lock "freeipa_ready";
+    mutex_unlock "freeipa_ready";
+    # run firefox and login to cockpit
+    $self->start_cockpit(1);
+    assert_and_click "cockpit_join_domain_button";
+    assert_screen "cockpit_join_domain";
+    send_key "tab";
+    wait_still_screen 1;
+    type_string "ipa001.domain.local";
+    wait_still_screen 1;
+    send_key "tab";
+    send_key "tab";
+    wait_still_screen 1;
+    type_string "admin";
+    wait_still_screen 1;
+    send_key "tab";
+    type_string "monkeys123";
+    wait_still_screen 1;
+    assert_and_click "cockpit_join_button";
+    # check we hit the progress screen, so we fail faster if it's
+    # broken
+    assert_screen "cockpit_join_progress";
+    # join involves package installs, so it may take some time
+    assert_screen "cockpit_join_complete", 300;
+    # quit browser to return to console
+    send_key "ctrl-q";
+    # we don't get back to a prompt instantly and keystrokes while X
+    # is still shutting down are swallowed, so wait_still_screen before
+    # finishing (and handing off to freeipa_client_postinstall)
+    wait_still_screen 5;
+}
+
+sub test_flags {
+    # without anything - rollback to 'lastgood' snapshot if failed
+    # 'fatal' - whole test suite is in danger if this fails
+    # 'milestone' - after this test succeeds, update 'lastgood'
+    # 'important' - if this fails, set the overall state to 'fail'
+    return { fatal => 1 };
+}
+
+1;
+
+# vim: set sw=4 et:

tests/role_deploy_domain_controller.pm

@@ -8,27 +8,68 @@ sub run {
     my $self=shift;
     # boot with kernel params to ensure interface is 'eth0' and not whatever
     # systemd feels like calling it today
-    $self->do_bootloader(1, "net.ifnames=0 biosdevname=0");
+    $self->do_bootloader(postinstall=>1, params=>"net.ifnames=0 biosdevname=0");
     $self->boot_to_login_screen("text_console_login", 5, 60);
     # login
     $self->root_console();
-    # set hostname
-    assert_script_run 'hostnamectl set-hostname ipa001.domain.local';
-    # add entry to /etc/hosts
-    assert_script_run 'echo "10.0.2.100 ipa001.domain.local ipa001" >> /etc/hosts';
-    # bring up network. DEFROUTE is *vital* here
-    assert_script_run 'printf "DEVICE=eth0\nBOOTPROTO=none\nIPADDR=10.0.2.100\nGATEWAY=10.0.2.2\nPREFIX=24\nDEFROUTE=yes" > /etc/sysconfig/network-scripts/ifcfg-eth0';
-    script_run "systemctl restart NetworkManager.service";
+    # clone host's /etc/hosts (for phx2 internal routing to work)
+    # must come *before* setup_tap_static or else it would overwrite
+    # its changes
+    $self->clone_host_file("/etc/hosts");
+    # set up networking
+    $self->setup_tap_static("10.0.2.100", "ipa001.domain.local");
     # clone host's resolv.conf to get name resolution
-    $self->clone_host_resolv();
+    $self->clone_host_file("/etc/resolv.conf");
     # we don't want updates-testing for validation purposes
     assert_script_run 'dnf config-manager --set-disabled updates-testing';
     # we need a lot of entropy for this, and we don't care how good
     # it is, so let's use haveged
     assert_script_run 'dnf -y install haveged', 120;
     assert_script_run 'systemctl start haveged.service';
-    # deploy the domain controller role
-    assert_script_run 'echo \'{"admin_password":"monkeys123"}\' | rolectl deploy domaincontroller --name=domain.local --settings-stdin', 1200;
+    # read DNS server IPs from host's /etc/resolv.conf for passing to
+    # rolectl
+    my @forwards;
+    open(FH, '<', "/etc/resolv.conf");
+    while (<FH>) {
+        if ($_ =~ m/^nameserver +(.+)/) {
+            push @forwards, $1;
+        }
+    }
+    # we are now gonna work around a stupid bug in rolekit. we want to
+    # pass it a list of ipv4 DNS forwarders and have no ipv6 DNS
+    # forwarders. but it won't allow you to have a dns_forwarders array
+    # with a "ipv4" list but no "ipv6" list, any values in the "ipv6"
+    # list must be contactable (so we can't use real IPv6 DNS servers
+    # as we have no IPv6 connectivity), and if you use an empty list
+    # as the "ipv6" value you often hit a weird DBus error "unable to
+    # guess signature from an empty list". Fortunately, rolekit doesn't
+    # actually check that the values in the lists are really IPv6 /
+    # IPv4, it just turns all the values in each list into --forwarder
+    # args for ipa-server-install. So we can just stuff IPv4 values
+    # into both lists. rolekit bug:
+    # https://github.com/libre-server/rolekit/issues/64
+    # it should be fixed relatively soon.
+    my $fourlist;
+    my $sixlist;
+    if (scalar @forwards == 1) {
+        # we've only got one server, so dupe it, best we can do
+        $fourlist = '["' . $forwards[0] . '"]';
+        $sixlist = $fourlist;
+    }
+    else {
+        # put the first value in the 'IPv4' list and all the others in
+        # the 'IPv6' list
+        $fourlist = '["' . shift(@forwards) . '"]';
+        $sixlist = '["' . join('","', @forwards) . '"]';
+    }
+    # deploy the domain controller role, specifying an admin password
+    # and the list of DNS server IPs as JSON via stdin. If we don't do
+    # this, rolectl defaults to using the root servers as forwarders
+    # (it does not copy the settings from resolv.conf), which give the
+    # public results for mirrors.fedoraproject.org, some of which
+    # things running in phx2 cannot reach; we must make sure the phx2
+    # deployments use the phx2 nameservers.
+    assert_script_run 'echo \'{"admin_password":"monkeys123","dns_forwarders":{"ipv4":' . $fourlist . ',"ipv6":' . $sixlist .'}}\' | rolectl deploy domaincontroller --name=domain.local --settings-stdin', 1200;
     # check the role status, should be 'running'
     validate_script_output 'rolectl status domaincontroller/domain.local', sub { $_ =~ m/^running/ };
     # check the admin password is listed in 'settings'
@@ -50,6 +91,9 @@ sub run {
     assert_script_run 'ipa hbacrule-add-user testrule --users=test1';
     # disable the default 'everyone everywhere' rule
     assert_script_run 'ipa hbacrule-disable allow_all';
+    # kinit as each user and set a new password
+    assert_script_run 'printf "correcthorse\nbatterystaple\nbatterystaple" | kinit test1@DOMAIN.LOCAL';
+    assert_script_run 'printf "correcthorse\nbatterystaple\nbatterystaple" | kinit test2@DOMAIN.LOCAL';
     # we're all ready for other jobs to run!
     mutex_create('freeipa_ready');
     wait_for_children;