add FreeIPA server role deploy and kickstart enrolment tests

Summary:
These require openQA tap networking to allow the server and
client boxes to communicate, and require masquerading (NAT) so
the server at least can reach a repository (dnf/rolekit really,
really do not want to work without a repo connection).

They use the 'parallel' test support to have the server deploy
run first while the client enrol test waits at the grub menu
until the server is done before it goes ahead.

This is all deployed and working on stg. The really tricky bit
was getting all the openvswitch and firewall config right in
ansible.

We *could* do the server deploy test as a follow-on from the
default install test to save the install, but then we'd have to
teach it to change the hostname and set up static networking
post-install. I'm not sure if it's worth doing that.

This requires the corresponding openqa_fedora_tools commit that
adds the hard disks (containing the kickstarts - it's possible
to get them from remote during install, but we have to set up
name resolution or hard code the IP of the server).

Test Plan:
Deploy this and the openqa_fedora_tools commit,
generate the disks, configure the networking (good luck! See
the docs in openqa_fedora_tools) and see if you can run the
tests. If you're using Docker, uh...sorry. You somehow need to
set things up so the workers can use tap interfaces that can
talk to each other and are NATed to the outside world. Have fun.
I can talk you through it on IRC...

Reviewers: jskladan, garretraziel

Reviewed By: garretraziel

Subscribers: tflink

Differential Revision: https://phab.qadevel.cloud.fedoraproject.org/D831

README.md

@@ -84,6 +84,7 @@ these functions:
        long still screen should be displayed until openQA decides that system is booted and third is timeout how long
        it should wait for still screen to appear. Example usage: `$self->boot_to_login_screen("graphical_login", 30);`
        will wait until screen is not moving for 30 seconds and then checks, whether `graphical_login` needle is displayed.
+    - `clone_host_resolv()` copies the contents of the host's `/etc/resolv.conf` into the guest, overwriting any existing contents. This is mainly intended for use by openvswitch guests which need external connectivity.
 - `anacondatest` should be used in tests where Anaconda is running. It uploads Anaconda logs (for example
 `anaconda.log` or `packaging.log`) in `post_fail_hook()`. It also provides these convenient methods for Anaconda:
     - `root_console()` tries to login is as a root. It decides to what TTY to switch into and then calls `console_login()`

lib/fedorabase.pm

@@ -1,5 +1,6 @@
 package fedorabase;
 use base 'basetest';
+use lockapi;
 
 # base class for all Fedora tests
 
@@ -75,6 +76,64 @@ sub console_login {
     $args{check} && die "Failed to reach console!"
 }
 
+sub do_bootloader {
+    # Handle bootloader screen. 'bootloader' is syslinux or grub.
+    # 'uefi' is whether this is a UEFI install, will get_var UEFI if
+    # not explicitly set. 'postinstall' is whether we're on an
+    # installed system or at the installer (this matters for how many
+    # times we press 'down' to find the kernel line when typing args).
+    # 'args' is a string of extra kernel args, if desired. 'mutex' is
+    # a parallel test mutex lock to wait for before proceeding, if
+    # desired. 'first' is whether to hit 'up' a couple of times to
+    # make sure we boot the first menu entry.
+    my ($self, $postinstall, $args, $mutex, $first, $bootloader, $uefi) = @_;
+    $uefi //= get_var("UEFI");
+    $postinstall //= 0;
+    # if not postinstall and not UEFI, syslinux
+    $bootloader //= ($uefi || $postinstall) ? "grub" : "syslinux";
+    $args //= "";
+    $mutex //= "";
+    $first //= 1;
+    if ($uefi) {
+        # we don't just tag all screens with 'bootloader' because we
+        # want to be sure we actually did a UEFI boot
+        assert_screen "bootloader_uefi", 30;
+    } else {
+        assert_screen "bootloader", 30;
+    }
+    if ($mutex) {
+        # cancel countdown
+        send_key "left";
+        mutex_lock $mutex;
+        mutex_unlock $mutex;
+    }
+    if ($first) {
+        # press up a couple of times to make sure we're at first entry
+        send_key "up";
+        send_key "up";
+    }
+    if ($args) {
+        if ($bootloader eq "syslinux") {
+            send_key "tab";
+        }
+        else {
+            send_key "e";
+            # ternary: 13 'downs' to reach the kernel line for installed
+            # system, 2 for UEFI installer
+            my $presses = $postinstall ? 13 : 2;
+            foreach my $i (1..$presses) {
+                send_key "down";
+            }
+            send_key "end";
+        }
+        type_string " $args";
+    }
+    # ctrl-X boots from grub editor mode
+    send_key "ctrl-x";
+    # return boots all other cases
+    send_key "ret";
+}
+
 sub boot_to_login_screen {
     my $self = shift;
     my $boot_done_screen = shift; # what to expect when system is booted (e. g. GDM), can be ""
@@ -94,6 +153,22 @@ sub get_milestone {
     return '';
 }
 
+sub clone_host_resolv {
+    # this is pretty crazy, but SUSE do almost the same thing...
+    # it's for openvswitch jobs to clone the host's resolv.conf, so
+    # we don't have to hard code 8.8.8.8 or have the templates pass
+    # in an address or something
+    my $self = shift;
+    my $resolv = '';
+    open(FH, '<', "/etc/resolv.conf");
+    while (<FH>) {
+        $resolv .= $_;
+    }
+    assert_script_run "printf '$resolv' > /etc/resolv.conf";
+    # for debugging...
+    assert_script_run "cat /etc/resolv.conf";
+}
+
 1;
 
 # vim: set sw=4 et:

needles/bootloader_bios_installed.json

@@ -0,0 +1,15 @@
+{
+    "area": [
+        {
+            "height": 15,
+            "type": "match",
+            "width": 160,
+            "xpos": 50,
+            "ypos": 335
+        }
+    ],
+    "tags": [
+        "bootloader",
+        "ENV-DISTRI-fedora"
+    ]
+}

needles/login_permission_denied.json

@@ -0,0 +1,15 @@
+{
+    "area": [
+        {
+            "height": 13,
+            "type": "match",
+            "width": 136,
+            "xpos": 0,
+            "ypos": 144
+        }
+    ],
+    "tags": [
+        "login_permission_denied",
+        "ENV-DISTRI-fedora"
+    ]
+}

needles/text_console_login.json

@@ -1,17 +1,17 @@
 {
-  "tags": [
-    "text_console_login",
-    "ENV-DISTRI-fedora",
-    "ENV-FLAVOR-develop"
-  ],
-  "properties": [],
-  "area": [
-    {
-      "width": 128,
-      "xpos": 0,
-      "type": "match",
-      "ypos": 49,
-      "height": 30
-    }
-  ]
+    "area": [
+        {
+            "height": 30,
+            "type": "match",
+            "width": 56,
+            "xpos": 72,
+            "ypos": 49
+        }
+    ],
+    "properties": [],
+    "tags": [
+        "text_console_login",
+        "ENV-DISTRI-fedora",
+        "ENV-FLAVOR-develop"
+    ]
 }

needles/user_logged_in-noprofile.json

@@ -0,0 +1,16 @@
+{
+    "area": [
+        {
+            "height": 16,
+            "type": "match",
+            "width": 65,
+            "xpos": 0,
+            "ypos": 143
+        }
+    ],
+    "tags": [
+        "user_logged_in",
+        "user_console",
+        "ENV-DISTRI-fedora"
+    ]
+}

templates

@@ -259,6 +259,28 @@
                                     },
                       test_suite => { name => "install_package_set_minimal" },
                     },
+                    {
+                      machine    => { name => "64bit" },
+                      prio       => 40,
+                      product    => {
+                                      arch    => "x86_64",
+                                      distri  => "fedora",
+                                      flavor  => "Server-dvd-iso",
+                                      version => "*",
+                                    },
+                      test_suite => { name => "server_role_deploy_domain_controller" },
+                    },
+                    {
+                      machine    => { name => "64bit" },
+                      prio       => 40,
+                      product    => {
+                                      arch    => "x86_64",
+                                      distri  => "fedora",
+                                      flavor  => "Server-dvd-iso",
+                                      version => "*",
+                                    },
+                      test_suite => { name => "server_realmd_join_kickstart" },
+                    },
                     {
                       machine    => { name => "64bit" },
                       prio       => 20,
@@ -1261,7 +1283,7 @@
                         { key => "KICKSTART", value => "1" },
                         { key => "GRUB", value => "inst.ks=hd:vdb1:/root-user-crypted-net.ks" },
                         { key => "NUMDISKS", value => "2" },
-                        { key => "HDD_2", value => "disk_ks.img" },
+                        { key => "HDD_2", value => "disk_ks_2.img" },
                         { key => "ROOT_PASSWORD", value => "111111" },
                         { key => "USER_LOGIN", value => "test" },
                         { key => "USER_PASSWORD", value => "test" },
@@ -1458,5 +1480,31 @@
                         { key => "USER_LOGIN", value => "false" },
                       ],
                     },
+                    {
+                      name => "server_role_deploy_domain_controller",
+                      settings => [
+                        { key => "ENTRYPOINT", value => "role_deploy_domain_controller" },
+                        { key => "START_AFTER_TEST", value => "install_default_upload" },
+                        { key => "BOOTFROM", value => "c" },
+                        { key => "HDD_1", value => "disk_%FLAVOR%_%MACHINE%.qcow2" },
+                        { key => "GRUB", value => "net.ifnames=0 biosdevname=0" },
+                        { key => "NICTYPE", value => "tap" },
+                      ],
+                    },
+                    {
+                      name => "server_realmd_join_kickstart",
+                      settings => [
+                        { key => "KICKSTART", value => "1" },
+                        { key => "GRUB", value => "inst.ks=hd:vdb1:/freeipaclient.ks" },
+                        { key => "NUMDISKS", value => "2" },
+                        { key => "HDD_2", value => "disk_ks_2.img" },
+                        { key => "POSTINSTALL", value => "freeipa_client" },
+                        { key => "USER_LOGIN", value => "false" },
+                        { key => "ROOT_PASSWORD", value => "anaconda" },
+                        { key => "PARALLEL_WITH", value => "server_role_deploy_domain_controller" },
+                        { key => "INSTALL_UNLOCK", value => "freeipa_ready" },
+                        { key => "NICTYPE", value => "tap" },
+                      ],
+                    },
                   ],
 }

tests/_boot_to_anaconda.pm

@@ -2,66 +2,33 @@ use base "anacondatest";
 use strict;
 use testapi;
 
-# get_kernel_line switches to menu edit screen and sets the cursor to the end of kernel line
-sub get_kernel_line {
-    if( get_var("UEFI")){
-        send_key "e";
-        send_key "down";
-        send_key "down";
-        send_key "end";
-    } else {
-        send_key "tab";
-    }
-}
-
 sub run {
-    # Wait for bootloader to appear
-    if( get_var("UEFI")){
-        assert_screen "bootloader_uefi", 30;
-    } else {
-        assert_screen "bootloader", 30;
+    my $self = shift;
+    # construct the kernel args. the trick here is to wind up with
+    # spaced args if GRUB or GRUBADD is set, and just spaces if not,
+    # then check if we got all spaces. We wind up with a harmless
+    # extra space if GRUBADD is set but GRUB is not.
+    my $args = "";
+    $args .= get_var("GRUB", "") . " ";
+    $args .= get_var("GRUBADD", "") . " ";
+    # Construct inst.repo arg for REPOSITORY_VARIATION
+    my $repourl = get_var("REPOSITORY_VARIATION");
+    if ($repourl) {
+        my $version = lc(get_var("VERSION", ""));
+        my $arch = get_var("ARCH", "");
+        $repourl .= "/$version/Everything/$arch/os";
+        $args .= "inst.repo=$repourl";
     }
+    # ternary: set $args to "" if it contains only spaces
+    $args = $args =~ /^\s+$/ ? "" : $args;
 
-    # Make sure we skip media check if it's selected by default. Standard
-    # 'boot installer' menu entry is always first.
-    send_key "up";
-    send_key "up";
+    # set mutex wait if necessary
+    my $mutex = get_var("INSTALL_UNLOCK");
 
-    # if variable GRUB is set, add its value into kernel line in grub
-    if( get_var("GRUB")){
-        get_kernel_line;
-        type_string " ".get_var("GRUB");
-
-    }
-
-    # if GRUBADD is set, add that to kernel line too. this is for doing
-    # stuff like running the tests with an updates.img to test some
-    # anaconda change
-    if (get_var("GRUBADD")) {
-        # unless GRUB was also set, we need to get to the kernel line now
-        get_kernel_line unless (get_var("GRUB"));
-        type_string " ".get_var("GRUBADD");
-    }
-
-    # if variable REPOSITORY_VARIATION is set, construct inst.repo url and add it to kernel line
-    if (get_var("REPOSITORY_VARIATION")){
-        unless (get_var("GRUB") || get_var("GRUBADD")) {
-            get_kernel_line;
-        }
-        my $repourl = "";
-
-        # REPOSITORY_VARIATION should be set to repository URL without version and architecture
-        # appended (it will be appended automatically)
-        $repourl = get_var("REPOSITORY_VARIATION")."/".lc(get_var("VERSION"))."/Everything/".get_var("ARCH")."/os";
-        type_string " inst.repo=".$repourl;
-    }
-
-    # now we are on the correct "boot" menu item
-    # hit Ctrl+x for the case when the uefi kernel line was edited
-    send_key "ctrl-x";
-    # Return starts boot in all other cases
-    send_key "ret";
+    # call do_bootloader with postinstall=0, the args, and the mutex
+    $self->do_bootloader(0, $args, $mutex);
 
+    # proceed to installer
     unless (get_var("KICKSTART"))
     {
         # on lives, we have to explicitly launch anaconda

tests/freeipa_client_postinstall.pm

@@ -0,0 +1,45 @@
+use base "installedtest";
+use strict;
+use testapi;
+
+sub run {
+    my $self=shift;
+    # check domain is listed in 'realm list'
+    validate_script_output 'realm list', sub { $_ =~ m/domain-name: domain\.local.*configured: kerberos-member/s };
+    # check we can see the admin user in getent
+    assert_script_run 'getent passwd admin@DOMAIN.LOCAL';
+    # check keytab entries
+    validate_script_output 'klist -k', sub { $_ =~ m/client001\.domain\.local\@DOMAIN.LOCAL/ };
+    # check we can kinit with the host principal
+    assert_script_run 'kinit -k host/client001.domain.local@DOMAIN.LOCAL';
+    # kinit as each user and set a new password
+    assert_script_run 'printf "correcthorse\nbatterystaple\nbatterystaple" | kinit test1@DOMAIN.LOCAL';
+    assert_script_run 'printf "correcthorse\nbatterystaple\nbatterystaple" | kinit test2@DOMAIN.LOCAL';
+    # switch to tty3
+    send_key "ctrl-alt-f3";
+    # try and login as test1, should work
+    $self->console_login(user=>'test1@DOMAIN.LOCAL', password=>'batterystaple');
+    type_string "exit\n";
+    # try and login as test2, should fail. we cannot use console_login
+    # as it takes 10 seconds to complete when login fails, and
+    # "permission denied" message doesn't last that long
+    sleep 2;
+    assert_screen "text_console_login";
+    type_string "test2\@DOMAIN.LOCAL\n";
+    assert_screen "console_password_required";
+    type_string "batterystaple\n";
+    assert_screen "login_permission_denied";
+}
+
+
+sub test_flags {
+    # without anything - rollback to 'lastgood' snapshot if failed
+    # 'fatal' - whole test suite is in danger if this fails
+    # 'milestone' - after this test succeeds, update 'lastgood'
+    # 'important' - if this fails, set the overall state to 'fail'
+    return { fatal => 1 };
+}
+
+1;
+
+# vim: set sw=4 et:

tests/role_deploy_domain_controller.pm

@@ -0,0 +1,77 @@
+use base "installedtest";
+use strict;
+use testapi;
+use lockapi;
+use mmapi;
+
+sub run {
+    my $self=shift;
+    # boot with kernel params to ensure interface is 'eth0' and not whatever
+    # systemd feels like calling it today
+    $self->do_bootloader(1, "net.ifnames=0 biosdevname=0");
+    $self->boot_to_login_screen("text_console_login", 5, 60);
+    # login
+    $self->root_console();
+    # set hostname
+    assert_script_run 'hostnamectl set-hostname ipa001.domain.local';
+    # add entry to /etc/hosts
+    assert_script_run 'echo "10.0.2.100 ipa001.domain.local ipa001" >> /etc/hosts';
+    # bring up network. DEFROUTE is *vital* here
+    assert_script_run 'printf "DEVICE=eth0\nBOOTPROTO=none\nIPADDR=10.0.2.100\nGATEWAY=10.0.2.2\nPREFIX=24\nDEFROUTE=yes" > /etc/sysconfig/network-scripts/ifcfg-eth0';
+    script_run "systemctl restart NetworkManager.service";
+    # clone host's resolv.conf to get name resolution
+    $self->clone_host_resolv();
+    # we don't want updates-testing for validation purposes
+    assert_script_run 'dnf config-manager --set-disabled updates-testing';
+    # we need a lot of entropy for this, and we don't care how good
+    # it is, so let's use haveged
+    assert_script_run 'dnf -y install haveged', 120;
+    assert_script_run 'systemctl start haveged.service';
+    # deploy the domain controller role
+    assert_script_run 'echo \'{"admin_password":"monkeys123"}\' | rolectl deploy domaincontroller --name=domain.local --settings-stdin', 1200;
+    # check the role status, should be 'running'
+    validate_script_output 'rolectl status domaincontroller/domain.local', sub { $_ =~ m/^running/ };
+    # check the admin password is listed in 'settings'
+    validate_script_output 'rolectl settings domaincontroller/domain.local', sub {$_ =~m/dm_password = \w{5,}/ };
+    # sanitize the settings
+    assert_script_run 'rolectl sanitize domaincontroller/domain.local';
+    # check the password now shows as 'None'
+    validate_script_output 'rolectl settings domaincontroller/domain.local', sub {$_ =~ m/dm_password = None/ };
+    # kinit as admin
+    assert_script_run 'echo "monkeys123" | kinit admin';
+    # set up an OTP for client001 enrolment (it will enrol with a kickstart)
+    assert_script_run 'ipa host-add client001.domain.local --password=monkeys --force';
+    # create two user accounts, test1 and test2
+    assert_script_run 'echo "correcthorse" | ipa user-add test1 --first test --last one --password';
+    assert_script_run 'echo "correcthorse" | ipa user-add test2 --first test --last two --password';
+    # add a rule allowing access to all hosts and services
+    assert_script_run 'ipa hbacrule-add testrule --servicecat=all --hostcat=all';
+    # add test1 (but not test2) to the rule
+    assert_script_run 'ipa hbacrule-add-user testrule --users=test1';
+    # disable the default 'everyone everywhere' rule
+    assert_script_run 'ipa hbacrule-disable allow_all';
+    # we're all ready for other jobs to run!
+    mutex_create('freeipa_ready');
+    wait_for_children;
+    # once child jobs are done, stop the role
+    assert_script_run 'rolectl stop domaincontroller/domain.local';
+    # check role is stopped
+    validate_script_output 'rolectl status domaincontroller/domain.local', sub { $_ =~ m/^ready-to-start/ };
+    # decommission the role
+    assert_script_run 'rolectl decommission domaincontroller/domain.local', 120;
+    # check role is decommissioned
+    validate_script_output 'rolectl list instances', sub { $_ eq "" };
+}
+
+
+sub test_flags {
+    # without anything - rollback to 'lastgood' snapshot if failed
+    # 'fatal' - whole test suite is in danger if this fails
+    # 'milestone' - after this test succeeds, update 'lastgood'
+    # 'important' - if this fails, set the overall state to 'fail'
+    return { fatal => 1 };
+}
+
+1;
+
+# vim: set sw=4 et: