Merge branch 'worker_host' of ssh://git.resf.org:22220/infrastructure/ansible-openqa-management into worker_host

handlers/main.yml

@@ -13,3 +13,25 @@
   # range "end" parameter is exclusive, so add 1
   loop: "{{ range(1, (openqa_worker_count | int + 1)) | list }}"
   ignore_errors: "{{ ansible_check_mode }}"
+
+- name: Restart openqa services
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: restarted
+  loop: "{{ openqa_services }}"
+  ignore_errors: "{{ ansible_check_mode }}"
+
+- name: Restart os-autoinst-openvswitch
+  ansible.builtin.systemd:
+    name: os-autoinst-openvswitch
+    state: restarted
+    enabled: true
+  ignore_errors: "{{ ansible_check_mode }}"
+
+- name: Restart httpd
+  ansible.builtin.service:
+    name: httpd
+    state: restarted
+    enabled: true
+  ignore_errors: "{{ ansible_check_mode }}"
+...

init-rocky-openqa-developer-host.yml

@@ -44,6 +44,9 @@
     - name: Install and configure OpenQA
       ansible.builtin.import_tasks: tasks/openqa.yml
 
+    - name: Apply Rocky Linux OpenQA Branding
+      ansible.builtin.import_tasks: tasks/openqa_branding.yml
+
   post_tasks:
     - name: Touching run file that ansible has ran here
       ansible.builtin.file:

remove-rocky-openqa-developer-host.yml

@@ -0,0 +1,41 @@
+# Delete local OpenQA testing environment
+# This playbook is *NOT* intended for WAN-facing systems!
+# Created: @akatch
+---
+- name: Rocky OpenQA Runbook
+  hosts: localhost
+  connection: local
+  become: true
+  vars_files:
+    - vars/openqa.yml
+
+  # This is to try to avoid the handler issue in pre/post tasks
+  handlers:
+    - name: Import handlers
+      ansible.builtin.import_tasks: handlers/main.yml
+
+  pre_tasks:
+    - name: Check if ansible cannot be run here
+      ansible.builtin.stat:
+        path: /etc/no-ansible
+      register: no_ansible
+
+    - name: Verify if we can run ansible
+      ansible.builtin.assert:
+        that:
+          - "not no_ansible.stat.exists"
+        success_msg: "We are able to run on this node"
+        fail_msg: "/etc/no-ansible exists - skipping run on this node"
+
+  tasks:
+    - name: Remove OpenQA installation from this system
+      ansible.builtin.import_tasks: tasks/remove_openqa.yml
+
+  post_tasks:
+    - name: Touching run file that ansible has ran here
+      ansible.builtin.file:
+        path: /var/log/ansible.run
+        state: touch
+        mode: '0644'
+        owner: root
+        group: root

remove-rocky-openqa-multivm-networking.yml

@@ -0,0 +1,54 @@
+# Sets up local OpenQA testing environment
+# This playbook is *NOT* intended for WAN-facing systems!
+#
+# Usages:
+#   # Install and configure an openQA developer host, download all current Rocky ISOs,
+#   # and POST a test job
+#   ansible-playbook playbooks/init-rocky-openqa-developer-host.yml
+#
+#   # Only perform ISO download tasks
+#   ansible-playbook playbooks/init-rocky-openqa-developer-host.yml --tags=download_isos
+#
+#   # Only perform configuration, do not download ISOs or POST a job
+#   ansible-playbook playbooks/init-rocky-openqa-developer-host.yml --tags=configure
+#
+# Created: @akatch
+---
+- name: Rocky OpenQA Runbook
+  hosts: localhost
+  connection: local
+  become: true
+  vars_files:
+    - vars/openqa.yml
+
+  # This is to try to avoid the handler issue in pre/post tasks
+  handlers:
+    - name: Import handlers
+      ansible.builtin.import_tasks: handlers/main.yml
+
+  pre_tasks:
+    - name: Check if ansible cannot be run here
+      ansible.builtin.stat:
+        path: /etc/no-ansible
+      register: no_ansible
+
+    - name: Verify if we can run ansible
+      ansible.builtin.assert:
+        that:
+          - "not no_ansible.stat.exists"
+        success_msg: "We are able to run on this node"
+        fail_msg: "/etc/no-ansible exists - skipping run on this node"
+
+  tasks:
+    - name: Remove openqa multivm networking configs
+      ansible.builtin.import_tasks: tasks/remove_openqa-multivm-networking.yml
+
+  post_tasks:
+    - name: Touching run file that ansible has ran here
+      ansible.builtin.file:
+        path: /var/log/ansible.run
+        state: touch
+        mode: '0644'
+        owner: root
+        group: root
+...

tasks/openqa-multivm-networking.yml

@@ -0,0 +1,111 @@
+---
+# {{ openqa_multivm_bridge_interface }} should not exist or we should use a different name
+- name: Assert bridge interface does not exist
+  ansible.builtin.assert:
+    that:
+      - 'openqa_multivm_bridge_interface not in ansible_interfaces'
+    success_msg: 'interface does not exist, can proceed'
+    fail_msg: '{{ openqa_multivm_bridge_interface }} already exists, please supply an alternative'
+
+- name: Install multivm networking packages
+  ansible.builtin.dnf:
+    pkg:
+      - os-autoinst-openvswitch
+      - tunctl
+
+- name: Create /etc/sysconfig/os-autoinst-openvswitch
+  ansible.builtin.copy:
+    src: etc/sysconfig/os-autoinst-openvswitch.j2
+    dest: /etc/sysconfig/os-autoinst-openvswitch
+    mode: '0644'
+  notify: Restart os-autoinst-openvswitch
+
+- name: Create bridge interface configuration
+  ansible.builtin.copy:
+    src: etc/sysconfig/network-scripts/ifcfg-br.j2
+    dest: /etc/sysconfig/network-scripts/ifcfg-{{ openqa_multivm_bridge_interface }}
+    mode: '0644'
+
+- name: Create worker tap interface configs
+  ansible.builtin.copy:
+    src: etc/sysconfig/network-scripts/ifcfg-tap.j2
+    dest: /etc/sysconfig/network-scripts/ifcfg-tap{{ item }}
+    mode: '0644'
+  loop: "{{ range(openqa_worker_count) | list }}"
+
+- name: Update /sbin/ifup-pre-local
+  ansible.builtin.template:
+    src: sbin/ifup-pre-local.j2
+    dest: /sbin/ifup-pre-local
+    mode: 'ug+x'
+
+- name: Enable bridge interface for internal zone
+  ansible.posix.firewalld:
+    permanent: true
+    interface: '{{ openqa_multivm_bridge_interface }}'
+    state: enabled
+    zone: internal
+  notify: Reload firewalld
+
+- name: Enable masquerade for public and internal zones
+  ansible.posix.firewalld:
+    masquerade: true
+    permanent: true
+    state: enabled
+    zone: '{{ item }}'
+  loop:
+    - public
+    - internal
+  notify: Reload firewalld
+
+- name: Enable ipv4 IP forwarding
+  ansible.posix.sysctl:
+    name: net.ipv4.ip_forward
+    value: '1'
+    state: present
+    sysctl_file: /etc/sysctl.d/ip-forward.conf
+    sysctl_set: true
+
+- name: Set-target ACCEPT on public zone
+  ansible.posix.firewalld:
+    permanent: true
+    state: present
+    zone: public
+    target: ACCEPT
+  notify: Reload firewalld
+
+# Only needed for multi-host setups
+- name: Add port for GRE tunnel
+  ansible.posix.firewalld:
+    permanent: true
+    port: 1723/tcp
+    state: enabled
+
+- name: Enable openvswitch services
+  ansible.builtin.systemd_service:
+    name: "{{ item }}"
+    state: started
+    enabled: true
+  loop:
+    - openvswitch
+    - os-autoinst-openvswitch
+  ignore_errors: "{{ ansible_check_mode }}"
+
+- name: Set WORKER_CLASS for tap interfaces
+  community.general.ini_file:
+    path: /etc/openqa/workers.ini
+    section: global
+    option: WORKER_CLASS
+    value: qemu_x86_64,tap
+    state: present
+    mode: '0644'
+  notify: Restart openqa services
+
+- name: Enable bridge interface for openvswitch
+  ansible.builtin.command: ovs-vsctl add-br {{ openqa_multivm_bridge_interface }}
+  changed_when: true
+
+- name: Enable capability
+  ansible.builtin.command: setcap CAP_NET_ADMIN=ep /usr/bin/qemu-system-x86_64
+  changed_when: true
+...

tasks/openqa.yml

@@ -11,15 +11,16 @@
     remote_src: true
     src: /etc/httpd/conf.d/{{ item }}.template
     dest: /etc/httpd/conf.d/{{ item }}
-    mode: '0644'
+    mode: "0644"
     owner: root
     group: root
   loop:
     - openqa.conf
     - openqa-ssl.conf
-  notify: restart_httpd
+  notify: Restart httpd
   tags:
     - configure
+  ignore_errors: "{{ ansible_check_mode }}"
 
 - name: Template OpenQA configuration files
   ansible.builtin.template:
@@ -33,9 +34,11 @@
     - client.conf
   tags:
     - configure
+  notify: Restart openQA workers
 
 - name: Get service facts
   ansible.builtin.service_facts:
+  check_mode: false
 
 - name: Check for non-empty postgres data directory
   ansible.builtin.stat:
@@ -47,6 +50,7 @@
   when: not ( ansible_facts.services["postgresql.service"]["state"] == "running" )
         and not postgres_data_dir.stat.exists
   changed_when: true
+  ignore_errors: "{{ ansible_check_mode }}"
 
 - name: Enable and start postgresql service
   ansible.builtin.systemd:
@@ -55,6 +59,7 @@
     enabled: true
   when: not ( ansible_facts.services["postgresql.service"]["state"] == "running" )
         and not postgres_data_dir.stat.exists
+  ignore_errors: "{{ ansible_check_mode }}"
 
 - name: Configure SELinux to allow httpd connection to network
   ansible.posix.seboolean:
@@ -72,6 +77,7 @@
   loop: "{{ openqa_services }}"
   tags:
     - configure
+  ignore_errors: "{{ ansible_check_mode }}"
 
 - name: Create openqa-vnc firewalld service
   ansible.builtin.template:
@@ -82,6 +88,11 @@
     mode: "0644"
   tags:
     - configure
+  notify: Reload firewalld
+
+- name: Systemctl daemon-reload
+  ansible.builtin.systemd:
+    daemon_reload: true
 
 - name: Load openqa-vnc firewalld service
   ansible.builtin.systemd:
@@ -131,63 +142,18 @@
     recurse: true
     owner: "{{ openqa_user }}"
     group: "{{ openqa_group }}"
-    mode: "u+rwX,g+rwX,o+rX,o-w"
+    mode: "0775"
   tags:
     - configure
 
-# fifloader.py will fail if the Demo user is not logged in
-- name: Authenticate to web UI the first time
-  ansible.builtin.uri:
-    url: "http://{{ openqa_host }}/login"
-
-- name: Run fifloader.py
-  ansible.builtin.command: ./fifloader.py -l -c templates.fif.json templates-updates.fif.json
-  changed_when: "1 != 1"
-  args:
-    chdir: "{{ openqa_homedir }}/share/tests/rocky"
-
-- name: Create ISO directory
+- name: Create asset directories
   ansible.builtin.file:
-    path: "{{ openqa_homedir }}/share/factory/iso/fixed"
+    path: "{{ openqa_homedir }}/share/factory/{{ item }}/fixed"
     state: directory
     owner: "{{ openqa_user }}"
     group: "{{ openqa_group }}"
     mode: "0775"
-  tags:
-    - download_isos
-
-- name: Download ISOs
-  ansible.builtin.get_url:
-    dest: "{{ openqa_homedir }}/share/factory/iso/fixed/{{ item.name }}"
-    url: "{{ rocky_iso_download_url }}/{{ item.name }}"
-    checksum: "{{ item.checksum }}"
-    owner: "{{ openqa_user }}"
-    group: "{{ openqa_group }}"
-    tmp_dest: "/var/tmp"
-    mode: "0644"
-  loop: "{{ openqa_isos }}"
-  tags:
-    - download_isos
-
-- name: Start OpenQA workers
-  ansible.builtin.systemd:
-    name: "openqa-worker@{{ item }}"
-    state: started
-    enabled: true
-  # range 'end' parameter is exclusive, so add 1
-  loop: "{{ range(1, (openqa_worker_count | int + 1)) | list }}"
-  tags:
-    - start_workers
-    - configure
-
-- name: POST a job
-  ansible.builtin.command: |
-    openqa-cli api -X POST isos \
-      ISO=Rocky-{{ rocky_version }}-{{ rocky_arch }}-minimal.iso \
-      ARCH={{ rocky_arch }} \
-      DISTRI=rocky \
-      FLAVOR=minimal-iso \
-      VERSION={{ rocky_version }} \
-      BUILD="{{ '%Y%m%d.%H%M%S' | strftime }}.0"
-  changed_when: "1 != 1"
+  loop:
+    - iso
+    - hdd
 ...

tasks/remove_openqa-multivm-networking.yml

@@ -0,0 +1,92 @@
+---
+- name: Remove files
+  ansible.builtin.file:
+    path: '{{ item }}'
+    state: absent
+  loop:
+    - /etc/sysconfig/os-autoinst-openvswitch
+    - /etc/sysconfig/network-scripts/ifcfg-{{ openqa_multivm_bridge_interface }}
+
+- name: Remove tap interface configurations
+  ansible.builtin.file:
+    path: /etc/sysconfig/network-scripts/ifcfg-tap{{ item }}
+    state: absent
+  loop: "{{ range(openqa_worker_count | int) | list }}"
+
+- name: Delete bridge interface
+  ansible.builtin.command: ovs-vsctl del-br {{ openqa_multivm_bridge_interface }}
+  changed_when: true
+
+- name: Disable openvswitch services
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: stopped
+    enabled: false
+  loop:
+    - os-autoinst-openvswitch
+    - openvswitch
+
+- name: Remove packages
+  ansible.builtin.dnf:
+    pkg:
+      - os-autoinst-openvswitch
+      - tunctl
+      - network-scripts
+    state: absent
+
+- name: Remove /sbin/ifup-pre-local
+  ansible.builtin.file:
+    path: /sbin/ifup-pre-local
+    state: absent
+
+- name: Disable bridge interface for internal zone
+  ansible.posix.firewalld:
+    permanent: true
+    interface: br0
+    state: disabled
+    zone: internal
+  notify: reload_firewalld
+
+- name: Disable masquerade for public and internal zones
+  ansible.posix.firewalld:
+    masquerade: true
+    permanent: true
+    state: disabled
+    zone: '{{ item }}'
+  loop:
+    - public
+    - internal
+  notify: reload_firewalld
+
+- name: Disable ipv4 IP forwarding
+  ansible.posix.sysctl:
+    name: net.ipv4.ip_forward
+    value: '1'
+    state: absent
+    sysctl_file: /etc/sysctl.d/ip-forward.conf
+    sysctl_set: true
+
+- name: Set-target ACCEPT on public zone
+  ansible.posix.firewalld:
+    permanent: true
+    state: absent
+    zone: public
+    target: ACCEPT
+  notify: reload_firewalld
+
+- name: Remove port for GRE tunnel
+  ansible.posix.firewalld:
+    permanent: true
+    port: 1723/tcp
+    state: disabled
+  notify: reload_firewalld
+
+- name: Set WORKER_CLASS for tap interfaces
+  community.general.ini_file:
+    path: /etc/openqa/workers.ini
+    section: global
+    option: WORKER_CLASS
+    value: qemu_x86_64,tap
+    state: absent
+    mode: '0644'
+...

tasks/remove_openqa.yml

@@ -0,0 +1,42 @@
+---
+- name: Uninstall OpenQA packages
+  ansible.builtin.yum:
+    name: "{{ openqa_packages }}"
+    state: absent
+
+- name: Delete OpenQA files and directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - "{{ openqa_homedir }}"
+    - /var/lib/pgsql
+    - /etc/openqa
+    - /etc/httpd/conf.d/openqa.conf
+    - /etc/httpd/conf.d/openqa-ssl.conf
+
+- name: Disable httpd_can_network_connect
+  ansible.posix.seboolean:
+    name: httpd_can_network_connect
+    state: false
+    persistent: true
+
+- name: Deny traffic for services
+  ansible.posix.firewalld:
+    service: "{{ item }}"
+    permanent: true
+    state: disabled
+  loop:
+    - http
+    - openqa-vnc
+
+- name: Deny VNC traffic for local workers
+  ansible.posix.firewalld:
+    port: "{{ openqa_min_vnc_port }}-{{ openqa_max_vnc_port }}/tcp"
+    permanent: true
+    state: disabled
+
+- name: Reload FirewallD
+  ansible.builtin.systemd:
+    name: firewalld
+    state: reloaded

templates/etc/sysconfig/network-scripts/ifcfg-br.j2

@@ -0,0 +1,10 @@
+DEVICETYPE='ovs'
+TYPE='OVSBridge'
+BOOTPROTO='static'
+IPADDR='172.16.2.2'
+NETMASK='255.254.0.0'
+DEVICE={{ openqa_multivm_bridge_interface }}
+STP=off
+ONBOOT='yes'
+NAME='{{ openqa_multivm_bridge_interface }}'
+HOTPLUG='no'

templates/etc/sysconfig/network-scripts/ifcfg-tap.j2

@@ -0,0 +1,7 @@
+DEVICETYPE='ovs'
+TYPE='OVSPort'
+OVS_BRIDGE='{{ openqa_multivm_bridge_interface }}'
+DEVICE='tap{{ item }}'
+ONBOOT='yes'
+BOOTPROTO='none'
+HOTPLUG='no'

templates/etc/sysconfig/os-autoinst-openvswitch.j2

@@ -0,0 +1,3 @@
+OS_AUTOINST_BRIDGE_LOCAL_IP=172.16.2.2
+OS_AUTOINST_BRIDGE_REWRITE_TARGET=172.17.0.0
+OS_AUTOINST_USE_BRIDGE={{ openqa_multivm_bridge_interface }}

templates/sbin/ifup-pre-local.j2

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+if=$(echo "$1" | sed -e 's,ifcfg-,,')
+iftype=$(echo "$if" | sed -e 's,[0-9]\+$,,')
+
+# if the interface being brought up is tap[n], create
+# the tap device first
+if [ "$iftype" == "tap" ]; then
+    tunctl -u _openqa-worker -p -t "$if"
+fi
+
+# if the interface being brough up is {{ openqa_multivm_bridge_interface }}, create
+# the gre tunnels
+if [ "$if" == "{{ openqa_multivm_bridge_interface }}" ]; then
+    ovs-vsctl set bridge {{ openqa_multivm_bridge_interface }} stp_enable=true
+    # This is only needed for multi-host setups
+{% for w in range(1, openqa_worker_count+1) %}
+    #ovs-vsctl --may-exist add-port {{ openqa_multivm_bridge_interface }} gre{{ w }} -- set interface gre{{ w }} type=gre options:remote_ip=172.16.2.{{ 2 + w|int }}
+{% endfor %}
+fi