WIP: Automation for configuring multivm networking

remove-rocky-openqa-multivm-networking.yml

@@ -0,0 +1,54 @@
+# Sets up local OpenQA testing environment
+# This playbook is *NOT* intended for WAN-facing systems!
+#
+# Usages:
+#   # Install and configure an openQA developer host, download all current Rocky ISOs,
+#   # and POST a test job
+#   ansible-playbook playbooks/init-rocky-openqa-developer-host.yml
+#
+#   # Only perform ISO download tasks
+#   ansible-playbook playbooks/init-rocky-openqa-developer-host.yml --tags=download_isos
+#
+#   # Only perform configuration, do not download ISOs or POST a job
+#   ansible-playbook playbooks/init-rocky-openqa-developer-host.yml --tags=configure
+#
+# Created: @akatch
+---
+- name: Rocky OpenQA Runbook
+  hosts: localhost
+  connection: local
+  become: true
+  vars_files:
+    - vars/openqa.yml
+
+  # This is to try to avoid the handler issue in pre/post tasks
+  handlers:
+    - name: Import handlers
+      ansible.builtin.import_tasks: handlers/main.yml
+
+  pre_tasks:
+    - name: Check if ansible cannot be run here
+      ansible.builtin.stat:
+        path: /etc/no-ansible
+      register: no_ansible
+
+    - name: Verify if we can run ansible
+      ansible.builtin.assert:
+        that:
+          - "not no_ansible.stat.exists"
+        success_msg: "We are able to run on this node"
+        fail_msg: "/etc/no-ansible exists - skipping run on this node"
+
+  tasks:
+    - name: Remove openqa multivm networking configs
+      ansible.builtin.import_tasks: tasks/remove_openqa-multivm-networking.yml
+
+  post_tasks:
+    - name: Touching run file that ansible has ran here
+      ansible.builtin.file:
+        path: /var/log/ansible.run
+        state: touch
+        mode: '0644'
+        owner: root
+        group: root
+...

tasks/openqa-multivm-networking.yml

@@ -0,0 +1,133 @@
+---
+# {{ openqa_multivm_bridge_interface }} should not exist or we should use a different name
+- name: Assert bridge interface does not exist
+  ansible.builtin.assert:
+    that:
+      - 'openqa_multivm_bridge_interface not in ansible_interfaces'
+    success_msg: 'interface does not exist, can proceed'
+    fail_msg: '{{ openqa_multivm_bridge_interface }} already exists, please supply an alternative'
+
+- name: Install multivm networking packages
+  ansible.builtin.dnf:
+    pkg:
+      - os-autoinst-openvswitch
+      - tunctl
+      - network-scripts
+
+- name: Create /etc/sysconfig/os-autoinst-openvswitch
+  ansible.builtin.copy:
+    dest: /etc/sysconfig/os-autoinst-openvswitch
+    mode: '0644'
+    content: |
+      OS_AUTOINST_BRIDGE_LOCAL_IP=172.16.2.2
+      OS_AUTOINST_BRIDGE_REWRITE_TARGET=172.17.0.0
+      OS_AUTOINST_USE_BRIDGE={{ openqa_multivm_bridge_interface }}
+  notify: restart_os-autoinst-openvswitch
+
+- name: Create bridge interface configuration
+  ansible.builtin.copy:
+    dest: /etc/sysconfig/network-scripts/ifcfg-{{ openqa_multivm_bridge_interface }}
+    mode: '0644'
+    content: |
+      DEVICETYPE='ovs'
+      TYPE='OVSBridge'
+      BOOTPROTO='static'
+      IPADDR='172.16.2.2'
+      NETMASK='255.254.0.0'
+      DEVICE={{ openqa_multivm_bridge_interface }}
+      STP=off
+      ONBOOT='yes'
+      NAME='{{ openqa_multivm_bridge_interface }}'
+      HOTPLUG='no'
+
+- name: Create worker tap interface configs
+  ansible.builtin.copy:
+    dest: /etc/sysconfig/network-scripts/ifcfg-tap{{ item }}
+    mode: '0644'
+    content: |
+      DEVICETYPE='ovs'
+      TYPE='OVSPort'
+      OVS_BRIDGE='{{ openqa_multivm_bridge_interface }}'
+      DEVICE='tap{{ item }}'
+      ONBOOT='yes'
+      BOOTPROTO='none'
+      HOTPLUG='no'
+  loop: "{{ range(openqa_worker_count) | list }}"
+
+- name: Update /sbin/ifup-pre-local
+  ansible.builtin.template:
+    src: sbin/ifup-pre-local.j2
+    dest: /sbin/ifup-pre-local
+    mode: 'ug+x'
+
+- name: Enable bridge interface for internal zone
+  ansible.posix.firewalld:
+    permanent: true
+    interface: '{{ openqa_multivm_bridge_interface }}'
+    state: enabled
+    zone: internal
+  notify: reload_firewalld
+
+- name: Enable masquerade for public and internal zones
+  ansible.posix.firewalld:
+    masquerade: true
+    permanent: true
+    state: enabled
+    zone: '{{ item }}'
+  loop:
+    - public
+    - internal
+  notify: reload_firewalld
+
+- name: Enable ipv4 IP forwarding
+  ansible.posix.sysctl:
+    name: net.ipv4.ip_forward
+    value: '1'
+    state: present
+    sysctl_file: /etc/sysctl.d/ip-forward.conf
+    sysctl_set: true
+
+- name: Set-target ACCEPT on public zone
+  ansible.posix.firewalld:
+    permanent: true
+    state: present
+    zone: public
+    target: ACCEPT
+  notify: reload_firewalld
+
+# Only needed for multi-host setups
+- name: Add port for GRE tunnel
+  ansible.posix.firewalld:
+    permanent: true
+    port: 1723/tcp
+    state: enabled
+
+- name: Enable openvswitch services
+  ansible.builtin.systemd_service:
+    name: "{{ item }}"
+    state: started
+    enabled: true
+  loop:
+    - openvswitch
+    - network
+    - os-autoinst-openvswitch
+  ignore_errors: "{{ ansible_check_mode }}"
+
+- name: Set WORKER_CLASS for tap interfaces
+  community.general.ini_file:
+    path: /etc/openqa/workers.ini
+    section: global
+    option: WORKER_CLASS
+    value: qemu_x86_64,tap
+    state: present
+    mode: '0644'
+  notify: restart_openqa_services
+
+- name: Enable bridge interface for openvswitch
+  ansible.builtin.command: ovs-vsctl add-br {{ openqa_multivm_bridge_interface }}
+  changed_when: true
+
+- name: Enable capability
+  ansible.builtin.command: setcap CAP_NET_ADMIN=ep /usr/bin/qemu-system-x86_64
+  changed_when: true
+...

tasks/remove_openqa-multivm-networking.yml

@@ -0,0 +1,92 @@
+---
+- name: Remove files
+  ansible.builtin.file:
+    path: '{{ item }}'
+    state: absent
+  loop:
+    - /etc/sysconfig/os-autoinst-openvswitch
+    - /etc/sysconfig/network-scripts/ifcfg-{{ openqa_multivm_bridge_interface }}
+
+- name: Remove tap interface configurations
+  ansible.builtin.file:
+    path: /etc/sysconfig/network-scripts/ifcfg-tap{{ item }}
+    state: absent
+  loop: "{{ range(openqa_worker_count | int) | list }}"
+
+- name: Delete bridge interface
+  ansible.builtin.command: ovs-vsctl del-br {{ openqa_multivm_bridge_interface }}
+  changed_when: true
+
+- name: Disable openvswitch services
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: stopped
+    enabled: false
+  loop:
+    - os-autoinst-openvswitch
+    - openvswitch
+
+- name: Remove packages
+  ansible.builtin.dnf:
+    pkg:
+      - os-autoinst-openvswitch
+      - tunctl
+      - network-scripts
+    state: absent
+
+- name: Remove /sbin/ifup-pre-local
+  ansible.builtin.file:
+    path: /sbin/ifup-pre-local
+    state: absent
+
+- name: Disable bridge interface for internal zone
+  ansible.posix.firewalld:
+    permanent: true
+    interface: br0
+    state: disabled
+    zone: internal
+  notify: reload_firewalld
+
+- name: Disable masquerade for public and internal zones
+  ansible.posix.firewalld:
+    masquerade: true
+    permanent: true
+    state: disabled
+    zone: '{{ item }}'
+  loop:
+    - public
+    - internal
+  notify: reload_firewalld
+
+- name: Disable ipv4 IP forwarding
+  ansible.posix.sysctl:
+    name: net.ipv4.ip_forward
+    value: '1'
+    state: absent
+    sysctl_file: /etc/sysctl.d/ip-forward.conf
+    sysctl_set: true
+
+- name: Set-target ACCEPT on public zone
+  ansible.posix.firewalld:
+    permanent: true
+    state: absent
+    zone: public
+    target: ACCEPT
+  notify: reload_firewalld
+
+- name: Remove port for GRE tunnel
+  ansible.posix.firewalld:
+    permanent: true
+    port: 1723/tcp
+    state: disabled
+  notify: reload_firewalld
+
+- name: Set WORKER_CLASS for tap interfaces
+  community.general.ini_file:
+    path: /etc/openqa/workers.ini
+    section: global
+    option: WORKER_CLASS
+    value: qemu_x86_64,tap
+    state: absent
+    mode: '0644'
+...

tasks/remove_openqa.yml

@@ -0,0 +1,42 @@
+---
+- name: Uninstall OpenQA packages
+  ansible.builtin.yum:
+    name: "{{ openqa_packages }}"
+    state: absent
+
+- name: Delete OpenQA files and directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - "{{ openqa_homedir }}"
+    - /var/lib/pgsql
+    - /etc/openqa
+    - /etc/httpd/conf.d/openqa.conf
+    - /etc/httpd/conf.d/openqa-ssl.conf
+
+- name: Disable httpd_can_network_connect
+  ansible.posix.seboolean:
+    name: httpd_can_network_connect
+    state: false
+    persistent: true
+
+- name: Deny traffic for services
+  ansible.posix.firewalld:
+    service: "{{ item }}"
+    permanent: true
+    state: disabled
+  loop:
+    - http
+    - openqa-vnc
+
+- name: Deny VNC traffic for local workers
+  ansible.posix.firewalld:
+    port: "{{ openqa_min_vnc_port }}-{{ openqa_max_vnc_port }}/tcp"
+    permanent: true
+    state: disabled
+
+- name: Reload FirewallD
+  ansible.builtin.systemd:
+    name: firewalld
+    state: reloaded

templates/sbin/ifup-pre-local.j2

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+if=$(echo "$1" | sed -e 's,ifcfg-,,')
+iftype=$(echo "$if" | sed -e 's,[0-9]\+$,,')
+
+# if the interface being brought up is tap[n], create
+# the tap device first
+if [ "$iftype" == "tap" ]; then
+    tunctl -u _openqa-worker -p -t "$if"
+fi
+
+# if the interface being brough up is {{ openqa_multivm_bridge_interface }}, create
+# the gre tunnels
+if [ "$if" == "{{ openqa_multivm_bridge_interface }}" ]; then
+    ovs-vsctl set bridge {{ openqa_multivm_bridge_interface }} stp_enable=true
+    # This is only needed for multi-host setups
+{% for w in range(1, openqa_worker_count+1) %}
+    #ovs-vsctl --may-exist add-port {{ openqa_multivm_bridge_interface }} gre{{ w }} -- set interface gre{{ w }} type=gre options:remote_ip=172.16.2.{{ 2 + w|int }}
+{% endfor %}
+fi