Merge pull request #14958 from rocky-linux/develop

RabbitMQ Additions, GitLab Fixes, Additional IPA Accounts

ansible/inventories/production/group_vars/rabbitmq/main.yml

@@ -0,0 +1,5 @@
+---
+# RabbitMQ Staging Vars
+rabbitmq_cluster_name: "rabbitprod"
+rabbitmq_cluster_list: "{{ groups['rabbitmq'] }}"
+rabbitmq_env: "production"

ansible/inventories/staging/group_vars/rabbitmq/main.yml

@@ -0,0 +1,5 @@
+---
+# RabbitMQ Staging Vars
+rabbitmq_cluster_name: "rabbitstage"
+rabbitmq_cluster_list: "{{ groups['rabbitmq'] }}"
+rabbitmq_env: "staging"

ansible/playbooks/adhoc-ipadns.yml

@@ -0,0 +1,27 @@
+---
+# This playbook is meant to be used with callable variables, like adhoc or AWX.
+# What: Creates dns zones in the idm infrastructure based on the variables
+#       provided.
+
+- name: Create a DNS Zone
+  hosts: ipaserver
+  become: false
+  gather_facts: false
+  vars_files:
+  - vars/encpass.yml
+
+  tasks:
+    - name: "Checking for user variables"
+      assert:
+        that:
+          - ipaadmin_password | mandatory
+          - ipa_zone | mandatory
+        success_msg: "Required variables provided"
+        fail_msg: "We are missing zone information or ipa admin password"
+
+    - name: "Creating DNS Zone"
+      freeipa.ansible_freeipa.ipadnszone:
+        ipaadmin_password: "{{ ipaadmin_password }}"
+        name: "{{ ipa_zone }}"
+      tags:
+        - dns

ansible/playbooks/adhoc-ipagroups.yml

@@ -14,17 +14,17 @@
       assert:
         that:
           - ipaadmin_password | mandatory
-          - ipaGroup | mandatory
-          - ipaDescription | mandatory
-          - ipaPosix | mandatory
+          - ipa_group | mandatory
+          - ipa_description | mandatory
+          - ipa_posix | mandatory
         success_msg: "Required variables provided"
         fail_msg: "We are missing group information or ipa admin password"
 
     - name: "Creating Mandatory Groups"
       freeipa.ansible_freeipa.ipagroup:
         ipaadmin_password: "{{ ipaadmin_password }}"
-        name: "{{ ipaGroup }}"
-        description: "{{ ipaDescription }}"
-        nonposix: "{{ ipaPosix }}"
+        name: "{{ ipa_group }}"
+        description: "{{ ipa_description }}"
+        nonposix: "{{ ipa_posix }}"
       tags:
         - groups

ansible/playbooks/adhoc-ipausers.yml

@@ -14,25 +14,25 @@
       assert:
         that:
           - ipaadmin_password | mandatory
-          - ipaName | mandatory
-          - ipaFirst | mandatory
-          - ipaLast | mandatory
-          - ipaEmail | mandatory
-          - ipaPassword | mandatory
-          - ipaTitle | mandatory
+          - ipa_name | mandatory
+          - ipa_first | mandatory
+          - ipa_last | mandatory
+          - ipa_email | mandatory
+          - ipa_password | mandatory
+          - ipa_title | mandatory
         success_msg: "Required variables provided"
         fail_msg: "We are missing user information or ipa admin password"
 
     - name: "Creating User Account"
       freeipa.ansible_freeipa.ipauser:
         ipaadmin_password: "{{ ipaadmin_password }}"
-        name: "{{ ipaName }}"
-        first: "{{ ipaFirst }}"
-        last: "{{ ipaLast }}"
-        email: "{{ ipaEmail }}"
-        password: "{{ ipaPassword }}"
-        title: "{{ ipaTitle }}"
-        loginshell: "{{ ipaLoginshell|default('/sbin/nologin', True) }}"
+        name: "{{ ipa_name }}"
+        first: "{{ ipa_first }}"
+        last: "{{ ipa_last }}"
+        email: "{{ ipa_email }}"
+        password: "{{ ipa_password }}"
+        title: "{{ ipa_title }}"
+        loginshell: "{{ ipa_loginshell|default('/sbin/nologin', True) }}"
         update_password: on_create
       tags:
         - users

ansible/playbooks/adhoc-rabbitmqqueue.yml

@@ -0,0 +1,84 @@
+---
+# This playbook is meant to be used with callable variables, like adhoc or AWX.
+# What: Creates RabbitMQ Users
+# Required parameters:
+#   -> username: The username to create in RabbitMQ, which should match an LDAP
+#                name or the CN of a certificate. Note that if it's a hostname
+#                it must be the FQDN.
+#   -> queue_name: Name of the queue to create. This should be setup with a
+#                  prefix_suffix name, where prefix is the username, and
+#                  the suffix is a service name.
+#   -> routing_keys: A list to be used as routing keys.
+# Optional:
+#   -> write_queues: A list of queues name prefixes that which the user will
+#                    be allowed to publish.
+#   -> thresholds: A dictionary with two keys "warning" and "critical" - The
+#                  values are numbers. In the event we have a monitoring system
+#                  this can be a number of messages that could cause an alert.
+#   -> vhost: The vhost this queue will be part of. The default is /pubsub.
+
+- name: Create a User
+  hosts: rabbitmq
+  become: false
+  gather_facts: false
+  vars_files:
+  - vars/encpass.yml
+  - vars/rabbitmq.yml
+
+  tasks:
+    - name: "Checking for user variables"
+      assert:
+        that:
+          - username != "admin"
+          - username != "guest"
+          - username != "mq-monitoring"
+        success_msg: "Required variables provided"
+        fail_msg: "Username is reserved"
+      tags:
+        - rabbitmq
+
+    - name: "Validate username queue name"
+      assert:
+        that:
+          - "queue_name.startswith(username)"
+      tags:
+        - rabbitmq
+
+    - name: "Creating User Account"
+      community.rabbitmq.rabbitmq_user:
+        user: "{{ username }}"
+        vhost: "{{ vhost|default('/pubsub') }}"
+        read_priv: "^(zmq\\.topic)|^(amq\\.topic)|({{ username  }}.*)$"
+        write_priv: "^(amq\\.topic)|({{ username  }}.*){% for queue in write_queues|default([]) %}|({{ queue  }}.*){% endfor %}$"
+        configure_priv: "^$"
+        state: present
+      tags:
+        - rabbitmq
+
+    - name: "Create {{ queue_name }}"
+      delegate_to: "{{ rabbitmq_cluster_list[0] }}"
+      community.rabbitmq.rabbitmq_queue:
+        name: "{{ queue_name }}"
+        vhost: "{{ vhost|default('/pubsub') }}"
+        auto_delete: false
+        durable: true
+        message_ttl: "{{ message_ttl|default('null') }}"
+        state: present
+        login_user: admin
+        login_password: "{{ rabbitmq_admin_password }}"
+      tags:
+        - rabbitmq
+
+    - name: "Bind {{ queue_name }} to amq.topic exchange"
+      delegate_to: "{{ rabbitmq_cluster_list[0] }}"
+      community.rabbitmq.rabbitmq_binding:
+        name: "amq.topic"
+        destination: "{{ queue_name }}"
+        destination_type: queue
+        routing_key: "{{ item }}"
+        vhost: "{{ vhost|default('/pubsub') }}"
+        state: present
+        login_user: admin
+        login_password: "{{ rabbitmq_admin_password }}"
+      tags:
+        - rabbitmq

ansible/playbooks/adhoc-rabbitmquser.yml

@@ -0,0 +1,35 @@
+---
+# This playbook is meant to be used with callable variables, like adhoc or AWX.
+# What: Creates RabbitMQ Users
+# The username is the required parameter
+
+- name: Create a User
+  hosts: rabbitmq
+  become: false
+  gather_facts: false
+  vars_files:
+  - vars/encpass.yml
+  - vars/rabbitmq.yml
+
+  tasks:
+    - name: "Checking for user variables"
+      assert:
+        that:
+          - username != "admin"
+          - username != "guest"
+          - username != "mq-monitoring"
+        success_msg: "Required variables provided"
+        fail_msg: "Username is reserved"
+      tags:
+        - rabbitmq
+
+    - name: "Creating User Account"
+      community.rabbitmq.rabbitmq_user:
+        user: "{{ username }}"
+        vhost: "{{ vhost }}"
+        read_priv: "^$"
+        write_priv: "amq\\.topic"
+        configure_priv: "^$"
+        state: present
+      tags:
+        - rabbitmq

ansible/playbooks/role-gitlab-ee.yml

@@ -4,6 +4,7 @@
   hosts: gitlabservers
   become: true
   vars_files:
+  - vars/common.yml
   - vars/gitlab.yml
 
   # This is to try to avoid the handler issue in pre/post tasks

ansible/playbooks/role-rocky-rabbitmq.yml

@@ -0,0 +1,76 @@
+---
+# Stands up a RabbitMQ Cluster
+- name: Configure RabbitMQ
+  hosts: rabbitmq
+  become: true
+  vars_files:
+  - vars/common.yml
+  - vars/encpass.yml
+  - vars/rabbitmq.yml
+
+  # This is to try to avoid the handler issue in pre/post tasks
+  handlers:
+    - import_tasks: handlers/main.yml
+
+  pre_tasks:
+    - name: Check if ansible cannot be run here
+      stat:
+        path: /etc/no-ansible
+      register: no_ansible
+
+    - name: Verify if we can run ansible
+      assert:
+        that:
+          - "not no_ansible.stat.exists"
+        msg: "/etc/no-ansible exists - skipping run on this node"
+
+    # We have separate passwords per rabbitmq env
+    - name: Import rabbitmq passwords
+      include_vars:
+        file: "vars/rabbitmq_{{ rabbitmq_env }}.yml"
+
+    # EPEL and PowerTools are required for ipsilon to function
+    # I also couldn't find an ansible built-in to do this
+    - name: Enable the PowerTools repository
+      ini_file:
+        dest: /etc/yum.repos.d/CentOS-Linux-PowerTools.repo
+        section: powertools
+        option: enabled
+        value: 1
+        owner: root
+        group: root
+        mode: '0644'
+
+    # The CentOS extras repos has epel-release provided
+    - name: Enable the EPEL repository
+      yum:
+        name: epel-release
+        state: present
+      tags:
+        - packages
+
+    # This will change eventually to a rocky-release-messaging repo or to a
+    # rocky-release-rabbitmq repo
+    - name: Install centos rabbitmq
+      yum:
+        name: centos-release-rabbitmq
+        state: present
+      tags:
+        - packages
+
+  roles:
+    - role: rockylinux.ipagetcert
+      state: present
+      when: rabbitmq_private
+
+    - role: rockylinux.rabbitmq
+      state: present
+
+  post_tasks:
+    - name: Touching run file that ansible has ran here
+      file:
+        path: /var/log/ansible.run
+        state: touch
+        mode: '0644'
+        owner: root
+        group: root

ansible/playbooks/tasks/gitlab-reconfigure.yml

@@ -69,7 +69,7 @@
 - name: Apply fcontext to GitLab unix socket for nginx
   command: restorecon -v /var/opt/gitlab/gitlab-workhorse/sockets/socket
   register: restorecon_result
-  changed_when: "restorecon_result == 0"
+  changed_when: "restorecon_result.rc == 0"
 
 - name: Add firewall rules - http/s
   ansible.posix.firewalld:

ansible/playbooks/tasks/rabbitmq-reconfigure.yml

@@ -0,0 +1,2 @@
+---
+# RabbitMQ Additional Changes

ansible/playbooks/templates/etc/gitlab/rocky_gitlab.rb

@@ -40,7 +40,7 @@ gitlab_rails['ldap_servers'] = YAML.load <<-'EOS'
     password: '{{ gitlab_ldap_password }}'
     allow_username_or_email_login: true
     base: '{{ gitlab_ldap_base }}'
-    user_filter: ''
+    user_filter: '{{ gitlab_ldap_user_filter }}'
     group_base: '{{ gitlab_ldap_group_dn }}'
     admin_group: '{{ gitlab_ldap_admin_group }}'
     sync_ssh_keys: true
@@ -122,3 +122,13 @@ registry_nginx['ssl_certificate_key'] = "{{ gitlab_registry_nginx_ssl_certificat
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/README.md#changing-gitlab-yml-settings
 nginx['enable'] = false
 nginx['external_users'] = ['nginx']
+
+{% if gitlab_external_db %}
+postgresql['enable'] = false
+gitlab_rails['db_adapter'] = 'postgresql'
+gitlab_rails['db_encoding'] = 'unicode'
+gitlab_rails['db_host'] = '{{ gitlab_external_db_host }}'
+gitlab_rails['db_port'] = '{{ gitlab_external_db_port }}'
+gitlab_rails['db_username'] = '{{ gitlab_external_db_user }}'
+gitlab_rails['db_password'] = '{{ gitlab_external_db_password }}'
+{% endif %}

ansible/playbooks/vars/common.yml

@@ -0,0 +1,8 @@
+---
+rocky_ldap_bind_dn: "uid=binder,cn=sysaccounts,cn=etc,dc=rockylinux,dc=org"
+rocky_ldap_bind_pw: "ThisIsNotThePassword!"
+rocky_ldap_user_basedn: "cn=users,cn=accounts,dc=rockylinux,dc=org"
+rocky_ldap_group_basedn: "cn=groups,cn=accounts,dc=rockylinux,dc=org"
+rocky_ldap_account_basedn: "cn=accounts,dc=rockylinux,dc=org"
+# Requires jinja 2.9+
+rocky_ipaserver_list: "{{ groups['ipaserver'] + groups['ipareplicas'] }}"

ansible/playbooks/vars/encpass.yml

@@ -16,3 +16,6 @@ ipsilon_db_password: !vault |
 koji_db_pass: !vault |
           $ANSIBLE_VAULT;1.1;AES256
           REDACTED
+pubsub_federation_pass: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          REDACTED

ansible/playbooks/vars/gitlab.yml

@@ -21,15 +21,16 @@ gitlab_ssl_key: "/etc/nginx/ssl/{{ gitlab_domain }}.key"
 
 # LDAP Configuration
 gitlab_ldap_enabled: "true"
-gitlab_ldap_host: "ipa001.rockylinux.org"
+gitlab_ldap_host: "{{ rocky_ipaserver_list[0] }}"
 gitlab_ldap_port: "389"
 gitlab_ldap_uid: "uid"
 gitlab_ldap_method: "start_tls"
-gitlab_ldap_bind_dn: "uid=binder,cn=sysaccounts,cn=etc,dc=rockylinux,dc=org"
-gitlab_ldap_password: "ThisIsNotThePassword!"
-gitlab_ldap_base: "cn=users,cn=accounts,dc=rockylinux,dc=org"
-gitlab_ldap_group_dn: "cn=groups,cn=accounts,dc=rockylinux,dc=org"
+gitlab_ldap_bind_dn: "{{ rocky_ldap_bind_dn }}"
+gitlab_ldap_password: "{{ rocky_ldap_bind_pw }}"
+gitlab_ldap_base: "{{ rocky_ldap_user_basedn }}"
+gitlab_ldap_group_dn: "{{ rocky_ldap_group_basedn }}"
 gitlab_ldap_admin_group: "cn=gitadm,cn=groups,cn=accounts,dc=rockylinux,dc=org"
+gitlab_ldap_user_filter: "(&(objectClass=posixAccount)(memberOf=cn=gitusers,cn=groups,cn=accounts,dc=rockylinux,dc=org))"
 
 gitlab_time_zone: "UTC"
 
@@ -61,3 +62,8 @@ gitlab_nginx_listen_port: 8080
 gitlab_nginx_listen_https: "false"
 
 gitlab_default_theme: 2
+
+gitlab_external_db: false
+gitlab_external_db_host: db.rockylinux.org
+gitlab_external_db_user: gitlab
+gitlab_external_db_password: gitlab

ansible/playbooks/vars/groups.yml

@@ -42,9 +42,32 @@ ipagroups:
       - rfelsburg
       - tg
       - hbjy
+  - group: gitusers
+    description: Rocky Linux GitLab Users
+    user:
+      - label
+      - neil
+      - rlh
+      - rfelsburg
+      - tg
+      - hbjy
+      - rockyautomation
   - group: services
     description: Rocky Linux Service Accounts
     user:
       - hostman
       - kerbman
       - rockykoji
+      - pubsub_federation
+      - rockypubsub
+      - rockyautomation
+  - group: iam
+    description: Rocky Linux Identity Management
+    user:
+      - label
+  - group: releng
+    description: Rocky Linux Release Engineering
+    user:
+      - label
+  - group: mq_pub_readonly
+    description: RabbitMQ ReadOnly

ansible/playbooks/vars/rabbitmq.yml

@@ -0,0 +1,39 @@
+---
+# rabbitmq settings
+rabbitmq_tls_ca_cert: "/etc/pki/tls/certs/ca-bundle.crt"
+rabbitmq_tls_cert: "/etc/pki/tls/certs/{{ ansible_fqdn }}.crt"
+rabbitmq_tls_key: "/etc/pki/tls/private/{{ ansible_fqdn }}.key"
+
+# These should be in a vault, with a different value. Generated by:
+# dd if=/dev/urandom bs=30 count=1 | base64
+#rabbitmq_cookie: ...
+
+# Admin passwords - these should be in a vault
+#rabbitmq_admin_password: ...
+
+# rabbitmq cluster list and information should be defined in hostvars to ensure
+# that the configuration is idempotent.
+#rabbitmq_cluster_name:
+#rabbitmq_env:
+
+# Federation / Public Queues
+rabbitmq_enable_public: false
+#pubsub_federation_pass:
+
+# THIS IS DYNAMIC. IT'S ADVISED IT NOT BE STATIC.
+# This should be changed depending on how inventory is managed. For example, if
+# it's not possible to have "staging inventory" as opposed to a "production"
+# inventory, you would likely have a different name than just "rabbitmq". It is
+# also possible there will be more than one cluster, so these must be taken
+# into account when setting this variable.
+rabbitmq_cluster_list: "{{ groups['rabbitmq'] }}"
+rabbitmq_ldap_servers: "{{ rocky_ipaserver_list }}"
+
+# Messaging queues are generally private
+rabbitmq_private: true
+ipa_getcert_requested_hostnames:
+  - name: "{{ ansible_fqdn }}"
+    owner: rabbitmq
+    key_location: "{{ rabbitmq_tls_key }}"
+    cert_location: "{{ rabbitmq_tls_cert }}"
+    postcmd: "/bin/systemctl restart rabbitmq-server"

ansible/playbooks/vars/rabbitmq_production.yml

@@ -0,0 +1,4 @@
+---
+# This will need to be vaulted
+rabbitmq_admin_password: ThisIsNotThePassword!
+rabbitmq_cookie: "X4MYneML6Ppp+ajPuG/qdD64ZjdVT1OJF8XUG/o+"

ansible/playbooks/vars/rabbitmq_staging.yml

@@ -0,0 +1,4 @@
+---
+# This will need to be vaulted
+rabbitmq_admin_password: ThisIsNotThePassword!
+rabbitmq_cookie: "X4MYneML6Ppp+ajPuG/qdD64ZjdVT1OJF8XUG/o+"

ansible/playbooks/vars/svcusers.yml

@@ -21,3 +21,24 @@ svcusers:
     password: ThisIsNotMyPassword1!
     title: System Account - Koji Manager
     loginshell: /sbin/nologin
+  - name: pubsub_federation
+    first: pubsub
+    last: federation
+    email: infrastructure@rockylinux.org
+    password: ThisIsNotMyPassword1!
+    title: System Account - pubsub federator
+    loginshell: /sbin/nologin
+  - name: rockypubsub
+    first: rocky
+    last: pubsub
+    email: infrastructure@rockylinux.org
+    password: ThisIsNotMyPassword1!
+    title: System Account - pubsub
+    loginshell: /sbin/nologin
+  - name: rockyautomation
+    first: Rocky
+    last: Automation
+    email: infrastructure@rockylinux.org
+    password: ThisIsNotMyPassword1!
+    title: System Account - Automation
+    loginshell: /sbin/nologin

ansible/roles/requirements.yml

@@ -16,8 +16,8 @@ roles:
   - name: rockylinux.kojihub
     src: https://github.com/rocky-linux/ansible-role-kojihub
     version: main
-  - name: rockylinux.mqtt
-    src: https://github.com/rocky-linux/ansible-role-mqtt
+  - name: rockylinux.rabbitmq
+    src: https://github.com/rocky-linux/ansible-role-rabbitmq
     version: main
 
 collections:
@@ -26,5 +26,7 @@ collections:
     version: 0.3.1
   - name: community.general
   - name: community.mysql
+  - name: community.rabbitmq
   - name: ansible.posix
   - name: ktdreyer.koji_ansible
+  - name: netbox.netbox