koji!

README.md

@@ -1,7 +1,11 @@
 CI Badge
 
-# Ansible template role
+# Kojihub Role
-basic Role to use going forward because I forget pieces
+This role installs kojihub and kojiweb. Note that it does NOT install the database. The database must be installed from a different role or playbook method. This role also assumes you are using a Kerberos infrastructure, such as FreeIPA.
+
+FAS is not yet implemented.
+
+Ansible 2.10 users: You will need the community.general collection installed.
 
 ## Getting started
 Ensure all dependencies are installed and then follow the below process

defaults/main.yml

@@ -1,2 +1,68 @@
 ---
 # ansible default variables - most variables live here
+koji_hub_packages:
+  - koji
+  - koji-hub
+  - koji-hub-plugins
+  - koji-web
+  - koji-utils
+  - git
+  - gnupg2
+  - python3-paho-mqtt
+  - nfs-utils
+
+koji_default_directories:
+  - packages
+  - repos
+  - work
+  - scratch
+  - repos-dist
+
+koji_db_name: koji
+koji_db_user: koji
+koji_db_pass: ThisIsNotThePassword!
+koji_db_host: localhost
+
+# Web
+koji_sitename: Rocky Linux Build Service
+koji_theme: false
+koji_theme_name: rocky
+koji_theme_file: rocky.tar.gz
+koji_web_url: https://koji.rockylinux.org/koji
+koji_hub_url: https://koji.rockylinux.org/kojihub
+koji_files_url: https://koji.rockylinux.org/kojifiles
+
+# This should be changed before deployment
+koji_hub_secret: cK5XCuzMSXJfgA7yFvXkGwFu
+koji_web_cacert: /etc/pki/tls/certs/chain.crt
+koji_web_tls_cert: /etc/pki/tls/certs/koji.rockylinux.org.crt
+koji_web_tls_key: /etc/pki/tls/private/koji.rockylinux.org.key
+
+# Kojira
+koji_kojira: true
+koji_kojira_user: kojira
+koji_kojira_principal: koji/kojiria@ROCKYLINUX.ORG
+koji_kojira_keytab: /etc/kojira.keytab
+
+# Storage
+koji_mount: /mnt/koji
+koji_nfs_path: nfs.rockylinux.org:/exports/koji
+
+# Koji Admin
+koji_admin_client: true
+koji_admin_user: rockykoji
+koji_admin_keytab: rockykoji@ROCKYLINUX.ORG
+
+# Koji FAS Syncing
+# This isn't implemented yet
+koji_fas_sync: false
+koji_fas_url: https://accounts.rockylinux.org
+
+# Koji Plugins
+koji_hub_plugins: false
+koji_hub_plugins_list: []
+
+koji_hub_plugin_mqtt_host: mqtt.rockylinux.org
+koji_hub_plugin_mqtt_topic: koji
+koji_hub_plugin_mqtt_excluded_tags:
+  - testing-tag

handlers/main.yml

@@ -0,0 +1,10 @@
+---
+- name: restart_kojira
+  service:
+    name: kojira
+    state: restarted
+
+- name: restart_httpd
+  service:
+    name: httpd
+    state: restarted

tasks/db.yml

@@ -0,0 +1,26 @@
+---
+# Note: We do not install postgresql. It's up to you to do so.
+- name: Template for koji admin and kojira
+  template:
+    src: koji-pgsql.sql.j2
+    dest: /var/tmp/koji-pgsql.sql
+    mode: '0644'
+
+- name: Load schema to postgresql database
+  community.general.postgresql_db:
+    name: "{{ koji_db_name }}"
+    target: /usr/share/doc/koji/docs/schema.sql
+    owner: "{{ koji_db_user }}"
+    state: restore
+    login_host: "{{ koji_db_host }}"
+    login_user: "{{ koji_db_user }}"
+    login_password: "{{ koji_db_pass }}"
+
+- name: Apply the postgres template
+  community.general.postgresql_db:
+    name: "{{ koji_db_name }}"
+    target: /var/tmp/koji-pgsql.sql
+    state: restore
+    login_host: "{{ koji_db_host }}"
+    login_user: "{{ koji_db_user }}"
+    login_password: "{{ koji_db_pass }}"

tasks/koji-admin.yml

@@ -0,0 +1,45 @@
+---
+# Create the koji admin user
+- name: Create local koji admin user
+  user: "{{ koji_admin_user }}"
+
+- name: Create koji config directory
+  file:
+    path: "/home/{{ koji_admin_user }}/.koji"
+    state: directory
+    owner: "{{ koji_admin_user }}"
+    group: "{{ koji_admin_user }}"
+    recurse: true
+
+- name: Reset permissions
+  file:
+    path: "/home/{{ koji_admin_user }}"
+    state: directory
+    owner: "{{ koji_admin_user }}"
+    group: "{{ koji_admin_user }}"
+    mode: '0700'
+
+- name: Configure the koji client
+  template:
+    src: koji-client-config.j2
+    dest: "/home/{{ koji_admin_user }}/.koji/config"
+    owner: "{{ koji_admin_user }}"
+    group: "{{ koji_admin_user }}"
+    mode: '0644'
+
+- name: Ensuring we have our scripts store
+  file:
+    path: /opt/rocky-tools/scripts
+    state: directory
+    owner: "{{ koji_admin_user }}"
+    group: "{{ koji_admin_user }}"
+    mode: '0750'
+    recurse: true
+
+# name: Cron job to rebuild repos
+# cron:
+#   name: "Regenerate repos"
+#   job: "/opt/rocky-tools/scripts/regen_build_repos.sh > /dev/null 2>&1"
+#   minute: "5"
+#   hour: "3"
+#   user: "{{ koji_admin_user }}"

tasks/kojira.yml

@@ -0,0 +1,15 @@
+---
+# Kojira
+- name: Configure kojira
+  template:
+    src: etc/kojira/kojira.conf.j2
+    dest: /etc/kojira/kojira.conf
+    mode: '0644'
+  notify:
+    - restart_kojira
+
+- name: Ensure kojira is running
+  service:
+    name: kojira
+    state: started
+    enabled: yes

tasks/main.yml

@@ -1,2 +1,60 @@
 ---
-# tasks
+# Koji hub and web installation
+# This is not for builder nodes
+
+- name: Apply required SELinux booleans
+  import_tasks: selinux_boolean.yml
+
+- name: Ensure NFS is mounted
+  import_tasks: nfs.yml
+
+- name: Install required packages
+  yum:
+    name: {{ koji_hub_packages }}
+    state: present
+
+- name: Configure koji database
+  import_tasks: db.yml
+
+- name: Configure koji admin
+  import_tasks: koji-admin.yml
+  when: koji_admin_client
+
+- name: Configure plugins
+  import_tasks: plugins.yml
+  when: koji_hub_plugins
+
+- name: Configure kojira
+  import_tasks: kojira.yml
+
+- name: Configure kojihub and web
+  template:
+    src: "{{ item }}.j2"
+    dest: "/{{ item }}"
+    mode: '0644'
+  with_items:
+    - etc/koji-hub/hub.conf
+    - etc/kojiweb/web.conf
+  notify:
+    - restart_httpd
+
+- name: Configure httpd for hub and web
+  template:
+    src: "etc/httpd/conf.d/{{ item }}.j2"
+    dest: "/etc/httpd/conf.d/{{ item }}"
+    mode: '0644'
+  with_items:
+    - kojihub.conf
+    - kojiweb.conf
+  notify:
+    - restart_httpd
+
+- name: Deploy custom theme for koji
+  unarchive:
+    src: "{{ koji_theme_file }}"
+    dest: /
+  when: koji_theme
+
+- name: User Sync from FAS
+  import_tasks: user-sync.yml
+  when: koji_fas_sync

tasks/nfs.yml

@@ -0,0 +1,21 @@
+---
+
+- name: Ensure the koji mountpoint exists
+  file:
+    path: "{{ koji_mount }}"
+    state: directory
+
+- name: Mount the NFS store
+  mount:
+    path: "{{ koji_mount }}"
+    src: "{{ koji_nfs_path }}"
+    fstype: nfs
+    state: mounted
+
+- name: Create required default directories
+  file:
+    path: "{{ koji_mount }}/{{ item }}"
+    state: directory
+    owner: apache
+    group: apache
+  with_items: "{{ koji_default_directories }}"

tasks/plugins.yml

@@ -0,0 +1,15 @@
+---
+# Plugins
+- name: Deploying enabled plugins
+  copy:
+    src: "plugins/{{ item }}.py"
+    dest: "/usr/lib/koji-hub-plugins/{{ item }}.py"
+    mode: 0755
+  with_items: "{{ koji_hub_plugins_list }}"
+
+- name: Configuring enabled plugins
+  template:
+    src: "etc/koji-hub/plugins/{{ item }}.conf.j2"
+    dest: "/etc/koji-hub/plugins/{{ item }}.conf"
+    mode: 0644
+  with_items: "{{ koji_hub_plugins_list }}"

tasks/selinux_boolean.yml

@@ -0,0 +1,12 @@
+---
+
+- name: Enabling SELinux booleans
+  seboolean:
+    name: "{{ item }}"
+    persistent: true
+    state: true
+  with_items:
+    - httpd_can_network_connect_db
+    - httpd_can_network_connect
+    - allow_httpd_anon_write
+    - httpd_use_nfs

tasks/users-sync.yml

@@ -0,0 +1,2 @@
+---
+# No tasks

templates/etc/httpd/conf.d/kojihub.conf.j2

@@ -0,0 +1,66 @@
+#
+# koji-hub is an xmlrpc interface to the Koji database
+#
+
+Alias /kojihub /usr/share/koji-hub/kojixmlrpc.py
+
+<Directory "/usr/share/koji-hub">
+    Options ExecCGI
+    SetHandler wsgi-script
+    WSGIApplicationGroup %{GLOBAL}
+    # ^ works around a hub issue with OpenSSL
+    # see: https://cryptography.io/en/latest/faq/#starting-cryptography-using-mod-wsgi-produces-an-internalerror-during-a-call-in-register-osrandom-engine
+    WSGIScriptReloading Off
+    # ^ reloading breaks hub "firstcall" check
+    # see: https://pagure.io/koji/issue/875
+    <IfVersion < 2.4>
+        Order allow,deny
+        Allow from all
+    </IfVersion>
+    <IfVersion >= 2.4>
+        Require all granted
+    </IfVersion>
+</Directory>
+
+# Also serve /mnt/koji
+Alias /kojifiles "/mnt/koji/"
+
+<Directory "/mnt/koji">
+    Options Indexes SymLinksIfOwnerMatch
+    #If your top /mnt/koji directory is not owned by the httpd user, then
+    #you will need to follow all symlinks instead, e.g.
+    #Options Indexes FollowSymLinks
+    AllowOverride None
+    IndexOptions +NameWidth=*
+    <IfVersion < 2.4>
+        Order allow,deny
+        Allow from all
+    </IfVersion>
+    <IfVersion >= 2.4>
+        Require all granted
+    </IfVersion>
+</Directory>
+
+# uncomment this to enable authentication via SSL client certificates
+# <Location /kojihub/ssllogin>
+#         SSLVerifyClient require
+#         SSLVerifyDepth  10
+#         SSLOptions +StdEnvVars
+# </Location>
+
+# If you need to support koji < 1.4.0 clients using SSL authentication, then use the following instead:
+# <Location /kojihub>
+#         SSLOptions +StdEnvVars
+# </Location>
+# In this case, you will need to enable these options globally (in ssl.conf):
+# SSLVerifyClient require
+# SSLVerifyDepth  10
+
+# uncomment this to enable authentication via GSSAPI
+<Location /kojihub/ssllogin>
+        AuthType GSSAPI
+        AuthName "GSSAPI Single Sign On Login"
+        GssapiCredStore keytab:/etc/koji.keytab
+        Require valid-user
+</Location>
+

templates/etc/httpd/conf.d/kojiweb.conf.j2

@@ -0,0 +1,79 @@
+#We use wsgi by default
+Alias /koji "/usr/share/koji-web/scripts/wsgi_publisher.py"
+#(configuration goes in /etc/kojiweb/web.conf)
+
+RewriteEngine on
+RewriteCond %{HTTPS} off
+RewriteRule .* https://%{HTTP_HOST}%{REQUEST_URI} [R=302,L]
+RewriteRule  ^/$    /koji [R,L] 
+
+ Header always set X-Frame-Options "SAMEORIGIN"
+ Header always set X-Xss-Protection "1; mode=block"
+ Header always set X-Content-Type-Options "nosniff"
+ Header always set Referrer-Policy "same-origin"
+
+Alias /repos {{ koji_mount }}/repos
+<Directory "{{ koji_mount }}/repos">
+   Options Indexes FollowSymLinks
+    AllowOverride None
+#    HeaderName /header/header.html
+    <IfVersion < 2.4>
+    Order allow,deny
+    Allow from all
+    </IfVersion>
+    <IfVersion >= 2.4>
+    IndexOptions FancyIndexing VersionSort NameWidth=* HTMLTable Charset=UTF-8
+    Require all granted
+    </IfVersion>
+</Directory>
+
+# Python 3 Cheetah expectes unicode everywhere, apache's default lang is C
+# which is not sufficient to open our templates
+WSGIDaemonProcess koji lang=C.UTF-8
+WSGIProcessGroup koji
+
+<Directory "/usr/share/koji-web/scripts/">
+    Options ExecCGI
+    SetHandler wsgi-script
+    WSGIApplicationGroup %{GLOBAL}
+    # ^ works around an OpenSSL issue
+    # see: https://cryptography.io/en/latest/faq/#starting-cryptography-using-mod-wsgi-produces-an-internalerror-during-a-call-in-register-osrandom-engine
+    <IfVersion < 2.4>
+        Order allow,deny
+        Allow from all
+    </IfVersion>
+    <IfVersion >= 2.4>
+        Require all granted
+    </IfVersion>
+</Directory>
+
+# uncomment this to enable authentication via Kerberos
+<Location /koji/login>
+    AuthType GSSAPI
+    AuthName "Koji Web UI"
+    GssapiCredStore keytab:/etc/koji.keytab
+    Require valid-user
+    ErrorDocument 401 /koji-static/errors/unauthorized.html
+</Location>
+
+# uncomment this to enable authentication via SSL client certificates
+# <Location /koji/login>
+#     SSLVerifyClient require
+#     SSLVerifyDepth  10
+#     SSLOptions +StdEnvVars
+# </Location>
+
+Alias /koji-static/ "/usr/share/koji-web/static/"
+
+<Directory "/usr/share/koji-web/static/">
+    Options None
+    AllowOverride None
+    <IfVersion < 2.4>
+        Order allow,deny
+        Allow from all
+    </IfVersion>
+    <IfVersion >= 2.4>
+        Require all granted
+    </IfVersion>
+</Directory>
+

templates/etc/koji-hub/hub.conf.j2

@@ -0,0 +1,81 @@
+[hub]
+
+## ConfigParser style config file, similar to ini files
+## http://docs.python.org/library/configparser.html
+##
+## Note that multiline values can be set by indenting subsequent lines
+## (which means you should not indent regular lines)
+
+## Basic options ##
+DBName = {{ koji_db_name }}
+DBUser = {{ koji_db_user }}
+DBHost = {{ koji_db_host }}
+DBPass = {{ koji_db_pass }}
+KojiDir = {{ koji_mount }}
+
+AuthPrincipal host/kojihub@ROCKYLINUX.ORG
+AuthKeytab /etc/koji.keytab
+ProxyPrincipals koji/kojiweb@ROCKYLINUX.ORG
+HostPrincipalFormat compile/%s@ROCKYLINUX.ORG
+
+##  Other options  ##
+LoginCreatesUser = Off
+KojiWebURL = {{ koji_web_url }}
+
+# The domain name that will be appended to Koji usernames
+# when creating email notifications
+#EmailDomain = example.com
+# whether to send the task owner and package owner email or not on success.  this still goes to watchers
+NotifyOnSuccess = True
+## Disables all notifications
+# DisableNotifications = False
+DisableNotifications = True
+
+## Extended features
+## Support Maven builds
+# EnableMaven = False
+## Support Windows builds
+# EnableWin = False
+
+## Koji hub plugins
+## The path where plugins are found
+# PluginPath = /usr/lib/koji-hub-plugins
+## A space-separated list of plugins to load
+# Plugins = echo
+{% if koji_hub_plugins %}
+Plugins = {% for plugin in koji_hub_plugins_list + koji_hub_noconfig_plugins_list %}{{ plugin }} {% endfor %}
+{% endif %}
+
+## If KojiDebug is on, the hub will be /very/ verbose and will report exception
+## details to clients for anticipated errors (i.e. koji's own exceptions --
+## subclasses of koji.GenericError).
+# KojiDebug = On
+
+## Determines how much detail about exceptions is reported to the client (via faults)
+## Meaningful values:
+##   normal - a basic traceback (format_exception)
+##   extended - an extended traceback (format_exc_plus)
+##   anything else - no traceback, just the error message
+## The extended traceback is intended for debugging only and should NOT be
+## used in production, since it may contain sensitive information.
+# KojiTraceback = normal
+
+## These options are intended for planned outages
+# ServerOffline = False
+# OfflineMessage = temporary outage
+# LockOut = False
+## If ServerOffline is True, the server will always report a ServerOffline fault (with
+## OfflineMessage as the fault string).
+## If LockOut is True, the server will report a ServerOffline fault for all non-admin
+## requests.
+
+[policy]
+tag =
+    all :: allow
+
+package_list =
+    all :: allow
+
+build_from_srpm =
+      tag * :: allow
+      has_perm build :: allow

templates/etc/kojira/kojira.conf.j2

@@ -0,0 +1,49 @@
+[kojira]
+; For user/pass authentication
+; user=kojira
+; password=kojira
+
+; The URL for the koji hub server
+server={{ koji_hub_url }}
+
+; The directory containing the repos/ directory
+topdir={{ koji_mount }}
+
+; Logfile
+logfile=/var/log/kojira.log
+
+with_src=no
+
+;configuration for Kerberos authentication
+
+;the kerberos principal to use
+principal = {{ koji_kojira_principal }}
+
+;location of the keytab
+keytab = {{ koji_kojira_keytab }}
+
+;configuration for SSL authentication
+
+;client certificate
+;cert = /etc/kojira/client.crt
+
+;certificate of the CA that issued the HTTP server certificate
+;serverca = /etc/kojira/serverca.crt
+
+;how soon (in seconds) to clean up expired repositories. 1 week default
+;deleted_repo_lifetime = 604800
+
+;how soon (in seconds) to clean up dist repositories. 1 week default here too
+;dist_repo_lifetime = 604800
+
+;turn on debugging statements in the log
+;debug = false
+
+; ignored repositories according to glob. Multiple masks separated by space.
+; ignore_tags =
+
+; Monitor external repos and trigger the appropriate Koji repo regenerations
+; when they change. Note that you need to have your database set to use UTC,
+; as otherwise you can end with weird behaviour. For details see
+; https://pagure.io/koji/issue/2159
+; check_external_repos = false

templates/etc/kojiweb/web.conf.j2

@@ -0,0 +1,23 @@
+[web]
+SiteName = {{ koji_sitename }}
+{% if koji_custom_theme %}
+KojiTheme = {{ koji_theme }}
+{% endif %}
+
+# Key urls
+KojiHubURL = {{ koji_hub_url }}
+KojiFilesURL = {{ koji_files_url }}
+
+# Kerberos authentication options
+WebPrincipal = koji/web@ROCKYLINUX.ORG
+WebKeytab = /etc/httpd.keytab
+WebCCache = /var/tmp/kojiweb.ccache
+# The service name of the principal being used by the hub
+KrbService = host
+
+LoginTimeout = 72
+
+# This must be changed and uncommented before deployment
+Secret = {{ koji_hub_secret }}
+
+LibPath = /usr/share/koji-web/lib

templates/koji-client-config.j2

@@ -0,0 +1,13 @@
+[koji]
+
+;url of XMLRPC server
+server = {{ koji_hub_url }}
+
+;url of web interface
+weburl = {{ koji_web_url }}
+
+;url of package download site
+topurl = {{ koji_files_url }}
+
+;path to the koji top directory
+topdir = {{ koji_mount }}

templates/koji-pgsql-kojira.sql.j2

@@ -0,0 +1,2 @@
+insert into users (name, status, usertype) values ('{{ koji_kojira_user }}', 0, 0);
+INSERT INTO user_perms (user_id, perm_id, creator_id) VALUES (2, 3, 1);

templates/koji-pgsql.sql.j2

@@ -0,0 +1,5 @@
+with user_id as (insert into users (name, status, usertype) values ('{{ koji_admin_user }}', 0, 0) returning id)
+insert into user_krb_principals (user_id, krb_principal) values ((select id from user_id),'{{ koji_admin_keytab }}');
+insert into user_perms (user_id, perm_id, creator_id) values (1, 1, 1);
+insert into users (name, status, usertype) values ('{{ koji_kojira_user }}', 0, 0);
+INSERT INTO user_perms (user_id, perm_id, creator_id) VALUES (2, 3, 1);