From 0bccf61c877c6f3dd15a29258df5213232ad838c Mon Sep 17 00:00:00 2001 From: nazunalika Date: Tue, 22 Dec 2020 20:52:59 -0700 Subject: [PATCH] koji! --- README.md | 8 ++- defaults/main.yml | 68 +++++++++++++++++- handlers/main.yml | 10 +++ tasks/db.yml | 26 +++++++ tasks/koji-admin.yml | 45 ++++++++++++ tasks/kojira.yml | 15 ++++ tasks/main.yml | 60 +++++++++++++++- tasks/nfs.yml | 21 ++++++ tasks/plugins.yml | 15 ++++ tasks/selinux_boolean.yml | 12 ++++ tasks/users-sync.yml | 2 + templates/etc/httpd/conf.d/kojihub.conf.j2 | 66 ++++++++++++++++++ templates/etc/httpd/conf.d/kojiweb.conf.j2 | 79 +++++++++++++++++++++ templates/etc/koji-hub/hub.conf.j2 | 81 ++++++++++++++++++++++ templates/etc/kojira/kojira.conf.j2 | 49 +++++++++++++ templates/etc/kojiweb/web.conf.j2 | 23 ++++++ templates/koji-client-config.j2 | 13 ++++ templates/koji-pgsql-kojira.sql.j2 | 2 + templates/koji-pgsql.sql.j2 | 5 ++ 19 files changed, 596 insertions(+), 4 deletions(-) create mode 100644 handlers/main.yml create mode 100644 tasks/db.yml create mode 100644 tasks/koji-admin.yml create mode 100644 tasks/kojira.yml create mode 100644 tasks/nfs.yml create mode 100644 tasks/plugins.yml create mode 100644 tasks/selinux_boolean.yml create mode 100644 tasks/users-sync.yml create mode 100644 templates/etc/httpd/conf.d/kojihub.conf.j2 create mode 100644 templates/etc/httpd/conf.d/kojiweb.conf.j2 create mode 100644 templates/etc/koji-hub/hub.conf.j2 create mode 100644 templates/etc/kojira/kojira.conf.j2 create mode 100644 templates/etc/kojiweb/web.conf.j2 create mode 100644 templates/koji-client-config.j2 create mode 100644 templates/koji-pgsql-kojira.sql.j2 create mode 100644 templates/koji-pgsql.sql.j2 diff --git a/README.md b/README.md index 13ffe68..e7124be 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,11 @@ CI Badge -# Ansible template role -basic Role to use going forward because I forget pieces +# Kojihub Role +This role installs kojihub and kojiweb. Note that it does NOT install the database. The database must be installed from a different role or playbook method. This role also assumes you are using a Kerberos infrastructure, such as FreeIPA. + +FAS is not yet implemented. + +Ansible 2.10 users: You will need the community.general collection installed. ## Getting started Ensure all dependencies are installed and then follow the below process diff --git a/defaults/main.yml b/defaults/main.yml index 7faaf71..93531a7 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,2 +1,68 @@ --- -# ansible default variables - most variables live here \ No newline at end of file +# ansible default variables - most variables live here +koji_hub_packages: + - koji + - koji-hub + - koji-hub-plugins + - koji-web + - koji-utils + - git + - gnupg2 + - python3-paho-mqtt + - nfs-utils + +koji_default_directories: + - packages + - repos + - work + - scratch + - repos-dist + +koji_db_name: koji +koji_db_user: koji +koji_db_pass: ThisIsNotThePassword! +koji_db_host: localhost + +# Web +koji_sitename: Rocky Linux Build Service +koji_theme: false +koji_theme_name: rocky +koji_theme_file: rocky.tar.gz +koji_web_url: https://koji.rockylinux.org/koji +koji_hub_url: https://koji.rockylinux.org/kojihub +koji_files_url: https://koji.rockylinux.org/kojifiles + +# This should be changed before deployment +koji_hub_secret: cK5XCuzMSXJfgA7yFvXkGwFu +koji_web_cacert: /etc/pki/tls/certs/chain.crt +koji_web_tls_cert: /etc/pki/tls/certs/koji.rockylinux.org.crt +koji_web_tls_key: /etc/pki/tls/private/koji.rockylinux.org.key + +# Kojira +koji_kojira: true +koji_kojira_user: kojira +koji_kojira_principal: koji/kojiria@ROCKYLINUX.ORG +koji_kojira_keytab: /etc/kojira.keytab + +# Storage +koji_mount: /mnt/koji +koji_nfs_path: nfs.rockylinux.org:/exports/koji + +# Koji Admin +koji_admin_client: true +koji_admin_user: rockykoji +koji_admin_keytab: rockykoji@ROCKYLINUX.ORG + +# Koji FAS Syncing +# This isn't implemented yet +koji_fas_sync: false +koji_fas_url: https://accounts.rockylinux.org + +# Koji Plugins +koji_hub_plugins: false +koji_hub_plugins_list: [] + +koji_hub_plugin_mqtt_host: mqtt.rockylinux.org +koji_hub_plugin_mqtt_topic: koji +koji_hub_plugin_mqtt_excluded_tags: + - testing-tag diff --git a/handlers/main.yml b/handlers/main.yml new file mode 100644 index 0000000..ed6957e --- /dev/null +++ b/handlers/main.yml @@ -0,0 +1,10 @@ +--- +- name: restart_kojira + service: + name: kojira + state: restarted + +- name: restart_httpd + service: + name: httpd + state: restarted diff --git a/tasks/db.yml b/tasks/db.yml new file mode 100644 index 0000000..a46287b --- /dev/null +++ b/tasks/db.yml @@ -0,0 +1,26 @@ +--- +# Note: We do not install postgresql. It's up to you to do so. +- name: Template for koji admin and kojira + template: + src: koji-pgsql.sql.j2 + dest: /var/tmp/koji-pgsql.sql + mode: '0644' + +- name: Load schema to postgresql database + community.general.postgresql_db: + name: "{{ koji_db_name }}" + target: /usr/share/doc/koji/docs/schema.sql + owner: "{{ koji_db_user }}" + state: restore + login_host: "{{ koji_db_host }}" + login_user: "{{ koji_db_user }}" + login_password: "{{ koji_db_pass }}" + +- name: Apply the postgres template + community.general.postgresql_db: + name: "{{ koji_db_name }}" + target: /var/tmp/koji-pgsql.sql + state: restore + login_host: "{{ koji_db_host }}" + login_user: "{{ koji_db_user }}" + login_password: "{{ koji_db_pass }}" diff --git a/tasks/koji-admin.yml b/tasks/koji-admin.yml new file mode 100644 index 0000000..af262f3 --- /dev/null +++ b/tasks/koji-admin.yml @@ -0,0 +1,45 @@ +--- +# Create the koji admin user +- name: Create local koji admin user + user: "{{ koji_admin_user }}" + +- name: Create koji config directory + file: + path: "/home/{{ koji_admin_user }}/.koji" + state: directory + owner: "{{ koji_admin_user }}" + group: "{{ koji_admin_user }}" + recurse: true + +- name: Reset permissions + file: + path: "/home/{{ koji_admin_user }}" + state: directory + owner: "{{ koji_admin_user }}" + group: "{{ koji_admin_user }}" + mode: '0700' + +- name: Configure the koji client + template: + src: koji-client-config.j2 + dest: "/home/{{ koji_admin_user }}/.koji/config" + owner: "{{ koji_admin_user }}" + group: "{{ koji_admin_user }}" + mode: '0644' + +- name: Ensuring we have our scripts store + file: + path: /opt/rocky-tools/scripts + state: directory + owner: "{{ koji_admin_user }}" + group: "{{ koji_admin_user }}" + mode: '0750' + recurse: true + +# name: Cron job to rebuild repos +# cron: +# name: "Regenerate repos" +# job: "/opt/rocky-tools/scripts/regen_build_repos.sh > /dev/null 2>&1" +# minute: "5" +# hour: "3" +# user: "{{ koji_admin_user }}" diff --git a/tasks/kojira.yml b/tasks/kojira.yml new file mode 100644 index 0000000..cccab22 --- /dev/null +++ b/tasks/kojira.yml @@ -0,0 +1,15 @@ +--- +# Kojira +- name: Configure kojira + template: + src: etc/kojira/kojira.conf.j2 + dest: /etc/kojira/kojira.conf + mode: '0644' + notify: + - restart_kojira + +- name: Ensure kojira is running + service: + name: kojira + state: started + enabled: yes diff --git a/tasks/main.yml b/tasks/main.yml index 2583e53..8459bd3 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,2 +1,60 @@ --- -# tasks \ No newline at end of file +# Koji hub and web installation +# This is not for builder nodes + +- name: Apply required SELinux booleans + import_tasks: selinux_boolean.yml + +- name: Ensure NFS is mounted + import_tasks: nfs.yml + +- name: Install required packages + yum: + name: {{ koji_hub_packages }} + state: present + +- name: Configure koji database + import_tasks: db.yml + +- name: Configure koji admin + import_tasks: koji-admin.yml + when: koji_admin_client + +- name: Configure plugins + import_tasks: plugins.yml + when: koji_hub_plugins + +- name: Configure kojira + import_tasks: kojira.yml + +- name: Configure kojihub and web + template: + src: "{{ item }}.j2" + dest: "/{{ item }}" + mode: '0644' + with_items: + - etc/koji-hub/hub.conf + - etc/kojiweb/web.conf + notify: + - restart_httpd + +- name: Configure httpd for hub and web + template: + src: "etc/httpd/conf.d/{{ item }}.j2" + dest: "/etc/httpd/conf.d/{{ item }}" + mode: '0644' + with_items: + - kojihub.conf + - kojiweb.conf + notify: + - restart_httpd + +- name: Deploy custom theme for koji + unarchive: + src: "{{ koji_theme_file }}" + dest: / + when: koji_theme + +- name: User Sync from FAS + import_tasks: user-sync.yml + when: koji_fas_sync diff --git a/tasks/nfs.yml b/tasks/nfs.yml new file mode 100644 index 0000000..ca820ab --- /dev/null +++ b/tasks/nfs.yml @@ -0,0 +1,21 @@ +--- + +- name: Ensure the koji mountpoint exists + file: + path: "{{ koji_mount }}" + state: directory + +- name: Mount the NFS store + mount: + path: "{{ koji_mount }}" + src: "{{ koji_nfs_path }}" + fstype: nfs + state: mounted + +- name: Create required default directories + file: + path: "{{ koji_mount }}/{{ item }}" + state: directory + owner: apache + group: apache + with_items: "{{ koji_default_directories }}" diff --git a/tasks/plugins.yml b/tasks/plugins.yml new file mode 100644 index 0000000..c64696e --- /dev/null +++ b/tasks/plugins.yml @@ -0,0 +1,15 @@ +--- +# Plugins +- name: Deploying enabled plugins + copy: + src: "plugins/{{ item }}.py" + dest: "/usr/lib/koji-hub-plugins/{{ item }}.py" + mode: 0755 + with_items: "{{ koji_hub_plugins_list }}" + +- name: Configuring enabled plugins + template: + src: "etc/koji-hub/plugins/{{ item }}.conf.j2" + dest: "/etc/koji-hub/plugins/{{ item }}.conf" + mode: 0644 + with_items: "{{ koji_hub_plugins_list }}" diff --git a/tasks/selinux_boolean.yml b/tasks/selinux_boolean.yml new file mode 100644 index 0000000..e78f018 --- /dev/null +++ b/tasks/selinux_boolean.yml @@ -0,0 +1,12 @@ +--- + +- name: Enabling SELinux booleans + seboolean: + name: "{{ item }}" + persistent: true + state: true + with_items: + - httpd_can_network_connect_db + - httpd_can_network_connect + - allow_httpd_anon_write + - httpd_use_nfs diff --git a/tasks/users-sync.yml b/tasks/users-sync.yml new file mode 100644 index 0000000..dfde0e7 --- /dev/null +++ b/tasks/users-sync.yml @@ -0,0 +1,2 @@ +--- +# No tasks diff --git a/templates/etc/httpd/conf.d/kojihub.conf.j2 b/templates/etc/httpd/conf.d/kojihub.conf.j2 new file mode 100644 index 0000000..4728a27 --- /dev/null +++ b/templates/etc/httpd/conf.d/kojihub.conf.j2 @@ -0,0 +1,66 @@ +# +# koji-hub is an xmlrpc interface to the Koji database +# + +Alias /kojihub /usr/share/koji-hub/kojixmlrpc.py + + + Options ExecCGI + SetHandler wsgi-script + WSGIApplicationGroup %{GLOBAL} + # ^ works around a hub issue with OpenSSL + # see: https://cryptography.io/en/latest/faq/#starting-cryptography-using-mod-wsgi-produces-an-internalerror-during-a-call-in-register-osrandom-engine + WSGIScriptReloading Off + # ^ reloading breaks hub "firstcall" check + # see: https://pagure.io/koji/issue/875 + + Order allow,deny + Allow from all + + = 2.4> + Require all granted + + + +# Also serve /mnt/koji +Alias /kojifiles "/mnt/koji/" + + + Options Indexes SymLinksIfOwnerMatch + #If your top /mnt/koji directory is not owned by the httpd user, then + #you will need to follow all symlinks instead, e.g. + #Options Indexes FollowSymLinks + AllowOverride None + IndexOptions +NameWidth=* + + Order allow,deny + Allow from all + + = 2.4> + Require all granted + + + +# uncomment this to enable authentication via SSL client certificates +# +# SSLVerifyClient require +# SSLVerifyDepth 10 +# SSLOptions +StdEnvVars +# + +# If you need to support koji < 1.4.0 clients using SSL authentication, then use the following instead: +# +# SSLOptions +StdEnvVars +# +# In this case, you will need to enable these options globally (in ssl.conf): +# SSLVerifyClient require +# SSLVerifyDepth 10 + +# uncomment this to enable authentication via GSSAPI + + AuthType GSSAPI + AuthName "GSSAPI Single Sign On Login" + GssapiCredStore keytab:/etc/koji.keytab + Require valid-user + + diff --git a/templates/etc/httpd/conf.d/kojiweb.conf.j2 b/templates/etc/httpd/conf.d/kojiweb.conf.j2 new file mode 100644 index 0000000..ef55ac6 --- /dev/null +++ b/templates/etc/httpd/conf.d/kojiweb.conf.j2 @@ -0,0 +1,79 @@ +#We use wsgi by default +Alias /koji "/usr/share/koji-web/scripts/wsgi_publisher.py" +#(configuration goes in /etc/kojiweb/web.conf) + +RewriteEngine on +RewriteCond %{HTTPS} off +RewriteRule .* https://%{HTTP_HOST}%{REQUEST_URI} [R=302,L] +RewriteRule ^/$ /koji [R,L] + + Header always set X-Frame-Options "SAMEORIGIN" + Header always set X-Xss-Protection "1; mode=block" + Header always set X-Content-Type-Options "nosniff" + Header always set Referrer-Policy "same-origin" + +Alias /repos {{ koji_mount }}/repos + + Options Indexes FollowSymLinks + AllowOverride None +# HeaderName /header/header.html + + Order allow,deny + Allow from all + + = 2.4> + IndexOptions FancyIndexing VersionSort NameWidth=* HTMLTable Charset=UTF-8 + Require all granted + + + +# Python 3 Cheetah expectes unicode everywhere, apache's default lang is C +# which is not sufficient to open our templates +WSGIDaemonProcess koji lang=C.UTF-8 +WSGIProcessGroup koji + + + Options ExecCGI + SetHandler wsgi-script + WSGIApplicationGroup %{GLOBAL} + # ^ works around an OpenSSL issue + # see: https://cryptography.io/en/latest/faq/#starting-cryptography-using-mod-wsgi-produces-an-internalerror-during-a-call-in-register-osrandom-engine + + Order allow,deny + Allow from all + + = 2.4> + Require all granted + + + +# uncomment this to enable authentication via Kerberos + + AuthType GSSAPI + AuthName "Koji Web UI" + GssapiCredStore keytab:/etc/koji.keytab + Require valid-user + ErrorDocument 401 /koji-static/errors/unauthorized.html + + +# uncomment this to enable authentication via SSL client certificates +# +# SSLVerifyClient require +# SSLVerifyDepth 10 +# SSLOptions +StdEnvVars +# + +Alias /koji-static/ "/usr/share/koji-web/static/" + + + Options None + AllowOverride None + + Order allow,deny + Allow from all + + = 2.4> + Require all granted + + + diff --git a/templates/etc/koji-hub/hub.conf.j2 b/templates/etc/koji-hub/hub.conf.j2 new file mode 100644 index 0000000..4601ae2 --- /dev/null +++ b/templates/etc/koji-hub/hub.conf.j2 @@ -0,0 +1,81 @@ +[hub] + +## ConfigParser style config file, similar to ini files +## http://docs.python.org/library/configparser.html +## +## Note that multiline values can be set by indenting subsequent lines +## (which means you should not indent regular lines) + +## Basic options ## +DBName = {{ koji_db_name }} +DBUser = {{ koji_db_user }} +DBHost = {{ koji_db_host }} +DBPass = {{ koji_db_pass }} +KojiDir = {{ koji_mount }} + +AuthPrincipal host/kojihub@ROCKYLINUX.ORG +AuthKeytab /etc/koji.keytab +ProxyPrincipals koji/kojiweb@ROCKYLINUX.ORG +HostPrincipalFormat compile/%s@ROCKYLINUX.ORG + +## Other options ## +LoginCreatesUser = Off +KojiWebURL = {{ koji_web_url }} + +# The domain name that will be appended to Koji usernames +# when creating email notifications +#EmailDomain = example.com +# whether to send the task owner and package owner email or not on success. this still goes to watchers +NotifyOnSuccess = True +## Disables all notifications +# DisableNotifications = False +DisableNotifications = True + +## Extended features +## Support Maven builds +# EnableMaven = False +## Support Windows builds +# EnableWin = False + +## Koji hub plugins +## The path where plugins are found +# PluginPath = /usr/lib/koji-hub-plugins +## A space-separated list of plugins to load +# Plugins = echo +{% if koji_hub_plugins %} +Plugins = {% for plugin in koji_hub_plugins_list + koji_hub_noconfig_plugins_list %}{{ plugin }} {% endfor %} +{% endif %} + +## If KojiDebug is on, the hub will be /very/ verbose and will report exception +## details to clients for anticipated errors (i.e. koji's own exceptions -- +## subclasses of koji.GenericError). +# KojiDebug = On + +## Determines how much detail about exceptions is reported to the client (via faults) +## Meaningful values: +## normal - a basic traceback (format_exception) +## extended - an extended traceback (format_exc_plus) +## anything else - no traceback, just the error message +## The extended traceback is intended for debugging only and should NOT be +## used in production, since it may contain sensitive information. +# KojiTraceback = normal + +## These options are intended for planned outages +# ServerOffline = False +# OfflineMessage = temporary outage +# LockOut = False +## If ServerOffline is True, the server will always report a ServerOffline fault (with +## OfflineMessage as the fault string). +## If LockOut is True, the server will report a ServerOffline fault for all non-admin +## requests. + +[policy] +tag = + all :: allow + +package_list = + all :: allow + +build_from_srpm = + tag * :: allow + has_perm build :: allow diff --git a/templates/etc/kojira/kojira.conf.j2 b/templates/etc/kojira/kojira.conf.j2 new file mode 100644 index 0000000..b0bd12f --- /dev/null +++ b/templates/etc/kojira/kojira.conf.j2 @@ -0,0 +1,49 @@ +[kojira] +; For user/pass authentication +; user=kojira +; password=kojira + +; The URL for the koji hub server +server={{ koji_hub_url }} + +; The directory containing the repos/ directory +topdir={{ koji_mount }} + +; Logfile +logfile=/var/log/kojira.log + +with_src=no + +;configuration for Kerberos authentication + +;the kerberos principal to use +principal = {{ koji_kojira_principal }} + +;location of the keytab +keytab = {{ koji_kojira_keytab }} + +;configuration for SSL authentication + +;client certificate +;cert = /etc/kojira/client.crt + +;certificate of the CA that issued the HTTP server certificate +;serverca = /etc/kojira/serverca.crt + +;how soon (in seconds) to clean up expired repositories. 1 week default +;deleted_repo_lifetime = 604800 + +;how soon (in seconds) to clean up dist repositories. 1 week default here too +;dist_repo_lifetime = 604800 + +;turn on debugging statements in the log +;debug = false + +; ignored repositories according to glob. Multiple masks separated by space. +; ignore_tags = + +; Monitor external repos and trigger the appropriate Koji repo regenerations +; when they change. Note that you need to have your database set to use UTC, +; as otherwise you can end with weird behaviour. For details see +; https://pagure.io/koji/issue/2159 +; check_external_repos = false diff --git a/templates/etc/kojiweb/web.conf.j2 b/templates/etc/kojiweb/web.conf.j2 new file mode 100644 index 0000000..9f4cf37 --- /dev/null +++ b/templates/etc/kojiweb/web.conf.j2 @@ -0,0 +1,23 @@ +[web] +SiteName = {{ koji_sitename }} +{% if koji_custom_theme %} +KojiTheme = {{ koji_theme }} +{% endif %} + +# Key urls +KojiHubURL = {{ koji_hub_url }} +KojiFilesURL = {{ koji_files_url }} + +# Kerberos authentication options +WebPrincipal = koji/web@ROCKYLINUX.ORG +WebKeytab = /etc/httpd.keytab +WebCCache = /var/tmp/kojiweb.ccache +# The service name of the principal being used by the hub +KrbService = host + +LoginTimeout = 72 + +# This must be changed and uncommented before deployment +Secret = {{ koji_hub_secret }} + +LibPath = /usr/share/koji-web/lib diff --git a/templates/koji-client-config.j2 b/templates/koji-client-config.j2 new file mode 100644 index 0000000..a26158a --- /dev/null +++ b/templates/koji-client-config.j2 @@ -0,0 +1,13 @@ +[koji] + +;url of XMLRPC server +server = {{ koji_hub_url }} + +;url of web interface +weburl = {{ koji_web_url }} + +;url of package download site +topurl = {{ koji_files_url }} + +;path to the koji top directory +topdir = {{ koji_mount }} diff --git a/templates/koji-pgsql-kojira.sql.j2 b/templates/koji-pgsql-kojira.sql.j2 new file mode 100644 index 0000000..3daa009 --- /dev/null +++ b/templates/koji-pgsql-kojira.sql.j2 @@ -0,0 +1,2 @@ +insert into users (name, status, usertype) values ('{{ koji_kojira_user }}', 0, 0); +INSERT INTO user_perms (user_id, perm_id, creator_id) VALUES (2, 3, 1); diff --git a/templates/koji-pgsql.sql.j2 b/templates/koji-pgsql.sql.j2 new file mode 100644 index 0000000..ee6ac83 --- /dev/null +++ b/templates/koji-pgsql.sql.j2 @@ -0,0 +1,5 @@ +with user_id as (insert into users (name, status, usertype) values ('{{ koji_admin_user }}', 0, 0) returning id) +insert into user_krb_principals (user_id, krb_principal) values ((select id from user_id),'{{ koji_admin_keytab }}'); +insert into user_perms (user_id, perm_id, creator_id) values (1, 1, 1); +insert into users (name, status, usertype) values ('{{ koji_kojira_user }}', 0, 0); +INSERT INTO user_perms (user_id, perm_id, creator_id) VALUES (2, 3, 1);