diff --git a/defaults/main.yml b/defaults/main.yml index 93531a7..07d836d 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -10,6 +10,8 @@ koji_hub_packages: - gnupg2 - python3-paho-mqtt - nfs-utils + - mod_ssl + - mod_auth_gssapi koji_default_directories: - packages @@ -41,17 +43,27 @@ koji_web_tls_key: /etc/pki/tls/private/koji.rockylinux.org.key # Kojira koji_kojira: true koji_kojira_user: kojira -koji_kojira_principal: koji/kojiria@ROCKYLINUX.ORG -koji_kojira_keytab: /etc/kojira.keytab +koji_kojira_principal: koji/kojira@ROCKYLINUX.ORG +koji_kojira_keytab: /etc/koji.keytab # Storage koji_mount: /mnt/koji -koji_nfs_path: nfs.rockylinux.org:/exports/koji +koji_nfs_path: nfs.rockylinux.org:/export/koji # Koji Admin koji_admin_client: true koji_admin_user: rockykoji -koji_admin_keytab: rockykoji@ROCKYLINUX.ORG +koji_admin_principal: rockykoji@ROCKYLINUX.ORG +koji_admin_localuser: true +koji_admin_localuser_name: koji + +# Hub Settings +koji_hub_principal: HTTP/{{ inventory_hostname }}@ROCKYLINUX.ORG +koji_hub_proxy_principals: koji/kojiweb@ROCKYLINUX.ORG +koji_hub_keytab: /etc/koji.keytab +koji_hub_principal_format: compile/%s@ROCKYLINUX.ORG +# This should be sufficient even for LE +koji_hub_ca: /etc/pki/tls/certs/ca-bundle.crt # Koji FAS Syncing # This isn't implemented yet @@ -60,9 +72,13 @@ koji_fas_url: https://accounts.rockylinux.org # Koji Plugins koji_hub_plugins: false -koji_hub_plugins_list: [] +koji_hub_plugins_list: + - rockymsg koji_hub_plugin_mqtt_host: mqtt.rockylinux.org koji_hub_plugin_mqtt_topic: koji +koji_hub_plugin_mqtt_ca: {{ koji_hub_ca }} +koji_hub_plugin_mqtt_tls_cert: /etc/pki/tls/certs/mqtt.pem +koji_hub_plugin_mqtt_tls_key: /etc/pki/tls/certs/mqtt.pem koji_hub_plugin_mqtt_excluded_tags: - testing-tag diff --git a/files/plugins/rockymsg.py b/files/plugins/rockymsg.py new file mode 100644 index 0000000..c462f56 --- /dev/null +++ b/files/plugins/rockymsg.py @@ -0,0 +1,162 @@ +# Koji callback sent to Rocky Linux mqtt +# +# Adapted from https://gitlab.cern.ch/linuxsupport/rpms/koji-hub-plugins-cern/blob/master/src/mash.py +# +# License: GPLv2 +# Authors: +# Alex (dot) Iribarren (at) cern (dot) ch (original script) +# Thomas (dot) Oulevey (at) cern (dot) ch (mqtt version) + +import koji +from koji import PluginError +from koji.context import context +from koji.plugin import callback, ignore_error +import kojihub +import ConfigParser +import logging +import base64, json +import os + +# mqtt client +import paho.mqtt.client as mqtt + +CONFIG_FILE = '/etc/koji-hub/plugins/rockymsg.conf' +PLUGIN_NAME = 'koji.plugin.rockymsg' +DEFAULT_ARCHES = 'x86_64' + +config = None +tagCache = {} + +def get_config(): + global config + if config: + return config + + config = ConfigParser.SafeConfigParser() + config.read(CONFIG_FILE) + + if not config.has_section('rockymsg'): + config.add_section('rockymsg') + if not config.has_option('rockymsg', 'host'): + logging.getLogger(PLUGIN_NAME).error('No mqtt host specified in config file!') + return None + if not config.has_option('rockymsg', 'port'): + logging.getLogger(PLUGIN_NAME).error('No mqtt port specified in config file!') + return None + if not config.has_option('rockymsg', 'topic'): + logging.getLogger(PLUGIN_NAME).error('No mqtt topic specified in config file!') + return None + if not config.has_option('rockymsg', 'ca_cert'): + logging.getLogger(PLUGIN_NAME).error('No mqtt cacert specified in config file!') + return None + if not config.has_option('rockymsg', 'tls_cert'): + logging.getLogger(PLUGIN_NAME).error('No mqtt tls_cert specified in config file!') + return None + if not config.has_option('rockymsg', 'tls_key'): + logging.getLogger(PLUGIN_NAME).error('No mqtt tls_key specified in config file!') + return None + if not config.has_option('rockymsg', 'tls_insecure'): + config.set('rockymsg' 'tls_insecure', 'False') + if not config.has_option('rockymsg', 'tls_version'): + config.set('rockymsg' 'tls_version', '2') + if not config.has_option('rockymsg', 'exclude_tags'): + config.set('rockymsg', 'exclude_tags', '') + + return config + +def mqtt_on_publish(client,userdata,result): + pass + +def _dispatch_on_topic(payload): + logger = logging.getLogger(PLUGIN_NAME) + + config = get_config() + if not config: + raise PluginError('Unable to use the bus, config not found') + + if not payload['tag']: + logger.info('No tag specified') + return None + + exclude_tags = config.get('rockymsg', 'exclude_tags') + if exclude_tags: + exclude_tags = [x.strip() for x in exclude_tags.split(',')] + else: + exclude_tags = [] + + if payload['tag'] in exclude_tags: + logger.info('Tag %s excluded' % payload['tag']) + return None + + mqtt_host = config.get('rockymsg', 'host') + mqtt_port = config.get('rockymsg', 'port') + mqtt_topic = config.get('rockymsg', 'topic') + mqtt_cacert = config.get('rockymsg', 'ca_cert') + mqtt_tls_cert = config.get('rockymsg', 'tls_cert') + mqtt_tls_key = config.get('rockymsg', 'tls_key') + mqtt_tls_insecure = config.get('rockymsg', 'tls_insecure') + mqtt_tls_version = config.get('rockymsg', 'tls_version') + + # Connect to the bus + try: + client = mqtt.Client() + except Exception as e: + logger.error('mqtt client error: %s' % e.message) + client.tls_set(ca_certs=mqtt_cacert, certfile=mqtt_tls_cert, keyfile=mqtt_tls_key, tls_version=2) + + client.tls_insecure_set('False') + try: + client.on_publish = mqtt_on_publish + client.connect(mqtt_host,mqtt_port) + except Exception as e: + logger.error('mqtt connection error: %s' % e.message) + + # Publish payload to the bus + # + ret = client.publish(mqtt_topic, json.dumps(payload)) + + # Disconnect from the bus + client.disconnect() + + return ret + +def _get_build_target(task_id): + try: + task = kojihub.Task(task_id) + info = task.getInfo(request=True) + request = info['request'] + if info['method'] in ('build', 'maven'): + # request is (source-url, build-target, map-of-other-options) + if request[1]: + return kojihub.get_build_target(request[1]) + elif info['method'] == 'winbuild': + # request is (vm-name, source-url, build-target, map-of-other-options) + if request[2]: + return kojihub.get_build_target(request[2]) + except Exception as e: + logger.error('Exception: %s', e) + + return None + + +@callback('postTag', 'postUntag') +#@ignore_error +def rockymsg(cbtype, *args, **kws): + logger = logging.getLogger(PLUGIN_NAME) + logger.debug('Called the %s callback, args: %s; kws: %s', cbtype, str(args), str(kws)) + + tag = kws['tag']['name'] + build_task_id = kws['build']['task_id'] + + build_target = _get_build_target(build_task_id) + logger.debug('Build target: %s', build_target) + + arches = DEFAULT_ARCHES + if build_target: + build_tag = kojihub.get_tag(build_target['build_tag_name']) + arches = build_tag['arches'] + + payload = { 'action': cbtype, 'tag': tag, 'arches': arches } + job = _dispatch_on_topic(payload) + if job: + logger.info('Sending payload: %s to mqtt - ret code: %s' % (payload, job)) diff --git a/tasks/db.yml b/tasks/db.yml index a46287b..f78ccd0 100644 --- a/tasks/db.yml +++ b/tasks/db.yml @@ -1,5 +1,5 @@ --- -# Note: We do not install postgresql. It's up to you to do so. +# Note: We do not install postgresql. It's up to you to do so, whether locally or not. - name: Template for koji admin and kojira template: src: koji-pgsql.sql.j2 diff --git a/tasks/koji-admin.yml b/tasks/koji-admin-ipa.yml similarity index 91% rename from tasks/koji-admin.yml rename to tasks/koji-admin-ipa.yml index af262f3..d9c493a 100644 --- a/tasks/koji-admin.yml +++ b/tasks/koji-admin-ipa.yml @@ -1,8 +1,5 @@ --- -# Create the koji admin user -- name: Create local koji admin user - user: "{{ koji_admin_user }}" - +# Setup the IPA service account - name: Create koji config directory file: path: "/home/{{ koji_admin_user }}/.koji" diff --git a/tasks/koji-admin-local.yml b/tasks/koji-admin-local.yml new file mode 100644 index 0000000..324c58a --- /dev/null +++ b/tasks/koji-admin-local.yml @@ -0,0 +1,46 @@ +--- +# Create the koji admin user +- name: Create local koji admin user + user: "{{ koji_admin_localuser_name }}" + comment: "Local Koji Admin" + +- name: Create koji config directory + file: + path: "/home/{{ koji_admin_localuser_name }}/.koji" + state: directory + owner: "{{ koji_admin_localuser_name }}" + group: "{{ koji_admin_localuser_name }}" + recurse: true + +- name: Reset permissions + file: + path: "/home/{{ koji_admin_localuser_name }}" + state: directory + owner: "{{ koji_admin_localuser_name }}" + group: "{{ koji_admin_localuser_name }}" + mode: '0700' + +- name: Configure the koji client + template: + src: koji-client-config.j2 + dest: "/home/{{ koji_admin_localuser_name }}/.koji/config" + owner: "{{ koji_admin_localuser_name }}" + group: "{{ koji_admin_localuser_name }}" + mode: '0644' + +- name: Ensuring we have our scripts store + file: + path: /opt/rocky-tools/scripts + state: directory + owner: "{{ koji_admin_localuser_name }}" + group: "{{ koji_admin_localuser_name }}" + mode: '0750' + recurse: true + +# name: Cron job to rebuild repos +# cron: +# name: "Regenerate repos" +# job: "/opt/rocky-tools/scripts/regen_build_repos.sh > /dev/null 2>&1" +# minute: "5" +# hour: "3" +# user: "{{ koji_admin_localuser_name }}" diff --git a/tasks/koji-gc.yml b/tasks/koji-gc.yml new file mode 100644 index 0000000..b170f24 --- /dev/null +++ b/tasks/koji-gc.yml @@ -0,0 +1,14 @@ +--- +# Configure gc +- name: Configure garbage collector + template: + src: etc/koji-gc/koji-gc.conf.j2 + dest: /etc/koji-gc/koji-gc.conf + owner: root + group: root + mode: '0644' + +- name: Enable the gc timer + service: + name: koji-gc.timer + enabled: true diff --git a/tasks/main.yml b/tasks/main.yml index 8459bd3..00363f6 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -16,33 +16,47 @@ - name: Configure koji database import_tasks: db.yml -- name: Configure koji admin - import_tasks: koji-admin.yml - when: koji_admin_client +- name: Configure local koji admin + import_tasks: koji-admin-local.yml + when: + - koji_admin_client + - koji_admin_localuser + +# This is specifically if we want the IPA account to also be an account on this +# system. ymmv. +- name: Configure ipa koji admin + import_tasks: koji-admin-ipa.yml + when: + - koji_admin_client + - not koji_admin_localuser - name: Configure plugins import_tasks: plugins.yml when: koji_hub_plugins -- name: Configure kojira - import_tasks: kojira.yml - - name: Configure kojihub and web template: src: "{{ item }}.j2" dest: "/{{ item }}" mode: '0644' + owner: root + group: root with_items: - etc/koji-hub/hub.conf - etc/kojiweb/web.conf notify: - restart_httpd +- name: Configure kojira + import_tasks: kojira.yml + - name: Configure httpd for hub and web template: src: "etc/httpd/conf.d/{{ item }}.j2" dest: "/etc/httpd/conf.d/{{ item }}" mode: '0644' + owner: root + group: root with_items: - kojihub.conf - kojiweb.conf @@ -55,6 +69,9 @@ dest: / when: koji_theme +- name: Configure garbage collector + import_tasks: koji-gc.yml + - name: User Sync from FAS import_tasks: user-sync.yml when: koji_fas_sync diff --git a/templates/etc/httpd/conf.d/kojihub.conf.j2 b/templates/etc/httpd/conf.d/kojihub.conf.j2 index 4728a27..d7286de 100644 --- a/templates/etc/httpd/conf.d/kojihub.conf.j2 +++ b/templates/etc/httpd/conf.d/kojihub.conf.j2 @@ -22,14 +22,14 @@ Alias /kojihub /usr/share/koji-hub/kojixmlrpc.py -# Also serve /mnt/koji -Alias /kojifiles "/mnt/koji/" +# Also serve {{ koji_mount }} +Alias /kojifiles "{{ koji_mount }}/" - - Options Indexes SymLinksIfOwnerMatch + + #Options Indexes SymLinksIfOwnerMatch #If your top /mnt/koji directory is not owned by the httpd user, then #you will need to follow all symlinks instead, e.g. - #Options Indexes FollowSymLinks + Options Indexes FollowSymLinks AllowOverride None IndexOptions +NameWidth=* @@ -41,21 +41,6 @@ Alias /kojifiles "/mnt/koji/" -# uncomment this to enable authentication via SSL client certificates -# -# SSLVerifyClient require -# SSLVerifyDepth 10 -# SSLOptions +StdEnvVars -# - -# If you need to support koji < 1.4.0 clients using SSL authentication, then use the following instead: -# -# SSLOptions +StdEnvVars -# -# In this case, you will need to enable these options globally (in ssl.conf): -# SSLVerifyClient require -# SSLVerifyDepth 10 - # uncomment this to enable authentication via GSSAPI AuthType GSSAPI diff --git a/templates/etc/httpd/conf.d/kojiweb.conf.j2 b/templates/etc/httpd/conf.d/kojiweb.conf.j2 index ef55ac6..c8b06a5 100644 --- a/templates/etc/httpd/conf.d/kojiweb.conf.j2 +++ b/templates/etc/httpd/conf.d/kojiweb.conf.j2 @@ -12,21 +12,6 @@ RewriteRule ^/$ /koji [R,L] Header always set X-Content-Type-Options "nosniff" Header always set Referrer-Policy "same-origin" -Alias /repos {{ koji_mount }}/repos - - Options Indexes FollowSymLinks - AllowOverride None -# HeaderName /header/header.html - - Order allow,deny - Allow from all - - = 2.4> - IndexOptions FancyIndexing VersionSort NameWidth=* HTMLTable Charset=UTF-8 - Require all granted - - - # Python 3 Cheetah expectes unicode everywhere, apache's default lang is C # which is not sufficient to open our templates WSGIDaemonProcess koji lang=C.UTF-8 @@ -56,13 +41,6 @@ WSGIProcessGroup koji ErrorDocument 401 /koji-static/errors/unauthorized.html -# uncomment this to enable authentication via SSL client certificates -# -# SSLVerifyClient require -# SSLVerifyDepth 10 -# SSLOptions +StdEnvVars -# - Alias /koji-static/ "/usr/share/koji-web/static/" @@ -77,3 +55,18 @@ Alias /koji-static/ "/usr/share/koji-web/static/" +Alias /repos {{ koji_mount }}/repos + + Options Indexes FollowSymLinks + AllowOverride None + #HeaderName /header/header.html + + Order allow,deny + Allow from all + + = 2.4> + IndexOptions FancyIndexing VersionSort NameWidth=* HTMLTable Charset=UTF-8 + Require all granted + + + diff --git a/templates/etc/koji-gc/koji-gc.conf.j2 b/templates/etc/koji-gc/koji-gc.conf.j2 new file mode 100644 index 0000000..10388d7 --- /dev/null +++ b/templates/etc/koji-gc/koji-gc.conf.j2 @@ -0,0 +1,44 @@ +#test policy file +#earlier = higher precedence! + +[main] +key_aliases = + 30C9ECF8 fedora-test + 4F2A6FD2 fedora-gold + 897DA07A redhat-beta + 1AC70CE6 fedora-extras + +unprotected_keys = + fedora-test + fedora-extras + redhat-beta + +server = {{ koji_hub_url }} +weburl = {{ koji_web_url }} + +# The domain name that will be appended to Koji usernames +# when creating email notifications +#email_domain = fedoraproject.org + +# SMTP user and pass (uncomment and fill in if your smtp server requires authentication) +#smtp_user=user@example.com +#smtp_pass=CHANGEME + +[prune] +policy = + #stuff to protect + #note that tags with master lock engaged are already protected + tag *-updates :: keep + age < 1 day :: skip + sig fedora-gold :: skip + sig fedora-test && age < 12 weeks :: keep + + #stuff to chuck semi-rapidly + tag *-testing *-candidate :: { # nested rules + order >= 2 :: untag + order > 0 && age > 6 weeks :: untag + } #closing braces must be on a line by themselves (modulo comments/whitespace) + tag *-candidate && age > 60 weeks :: untag + + #default: keep the last 3 + order > 2 :: untag diff --git a/templates/etc/koji-hub/hub.conf.j2 b/templates/etc/koji-hub/hub.conf.j2 index 4601ae2..d8400ed 100644 --- a/templates/etc/koji-hub/hub.conf.j2 +++ b/templates/etc/koji-hub/hub.conf.j2 @@ -13,13 +13,13 @@ DBHost = {{ koji_db_host }} DBPass = {{ koji_db_pass }} KojiDir = {{ koji_mount }} -AuthPrincipal host/kojihub@ROCKYLINUX.ORG -AuthKeytab /etc/koji.keytab -ProxyPrincipals koji/kojiweb@ROCKYLINUX.ORG -HostPrincipalFormat compile/%s@ROCKYLINUX.ORG +AuthPrincipal {{ koji_hub_principal }} +AuthKeytab {{ koji_hub_keytab }} +ProxyPrincipals {{ koji_hub_proxy_principals }} +HostPrincipalFormat {{ koji_hub_principal_format }} ## Other options ## -LoginCreatesUser = Off +LoginCreatesUser = On KojiWebURL = {{ koji_web_url }} # The domain name that will be appended to Koji usernames diff --git a/templates/etc/koji-hub/plugins/rockymsg.conf.j2 b/templates/etc/koji-hub/plugins/rockymsg.conf.j2 new file mode 100644 index 0000000..4030036 --- /dev/null +++ b/templates/etc/koji-hub/plugins/rockymsg.conf.j2 @@ -0,0 +1,9 @@ +[rockymsg] +host = {{ koji_hub_plugin_mqtt_host }} +port = 8883 +tls_cert = {{ koji_hub_plugin_mqtt_tls_cert }} +tls_key = {{ koji_hub_plugin_mqtt_tls_key }} +ca_cert = {{ koji_hub_plugin_mqtt_ca }} +tls_insecure = False +tls_version = 2 +exclude_tags = {% for tag in koji_hub_plugin_mqtt_excluded_tags %}{{ tag }}{%- if not loop.last -%},{% endif %}{% endfor %} diff --git a/templates/etc/kojira/kojira.conf.j2 b/templates/etc/kojira/kojira.conf.j2 index b0bd12f..9604fd9 100644 --- a/templates/etc/kojira/kojira.conf.j2 +++ b/templates/etc/kojira/kojira.conf.j2 @@ -1,49 +1,7 @@ [kojira] -; For user/pass authentication -; user=kojira -; password=kojira - -; The URL for the koji hub server server={{ koji_hub_url }} - -; The directory containing the repos/ directory topdir={{ koji_mount }} - -; Logfile logfile=/var/log/kojira.log - -with_src=no - -;configuration for Kerberos authentication - -;the kerberos principal to use +;with_src=no principal = {{ koji_kojira_principal }} - -;location of the keytab keytab = {{ koji_kojira_keytab }} - -;configuration for SSL authentication - -;client certificate -;cert = /etc/kojira/client.crt - -;certificate of the CA that issued the HTTP server certificate -;serverca = /etc/kojira/serverca.crt - -;how soon (in seconds) to clean up expired repositories. 1 week default -;deleted_repo_lifetime = 604800 - -;how soon (in seconds) to clean up dist repositories. 1 week default here too -;dist_repo_lifetime = 604800 - -;turn on debugging statements in the log -;debug = false - -; ignored repositories according to glob. Multiple masks separated by space. -; ignore_tags = - -; Monitor external repos and trigger the appropriate Koji repo regenerations -; when they change. Note that you need to have your database set to use UTC, -; as otherwise you can end with weird behaviour. For details see -; https://pagure.io/koji/issue/2159 -; check_external_repos = false diff --git a/templates/etc/kojiweb/web.conf.j2 b/templates/etc/kojiweb/web.conf.j2 index 9f4cf37..4328a5a 100644 --- a/templates/etc/kojiweb/web.conf.j2 +++ b/templates/etc/kojiweb/web.conf.j2 @@ -8,12 +8,13 @@ KojiTheme = {{ koji_theme }} KojiHubURL = {{ koji_hub_url }} KojiFilesURL = {{ koji_files_url }} +# CA +KojiHubCA = {{ koji_hub_ca }} + # Kerberos authentication options -WebPrincipal = koji/web@ROCKYLINUX.ORG -WebKeytab = /etc/httpd.keytab +WebPrincipal = {{ koji_hub_proxy_principals }} +WebKeytab = {{ koji_hub_keytab }} WebCCache = /var/tmp/kojiweb.ccache -# The service name of the principal being used by the hub -KrbService = host LoginTimeout = 72 @@ -21,3 +22,4 @@ LoginTimeout = 72 Secret = {{ koji_hub_secret }} LibPath = /usr/share/koji-web/lib +LiteralFooter = True diff --git a/templates/koji-client-config.j2 b/templates/koji-client-config.j2 index a26158a..052ea05 100644 --- a/templates/koji-client-config.j2 +++ b/templates/koji-client-config.j2 @@ -11,3 +11,6 @@ topurl = {{ koji_files_url }} ;path to the koji top directory topdir = {{ koji_mount }} + +; https ca, not for ssl auth +serverca = {{ koji_hub_ca }} diff --git a/templates/koji-pgsql.sql.j2 b/templates/koji-pgsql.sql.j2 index ee6ac83..155d0c4 100644 --- a/templates/koji-pgsql.sql.j2 +++ b/templates/koji-pgsql.sql.j2 @@ -1,5 +1,5 @@ with user_id as (insert into users (name, status, usertype) values ('{{ koji_admin_user }}', 0, 0) returning id) -insert into user_krb_principals (user_id, krb_principal) values ((select id from user_id),'{{ koji_admin_keytab }}'); +insert into user_krb_principals (user_id, krb_principal) values ((select id from user_id),'{{ koji_admin_principal }}'); insert into user_perms (user_id, perm_id, creator_id) values (1, 1, 1); insert into users (name, status, usertype) values ('{{ koji_kojira_user }}', 0, 0); -INSERT INTO user_perms (user_id, perm_id, creator_id) VALUES (2, 3, 1); +INSERT INTO user_perms (user_id, perm_id, creator_id) VALUES (2, 10, 1);