numerous fixes to address krb

This commit is contained in:
nazunalika 2020-12-23 02:12:56 -07:00
parent 0bccf61c87
commit 8225aa2626
16 changed files with 358 additions and 112 deletions

View File

@ -10,6 +10,8 @@ koji_hub_packages:
- gnupg2 - gnupg2
- python3-paho-mqtt - python3-paho-mqtt
- nfs-utils - nfs-utils
- mod_ssl
- mod_auth_gssapi
koji_default_directories: koji_default_directories:
- packages - packages
@ -41,17 +43,27 @@ koji_web_tls_key: /etc/pki/tls/private/koji.rockylinux.org.key
# Kojira # Kojira
koji_kojira: true koji_kojira: true
koji_kojira_user: kojira koji_kojira_user: kojira
koji_kojira_principal: koji/kojiria@ROCKYLINUX.ORG koji_kojira_principal: koji/kojira@ROCKYLINUX.ORG
koji_kojira_keytab: /etc/kojira.keytab koji_kojira_keytab: /etc/koji.keytab
# Storage # Storage
koji_mount: /mnt/koji koji_mount: /mnt/koji
koji_nfs_path: nfs.rockylinux.org:/exports/koji koji_nfs_path: nfs.rockylinux.org:/export/koji
# Koji Admin # Koji Admin
koji_admin_client: true koji_admin_client: true
koji_admin_user: rockykoji koji_admin_user: rockykoji
koji_admin_keytab: rockykoji@ROCKYLINUX.ORG koji_admin_principal: rockykoji@ROCKYLINUX.ORG
koji_admin_localuser: true
koji_admin_localuser_name: koji
# Hub Settings
koji_hub_principal: HTTP/{{ inventory_hostname }}@ROCKYLINUX.ORG
koji_hub_proxy_principals: koji/kojiweb@ROCKYLINUX.ORG
koji_hub_keytab: /etc/koji.keytab
koji_hub_principal_format: compile/%s@ROCKYLINUX.ORG
# This should be sufficient even for LE
koji_hub_ca: /etc/pki/tls/certs/ca-bundle.crt
# Koji FAS Syncing # Koji FAS Syncing
# This isn't implemented yet # This isn't implemented yet
@ -60,9 +72,13 @@ koji_fas_url: https://accounts.rockylinux.org
# Koji Plugins # Koji Plugins
koji_hub_plugins: false koji_hub_plugins: false
koji_hub_plugins_list: [] koji_hub_plugins_list:
- rockymsg
koji_hub_plugin_mqtt_host: mqtt.rockylinux.org koji_hub_plugin_mqtt_host: mqtt.rockylinux.org
koji_hub_plugin_mqtt_topic: koji koji_hub_plugin_mqtt_topic: koji
koji_hub_plugin_mqtt_ca: {{ koji_hub_ca }}
koji_hub_plugin_mqtt_tls_cert: /etc/pki/tls/certs/mqtt.pem
koji_hub_plugin_mqtt_tls_key: /etc/pki/tls/certs/mqtt.pem
koji_hub_plugin_mqtt_excluded_tags: koji_hub_plugin_mqtt_excluded_tags:
- testing-tag - testing-tag

162
files/plugins/rockymsg.py Normal file
View File

@ -0,0 +1,162 @@
# Koji callback sent to Rocky Linux mqtt
#
# Adapted from https://gitlab.cern.ch/linuxsupport/rpms/koji-hub-plugins-cern/blob/master/src/mash.py
#
# License: GPLv2
# Authors:
# Alex (dot) Iribarren (at) cern (dot) ch (original script)
# Thomas (dot) Oulevey (at) cern (dot) ch (mqtt version)
import koji
from koji import PluginError
from koji.context import context
from koji.plugin import callback, ignore_error
import kojihub
import ConfigParser
import logging
import base64, json
import os
# mqtt client
import paho.mqtt.client as mqtt
CONFIG_FILE = '/etc/koji-hub/plugins/rockymsg.conf'
PLUGIN_NAME = 'koji.plugin.rockymsg'
DEFAULT_ARCHES = 'x86_64'
config = None
tagCache = {}
def get_config():
global config
if config:
return config
config = ConfigParser.SafeConfigParser()
config.read(CONFIG_FILE)
if not config.has_section('rockymsg'):
config.add_section('rockymsg')
if not config.has_option('rockymsg', 'host'):
logging.getLogger(PLUGIN_NAME).error('No mqtt host specified in config file!')
return None
if not config.has_option('rockymsg', 'port'):
logging.getLogger(PLUGIN_NAME).error('No mqtt port specified in config file!')
return None
if not config.has_option('rockymsg', 'topic'):
logging.getLogger(PLUGIN_NAME).error('No mqtt topic specified in config file!')
return None
if not config.has_option('rockymsg', 'ca_cert'):
logging.getLogger(PLUGIN_NAME).error('No mqtt cacert specified in config file!')
return None
if not config.has_option('rockymsg', 'tls_cert'):
logging.getLogger(PLUGIN_NAME).error('No mqtt tls_cert specified in config file!')
return None
if not config.has_option('rockymsg', 'tls_key'):
logging.getLogger(PLUGIN_NAME).error('No mqtt tls_key specified in config file!')
return None
if not config.has_option('rockymsg', 'tls_insecure'):
config.set('rockymsg' 'tls_insecure', 'False')
if not config.has_option('rockymsg', 'tls_version'):
config.set('rockymsg' 'tls_version', '2')
if not config.has_option('rockymsg', 'exclude_tags'):
config.set('rockymsg', 'exclude_tags', '')
return config
def mqtt_on_publish(client,userdata,result):
pass
def _dispatch_on_topic(payload):
logger = logging.getLogger(PLUGIN_NAME)
config = get_config()
if not config:
raise PluginError('Unable to use the bus, config not found')
if not payload['tag']:
logger.info('No tag specified')
return None
exclude_tags = config.get('rockymsg', 'exclude_tags')
if exclude_tags:
exclude_tags = [x.strip() for x in exclude_tags.split(',')]
else:
exclude_tags = []
if payload['tag'] in exclude_tags:
logger.info('Tag %s excluded' % payload['tag'])
return None
mqtt_host = config.get('rockymsg', 'host')
mqtt_port = config.get('rockymsg', 'port')
mqtt_topic = config.get('rockymsg', 'topic')
mqtt_cacert = config.get('rockymsg', 'ca_cert')
mqtt_tls_cert = config.get('rockymsg', 'tls_cert')
mqtt_tls_key = config.get('rockymsg', 'tls_key')
mqtt_tls_insecure = config.get('rockymsg', 'tls_insecure')
mqtt_tls_version = config.get('rockymsg', 'tls_version')
# Connect to the bus
try:
client = mqtt.Client()
except Exception as e:
logger.error('mqtt client error: %s' % e.message)
client.tls_set(ca_certs=mqtt_cacert, certfile=mqtt_tls_cert, keyfile=mqtt_tls_key, tls_version=2)
client.tls_insecure_set('False')
try:
client.on_publish = mqtt_on_publish
client.connect(mqtt_host,mqtt_port)
except Exception as e:
logger.error('mqtt connection error: %s' % e.message)
# Publish payload to the bus
#
ret = client.publish(mqtt_topic, json.dumps(payload))
# Disconnect from the bus
client.disconnect()
return ret
def _get_build_target(task_id):
try:
task = kojihub.Task(task_id)
info = task.getInfo(request=True)
request = info['request']
if info['method'] in ('build', 'maven'):
# request is (source-url, build-target, map-of-other-options)
if request[1]:
return kojihub.get_build_target(request[1])
elif info['method'] == 'winbuild':
# request is (vm-name, source-url, build-target, map-of-other-options)
if request[2]:
return kojihub.get_build_target(request[2])
except Exception as e:
logger.error('Exception: %s', e)
return None
@callback('postTag', 'postUntag')
#@ignore_error
def rockymsg(cbtype, *args, **kws):
logger = logging.getLogger(PLUGIN_NAME)
logger.debug('Called the %s callback, args: %s; kws: %s', cbtype, str(args), str(kws))
tag = kws['tag']['name']
build_task_id = kws['build']['task_id']
build_target = _get_build_target(build_task_id)
logger.debug('Build target: %s', build_target)
arches = DEFAULT_ARCHES
if build_target:
build_tag = kojihub.get_tag(build_target['build_tag_name'])
arches = build_tag['arches']
payload = { 'action': cbtype, 'tag': tag, 'arches': arches }
job = _dispatch_on_topic(payload)
if job:
logger.info('Sending payload: %s to mqtt - ret code: %s' % (payload, job))

View File

@ -1,5 +1,5 @@
--- ---
# Note: We do not install postgresql. It's up to you to do so. # Note: We do not install postgresql. It's up to you to do so, whether locally or not.
- name: Template for koji admin and kojira - name: Template for koji admin and kojira
template: template:
src: koji-pgsql.sql.j2 src: koji-pgsql.sql.j2

View File

@ -1,8 +1,5 @@
--- ---
# Create the koji admin user # Setup the IPA service account
- name: Create local koji admin user
user: "{{ koji_admin_user }}"
- name: Create koji config directory - name: Create koji config directory
file: file:
path: "/home/{{ koji_admin_user }}/.koji" path: "/home/{{ koji_admin_user }}/.koji"

View File

@ -0,0 +1,46 @@
---
# Create the koji admin user
- name: Create local koji admin user
user: "{{ koji_admin_localuser_name }}"
comment: "Local Koji Admin"
- name: Create koji config directory
file:
path: "/home/{{ koji_admin_localuser_name }}/.koji"
state: directory
owner: "{{ koji_admin_localuser_name }}"
group: "{{ koji_admin_localuser_name }}"
recurse: true
- name: Reset permissions
file:
path: "/home/{{ koji_admin_localuser_name }}"
state: directory
owner: "{{ koji_admin_localuser_name }}"
group: "{{ koji_admin_localuser_name }}"
mode: '0700'
- name: Configure the koji client
template:
src: koji-client-config.j2
dest: "/home/{{ koji_admin_localuser_name }}/.koji/config"
owner: "{{ koji_admin_localuser_name }}"
group: "{{ koji_admin_localuser_name }}"
mode: '0644'
- name: Ensuring we have our scripts store
file:
path: /opt/rocky-tools/scripts
state: directory
owner: "{{ koji_admin_localuser_name }}"
group: "{{ koji_admin_localuser_name }}"
mode: '0750'
recurse: true
# name: Cron job to rebuild repos
# cron:
# name: "Regenerate repos"
# job: "/opt/rocky-tools/scripts/regen_build_repos.sh > /dev/null 2>&1"
# minute: "5"
# hour: "3"
# user: "{{ koji_admin_localuser_name }}"

14
tasks/koji-gc.yml Normal file
View File

@ -0,0 +1,14 @@
---
# Configure gc
- name: Configure garbage collector
template:
src: etc/koji-gc/koji-gc.conf.j2
dest: /etc/koji-gc/koji-gc.conf
owner: root
group: root
mode: '0644'
- name: Enable the gc timer
service:
name: koji-gc.timer
enabled: true

View File

@ -16,33 +16,47 @@
- name: Configure koji database - name: Configure koji database
import_tasks: db.yml import_tasks: db.yml
- name: Configure koji admin - name: Configure local koji admin
import_tasks: koji-admin.yml import_tasks: koji-admin-local.yml
when: koji_admin_client when:
- koji_admin_client
- koji_admin_localuser
# This is specifically if we want the IPA account to also be an account on this
# system. ymmv.
- name: Configure ipa koji admin
import_tasks: koji-admin-ipa.yml
when:
- koji_admin_client
- not koji_admin_localuser
- name: Configure plugins - name: Configure plugins
import_tasks: plugins.yml import_tasks: plugins.yml
when: koji_hub_plugins when: koji_hub_plugins
- name: Configure kojira
import_tasks: kojira.yml
- name: Configure kojihub and web - name: Configure kojihub and web
template: template:
src: "{{ item }}.j2" src: "{{ item }}.j2"
dest: "/{{ item }}" dest: "/{{ item }}"
mode: '0644' mode: '0644'
owner: root
group: root
with_items: with_items:
- etc/koji-hub/hub.conf - etc/koji-hub/hub.conf
- etc/kojiweb/web.conf - etc/kojiweb/web.conf
notify: notify:
- restart_httpd - restart_httpd
- name: Configure kojira
import_tasks: kojira.yml
- name: Configure httpd for hub and web - name: Configure httpd for hub and web
template: template:
src: "etc/httpd/conf.d/{{ item }}.j2" src: "etc/httpd/conf.d/{{ item }}.j2"
dest: "/etc/httpd/conf.d/{{ item }}" dest: "/etc/httpd/conf.d/{{ item }}"
mode: '0644' mode: '0644'
owner: root
group: root
with_items: with_items:
- kojihub.conf - kojihub.conf
- kojiweb.conf - kojiweb.conf
@ -55,6 +69,9 @@
dest: / dest: /
when: koji_theme when: koji_theme
- name: Configure garbage collector
import_tasks: koji-gc.yml
- name: User Sync from FAS - name: User Sync from FAS
import_tasks: user-sync.yml import_tasks: user-sync.yml
when: koji_fas_sync when: koji_fas_sync

View File

@ -22,14 +22,14 @@ Alias /kojihub /usr/share/koji-hub/kojixmlrpc.py
</IfVersion> </IfVersion>
</Directory> </Directory>
# Also serve /mnt/koji # Also serve {{ koji_mount }}
Alias /kojifiles "/mnt/koji/" Alias /kojifiles "{{ koji_mount }}/"
<Directory "/mnt/koji"> <Directory "{{ koji_mount }}/">
Options Indexes SymLinksIfOwnerMatch #Options Indexes SymLinksIfOwnerMatch
#If your top /mnt/koji directory is not owned by the httpd user, then #If your top /mnt/koji directory is not owned by the httpd user, then
#you will need to follow all symlinks instead, e.g. #you will need to follow all symlinks instead, e.g.
#Options Indexes FollowSymLinks Options Indexes FollowSymLinks
AllowOverride None AllowOverride None
IndexOptions +NameWidth=* IndexOptions +NameWidth=*
<IfVersion < 2.4> <IfVersion < 2.4>
@ -41,21 +41,6 @@ Alias /kojifiles "/mnt/koji/"
</IfVersion> </IfVersion>
</Directory> </Directory>
# uncomment this to enable authentication via SSL client certificates
# <Location /kojihub/ssllogin>
# SSLVerifyClient require
# SSLVerifyDepth 10
# SSLOptions +StdEnvVars
# </Location>
# If you need to support koji < 1.4.0 clients using SSL authentication, then use the following instead:
# <Location /kojihub>
# SSLOptions +StdEnvVars
# </Location>
# In this case, you will need to enable these options globally (in ssl.conf):
# SSLVerifyClient require
# SSLVerifyDepth 10
# uncomment this to enable authentication via GSSAPI # uncomment this to enable authentication via GSSAPI
<Location /kojihub/ssllogin> <Location /kojihub/ssllogin>
AuthType GSSAPI AuthType GSSAPI

View File

@ -12,21 +12,6 @@ RewriteRule ^/$ /koji [R,L]
Header always set X-Content-Type-Options "nosniff" Header always set X-Content-Type-Options "nosniff"
Header always set Referrer-Policy "same-origin" Header always set Referrer-Policy "same-origin"
Alias /repos {{ koji_mount }}/repos
<Directory "{{ koji_mount }}/repos">
Options Indexes FollowSymLinks
AllowOverride None
# HeaderName /header/header.html
<IfVersion < 2.4>
Order allow,deny
Allow from all
</IfVersion>
<IfVersion >= 2.4>
IndexOptions FancyIndexing VersionSort NameWidth=* HTMLTable Charset=UTF-8
Require all granted
</IfVersion>
</Directory>
# Python 3 Cheetah expectes unicode everywhere, apache's default lang is C # Python 3 Cheetah expectes unicode everywhere, apache's default lang is C
# which is not sufficient to open our templates # which is not sufficient to open our templates
WSGIDaemonProcess koji lang=C.UTF-8 WSGIDaemonProcess koji lang=C.UTF-8
@ -56,13 +41,6 @@ WSGIProcessGroup koji
ErrorDocument 401 /koji-static/errors/unauthorized.html ErrorDocument 401 /koji-static/errors/unauthorized.html
</Location> </Location>
# uncomment this to enable authentication via SSL client certificates
# <Location /koji/login>
# SSLVerifyClient require
# SSLVerifyDepth 10
# SSLOptions +StdEnvVars
# </Location>
Alias /koji-static/ "/usr/share/koji-web/static/" Alias /koji-static/ "/usr/share/koji-web/static/"
<Directory "/usr/share/koji-web/static/"> <Directory "/usr/share/koji-web/static/">
@ -77,3 +55,18 @@ Alias /koji-static/ "/usr/share/koji-web/static/"
</IfVersion> </IfVersion>
</Directory> </Directory>
Alias /repos {{ koji_mount }}/repos
<Directory "{{ koji_mount }}/repos">
Options Indexes FollowSymLinks
AllowOverride None
#HeaderName /header/header.html
<IfVersion < 2.4>
Order allow,deny
Allow from all
</IfVersion>
<IfVersion >= 2.4>
IndexOptions FancyIndexing VersionSort NameWidth=* HTMLTable Charset=UTF-8
Require all granted
</IfVersion>
</Directory>

View File

@ -0,0 +1,44 @@
#test policy file
#earlier = higher precedence!
[main]
key_aliases =
30C9ECF8 fedora-test
4F2A6FD2 fedora-gold
897DA07A redhat-beta
1AC70CE6 fedora-extras
unprotected_keys =
fedora-test
fedora-extras
redhat-beta
server = {{ koji_hub_url }}
weburl = {{ koji_web_url }}
# The domain name that will be appended to Koji usernames
# when creating email notifications
#email_domain = fedoraproject.org
# SMTP user and pass (uncomment and fill in if your smtp server requires authentication)
#smtp_user=user@example.com
#smtp_pass=CHANGEME
[prune]
policy =
#stuff to protect
#note that tags with master lock engaged are already protected
tag *-updates :: keep
age < 1 day :: skip
sig fedora-gold :: skip
sig fedora-test && age < 12 weeks :: keep
#stuff to chuck semi-rapidly
tag *-testing *-candidate :: { # nested rules
order >= 2 :: untag
order > 0 && age > 6 weeks :: untag
} #closing braces must be on a line by themselves (modulo comments/whitespace)
tag *-candidate && age > 60 weeks :: untag
#default: keep the last 3
order > 2 :: untag

View File

@ -13,13 +13,13 @@ DBHost = {{ koji_db_host }}
DBPass = {{ koji_db_pass }} DBPass = {{ koji_db_pass }}
KojiDir = {{ koji_mount }} KojiDir = {{ koji_mount }}
AuthPrincipal host/kojihub@ROCKYLINUX.ORG AuthPrincipal {{ koji_hub_principal }}
AuthKeytab /etc/koji.keytab AuthKeytab {{ koji_hub_keytab }}
ProxyPrincipals koji/kojiweb@ROCKYLINUX.ORG ProxyPrincipals {{ koji_hub_proxy_principals }}
HostPrincipalFormat compile/%s@ROCKYLINUX.ORG HostPrincipalFormat {{ koji_hub_principal_format }}
## Other options ## ## Other options ##
LoginCreatesUser = Off LoginCreatesUser = On
KojiWebURL = {{ koji_web_url }} KojiWebURL = {{ koji_web_url }}
# The domain name that will be appended to Koji usernames # The domain name that will be appended to Koji usernames

View File

@ -0,0 +1,9 @@
[rockymsg]
host = {{ koji_hub_plugin_mqtt_host }}
port = 8883
tls_cert = {{ koji_hub_plugin_mqtt_tls_cert }}
tls_key = {{ koji_hub_plugin_mqtt_tls_key }}
ca_cert = {{ koji_hub_plugin_mqtt_ca }}
tls_insecure = False
tls_version = 2
exclude_tags = {% for tag in koji_hub_plugin_mqtt_excluded_tags %}{{ tag }}{%- if not loop.last -%},{% endif %}{% endfor %}

View File

@ -1,49 +1,7 @@
[kojira] [kojira]
; For user/pass authentication
; user=kojira
; password=kojira
; The URL for the koji hub server
server={{ koji_hub_url }} server={{ koji_hub_url }}
; The directory containing the repos/ directory
topdir={{ koji_mount }} topdir={{ koji_mount }}
; Logfile
logfile=/var/log/kojira.log logfile=/var/log/kojira.log
;with_src=no
with_src=no
;configuration for Kerberos authentication
;the kerberos principal to use
principal = {{ koji_kojira_principal }} principal = {{ koji_kojira_principal }}
;location of the keytab
keytab = {{ koji_kojira_keytab }} keytab = {{ koji_kojira_keytab }}
;configuration for SSL authentication
;client certificate
;cert = /etc/kojira/client.crt
;certificate of the CA that issued the HTTP server certificate
;serverca = /etc/kojira/serverca.crt
;how soon (in seconds) to clean up expired repositories. 1 week default
;deleted_repo_lifetime = 604800
;how soon (in seconds) to clean up dist repositories. 1 week default here too
;dist_repo_lifetime = 604800
;turn on debugging statements in the log
;debug = false
; ignored repositories according to glob. Multiple masks separated by space.
; ignore_tags =
; Monitor external repos and trigger the appropriate Koji repo regenerations
; when they change. Note that you need to have your database set to use UTC,
; as otherwise you can end with weird behaviour. For details see
; https://pagure.io/koji/issue/2159
; check_external_repos = false

View File

@ -8,12 +8,13 @@ KojiTheme = {{ koji_theme }}
KojiHubURL = {{ koji_hub_url }} KojiHubURL = {{ koji_hub_url }}
KojiFilesURL = {{ koji_files_url }} KojiFilesURL = {{ koji_files_url }}
# CA
KojiHubCA = {{ koji_hub_ca }}
# Kerberos authentication options # Kerberos authentication options
WebPrincipal = koji/web@ROCKYLINUX.ORG WebPrincipal = {{ koji_hub_proxy_principals }}
WebKeytab = /etc/httpd.keytab WebKeytab = {{ koji_hub_keytab }}
WebCCache = /var/tmp/kojiweb.ccache WebCCache = /var/tmp/kojiweb.ccache
# The service name of the principal being used by the hub
KrbService = host
LoginTimeout = 72 LoginTimeout = 72
@ -21,3 +22,4 @@ LoginTimeout = 72
Secret = {{ koji_hub_secret }} Secret = {{ koji_hub_secret }}
LibPath = /usr/share/koji-web/lib LibPath = /usr/share/koji-web/lib
LiteralFooter = True

View File

@ -11,3 +11,6 @@ topurl = {{ koji_files_url }}
;path to the koji top directory ;path to the koji top directory
topdir = {{ koji_mount }} topdir = {{ koji_mount }}
; https ca, not for ssl auth
serverca = {{ koji_hub_ca }}

View File

@ -1,5 +1,5 @@
with user_id as (insert into users (name, status, usertype) values ('{{ koji_admin_user }}', 0, 0) returning id) with user_id as (insert into users (name, status, usertype) values ('{{ koji_admin_user }}', 0, 0) returning id)
insert into user_krb_principals (user_id, krb_principal) values ((select id from user_id),'{{ koji_admin_keytab }}'); insert into user_krb_principals (user_id, krb_principal) values ((select id from user_id),'{{ koji_admin_principal }}');
insert into user_perms (user_id, perm_id, creator_id) values (1, 1, 1); insert into user_perms (user_id, perm_id, creator_id) values (1, 1, 1);
insert into users (name, status, usertype) values ('{{ koji_kojira_user }}', 0, 0); insert into users (name, status, usertype) values ('{{ koji_kojira_user }}', 0, 0);
INSERT INTO user_perms (user_id, perm_id, creator_id) VALUES (2, 3, 1); INSERT INTO user_perms (user_id, perm_id, creator_id) VALUES (2, 10, 1);