From 6cb0a5509da105e1abbbedc911a7f9ca922d6017 Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Wed, 28 Aug 2024 08:51:25 -0400 Subject: [PATCH] init --- .gitignore | 2 + Containerfile | 66 +++++++++++++ mirrormanager2.cfg | 225 +++++++++++++++++++++++++++++++++++++++++++++ nginx.conf | 14 +++ run.py | 5 + start-dev.sh | 28 ++++++ 6 files changed, 340 insertions(+) create mode 100644 .gitignore create mode 100644 Containerfile create mode 100644 mirrormanager2.cfg create mode 100644 nginx.conf create mode 100644 run.py create mode 100644 start-dev.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..94facff --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/tmp +/client_secrets.json diff --git a/Containerfile b/Containerfile new file mode 100644 index 0000000..fcf38a8 --- /dev/null +++ b/Containerfile @@ -0,0 +1,66 @@ +# Stage 1: Build stage with necessary build dependencies +FROM quay.io/fedora/python-312:latest AS build-stage +LABEL \ +name="python-312-with-rust" \ +vendor="Fedora Infrastructure" \ +license="MIT" + +USER root +# Add RPM-only modules and build dependencies +RUN dnf install -y \ + python3-pyrpmmd \ + python3-poetry \ + poetry \ + cargo \ + logrotate \ + git \ + gcc \ + gcc-c++ \ + libffi-devel \ + openssl-devel + +# Clone MirrorManager2 source code from the Git repo +RUN mkdir -p /opt/mirrormanager2 +WORKDIR /opt/mirrormanager2 +RUN git clone https://github.com/fedora-infra/mirrormanager2.git . + +RUN pip install --prefix=/install . + +# Stage 2: Final stage with runtime dependencies +FROM quay.io/fedora/python-312:latest +LABEL \ +name="python-312-with-rust" \ +vendor="Fedora Infrastructure" \ +license="MIT" + +USER root +# Add only runtime dependencies +RUN dnf install -y \ + python3-pyrpmmd \ + uwsgi \ + uwsgi-plugin-python3 \ + logrotate + +# Copy installed dependencies from the build stage +COPY --from=build-stage /install /usr/ + +# Copy in the tree +#COPY --from=build-stage /opt/mirrormanager2 /opt/mirrormanager2 +ADD run.py /opt/mirrormanager2/ + +# Set working directory +WORKDIR /opt/mirrormanager2 + +# Expose necessary ports +EXPOSE 5000 + +# Define entrypoint script to start the application +CMD [ "uwsgi", "--socket", "0.0.0.0:3031", \ + "--uid", "uwsgi", \ + "--plugins", "python3", \ + "--protocol", "uwsgi", \ + "--enable-threads", \ + "--master", \ + "-b", "65535", \ + "--wsgi-file", "/opt/mirrormanager2/run.py" ] + diff --git a/mirrormanager2.cfg b/mirrormanager2.cfg new file mode 100644 index 0000000..67b7c7c --- /dev/null +++ b/mirrormanager2.cfg @@ -0,0 +1,225 @@ +''' +MirrorManager2 sample configuration. +''' + +### +# Most important configuration items +### + + +# url to the database server: +SQLALCHEMY_DATABASE_URI = 'sqlite:////var/tmp/mirrormanager2_dev.sqlite' + +# the number of items to display on the search pages +# Default: ``50``. +#ITEMS_PER_PAGE = 50 + +# secret key used to generate unique csrf token +SECRET_KEY = "" + +# Seed used to make the password harder to brute force in case of leaking +# This should be kept really secret! +PASSWORD_SEED = "" + + + +### +# Other configuration items for the web-app +### + + +# Set the time after which the session expires. Flask's default is 31 days. +# Default: ``timedelta(hours=1)`` corresponds to 1 hour. +#from datetime import timedelta +#PERMANENT_SESSION_LIFETIME = timedelta(hours=1) + +# Folder containing the theme to use. +# Default: ``fedora``. +#THEME_FOLDER = "fedora" + +# Which authentication method to use, defaults to `fas` can be or `local` +# Default: ``fas``. +# Note that this previously used openid, now it uses openid connect oidc +#MM_AUTHENTICATION = "fas" + +OIDC_CLIENT_SECRETS = "/etc/mirrormanager/client_secrets.json" + +# If the authentication method is `fas`, groups in which should be the user +# to be recognized as an admin. +#ADMIN_GROUP = ["sysadmin-main"] + +# Email of the admin to which send notification or error +ADMIN_EMAIL = "admin@fedoraproject.org" + +# Email address used in the "From" field of the emails sent. +# Default: ``nobody@fedoraproject.org``. +#EMAIL_FROM = "nobody@fedoraproject.org" + +# SMTP server to use, +# Default: ``localhost``. +#SMTP_SERVER = "localhost" + +# If the SMTP server requires authentication, fill in the information here +# SMTP_USERNAME = 'username' +# SMTP_PASSWORD = 'password' + +# Countries which have to be excluded. +#EMBARGOED_COUNTRIES = ["CU", "IR", "KP", "SD", "SY"] + +# When this is set to True, an additional menu item is shown which +# displays the maps generated with mm2_generate-worldmap. +#SHOW_MAPS = True + +# Location of the static map displayed in the map tab. +#STATIC_MAP = "map.png" + +# Location of the interactive openstreetmap based map. +#INTERACTIVE_MAP = "mirrors.html" + +# The crawler can generate propagation statistics which can be +# converted into svg/pdf with mm2_propagation. These files +# can be displayed next to the statistics and maps tab if desired. +#SHOW_PROPAGATION = True + +# Where to look for the above mentioned propagation images. +#PROPAGATION_BASE = "/var/lib/mirrormanager/statistics/data/propagation" + +# Where the GeoIP database lives +#GEOIP_BASE = "/usr/share/GeoIP" + +# Disable master rsync server ACL +# Fedora does not use it and therefore it is set to False +#MASTER_RSYNC_ACL = False + +# When this is set to True, the session cookie will only be returned to the +# server via ssl (https). If you connect to the server via plain http, the +# cookie will not be sent. This prevents sniffing of the cookie contents. +# This may be set to False when testing your application but should always +# be set to True in production. +# Default: ``True``. +#MM_COOKIE_REQUIRES_HTTPS = True + +# The name of the cookie used to store the session id. +# Default: ``.MirrorManager``. +#MM_COOKIE_NAME = "MirrorManager" + +# If this variable is set (and the directory exists) the crawler +# will create per host log files in MM_LOG_DIR/crawler/.log +# which can the be used in the web interface by the mirror admins. +# Other parts besides the crawler are also using this variable to +# decide where to store log files. +#MM_LOG_DIR = "/var/log/mirrormanager" + +# This is used to exclude certain protocols to be entered +# for host category URLs at all. +# The following is the default for Fedora to exclude FTP based +# mirrors to be added. Removing this confguration option +# or setting it to "" removes any protocol restrictions. +#MM_PROTOCOL_REGEX = "^(?!ftp)(.*)$" + +# The netblock size parameters define which netblock sizes can be +# added by a site administrator. Larger networks can only be added by +# mirrormanager admins. +#MM_IPV4_NETBLOCK_SIZE = "/16" +#MM_IPV6_NETBLOCK_SIZE = "/32" + +# If not specified the application will rely on the root_url when sending +# emails, otherwise it will use this URL +# Default: ``None``. +#APPLICATION_URL = None + +# Boolean specifying wether to check the user's IP address when retrieving +# its session. This make things more secure (thus is on by default) but +# under certain setup it might not work (for example is there are proxies +# in front of the application). +#CHECK_SESSION_IP = True + +# Specify additional rsync parameters for the crawler +# # --timeout 14400: abort rsync crawl after 4 hours +# # --no-human-readable: because rsync made things pretty by default in 3.1.x +#CRAWLER_RSYNC_PARAMETERS = "--no-motd" + +# This is a list of directories which MirrorManager will ignore while guessing +# the version and architecture from a path. +#SKIP_PATHS_FOR_VERSION = [] + +### +# Configuration options used by the utilities +### + +# Specify whether the crawler should send a report by email +CRAWLER_SEND_EMAIL = True + +# Specify additional rsync parameters for the crawler +# --timeout 14400: abort rsync crawl after 4 hours +# Depending on the setup and the crawler frequency rsync's timeout option +# can be used decrease the probability of stale rsync processes +#CRAWLER_RSYNC_PARAMETERS = "--no-motd --timeout 14400" + +# If a host fails for CRAWLER_AUTO_DISABLE times in a row +# the host will be disable automatically (user_active) +#CRAWLER_AUTO_DISABLE = 4 + +# This is a list of directories which MirrorManager will ignore while guessing +# the version and architecture from a path. +#SKIP_PATHS_FOR_VERSION = ["pub/alt"] + +# Whether to use Fedora Messaging for notifications +#USE_FEDORA_MESSAGING = True + +UMDL_PREFIX = "/srv/" + +UMDL_MASTER_DIRECTORIES = [ + { + 'type': 'directory', + 'path': '../testdata/pub/epel/', + 'category': 'Fedora EPEL' + }, + { + 'type': 'directory', + 'path': '../testdata/pub/fedora/linux/', + 'category': 'Fedora Linux' + }, + { + 'type': 'directory', + 'path': '../testdata/pub/fedora-secondary/', + 'category': 'Fedora Secondary Arches' + }, + { + 'type': 'directory', + 'path': '../testdata/pub/archive/', + 'category': 'Fedora Archive' + }, + { + 'type': 'directory', + 'path': '../testdata/pub/alt/', + 'category': 'Fedora Other' + }, +# { +# 'type':'directory', +# 'path':'../testdata/pub/fedora/linux/', +# 'category':'Fedora Linux', +# 'excludes':['.*/core/?.*', '.*/extras/?.*', '.*/[7-8]/?.*' ] +# }, +# { +# 'type':'rsync', +# 'url':'rsync://archive.ubuntu.com/ubuntu/', +# 'category':'Ubuntu Archive' +# }, +# { +# 'type':'rsync', +# 'url':'rsync://releases.ubuntu.com/releases/', +# 'category':'Ubuntu CD Images' +# }, +# { +# 'type':'rsync', +# 'url':'rsync://ports.ubuntu.com/ubuntu-ports/', +# 'category':'Ubuntu Ports Archive' +# }, +# { +# 'type':'rsync', +# 'url':'rsync://security.ubuntu.com/ubuntu/', +# 'category':'Ubuntu Security Archive' +# }, +] + diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 0000000..4c5ab93 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,14 @@ +server { + listen 80; + root /usr/share/nginx/html; + location / { + try_files $uri @wsgi; + } + location @wsgi { + include uwsgi_params; + #uwsgi_pass_request_headers off; + uwsgi_store on; + uwsgi_buffering on; + uwsgi_pass 127.0.0.1:3031; + } +} diff --git a/run.py b/run.py new file mode 100644 index 0000000..632dce6 --- /dev/null +++ b/run.py @@ -0,0 +1,5 @@ +from werkzeug.middleware.proxy_fix import ProxyFix +from mirrormanager2.app import create_app +application = create_app() +application.wsgi_app = ProxyFix(application.wsgi_app, x_proto=1, x_host=1) + diff --git a/start-dev.sh b/start-dev.sh new file mode 100644 index 0000000..d9a22c1 --- /dev/null +++ b/start-dev.sh @@ -0,0 +1,28 @@ +POD=mirrormanager2 + +podman pod exists $POD || podman pod create -p 5000:80 -n $POD + +podman run \ + --pod $POD \ + --name nginx \ + --replace \ + -v $PWD/nginx.conf:/etc/nginx/conf.d/default.conf:ro \ + -d docker.io/library/nginx:1.13-alpine + +test -d tmp || mkdir tmp +test -f client_secrets.json || (echo "missing client_secrets" && exit 2) + +podman run \ + --pod $POD \ + --name mm2 \ + --replace \ + -e 'MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg' \ + -v $PWD/mirrormanager2.cfg:/etc/mirrormanager/mirrormanager2.cfg \ + -v $PWD/client_secrets.json:/etc/mirrormanager/client_secrets.json \ + -v $PWD/tmp:/var/tmp:rw \ + -d git.resf.org/infrastructure/mirrormanager2:latest + +# Setup sqlite database +echo "mirrormanager2 and nginx have been statred. You may need to run:" + +echo "podman exec mm2 /usr/bin/python3 -m flask -A mirrormanager2.app db sync"