Compare commits

..

No commits in common. "main" and "main" have entirely different histories.
main ... main

12 changed files with 307216 additions and 11999 deletions

View file

@ -1,5 +1,5 @@
# Stage 1: Build stage with necessary build dependencies
FROM quay.io/fedora/python-312:20240814 AS prebuild
FROM quay.io/fedora/python-312:latest AS build-stage
LABEL \
name="python-312-with-rust" \
vendor="Fedora Infrastructure" \
@ -19,50 +19,21 @@ RUN dnf install -y \
libffi-devel \
openssl-devel
# Stage 2: Build mirrormanager2 python code
FROM prebuild AS mirrormanager2-build
# Clone MirrorManager2 source code from the Git repo
RUN mkdir -p /opt/mirrormanager2
WORKDIR /opt/mirrormanager2
RUN git clone https://github.com/fedora-infra/mirrormanager2.git .
RUN sed -e 's/signed_fpca/signed_rosca/' -i mirrormanager2/perms.py mirrormanager2/auth.py
RUN pip install --prefix=/install .
RUN pip install --prefix=/install flask_session psycopg2
# Stage 3: Build generate-mirrorlist-cache and mirrorlist-server
FROM prebuild AS mirrorlist-build
RUN mkdir -p /opt/mirrorlist-server
WORKDIR /opt/mirrorlist-server
RUN git clone https://github.com/adrianreber/mirrorlist-server.git .
RUN echo -e '[profile.release-with-debug]\ninherits = "release"\ndebug = true\n' >> Cargo.toml
# NOTE(neil): 20241217 #![deny(warnings)] causes deprecated/removed lints to be errors...
RUN sed -i 's,#!\[deny(warnings)\],\#!\[allow(renamed_and_removed_lints)\],' src/bin/mirrorlist-server.rs src/bin/generate-mirrorlist-cache.rs
RUN cargo build --profile=release-with-debug
# Stage 4: Build scan-primary-mirror
FROM prebuild AS scan-primary-mirror-build
RUN mkdir -p /opt/scan-primary-mirror
WORKDIR /opt/scan-primary-mirror
RUN git clone https://github.com/adrianreber/scan-primary-mirror.git .
RUN RUSTFLAGS=-g cargo build --release
# Stage 5: Final stage with runtime dependencies
FROM quay.io/fedora/python-312:20240814 AS runtime
# Stage 2: Final stage with runtime dependencies
FROM quay.io/fedora/python-312:latest AS runtime
LABEL \
name="python-312-with-rust" \
vendor="Fedora Infrastructure" \
license="MIT"
USER root
# Add only runtime dependencies
RUN dnf install -y \
python3-pyrpmmd \
@ -70,41 +41,33 @@ RUN dnf install -y \
uwsgi-plugin-python3 \
logrotate
# Copy installation of mirrormanager from it's build stage
COPY --from=mirrormanager2-build /install /opt/app-root/
# Copy installed dependencies from the build stage
COPY --from=build-stage /install /usr/
# Copy mirrorlist-server and generate-mirrorlist-cache from mirrorlist-server build
COPY --from=mirrorlist-build /opt/mirrorlist-server/target/release-with-debug/generate-mirrorlist-cache /usr/local/bin/
COPY --from=mirrorlist-build /opt/mirrorlist-server/target/release-with-debug/mirrorlist-server /usr/local/bin/
# Copy scan-primary-mirror from its build step
COPY --from=scan-primary-mirror-build /opt/scan-primary-mirror/target/release/scan-primary-mirror /usr/local/bin/
# Copy in the wsgi entry point
# Copy in the tree
#COPY --from=build-stage /opt/mirrormanager2 /opt/mirrormanager2
ADD run.py /opt/mirrormanager2/
# Stage 6: flatten :)
FROM runtime as final
LABEL \
name="python-312-with-rust-mirrormanager" \
vendor="Rocky Linux Infrastructure" \
license="MIT"
WORKDIR /opt/mirrormanager2
EXPOSE 5000
# @TODO(neil): probably this shouldn't be here? maybe SSM -> parse out in mm2.cfg?
FROM runtime AS database
COPY client_secrets.json /etc/mirrormanager/
COPY mirrormanager2.cfg /etc/mirrormanager/
COPY static/rocky /opt/app-root/lib/python3.12/site-packages/mirrormanager2/static/rocky
COPY templates/rocky /opt/app-root/lib/python3.12/site-packages/mirrormanager2/templates/rocky
ENV MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg
RUN /usr/bin/python3 -m flask -A mirrormanager2.app db sync
FROM runtime as final
COPY --from=database /var/tmp/mirrormanager2_dev.sqlite /var/tmp/mirrormanager2_dev.sqlite
# Set working directory
WORKDIR /opt/mirrormanager2
# Expose necessary ports
EXPOSE 5000
COPY client_secrets.json /etc/mirrormanager/
COPY mirrormanager2.cfg /etc/mirrormanager/
ENV MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg
ENV SETUP_DB=false
RUN test "$SETUP_DB" || python -m flask -A mirrormanager2.app db sync
COPY static/rocky /usr/lib/python3.12/site-packages/mirrormanager2/static/rocky
COPY templates/rocky /usr/lib/python3.12/site-packages/mirrormanager2/templates/rocky
# Define entrypoint script to start the application
CMD [ "uwsgi", "--socket", "0.0.0.0:5000", \
@ -114,6 +77,5 @@ CMD [ "uwsgi", "--socket", "0.0.0.0:5000", \
"--enable-threads", \
"--master", \
"-b", "65535", \
"-H", "/opt/app-root/", \
"--wsgi-file", "/opt/mirrormanager2/run.py" ]

View file

@ -10,7 +10,7 @@ import os
# url to the database server:
SQLALCHEMY_DATABASE_URI = os.environ.get('MM2_SQLALCHEMY_DATABASE_URI', 'sqlite:////var/tmp/mirrormanager2_dev.sqlite')
SQLALCHEMY_DATABASE_URI = 'sqlite:////var/tmp/mirrormanager2_dev.sqlite'
# the number of items to display on the search pages
# Default: ``50``.
@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('MM2_SECRET_KEY')
# Seed used to make the password harder to brute force in case of leaking
# This should be kept really secret!
PASSWORD_SEED = os.environ.get('MM2_PASSWORD_SEED') # only valid for local auth
PASSWORD_SEED = os.environ.get('MM2_PASSWORD_SEED')
###
# Other configuration items for the web-app
@ -44,26 +44,17 @@ THEME_FOLDER = os.environ.get('MM2_THEME_FOLDER', 'fedora')
#MM_AUTHENTICATION = "fas"
OIDC_CLIENT_SECRETS = "/etc/mirrormanager/client_secrets.json"
OIDC_SCOPES = " ".join(
[
"openid",
"email",
"profile",
"https://id.fedoraproject.org/scope/groups",
"https://id.fedoraproject.org/scope/agreements",
]
)
# If the authentication method is `fas`, groups in which should be the user
# to be recognized as an admin.
ADMIN_GROUP = ["infrastructure"]
#ADMIN_GROUP = ["sysadmin-main"]
# Email of the admin to which send notification or error
ADMIN_EMAIL = "infrastructure@rockylinux.org"
ADMIN_EMAIL = "admin@fedoraproject.org"
# Email address used in the "From" field of the emails sent.
# Default: ``nobody@fedoraproject.org``.
EMAIL_FROM = "nobody@rockylinux.org"
#EMAIL_FROM = "nobody@fedoraproject.org"
# SMTP server to use,
# Default: ``localhost``.
@ -74,7 +65,7 @@ EMAIL_FROM = "nobody@rockylinux.org"
# SMTP_PASSWORD = 'password'
# Countries which have to be excluded.
EMBARGOED_COUNTRIES = ["CU", "IR", "KP", "SD", "SY", "RU"]
#EMBARGOED_COUNTRIES = ["CU", "IR", "KP", "SD", "SY"]
# When this is set to True, an additional menu item is shown which
# displays the maps generated with mm2_generate-worldmap.

89
run.py
View file

@ -1,90 +1,5 @@
from werkzeug.middleware.proxy_fix import ProxyFix
from mirrormanager2.app import create_app
from flask_session import Session
from cachelib.file import FileSystemCache
from flask import Flask
import os
import sys
import subprocess
from threading import Thread
application = create_app()
application.wsgi_app = ProxyFix(application.wsgi_app, x_proto=1, x_host=1)
def setup_env():
password = os.environ.get('MM2_DATABASE_PASSWORD')
if password:
user = os.environ.get('DB_USER')
host = os.environ.get('DB_HOST')
port = os.environ.get('DB_PORT')
name = os.environ.get('DB_NAME')
os.environ["MM2_SQLALCHEMY_DATABASE_URI"] = f"postgresql://{user}:{password}@{host}:{port}/{name}"
def mirrormanager_wsgi() -> Flask:
application = create_app()
application.debug = os.environ.get("MM2_DEBUG", False)
application.config['SESSION_TYPE'] = "cachelib"
application.config['SESSION_CACHELIB'] = FileSystemCache(cache_dir='/mnt/efs/fs/0/tmp/sessions', threshold=500)
Session(application)
application.wsgi_app = ProxyFix(application.wsgi_app, x_proto=1, x_host=1)
def execute(func: str) -> int:
def stream_output(pipe, output_stream):
while True:
data = pipe.read(1024) # Read in chunks of 1024 bytes
if not data:
break
output_stream.write(data)
output_stream.flush()
pipe.close()
argv = func.split()
cmd = argv.pop(0)
args = argv if argv else []
print(f'running: {cmd} with args: {args}')
process = subprocess.Popen([cmd] + args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=0) # Unbuffered binary mode
stdout_thread = Thread(target=stream_output, args=(process.stdout, sys.stdout.buffer))
stderr_thread = Thread(target=stream_output, args=(process.stderr, sys.stderr.buffer))
stdout_thread.start()
stderr_thread.start()
process.wait()
stdout_thread.join()
stderr_thread.join()
return process.returncode
if __name__ == "__main__":
setup_env()
# map cmd name to script/function
cmd_map = {
"check-propagation": "/usr/bin/echo undefined",
"crawl-mirrors": "/usr/bin/echo undefined",
"generate-worldmaps": "/opt/app-root/bin/mm2_generate-worldmap",
"scan-primary-mirror": "/usr/local/bin/scan-primary-mirror",
"update-clouds": "/usr/bin/echo undefined",
"update-geoip": "/usr/bin/echo undefined",
"generate-mirrorlist-cache": "/usr/local/bin/generate-mirrorlist-cache",
"update-netblocks": "/opt/app-root/bin/mm2_get-netblocks",
"update-EC2-netblocks": "/opt/app-root/bin/update-EC2-netblocks",
"mirrorlist-server": "/usr/local/bin/mirrorlist-server",
"mirrormanager-wsgi": mirrormanager_wsgi,
"debug-filesystem": "python3 -m http.server 8888 -d /mnt/efs/fs/0/"
}
arg = sys.argv[1] if len(sys.argv) > 1 else "mirrormanager-wsgi"
if arg and (cmd := cmd_map.get(arg, None)):
if not isinstance(cmd, str):
application = cmd()
exit(0)
sys.exit(execute(cmd))

28
start-dev.sh Executable file → Normal file
View file

@ -2,18 +2,17 @@ POD=mirrormanager2
podman pod exists $POD || podman pod create -p 5000:5000 -n $POD
podman container exists postgres || podman run --pod $POD \
--name postgres \
--replace -d \
-e POSTGRES_PASSWORD=mirrormanager \
-e POSTGRES_DB=mirrormanager \
-v pgdata:/var/lib/postgresql/data \
docker.io/library/postgres:15
#podman run \
# --pod $POD \
# --name nginx \
# --replace \
# -v $PWD/nginx.conf:/etc/nginx/conf.d/default.conf:ro \
# -d docker.io/library/nginx:1.13-alpine
test -d tmp || mkdir tmp
test -f client_secrets.json || (echo "missing client_secrets" && exit 2)
#podman build -t git.resf.org/infrastructure/mirrormanager2:dev -f Containerfile
podman build -t git.resf.org/infrastructure/mirrormanager2:dev -f Containerfile
podman rm --force mm2 -t 1
podman run \
--pod $POD \
@ -22,15 +21,12 @@ podman run \
-e 'MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg' \
-e "MM2_SECRET_KEY=$(openssl rand -hex 32)" \
-e "MM2_PASSWORD_SEED=$(openssl rand -hex 32)" \
-e 'MM2_SQLALCHEMY_DATABASE_URI=postgresql://postgres:mirrormanager@postgres:5432/mirrormanager' \
-e 'MM2_THEME_FOLDER=rocky' \
-e 'FLASK_DEBUG=1' \
-e 'MM2_DEBUG=1' \
-v $PWD/mirrormanager2.cfg:/etc/mirrormanager/mirrormanager2.cfg:z,ro \
-v $PWD/client_secrets.json:/etc/mirrormanager/client_secrets.json:z,ro \
-v $PWD/tmp:/var/tmp:z,rw \
-v $PWD/static:/opt/app-root/lib/python3.12/site-packages/mirrormanager2/static:z,ro \
-v $PWD/templates:/opt/app-root/lib/python3.12/site-packages/mirrormanager2/templates:z,ro \
-v $PWD/mirrormanager2.cfg:/etc/mirrormanager/mirrormanager2.cfg \
-v $PWD/client_secrets.json:/etc/mirrormanager/client_secrets.json \
-v $PWD/tmp:/var/tmp:rw \
-v $PWD/static:/usr/lib/python3.12/site-packages/mirrormanager2/static \
-v $PWD/templates:/usr/lib/python3.12/site-packages/mirrormanager2/templates \
-d git.resf.org/infrastructure/mirrormanager2:dev
# Setup sqlite database

View file

@ -31,9 +31,7 @@ form.icon button {
}
/* Stealed this from bootstrap */
label,
select,
button,
label, select, button,
input[type="button"],
input[type="reset"],
input[type="submit"],
@ -43,33 +41,31 @@ input[type="checkbox"] {
}
.blue {
color: #0066cc;
color:#0066CC;
}
#matrixtitle {
background-color: #2963a6;
color: #ffffff;
background-color: #2963A6;
color: #FFFFFF;
text-align: center;
}
#matrixheadings,
#matrixheadings th {
background-color: #dddddd;
border-bottom: 1px solid #bbbbbb;
#matrixheadings, #matrixheadings th {
background-color: #DDDDDD;
border-bottom: 1px solid #BBBBBB;
margin: 0;
color: #333333;
text-align: center;
}
.matrix_section,
.matrix_section td {
border-bottom: 1px solid #bbbbbb !important;
.matrix_section, .matrix_section td {
border-bottom: 1px solid #BBBBBB !important;
}
.matrix_even,
.matrix_even td {
background-color: #f0f0f0;
border-bottom: 1px solid #dddddd;
border-top: 1px solid #dddddd;
.matrix_even, .matrix_even td {
background-color: #F0F0F0;
border-bottom: 1px solid #DDDDDD;
border-top: 1px solid #DDDDDD;
}
.centered {
@ -80,9 +76,10 @@ input[type="checkbox"] {
}
.bodycontent {
min-height: 600px;
min-height:600px
}
table.mm2-table-small {
width: auto;
}

File diff suppressed because it is too large Load diff

View file

@ -30,9 +30,9 @@
{% endmacro %}
{% macro footer() %}
<div class="footer py-5">
<div class="footer py-5 text-white">
<div class="container">
<div class="row footerlinks">
<div class="row footerlinks justify-content-center">
<div class="col-sm-3 col-4 mt-3">
<div>
<dl>
@ -67,11 +67,11 @@
</div>
</div>
</div>
<div class="row footerlinks mt-5">
<div class="col-12">
<div class="row footerlinks">
<div class="col-12 text-center">
<p> © 2024 Rocky Enterprise Software Foundation, and others. </p>
</div>
<div class="col-12">
<div class="col-12 text-center">
{% set noggin_link %}
<a href="https://github.com/fedora-infra/noggin">noggin</a>
{% endset %}
@ -108,6 +108,7 @@ href="{{ url_for('static', filename='favicon.ico')}}"/>
<link rel="stylesheet" type="text/css" media="screen"
href="{{ url_for('static', filename='mirrormanager2.css') }}"/>
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='rocky.css') }}"/>
<link href="{{ url_for('static', filename='fedora-bootstrap/fedora-bootstrap.min.css') }}" rel="stylesheet" />
<link href="{{ url_for('static', filename='fonts/open-sans.css') }}" rel="stylesheet" />
<link href="{{ url_for('static', filename='fonts/font-awesome.css') }}" rel="stylesheet" />
{% endmacro %}
@ -123,8 +124,8 @@ href="{{ url_for('static', filename='mirrormanager2.css') }}"/>
{%endmacro%}
{% macro nav(is_admin) %}
<nav class="navbar navbar-expand-lg navbar-light masthead py-2 container">
<a class="navbar-brand" href="{{url_for('base.index')}}"><img src="{{ url_for('static', filename='mirrormanager-logo.png') }}" alt="Fedora Account System logo" height="60"></a>
<nav class="navbar navbar-expand-lg navbar-light masthead py-2">
<a class="navbar-brand" href="{{url_for('base.index')}}"><img src="{{ url_for('static', filename='mirrormanager-logo.png') }}" alt="Fedora Account System logo" height="40"></a>
<ul class="navbar-nav align-items-center ms-auto">
<li class="nav-item">
<a class="nav-link color-white" href="{{url_for('base.list_mirrors')}}">Mirrors</a>
@ -159,7 +160,7 @@ href="{{ url_for('static', filename='mirrormanager2.css') }}"/>
</div>
</li>
{% else %}
<a class="btn btn-primary ml-2 py-2 rounded-4" href="{{ url_for('auth.login') }}?next={{request.url}}">Login</a>
<a class="btn btn-primary ml-2" href="{{ url_for('auth.login') }}?next={{request.url}}">Login</a>
{% endif %}
</ul>
</nav>
@ -278,34 +279,3 @@ href="{{ url_for('static', filename='mirrormanager2.css') }}"/>
{{ form.csrf_token }}
</form>
{% endmacro %}
{% macro pagination_bar(result) %}
{% if result.total_pages > 1 %}
<nav aria-label="Pagination">
<ul class="pagination pagination-sm justify-content-center my-4">
{# Page list #}
{% for page_number in result.truncated_pages_list(margin=6) %}
{% if page_number == result.page_number %}
<li class="page-item active" aria-current="page">
<span class="page-link">
{{ page_number }}
<span class="sr-only">(current)</span>
</span>
</li>
{% elif page_number == None %}
<li class="page-item disabled">
<span class="page-link">...</span>
</li>
{% else %}
<li class="page-item">
<a class="page-link" href="{{ result.page_url(page_number) }}">
{{ page_number }}
</a>
</li>
{% endif %}
{% endfor %}
</ul>
</nav>
{% endif %}
{% endmacro %}

View file

@ -1,6 +1,6 @@
{% extends 'admin/master.html' %}
{% extends 'admin/admin_master.html' %}
{% block body %}
{% block content %}
<p>
These pages present the admin interface of MirrorManager.
</p>

View file

@ -231,26 +231,14 @@
</div>
{% endif %}
{% if category.directories %}
<div class="accordion accordion-flush" id="accordion-category-{{category.id}}">
<div class="accordion-item">
<h2 class="accordion-header fw-bold" id="heading-category-{{category.id}}">
<button class="accordion-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapse-category-{{category.id}}" aria-expanded="false" aria-controls="flush-collapseOne">
<strong>Up-to-Date Directories this host carries <span class="badge bg-primary"> {{category.directories|length}}</span></strong>
</button>
</h2>
<div id="collapse-category-{{category.id}}" class="accordion-collapse collapse" aria-labelledby="#heading-category-{{category.id}}" data-bs-parent="#accordion-category-{{category.id}}">
<div class="accordion-body">
<ul>
{% for dir in category.directories %}
{% if dir.up2date %}
<li>{{ dir.path }}</li>
{% endif %}
{% endfor %}
</ul>
</div>
</div>
</div>
</div>
<h3>Up-to-Date Directories this host carries</h3>
<ul>
{% for dir in category.directories %}
{% if dir.up2date %}
<li>{{ dir.path }}</li>
{% endif %}
{% endfor %}
</ul>
{% endif %}
</li>

View file

@ -4,7 +4,7 @@
{%block tag %}home{% endblock %}
{% block content %}
<h2 class="h1 mb-4">Rocky Public Active Mirrors</h2>
<h2>Rocky Public Active Mirrors</h2>
<p>
Rocky Linux is distributed to millions of systems globally.
@ -35,7 +35,7 @@ networks globally.
</p>
<table class="table table-sm mt-5">
<table class="table table-sm">
<tr id="matrixtitle">
<th colspan="{{ arches | length + 2 }}">
Mirror list filtering matrix

View file

@ -1,7 +1,5 @@
{% extends "master.html" %}
{% from "_macros.html" import pagination_bar %}
{% block title %}Mirrors{% endblock %}
{%block tag %}mirrors{% endblock %}
@ -31,8 +29,8 @@
<th>Internet2</th>
<th>Comment</th>
</tr>
{% for mirror in mirrors.items %}
<tr class="mirror-row">
{% for mirror in mirrors %}
<tr>
<td>{{ mirror.country }}</td>
<td>
<a href="{{ mirror.site.org_url }}">
@ -60,7 +58,6 @@
</tr>
{% endfor %}
</table>
{{pagination_bar(mirrors)}}
{% else %}
<p>
There are currently no active mirrors registered.

View file

@ -17,7 +17,7 @@ main {
/* Prose */
.prose code,
.prose strong {
@apply dark:text-gray-300;
@apply dark:text-gray-300;
}
.prose h1,
.prose h2,
@ -25,36 +25,23 @@ main {
.prose h4,
.prose h5,
.prose h6 {
@apply dark:text-white;
@apply dark:text-white;
}
.prose a {
@apply text-green-600;
@apply text-green-600;
}
.prose blockquote {
@apply dark:text-white;
@apply dark:text-white;
}
.prose img {
@apply rounded-md;
@apply rounded-md;
}
[type="text"],
[type="email"],
[type="url"],
[type="password"],
[type="number"],
[type="date"],
[type="datetime-local"],
[type="month"],
[type="search"],
[type="tel"],
[type="time"],
[type="week"],
[multiple],
textarea,
select {
@apply rounded-md bg-gray-100 dark:bg-gray-700
[type='text'], [type='email'], [type='url'], [type='password'], [type='number'], [type='date'], [type='datetime-local'], [type='month'], [type='search'], [type='tel'], [type='time'], [type='week'], [multiple], textarea, select {
@apply
rounded-md bg-gray-100 dark:bg-gray-700
dark:border-gray-600 border-gray-300 border-2
py-2 px-3
focus:outline-none focus:ring focus:ring-2 focus:border-green-600 focus:ring-green-500
@ -62,15 +49,15 @@ select {
}
.form-group label {
@apply text-sm font-medium mb-2;
@apply text-sm font-medium mb-2;
}
.form-group .check-box input {
@apply rounded-md bg-gray-100 dark:bg-gray-700
@apply rounded-md bg-gray-100 dark:bg-gray-700
dark:border-gray-600 border-gray-300 border-2
p-1 checked:bg-green-600 checked:border-transparent
focus:outline-none focus:ring focus:ring-1 focus:border-green-600 focus:ring-green-500
transition duration-300 ease-in-out;
}
.form-group .check-box label {
@apply mb-0 ml-2;
}
@apply mb-0 ml-2;
}