From 363191a9b5e48d3559011f4a55c0b9fb3f38c258 Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Fri, 20 Dec 2024 12:30:23 -0500 Subject: [PATCH] update theme for latest mirrormanager various fixes and updates for running in ECS, too --- Containerfile | 73 ++++++++++++++++++++++++++---------- mirrormanager2.cfg | 8 ++-- run.py | 8 ++++ start-dev.sh | 15 ++++++-- static/rocky/rocky.css | 3 +- templates/rocky/_macros.html | 32 ++++++++++++++++ templates/rocky/host.html | 28 ++++++++++---- templates/rocky/index.html | 8 +++- templates/rocky/mirrors.html | 7 +++- 9 files changed, 144 insertions(+), 38 deletions(-) diff --git a/Containerfile b/Containerfile index ebb5924..9f17bfd 100644 --- a/Containerfile +++ b/Containerfile @@ -1,5 +1,5 @@ # Stage 1: Build stage with necessary build dependencies -FROM quay.io/fedora/python-312:latest AS build-stage +FROM quay.io/fedora/python-312:20240814 AS prebuild LABEL \ name="python-312-with-rust" \ vendor="Fedora Infrastructure" \ @@ -19,6 +19,10 @@ RUN dnf install -y \ libffi-devel \ openssl-devel + +# Stage 2: Build mirrormanager2 python code +FROM prebuild AS mirrormanager2-build + # Clone MirrorManager2 source code from the Git repo RUN mkdir -p /opt/mirrormanager2 WORKDIR /opt/mirrormanager2 @@ -27,16 +31,38 @@ RUN git clone https://github.com/fedora-infra/mirrormanager2.git . RUN sed -e 's/signed_fpca/signed_rosca/' -i mirrormanager2/perms.py mirrormanager2/auth.py RUN pip install --prefix=/install . -RUN pip install --prefix=/install flask_session +RUN pip install --prefix=/install flask_session psycopg2 -# Stage 2: Final stage with runtime dependencies -FROM quay.io/fedora/python-312:latest AS runtime +# Stage 3: Build generate-mirrorlist-cache and mirrorlist-server +FROM prebuild AS mirrorlist-build + +RUN mkdir -p /opt/mirrorlist-server +WORKDIR /opt/mirrorlist-server +RUN git clone https://github.com/adrianreber/mirrorlist-server.git . +RUN echo -e '[profile.release-with-debug]\ninherits = "release"\ndebug = true\n' >> Cargo.toml + +# NOTE(neil): 20241217 #![deny(warnings)] causes deprecated/removed lints to be errors... +RUN sed -i 's,#!\[deny(warnings)\],\#!\[allow(renamed_and_removed_lints)\],' src/bin/mirrorlist-server.rs src/bin/generate-mirrorlist-cache.rs + +RUN cargo build --profile=release-with-debug + +# Stage 4: Build scan-primary-mirror +FROM prebuild AS scan-primary-mirror-build + +RUN mkdir -p /opt/scan-primary-mirror +WORKDIR /opt/scan-primary-mirror +RUN git clone https://github.com/adrianreber/scan-primary-mirror.git . +RUN RUSTFLAGS=-g cargo build --release + +# Stage 5: Final stage with runtime dependencies +FROM quay.io/fedora/python-312:20240814 AS runtime LABEL \ name="python-312-with-rust" \ vendor="Fedora Infrastructure" \ license="MIT" USER root + # Add only runtime dependencies RUN dnf install -y \ python3-pyrpmmd \ @@ -44,33 +70,41 @@ RUN dnf install -y \ uwsgi-plugin-python3 \ logrotate -# Copy installed dependencies from the build stage -COPY --from=build-stage /install /usr/ +# Copy installation of mirrormanager from it's build stage +COPY --from=mirrormanager2-build /install /opt/app-root/ -# Copy in the tree -#COPY --from=build-stage /opt/mirrormanager2 /opt/mirrormanager2 +# Copy mirrorlist-server and generate-mirrorlist-cache from mirrorlist-server build +COPY --from=mirrorlist-build /opt/mirrorlist-server/target/release-with-debug/generate-mirrorlist-cache /usr/local/bin/ +COPY --from=mirrorlist-build /opt/mirrorlist-server/target/release-with-debug/mirrorlist-server /usr/local/bin/ + +# Copy scan-primary-mirror from its build step +COPY --from=scan-primary-mirror-build /opt/scan-primary-mirror/target/release/scan-primary-mirror /usr/local/bin/ + +# Copy in the wsgi entry point ADD run.py /opt/mirrormanager2/ -FROM runtime AS database -COPY client_secrets.json /etc/mirrormanager/ -COPY mirrormanager2.cfg /etc/mirrormanager/ -ENV MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg -RUN /usr/bin/python3 -m flask -A mirrormanager2.app db sync - +# Stage 6: flatten :) FROM runtime as final -COPY --from=database /var/tmp/mirrormanager2_dev.sqlite /var/tmp/mirrormanager2_dev.sqlite -# Set working directory +LABEL \ +name="python-312-with-rust-mirrormanager" \ +vendor="Rocky Linux Infrastructure" \ +license="MIT" + WORKDIR /opt/mirrormanager2 -# Expose necessary ports EXPOSE 5000 + +# @TODO(neil): probably this shouldn't be here? maybe SSM -> parse out in mm2.cfg? COPY client_secrets.json /etc/mirrormanager/ COPY mirrormanager2.cfg /etc/mirrormanager/ +COPY static/rocky /opt/app-root/lib/python3.12/site-packages/mirrormanager2/static/rocky +COPY templates/rocky /opt/app-root/lib/python3.12/site-packages/mirrormanager2/templates/rocky + ENV MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg -COPY static/rocky /usr/lib/python3.12/site-packages/mirrormanager2/static/rocky -COPY templates/rocky /usr/lib/python3.12/site-packages/mirrormanager2/templates/rocky +ENV SETUP_DB=false +RUN test "$SETUP_DB" || python -m flask -A mirrormanager2.app db sync # Define entrypoint script to start the application CMD [ "uwsgi", "--socket", "0.0.0.0:5000", \ @@ -80,5 +114,6 @@ CMD [ "uwsgi", "--socket", "0.0.0.0:5000", \ "--enable-threads", \ "--master", \ "-b", "65535", \ + "-H", "/opt/app-root/", \ "--wsgi-file", "/opt/mirrormanager2/run.py" ] diff --git a/mirrormanager2.cfg b/mirrormanager2.cfg index 3201a4d..2a105ca 100644 --- a/mirrormanager2.cfg +++ b/mirrormanager2.cfg @@ -10,7 +10,7 @@ import os # url to the database server: -SQLALCHEMY_DATABASE_URI = 'sqlite:////var/tmp/mirrormanager2_dev.sqlite' +SQLALCHEMY_DATABASE_URI = os.environ.get('MM2_SQLALCHEMY_DATABASE_URI', 'sqlite:////var/tmp/mirrormanager2_dev.sqlite') # the number of items to display on the search pages # Default: ``50``. @@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('MM2_SECRET_KEY') # Seed used to make the password harder to brute force in case of leaking # This should be kept really secret! -PASSWORD_SEED = os.environ.get('MM2_PASSWORD_SEED') +PASSWORD_SEED = os.environ.get('MM2_PASSWORD_SEED') # only valid for local auth ### # Other configuration items for the web-app @@ -63,7 +63,7 @@ ADMIN_EMAIL = "infrastructure@rockylinux.org" # Email address used in the "From" field of the emails sent. # Default: ``nobody@fedoraproject.org``. -#EMAIL_FROM = "nobody@fedoraproject.org" +EMAIL_FROM = "nobody@rockylinux.org" # SMTP server to use, # Default: ``localhost``. @@ -74,7 +74,7 @@ ADMIN_EMAIL = "infrastructure@rockylinux.org" # SMTP_PASSWORD = 'password' # Countries which have to be excluded. -#EMBARGOED_COUNTRIES = ["CU", "IR", "KP", "SD", "SY"] +EMBARGOED_COUNTRIES = ["CU", "IR", "KP", "SD", "SY", "RU"] # When this is set to True, an additional menu item is shown which # displays the maps generated with mm2_generate-worldmap. diff --git a/run.py b/run.py index 7db3aa4..4bfdfa0 100644 --- a/run.py +++ b/run.py @@ -4,6 +4,14 @@ from flask_session import Session from cachelib.file import FileSystemCache import os +password = os.environ.get('MM2_DATABASE_PASSWORD') +if password: + user = os.environ.get('DB_USER') + host = os.environ.get('DB_HOST') + port = os.environ.get('DB_PORT') + name = os.environ.get('DB_NAME') + os.environ["MM2_SQLALCHEMY_DATABASE_URI"] = f"postgresql://{user}:{password}@{host}:{port}/{name}" + application = create_app() application.debug = os.environ.get("MM2_DEBUG", False) application.config['SESSION_TYPE'] = "cachelib" diff --git a/start-dev.sh b/start-dev.sh index cd9b601..ac702e8 100755 --- a/start-dev.sh +++ b/start-dev.sh @@ -2,10 +2,18 @@ POD=mirrormanager2 podman pod exists $POD || podman pod create -p 5000:5000 -n $POD +podman container exists postgres || podman run --pod $POD \ + --name postgres \ + --replace -d \ + -e POSTGRES_PASSWORD=mirrormanager \ + -e POSTGRES_DB=mirrormanager \ + -v pgdata:/var/lib/postgresql/data \ + docker.io/library/postgres:15 + test -d tmp || mkdir tmp test -f client_secrets.json || (echo "missing client_secrets" && exit 2) -podman build -t git.resf.org/infrastructure/mirrormanager2:dev -f Containerfile +#podman build -t git.resf.org/infrastructure/mirrormanager2:dev -f Containerfile podman rm --force mm2 -t 1 podman run \ --pod $POD \ @@ -14,14 +22,15 @@ podman run \ -e 'MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg' \ -e "MM2_SECRET_KEY=$(openssl rand -hex 32)" \ -e "MM2_PASSWORD_SEED=$(openssl rand -hex 32)" \ + -e 'MM2_SQLALCHEMY_DATABASE_URI=postgresql://postgres:mirrormanager@postgres:5432/mirrormanager' \ -e 'MM2_THEME_FOLDER=rocky' \ -e 'FLASK_DEBUG=1' \ -e 'MM2_DEBUG=1' \ -v $PWD/mirrormanager2.cfg:/etc/mirrormanager/mirrormanager2.cfg:z,ro \ -v $PWD/client_secrets.json:/etc/mirrormanager/client_secrets.json:z,ro \ -v $PWD/tmp:/var/tmp:z,rw \ - -v $PWD/static:/usr/lib/python3.12/site-packages/mirrormanager2/static:z,ro \ - -v $PWD/templates:/usr/lib/python3.12/site-packages/mirrormanager2/templates:z,ro \ + -v $PWD/static:/opt/app-root/lib/python3.12/site-packages/mirrormanager2/static:z,ro \ + -v $PWD/templates:/opt/app-root/lib/python3.12/site-packages/mirrormanager2/templates:z,ro \ -d git.resf.org/infrastructure/mirrormanager2:dev # Setup sqlite database diff --git a/static/rocky/rocky.css b/static/rocky/rocky.css index b229d01..4f05811 100644 --- a/static/rocky/rocky.css +++ b/static/rocky/rocky.css @@ -32,7 +32,8 @@ --bs-danger: #dc3545; --bs-light: #f8f9fa; --bs-dark: #212529; - --bs-primary-rgb: 60, 151, 214; + /*--bs-primary-rgb: 60, 151, 214;*/ + --bs-primary-rgb: 16, 185, 91; --bs-secondary-rgb: 108, 117, 125; --bs-success-rgb: 25, 135, 84; --bs-info-rgb: 13, 202, 240; diff --git a/templates/rocky/_macros.html b/templates/rocky/_macros.html index f459454..961390d 100644 --- a/templates/rocky/_macros.html +++ b/templates/rocky/_macros.html @@ -77,6 +77,7 @@ {% endset %}

mirrormanager + -- {{version}} -- Documentation -- Authors

@@ -277,3 +278,34 @@ href="{{ url_for('static', filename='mirrormanager2.css') }}"/> {{ form.csrf_token }} {% endmacro %} + + + {% macro pagination_bar(result) %} + {% if result.total_pages > 1 %} + + {% endif %} + {% endmacro %} diff --git a/templates/rocky/host.html b/templates/rocky/host.html index d783d86..48fdd8d 100644 --- a/templates/rocky/host.html +++ b/templates/rocky/host.html @@ -231,14 +231,26 @@ {% endif %} {% if category.directories %} -

Up-to-Date Directories this host carries

- +
+
+

+ +

+
+
+
    + {% for dir in category.directories %} + {% if dir.up2date %} +
  • {{ dir.path }}
    • + {% endif %} + {% endfor %} +
+
+
+
+
{% endif %} diff --git a/templates/rocky/index.html b/templates/rocky/index.html index 1cc0174..78594fb 100644 --- a/templates/rocky/index.html +++ b/templates/rocky/index.html @@ -34,8 +34,14 @@ I2 means both Internet2 and its peer high speed research and development networks globally.

+ - + + + + - {% for mirror in mirrors %} - + {% for mirror in mirrors.items %} + {% endfor %}
+ Mirror list filtering matrix +
Projects Versions diff --git a/templates/rocky/mirrors.html b/templates/rocky/mirrors.html index 10809ad..6366b91 100644 --- a/templates/rocky/mirrors.html +++ b/templates/rocky/mirrors.html @@ -1,5 +1,7 @@ {% extends "master.html" %} +{% from "_macros.html" import pagination_bar %} + {% block title %}Mirrors{% endblock %} {%block tag %}mirrors{% endblock %} @@ -29,8 +31,8 @@ Internet2 Comment
{{ mirror.country }} @@ -58,6 +60,7 @@
+{{pagination_bar(mirrors)}} {% else %}

There are currently no active mirrors registered.