update theme for latest mirrormanager

various fixes and updates for running in ECS, too

Containerfile

@@ -1,5 +1,5 @@
 # Stage 1: Build stage with necessary build dependencies
-FROM quay.io/fedora/python-312:latest AS build-stage
+FROM quay.io/fedora/python-312:20240814 AS prebuild
 LABEL \
 name="python-312-with-rust" \
 vendor="Fedora Infrastructure" \
@@ -19,6 +19,10 @@ RUN dnf install -y \
     libffi-devel \
     openssl-devel
 
+
+# Stage 2: Build mirrormanager2 python code
+FROM prebuild AS mirrormanager2-build
+
 # Clone MirrorManager2 source code from the Git repo
 RUN mkdir -p /opt/mirrormanager2
 WORKDIR /opt/mirrormanager2
@@ -27,16 +31,38 @@ RUN git clone https://github.com/fedora-infra/mirrormanager2.git .
 RUN sed -e 's/signed_fpca/signed_rosca/' -i mirrormanager2/perms.py mirrormanager2/auth.py
 RUN pip install --prefix=/install .
 
-RUN pip install --prefix=/install flask_session
+RUN pip install --prefix=/install flask_session psycopg2
 
-# Stage 2: Final stage with runtime dependencies
+# Stage 3: Build generate-mirrorlist-cache and mirrorlist-server
-FROM quay.io/fedora/python-312:latest AS runtime
+FROM prebuild AS mirrorlist-build
+
+RUN mkdir -p /opt/mirrorlist-server
+WORKDIR /opt/mirrorlist-server
+RUN git clone https://github.com/adrianreber/mirrorlist-server.git .
+RUN echo -e '[profile.release-with-debug]\ninherits = "release"\ndebug = true\n' >> Cargo.toml
+
+# NOTE(neil): 20241217 #![deny(warnings)] causes deprecated/removed lints to be errors...
+RUN sed -i 's,#!\[deny(warnings)\],\#!\[allow(renamed_and_removed_lints)\],' src/bin/mirrorlist-server.rs src/bin/generate-mirrorlist-cache.rs
+
+RUN cargo build --profile=release-with-debug
+
+# Stage 4: Build scan-primary-mirror
+FROM prebuild AS scan-primary-mirror-build
+
+RUN mkdir -p /opt/scan-primary-mirror
+WORKDIR /opt/scan-primary-mirror
+RUN git clone https://github.com/adrianreber/scan-primary-mirror.git .
+RUN RUSTFLAGS=-g cargo build --release
+
+# Stage 5: Final stage with runtime dependencies
+FROM quay.io/fedora/python-312:20240814 AS runtime
 LABEL \
 name="python-312-with-rust" \
 vendor="Fedora Infrastructure" \
 license="MIT"
 
 USER root
+
 # Add only runtime dependencies
 RUN dnf install -y \
     python3-pyrpmmd \
@@ -44,33 +70,41 @@ RUN dnf install -y \
     uwsgi-plugin-python3 \
     logrotate
 
-# Copy installed dependencies from the build stage
+# Copy installation of mirrormanager from it's build stage
-COPY --from=build-stage /install /usr/
+COPY --from=mirrormanager2-build /install /opt/app-root/
 
-# Copy in the tree
+# Copy mirrorlist-server and generate-mirrorlist-cache from mirrorlist-server build
-#COPY --from=build-stage /opt/mirrormanager2 /opt/mirrormanager2
+COPY --from=mirrorlist-build /opt/mirrorlist-server/target/release-with-debug/generate-mirrorlist-cache /usr/local/bin/
+COPY --from=mirrorlist-build /opt/mirrorlist-server/target/release-with-debug/mirrorlist-server /usr/local/bin/
+
+# Copy scan-primary-mirror from its build step
+COPY --from=scan-primary-mirror-build /opt/scan-primary-mirror/target/release/scan-primary-mirror /usr/local/bin/
+
+# Copy in the wsgi entry point
 ADD run.py /opt/mirrormanager2/
 
-FROM runtime AS database
+# Stage 6: flatten :)
-COPY client_secrets.json /etc/mirrormanager/
-COPY mirrormanager2.cfg /etc/mirrormanager/
-ENV MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg
-RUN /usr/bin/python3 -m flask -A mirrormanager2.app db sync
-
 FROM runtime as final
-COPY --from=database /var/tmp/mirrormanager2_dev.sqlite /var/tmp/mirrormanager2_dev.sqlite
 
-# Set working directory
+LABEL \
+name="python-312-with-rust-mirrormanager" \
+vendor="Rocky Linux Infrastructure" \
+license="MIT"
+
 WORKDIR /opt/mirrormanager2
 
-# Expose necessary ports
 EXPOSE 5000
+
+# @TODO(neil): probably this shouldn't be here? maybe SSM -> parse out in mm2.cfg?
 COPY client_secrets.json /etc/mirrormanager/
 COPY mirrormanager2.cfg /etc/mirrormanager/
+COPY static/rocky /opt/app-root/lib/python3.12/site-packages/mirrormanager2/static/rocky
+COPY templates/rocky /opt/app-root/lib/python3.12/site-packages/mirrormanager2/templates/rocky
+
 ENV MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg
 
-COPY static/rocky /usr/lib/python3.12/site-packages/mirrormanager2/static/rocky
+ENV SETUP_DB=false
-COPY templates/rocky /usr/lib/python3.12/site-packages/mirrormanager2/templates/rocky
+RUN test "$SETUP_DB" || python -m flask -A mirrormanager2.app db sync
 
 # Define entrypoint script to start the application
 CMD [ "uwsgi", "--socket", "0.0.0.0:5000", \
@@ -80,5 +114,6 @@ CMD [ "uwsgi", "--socket", "0.0.0.0:5000", \
                "--enable-threads", \
                "--master", \
                "-b", "65535", \
+               "-H", "/opt/app-root/", \
                "--wsgi-file", "/opt/mirrormanager2/run.py" ]
 

mirrormanager2.cfg

@@ -10,7 +10,7 @@ import os
 
 
 # url to the database server:
-SQLALCHEMY_DATABASE_URI = 'sqlite:////var/tmp/mirrormanager2_dev.sqlite'
+SQLALCHEMY_DATABASE_URI = os.environ.get('MM2_SQLALCHEMY_DATABASE_URI', 'sqlite:////var/tmp/mirrormanager2_dev.sqlite')
 
 # the number of items to display on the search pages
 # Default: ``50``.
@@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('MM2_SECRET_KEY')
 
 # Seed used to make the password harder to brute force in case of leaking
 # This should be kept really secret!
-PASSWORD_SEED = os.environ.get('MM2_PASSWORD_SEED')
+PASSWORD_SEED = os.environ.get('MM2_PASSWORD_SEED') # only valid for local auth 
 
 ###
 # Other configuration items for the web-app
@@ -63,7 +63,7 @@ ADMIN_EMAIL = "infrastructure@rockylinux.org"
 
 # Email address used in the "From" field of the emails sent.
 # Default: ``nobody@fedoraproject.org``.
-#EMAIL_FROM = "nobody@fedoraproject.org"
+EMAIL_FROM = "nobody@rockylinux.org"
 
 # SMTP server to use,
 # Default: ``localhost``.
@@ -74,7 +74,7 @@ ADMIN_EMAIL = "infrastructure@rockylinux.org"
 # SMTP_PASSWORD = 'password'
 
 # Countries which have to be excluded.
-#EMBARGOED_COUNTRIES = ["CU", "IR", "KP", "SD", "SY"]
+EMBARGOED_COUNTRIES = ["CU", "IR", "KP", "SD", "SY", "RU"]
 
 # When this is set to True, an additional menu item is shown which
 # displays the maps generated with mm2_generate-worldmap.

run.py

@@ -4,6 +4,14 @@ from flask_session import Session
 from cachelib.file import FileSystemCache
 import os
 
+password = os.environ.get('MM2_DATABASE_PASSWORD')
+if password:
+    user = os.environ.get('DB_USER')
+    host = os.environ.get('DB_HOST')
+    port = os.environ.get('DB_PORT')
+    name = os.environ.get('DB_NAME')
+    os.environ["MM2_SQLALCHEMY_DATABASE_URI"] = f"postgresql://{user}:{password}@{host}:{port}/{name}"
+
 application = create_app()
 application.debug = os.environ.get("MM2_DEBUG", False)
 application.config['SESSION_TYPE'] = "cachelib"

start-dev.sh

@@ -2,10 +2,18 @@ POD=mirrormanager2
 
 podman pod exists $POD || podman pod create -p 5000:5000 -n $POD
 
+podman container exists postgres || podman run --pod $POD \
+	--name postgres \
+	--replace -d \
+	-e POSTGRES_PASSWORD=mirrormanager \
+	-e POSTGRES_DB=mirrormanager \
+	-v pgdata:/var/lib/postgresql/data \
+	docker.io/library/postgres:15
+
 test -d tmp || mkdir tmp
 test -f client_secrets.json || (echo "missing client_secrets" && exit 2)
 
-podman build -t git.resf.org/infrastructure/mirrormanager2:dev -f Containerfile
+#podman build -t git.resf.org/infrastructure/mirrormanager2:dev -f Containerfile
 podman rm --force mm2 -t 1
 podman run \
     --pod $POD \
@@ -14,14 +22,15 @@ podman run \
     -e 'MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg' \
     -e "MM2_SECRET_KEY=$(openssl rand -hex 32)" \
     -e "MM2_PASSWORD_SEED=$(openssl rand -hex 32)" \
+    -e 'MM2_SQLALCHEMY_DATABASE_URI=postgresql://postgres:mirrormanager@postgres:5432/mirrormanager' \
     -e 'MM2_THEME_FOLDER=rocky' \
     -e 'FLASK_DEBUG=1' \
     -e 'MM2_DEBUG=1' \
     -v $PWD/mirrormanager2.cfg:/etc/mirrormanager/mirrormanager2.cfg:z,ro \
     -v $PWD/client_secrets.json:/etc/mirrormanager/client_secrets.json:z,ro \
     -v $PWD/tmp:/var/tmp:z,rw \
-    -v $PWD/static:/usr/lib/python3.12/site-packages/mirrormanager2/static:z,ro \
+    -v $PWD/static:/opt/app-root/lib/python3.12/site-packages/mirrormanager2/static:z,ro \
-    -v $PWD/templates:/usr/lib/python3.12/site-packages/mirrormanager2/templates:z,ro \
+    -v $PWD/templates:/opt/app-root/lib/python3.12/site-packages/mirrormanager2/templates:z,ro \
     -d git.resf.org/infrastructure/mirrormanager2:dev
 
 # Setup sqlite database

static/rocky/rocky.css

@@ -32,7 +32,8 @@
     --bs-danger: #dc3545;
     --bs-light: #f8f9fa;
     --bs-dark: #212529;
-    --bs-primary-rgb: 60, 151, 214;
+    /*--bs-primary-rgb: 60, 151, 214;*/
+    --bs-primary-rgb: 16, 185, 91;
     --bs-secondary-rgb: 108, 117, 125;
     --bs-success-rgb: 25, 135, 84;
     --bs-info-rgb: 13, 202, 240;

templates/rocky/_macros.html

@@ -77,6 +77,7 @@
                 {% endset %}
                 <p>
                 <a href="https://github.com/fedora-infra/mirrormanager2/">mirrormanager</a>
+                -- {{version}}
                 -- <a href="http://mirrormanager.rtfd.org" rel="noopener noreferrer"
                         target="_blank">Documentation</a>
                 -- <a href="http://mirrormanager.readthedocs.org/en/latest/contributors.html">Authors</a></p>
@@ -277,3 +278,34 @@ href="{{ url_for('static', filename='mirrormanager2.css') }}"/>
     {{ form.csrf_token }}
   </form>
     {% endmacro %}
+
+
+  {% macro pagination_bar(result) %}
+    {% if result.total_pages > 1 %}
+      <nav aria-label="Pagination">
+        <ul class="pagination pagination-sm justify-content-center my-4">
+          {# Page list #}
+          {% for page_number in result.truncated_pages_list(margin=6) %}
+            {% if page_number == result.page_number %}
+              <li class="page-item active" aria-current="page">
+                <span class="page-link">
+                  {{ page_number }}
+                  <span class="sr-only">(current)</span>
+                </span>
+              </li>
+            {% elif page_number == None %}
+            <li class="page-item disabled">
+              <span class="page-link">...</span>
+            </li>
+            {% else %}
+              <li class="page-item">
+                <a class="page-link" href="{{ result.page_url(page_number) }}">
+                  {{ page_number }}
+                </a>
+              </li>
+            {% endif %}
+          {% endfor %}
+        </ul>
+      </nav>
+    {% endif %}
+  {% endmacro %}

templates/rocky/host.html

@@ -231,7 +231,15 @@
           </div>
           {% endif %}
           {% if category.directories %}
-          <h3>Up-to-Date Directories this host carries</h3>
+          <div class="accordion accordion-flush" id="accordion-category-{{category.id}}">
+            <div class="accordion-item">
+              <h2 class="accordion-header fw-bold" id="heading-category-{{category.id}}">
+                <button class="accordion-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapse-category-{{category.id}}" aria-expanded="false" aria-controls="flush-collapseOne">
+                  <strong>Up-to-Date Directories this host carries <span class="badge bg-primary"> {{category.directories|length}}</span></strong>
+                </button>
+              </h2>
+              <div id="collapse-category-{{category.id}}" class="accordion-collapse collapse" aria-labelledby="#heading-category-{{category.id}}" data-bs-parent="#accordion-category-{{category.id}}">
+                <div class="accordion-body">
                   <ul>
                     {% for dir in category.directories %}
                       {% if dir.up2date %}
@@ -239,6 +247,10 @@
                       {% endif %}
                     {% endfor %}
                     </ul>
+                </div>
+              </div>
+            </div>
+          </div>
           {% endif %}
 
           </li>

templates/rocky/index.html

@@ -34,7 +34,13 @@ I2 means both Internet2 and its peer high speed research and development
 networks globally.
 </p>
 
+
 <table class="table table-sm mt-5">
+<tr id="matrixtitle">
+  <th colspan="{{ arches | length + 2 }}">
+    Mirror list filtering matrix
+  </th>
+</tr>
 <tr id="matrixheadings" >
   <th>Projects</th>
   <th>Versions</th>

templates/rocky/mirrors.html

@@ -1,5 +1,7 @@
 {% extends "master.html" %}
 
+{% from "_macros.html" import pagination_bar %}
+
 {% block title %}Mirrors{% endblock %}
 {%block tag %}mirrors{% endblock %}
 
@@ -29,8 +31,8 @@
     <th>Internet2</th>
     <th>Comment</th>
   </tr>
-  {% for mirror in mirrors %}
+  {% for mirror in mirrors.items %}
-  <tr>
+  <tr class="mirror-row">
     <td>{{ mirror.country }}</td>
     <td>
       <a href="{{ mirror.site.org_url }}">
@@ -58,6 +60,7 @@
   </tr>
   {% endfor %}
 </table>
+{{pagination_bar(mirrors)}}
 {% else %}
 <p>
   There are currently no active mirrors registered.