Containerfile

@@ -1,5 +1,5 @@
 # Stage 1: Build stage with necessary build dependencies
-FROM quay.io/fedora/python-312:20240814 AS prebuild
+FROM quay.io/fedora/python-312:latest AS build-stage
 LABEL \
 name="python-312-with-rust" \
 vendor="Fedora Infrastructure" \
@@ -19,10 +19,6 @@ RUN dnf install -y \
     libffi-devel \
     openssl-devel
 
-
-# Stage 2: Build mirrormanager2 python code
-FROM prebuild AS mirrormanager2-build
-
 # Clone MirrorManager2 source code from the Git repo
 RUN mkdir -p /opt/mirrormanager2
 WORKDIR /opt/mirrormanager2
@@ -31,38 +27,16 @@ RUN git clone https://github.com/fedora-infra/mirrormanager2.git .
 RUN sed -e 's/signed_fpca/signed_rosca/' -i mirrormanager2/perms.py mirrormanager2/auth.py
 RUN pip install --prefix=/install .
 
-RUN pip install --prefix=/install flask_session psycopg2
+RUN pip install --prefix=/install flask_session
 
-# Stage 3: Build generate-mirrorlist-cache and mirrorlist-server
+# Stage 2: Final stage with runtime dependencies
-FROM prebuild AS mirrorlist-build
+FROM quay.io/fedora/python-312:latest AS runtime
-
-RUN mkdir -p /opt/mirrorlist-server
-WORKDIR /opt/mirrorlist-server
-RUN git clone https://github.com/adrianreber/mirrorlist-server.git .
-RUN echo -e '[profile.release-with-debug]\ninherits = "release"\ndebug = true\n' >> Cargo.toml
-
-# NOTE(neil): 20241217 #![deny(warnings)] causes deprecated/removed lints to be errors...
-RUN sed -i 's,#!\[deny(warnings)\],\#!\[allow(renamed_and_removed_lints)\],' src/bin/mirrorlist-server.rs src/bin/generate-mirrorlist-cache.rs
-
-RUN cargo build --profile=release-with-debug
-
-# Stage 4: Build scan-primary-mirror
-FROM prebuild AS scan-primary-mirror-build
-
-RUN mkdir -p /opt/scan-primary-mirror
-WORKDIR /opt/scan-primary-mirror
-RUN git clone https://github.com/adrianreber/scan-primary-mirror.git .
-RUN RUSTFLAGS=-g cargo build --release
-
-# Stage 5: Final stage with runtime dependencies
-FROM quay.io/fedora/python-312:20240814 AS runtime
 LABEL \
 name="python-312-with-rust" \
 vendor="Fedora Infrastructure" \
 license="MIT"
 
 USER root
-
 # Add only runtime dependencies
 RUN dnf install -y \
     python3-pyrpmmd \
@@ -70,41 +44,33 @@ RUN dnf install -y \
     uwsgi-plugin-python3 \
     logrotate
 
-# Copy installation of mirrormanager from it's build stage
+# Copy installed dependencies from the build stage
-COPY --from=mirrormanager2-build /install /opt/app-root/
+COPY --from=build-stage /install /usr/
 
-# Copy mirrorlist-server and generate-mirrorlist-cache from mirrorlist-server build
+# Copy in the tree
-COPY --from=mirrorlist-build /opt/mirrorlist-server/target/release-with-debug/generate-mirrorlist-cache /usr/local/bin/
+#COPY --from=build-stage /opt/mirrormanager2 /opt/mirrormanager2
-COPY --from=mirrorlist-build /opt/mirrorlist-server/target/release-with-debug/mirrorlist-server /usr/local/bin/
-
-# Copy scan-primary-mirror from its build step
-COPY --from=scan-primary-mirror-build /opt/scan-primary-mirror/target/release/scan-primary-mirror /usr/local/bin/
-
-# Copy in the wsgi entry point
 ADD run.py /opt/mirrormanager2/
 
-# Stage 6: flatten :)
+FROM runtime AS database
-FROM runtime as final
-
-LABEL \
-name="python-312-with-rust-mirrormanager" \
-vendor="Rocky Linux Infrastructure" \
-license="MIT"
-
-WORKDIR /opt/mirrormanager2
-
-EXPOSE 5000
-
-# @TODO(neil): probably this shouldn't be here? maybe SSM -> parse out in mm2.cfg?
 COPY client_secrets.json /etc/mirrormanager/
 COPY mirrormanager2.cfg /etc/mirrormanager/
-COPY static/rocky /opt/app-root/lib/python3.12/site-packages/mirrormanager2/static/rocky
+ENV MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg
-COPY templates/rocky /opt/app-root/lib/python3.12/site-packages/mirrormanager2/templates/rocky
+RUN /usr/bin/python3 -m flask -A mirrormanager2.app db sync
 
+FROM runtime as final
+COPY --from=database /var/tmp/mirrormanager2_dev.sqlite /var/tmp/mirrormanager2_dev.sqlite
+
+# Set working directory
+WORKDIR /opt/mirrormanager2
+
+# Expose necessary ports
+EXPOSE 5000
+COPY client_secrets.json /etc/mirrormanager/
+COPY mirrormanager2.cfg /etc/mirrormanager/
 ENV MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg
 
-ENV SETUP_DB=false
+COPY static/rocky /usr/lib/python3.12/site-packages/mirrormanager2/static/rocky
-RUN test "$SETUP_DB" || python -m flask -A mirrormanager2.app db sync
+COPY templates/rocky /usr/lib/python3.12/site-packages/mirrormanager2/templates/rocky
 
 # Define entrypoint script to start the application
 CMD [ "uwsgi", "--socket", "0.0.0.0:5000", \
@@ -114,6 +80,5 @@ CMD [ "uwsgi", "--socket", "0.0.0.0:5000", \
                "--enable-threads", \
                "--master", \
                "-b", "65535", \
-               "-H", "/opt/app-root/", \
                "--wsgi-file", "/opt/mirrormanager2/run.py" ]
 

mirrormanager2.cfg

@@ -10,7 +10,7 @@ import os
 
 
 # url to the database server:
-SQLALCHEMY_DATABASE_URI = os.environ.get('MM2_SQLALCHEMY_DATABASE_URI', 'sqlite:////var/tmp/mirrormanager2_dev.sqlite')
+SQLALCHEMY_DATABASE_URI = 'sqlite:////var/tmp/mirrormanager2_dev.sqlite'
 
 # the number of items to display on the search pages
 # Default: ``50``.
@@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('MM2_SECRET_KEY')
 
 # Seed used to make the password harder to brute force in case of leaking
 # This should be kept really secret!
-PASSWORD_SEED = os.environ.get('MM2_PASSWORD_SEED') # only valid for local auth 
+PASSWORD_SEED = os.environ.get('MM2_PASSWORD_SEED')
 
 ###
 # Other configuration items for the web-app
@@ -63,7 +63,7 @@ ADMIN_EMAIL = "infrastructure@rockylinux.org"
 
 # Email address used in the "From" field of the emails sent.
 # Default: ``nobody@fedoraproject.org``.
-EMAIL_FROM = "nobody@rockylinux.org"
+#EMAIL_FROM = "nobody@fedoraproject.org"
 
 # SMTP server to use,
 # Default: ``localhost``.
@@ -74,7 +74,7 @@ EMAIL_FROM = "nobody@rockylinux.org"
 # SMTP_PASSWORD = 'password'
 
 # Countries which have to be excluded.
-EMBARGOED_COUNTRIES = ["CU", "IR", "KP", "SD", "SY", "RU"]
+#EMBARGOED_COUNTRIES = ["CU", "IR", "KP", "SD", "SY"]
 
 # When this is set to True, an additional menu item is shown which
 # displays the maps generated with mm2_generate-worldmap.

run.py

@@ -2,88 +2,12 @@ from werkzeug.middleware.proxy_fix import ProxyFix
 from mirrormanager2.app import create_app
 from flask_session import Session
 from cachelib.file import FileSystemCache
-from flask import Flask
 import os
-import sys
-import subprocess
-from threading import Thread
 
+application = create_app()
+application.debug = os.environ.get("MM2_DEBUG", False)
+application.config['SESSION_TYPE'] = "cachelib"
+application.config['SESSION_CACHELIB'] = FileSystemCache(cache_dir='/tmp/sessions', threshold=500)
+Session(application)
+application.wsgi_app = ProxyFix(application.wsgi_app, x_proto=1, x_host=1)
 
-def setup_env():
-    password = os.environ.get('MM2_DATABASE_PASSWORD')
-    if password:
-        user = os.environ.get('DB_USER')
-        host = os.environ.get('DB_HOST')
-        port = os.environ.get('DB_PORT')
-        name = os.environ.get('DB_NAME')
-        os.environ["MM2_SQLALCHEMY_DATABASE_URI"] = f"postgresql://{user}:{password}@{host}:{port}/{name}"
-
-
-def mirrormanager_wsgi() -> Flask:
-    application = create_app()
-    application.debug = os.environ.get("MM2_DEBUG", False)
-    application.config['SESSION_TYPE'] = "cachelib"
-    application.config['SESSION_CACHELIB'] = FileSystemCache(cache_dir='/mnt/efs/fs/0/tmp/sessions', threshold=500)
-    Session(application)
-    application.wsgi_app = ProxyFix(application.wsgi_app, x_proto=1, x_host=1)
-
-
-def execute(func: str) -> int:
-    def stream_output(pipe, output_stream):
-        while True:
-            data = pipe.read(1024)  # Read in chunks of 1024 bytes
-            if not data:
-                break
-            output_stream.write(data)
-            output_stream.flush()
-        pipe.close()
-
-    argv = func.split()
-    cmd = argv.pop(0)
-    args = argv if argv else []
-
-    print(f'running: {cmd} with args: {args}')
-
-    process = subprocess.Popen([cmd] + args,
-                               stdout=subprocess.PIPE,
-                               stderr=subprocess.PIPE,
-                               bufsize=0)  # Unbuffered binary mode
-
-    stdout_thread = Thread(target=stream_output, args=(process.stdout, sys.stdout.buffer))
-    stderr_thread = Thread(target=stream_output, args=(process.stderr, sys.stderr.buffer))
-
-    stdout_thread.start()
-    stderr_thread.start()
-
-    process.wait()
-
-    stdout_thread.join()
-    stderr_thread.join()
-
-    return process.returncode
-
-
-if __name__ == "__main__":
-    setup_env()
-    # map cmd name to script/function
-    cmd_map = {
-        "check-propagation": "/usr/bin/echo undefined",
-        "crawl-mirrors": "/usr/bin/echo undefined",
-        "generate-worldmaps": "/opt/app-root/bin/mm2_generate-worldmap",
-        "scan-primary-mirror": "/usr/local/bin/scan-primary-mirror",
-        "update-clouds": "/usr/bin/echo undefined",
-        "update-geoip": "/usr/bin/echo undefined",
-        "generate-mirrorlist-cache": "/usr/local/bin/generate-mirrorlist-cache",
-        "update-netblocks": "/opt/app-root/bin/mm2_get-netblocks",
-        "update-EC2-netblocks": "/opt/app-root/bin/update-EC2-netblocks",
-        "mirrorlist-server": "/usr/local/bin/mirrorlist-server",
-        "mirrormanager-wsgi": mirrormanager_wsgi
-    }
-
-    arg = sys.argv[1] if len(sys.argv) > 1 else "mirrormanager-wsgi"
-    if arg and (cmd := cmd_map.get(arg, None)):
-        if not isinstance(cmd, str):
-            application = cmd()
-            exit(0)
-
-        sys.exit(execute(cmd))

start-dev.sh

@@ -2,18 +2,10 @@ POD=mirrormanager2
 
 podman pod exists $POD || podman pod create -p 5000:5000 -n $POD
 
-podman container exists postgres || podman run --pod $POD \
-	--name postgres \
-	--replace -d \
-	-e POSTGRES_PASSWORD=mirrormanager \
-	-e POSTGRES_DB=mirrormanager \
-	-v pgdata:/var/lib/postgresql/data \
-	docker.io/library/postgres:15
-
 test -d tmp || mkdir tmp
 test -f client_secrets.json || (echo "missing client_secrets" && exit 2)
 
-#podman build -t git.resf.org/infrastructure/mirrormanager2:dev -f Containerfile
+podman build -t git.resf.org/infrastructure/mirrormanager2:dev -f Containerfile
 podman rm --force mm2 -t 1
 podman run \
     --pod $POD \
@@ -22,15 +14,14 @@ podman run \
     -e 'MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg' \
     -e "MM2_SECRET_KEY=$(openssl rand -hex 32)" \
     -e "MM2_PASSWORD_SEED=$(openssl rand -hex 32)" \
-    -e 'MM2_SQLALCHEMY_DATABASE_URI=postgresql://postgres:mirrormanager@postgres:5432/mirrormanager' \
     -e 'MM2_THEME_FOLDER=rocky' \
     -e 'FLASK_DEBUG=1' \
     -e 'MM2_DEBUG=1' \
     -v $PWD/mirrormanager2.cfg:/etc/mirrormanager/mirrormanager2.cfg:z,ro \
     -v $PWD/client_secrets.json:/etc/mirrormanager/client_secrets.json:z,ro \
     -v $PWD/tmp:/var/tmp:z,rw \
-    -v $PWD/static:/opt/app-root/lib/python3.12/site-packages/mirrormanager2/static:z,ro \
+    -v $PWD/static:/usr/lib/python3.12/site-packages/mirrormanager2/static:z,ro \
-    -v $PWD/templates:/opt/app-root/lib/python3.12/site-packages/mirrormanager2/templates:z,ro \
+    -v $PWD/templates:/usr/lib/python3.12/site-packages/mirrormanager2/templates:z,ro \
     -d git.resf.org/infrastructure/mirrormanager2:dev
 
 # Setup sqlite database

static/rocky/rocky.css

@@ -32,8 +32,7 @@
     --bs-danger: #dc3545;
     --bs-light: #f8f9fa;
     --bs-dark: #212529;
-    /*--bs-primary-rgb: 60, 151, 214;*/
+    --bs-primary-rgb: 60, 151, 214;
-    --bs-primary-rgb: 16, 185, 91;
     --bs-secondary-rgb: 108, 117, 125;
     --bs-success-rgb: 25, 135, 84;
     --bs-info-rgb: 13, 202, 240;

templates/rocky/_macros.html

@@ -77,7 +77,6 @@
                 {% endset %}
                 <p>
                 <a href="https://github.com/fedora-infra/mirrormanager2/">mirrormanager</a>
-                -- {{version}}
                 -- <a href="http://mirrormanager.rtfd.org" rel="noopener noreferrer"
                         target="_blank">Documentation</a>
                 -- <a href="http://mirrormanager.readthedocs.org/en/latest/contributors.html">Authors</a></p>
@@ -278,34 +277,3 @@ href="{{ url_for('static', filename='mirrormanager2.css') }}"/>
     {{ form.csrf_token }}
   </form>
     {% endmacro %}
-
-
-  {% macro pagination_bar(result) %}
-    {% if result.total_pages > 1 %}
-      <nav aria-label="Pagination">
-        <ul class="pagination pagination-sm justify-content-center my-4">
-          {# Page list #}
-          {% for page_number in result.truncated_pages_list(margin=6) %}
-            {% if page_number == result.page_number %}
-              <li class="page-item active" aria-current="page">
-                <span class="page-link">
-                  {{ page_number }}
-                  <span class="sr-only">(current)</span>
-                </span>
-              </li>
-            {% elif page_number == None %}
-            <li class="page-item disabled">
-              <span class="page-link">...</span>
-            </li>
-            {% else %}
-              <li class="page-item">
-                <a class="page-link" href="{{ result.page_url(page_number) }}">
-                  {{ page_number }}
-                </a>
-              </li>
-            {% endif %}
-          {% endfor %}
-        </ul>
-      </nav>
-    {% endif %}
-  {% endmacro %}

templates/rocky/host.html

@@ -231,26 +231,14 @@
           </div>
           {% endif %}
           {% if category.directories %}
-          <div class="accordion accordion-flush" id="accordion-category-{{category.id}}">
+          <h3>Up-to-Date Directories this host carries</h3>
-            <div class="accordion-item">
+          <ul>
-              <h2 class="accordion-header fw-bold" id="heading-category-{{category.id}}">
+          {% for dir in category.directories %}
-                <button class="accordion-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapse-category-{{category.id}}" aria-expanded="false" aria-controls="flush-collapseOne">
+            {% if dir.up2date %}
-                  <strong>Up-to-Date Directories this host carries <span class="badge bg-primary"> {{category.directories|length}}</span></strong>
+              <li>{{ dir.path }}</li>
-                </button>
+            {% endif %}
-              </h2>
+          {% endfor %}
-              <div id="collapse-category-{{category.id}}" class="accordion-collapse collapse" aria-labelledby="#heading-category-{{category.id}}" data-bs-parent="#accordion-category-{{category.id}}">
+          </ul>
-                <div class="accordion-body">
-                  <ul>
-                    {% for dir in category.directories %}
-                      {% if dir.up2date %}
-                        <li>{{ dir.path }}</li>
-                      {% endif %}
-                    {% endfor %}
-                    </ul>
-                </div>
-              </div>
-            </div>
-          </div>
           {% endif %}
 
           </li>

templates/rocky/index.html

@@ -34,14 +34,8 @@ I2 means both Internet2 and its peer high speed research and development
 networks globally.
 </p>
 
-
 <table class="table table-sm mt-5">
-<tr id="matrixtitle">
+<tr id="matrixheadings">
-  <th colspan="{{ arches | length + 2 }}">
-    Mirror list filtering matrix
-  </th>
-</tr>
-<tr id="matrixheadings" >
   <th>Projects</th>
   <th>Versions</th>
   <th colspan="{{ arches | length}}">

templates/rocky/mirrors.html

@@ -1,7 +1,5 @@
 {% extends "master.html" %}
 
-{% from "_macros.html" import pagination_bar %}
-
 {% block title %}Mirrors{% endblock %}
 {%block tag %}mirrors{% endblock %}
 
@@ -31,8 +29,8 @@
     <th>Internet2</th>
     <th>Comment</th>
   </tr>
-  {% for mirror in mirrors.items %}
+  {% for mirror in mirrors %}
-  <tr class="mirror-row">
+  <tr>
     <td>{{ mirror.country }}</td>
     <td>
       <a href="{{ mirror.site.org_url }}">
@@ -60,7 +58,6 @@
   </tr>
   {% endfor %}
 </table>
-{{pagination_bar(mirrors)}}
 {% else %}
 <p>
   There are currently no active mirrors registered.