update run.py to enable a common entrypoint for the container

update theme for latest mirrormanager
various fixes and updates for running in ECS, too
2024-12-20 13:32:11 -05:00 · 2024-12-20 12:30:23 -05:00
9 changed files with 218 additions and 44 deletions
--- a/73
+++ b/73
@ -1,5 +1,5 @@
 # Stage 1: Build stage with necessary build dependencies
-FROM quay.io/fedora/python-312:latest AS build-stage
+FROM quay.io/fedora/python-312:20240814 AS prebuild
 LABEL \
 name="python-312-with-rust" \
 vendor="Fedora Infrastructure" \
@ -19,6 +19,10 @@ RUN dnf install -y \
    libffi-devel \
    openssl-devel

+
+# Stage 2: Build mirrormanager2 python code
+FROM prebuild AS mirrormanager2-build
+
 # Clone MirrorManager2 source code from the Git repo
 RUN mkdir -p /opt/mirrormanager2
 WORKDIR /opt/mirrormanager2
@ -27,16 +31,38 @@ RUN git clone https://github.com/fedora-infra/mirrormanager2.git .
 RUN sed -e 's/signed_fpca/signed_rosca/' -i mirrormanager2/perms.py mirrormanager2/auth.py
 RUN pip install --prefix=/install .

-RUN pip install --prefix=/install flask_session
+RUN pip install --prefix=/install flask_session psycopg2

-# Stage 2: Final stage with runtime dependencies
-FROM quay.io/fedora/python-312:latest AS runtime
+# Stage 3: Build generate-mirrorlist-cache and mirrorlist-server
+FROM prebuild AS mirrorlist-build
+
+RUN mkdir -p /opt/mirrorlist-server
+WORKDIR /opt/mirrorlist-server
+RUN git clone https://github.com/adrianreber/mirrorlist-server.git .
+RUN echo -e '[profile.release-with-debug]\ninherits = "release"\ndebug = true\n' >> Cargo.toml
+
+# NOTE(neil): 20241217 #![deny(warnings)] causes deprecated/removed lints to be errors...
+RUN sed -i 's,#!\[deny(warnings)\],\#!\[allow(renamed_and_removed_lints)\],' src/bin/mirrorlist-server.rs src/bin/generate-mirrorlist-cache.rs
+
+RUN cargo build --profile=release-with-debug
+
+# Stage 4: Build scan-primary-mirror
+FROM prebuild AS scan-primary-mirror-build
+
+RUN mkdir -p /opt/scan-primary-mirror
+WORKDIR /opt/scan-primary-mirror
+RUN git clone https://github.com/adrianreber/scan-primary-mirror.git .
+RUN RUSTFLAGS=-g cargo build --release
+
+# Stage 5: Final stage with runtime dependencies
+FROM quay.io/fedora/python-312:20240814 AS runtime
 LABEL \
 name="python-312-with-rust" \
 vendor="Fedora Infrastructure" \
 license="MIT"

 USER root
+
 # Add only runtime dependencies
 RUN dnf install -y \
    python3-pyrpmmd \
@ -44,33 +70,41 @@ RUN dnf install -y \
    uwsgi-plugin-python3 \
    logrotate

-# Copy installed dependencies from the build stage
-COPY --from=build-stage /install /usr/
+# Copy installation of mirrormanager from it's build stage
+COPY --from=mirrormanager2-build /install /opt/app-root/

-# Copy in the tree
-#COPY --from=build-stage /opt/mirrormanager2 /opt/mirrormanager2
+# Copy mirrorlist-server and generate-mirrorlist-cache from mirrorlist-server build
+COPY --from=mirrorlist-build /opt/mirrorlist-server/target/release-with-debug/generate-mirrorlist-cache /usr/local/bin/
+COPY --from=mirrorlist-build /opt/mirrorlist-server/target/release-with-debug/mirrorlist-server /usr/local/bin/
+
+# Copy scan-primary-mirror from its build step
+COPY --from=scan-primary-mirror-build /opt/scan-primary-mirror/target/release/scan-primary-mirror /usr/local/bin/
+
+# Copy in the wsgi entry point
 ADD run.py /opt/mirrormanager2/

-FROM runtime AS database
-COPY client_secrets.json /etc/mirrormanager/
-COPY mirrormanager2.cfg /etc/mirrormanager/
-ENV MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg
-RUN /usr/bin/python3 -m flask -A mirrormanager2.app db sync
-
+# Stage 6: flatten :)
 FROM runtime as final
-COPY --from=database /var/tmp/mirrormanager2_dev.sqlite /var/tmp/mirrormanager2_dev.sqlite

-# Set working directory
+LABEL \
+name="python-312-with-rust-mirrormanager" \
+vendor="Rocky Linux Infrastructure" \
+license="MIT"
+
 WORKDIR /opt/mirrormanager2

-# Expose necessary ports
 EXPOSE 5000
+
+# @TODO(neil): probably this shouldn't be here? maybe SSM -> parse out in mm2.cfg?
 COPY client_secrets.json /etc/mirrormanager/
 COPY mirrormanager2.cfg /etc/mirrormanager/
+COPY static/rocky /opt/app-root/lib/python3.12/site-packages/mirrormanager2/static/rocky
+COPY templates/rocky /opt/app-root/lib/python3.12/site-packages/mirrormanager2/templates/rocky
+
 ENV MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg

-COPY static/rocky /usr/lib/python3.12/site-packages/mirrormanager2/static/rocky
-COPY templates/rocky /usr/lib/python3.12/site-packages/mirrormanager2/templates/rocky
+ENV SETUP_DB=false
+RUN test "$SETUP_DB" || python -m flask -A mirrormanager2.app db sync

 # Define entrypoint script to start the application
 CMD [ "uwsgi", "--socket", "0.0.0.0:5000", \
@ -80,5 +114,6 @@ CMD [ "uwsgi", "--socket", "0.0.0.0:5000", \
               "--enable-threads", \
               "--master", \
               "-b", "65535", \
+               "-H", "/opt/app-root/", \
               "--wsgi-file", "/opt/mirrormanager2/run.py" ]

--- a/mirrormanager2.cfg
+++ b/mirrormanager2.cfg
@ -10,7 +10,7 @@ import os


 # url to the database server:
-SQLALCHEMY_DATABASE_URI = 'sqlite:////var/tmp/mirrormanager2_dev.sqlite'
+SQLALCHEMY_DATABASE_URI = os.environ.get('MM2_SQLALCHEMY_DATABASE_URI', 'sqlite:////var/tmp/mirrormanager2_dev.sqlite')

 # the number of items to display on the search pages
 # Default: ``50``.
@ -21,7 +21,7 @@ SECRET_KEY = os.environ.get('MM2_SECRET_KEY')

 # Seed used to make the password harder to brute force in case of leaking
 # This should be kept really secret!
-PASSWORD_SEED = os.environ.get('MM2_PASSWORD_SEED')
+PASSWORD_SEED = os.environ.get('MM2_PASSWORD_SEED') # only valid for local auth 

 ###
 # Other configuration items for the web-app
@ -63,7 +63,7 @@ ADMIN_EMAIL = "infrastructure@rockylinux.org"

 # Email address used in the "From" field of the emails sent.
 # Default: ``nobody@fedoraproject.org``.
-#EMAIL_FROM = "nobody@fedoraproject.org"
+EMAIL_FROM = "nobody@rockylinux.org"

 # SMTP server to use,
 # Default: ``localhost``.
@ -74,7 +74,7 @@ ADMIN_EMAIL = "infrastructure@rockylinux.org"
 # SMTP_PASSWORD = 'password'

 # Countries which have to be excluded.
-#EMBARGOED_COUNTRIES = ["CU", "IR", "KP", "SD", "SY"]
+EMBARGOED_COUNTRIES = ["CU", "IR", "KP", "SD", "SY", "RU"]

 # When this is set to True, an additional menu item is shown which
 # displays the maps generated with mm2_generate-worldmap.
--- a/run.py
+++ b/run.py
@ -2,12 +2,88 @@ from werkzeug.middleware.proxy_fix import ProxyFix
 from mirrormanager2.app import create_app
 from flask_session import Session
 from cachelib.file import FileSystemCache
+from flask import Flask
 import os
+import sys
+import subprocess
+from threading import Thread

-application = create_app()
-application.debug = os.environ.get("MM2_DEBUG", False)
-application.config['SESSION_TYPE'] = "cachelib"
-application.config['SESSION_CACHELIB'] = FileSystemCache(cache_dir='/tmp/sessions', threshold=500)
-Session(application)
-application.wsgi_app = ProxyFix(application.wsgi_app, x_proto=1, x_host=1)

+def setup_env():
+    password = os.environ.get('MM2_DATABASE_PASSWORD')
+    if password:
+        user = os.environ.get('DB_USER')
+        host = os.environ.get('DB_HOST')
+        port = os.environ.get('DB_PORT')
+        name = os.environ.get('DB_NAME')
+        os.environ["MM2_SQLALCHEMY_DATABASE_URI"] = f"postgresql://{user}:{password}@{host}:{port}/{name}"
+
+
+def mirrormanager_wsgi() -> Flask:
+    application = create_app()
+    application.debug = os.environ.get("MM2_DEBUG", False)
+    application.config['SESSION_TYPE'] = "cachelib"
+    application.config['SESSION_CACHELIB'] = FileSystemCache(cache_dir='/mnt/efs/fs/0/tmp/sessions', threshold=500)
+    Session(application)
+    application.wsgi_app = ProxyFix(application.wsgi_app, x_proto=1, x_host=1)
+
+
+def execute(func: str) -> int:
+    def stream_output(pipe, output_stream):
+        while True:
+            data = pipe.read(1024)  # Read in chunks of 1024 bytes
+            if not data:
+                break
+            output_stream.write(data)
+            output_stream.flush()
+        pipe.close()
+
+    argv = func.split()
+    cmd = argv.pop(0)
+    args = argv if argv else []
+
+    print(f'running: {cmd} with args: {args}')
+
+    process = subprocess.Popen([cmd] + args,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE,
+                               bufsize=0)  # Unbuffered binary mode
+
+    stdout_thread = Thread(target=stream_output, args=(process.stdout, sys.stdout.buffer))
+    stderr_thread = Thread(target=stream_output, args=(process.stderr, sys.stderr.buffer))
+
+    stdout_thread.start()
+    stderr_thread.start()
+
+    process.wait()
+
+    stdout_thread.join()
+    stderr_thread.join()
+
+    return process.returncode
+
+
+if __name__ == "__main__":
+    setup_env()
+    # map cmd name to script/function
+    cmd_map = {
+        "check-propagation": "/usr/bin/echo undefined",
+        "crawl-mirrors": "/usr/bin/echo undefined",
+        "generate-worldmaps": "/opt/app-root/bin/mm2_generate-worldmap",
+        "scan-primary-mirror": "/usr/local/bin/scan-primary-mirror",
+        "update-clouds": "/usr/bin/echo undefined",
+        "update-geoip": "/usr/bin/echo undefined",
+        "generate-mirrorlist-cache": "/usr/local/bin/generate-mirrorlist-cache",
+        "update-netblocks": "/opt/app-root/bin/mm2_get-netblocks",
+        "update-EC2-netblocks": "/opt/app-root/bin/update-EC2-netblocks",
+        "mirrorlist-server": "/usr/local/bin/mirrorlist-server",
+        "mirrormanager-wsgi": mirrormanager_wsgi
+    }
+
+    arg = sys.argv[1] if len(sys.argv) > 1 else "mirrormanager-wsgi"
+    if arg and (cmd := cmd_map.get(arg, None)):
+        if not isinstance(cmd, str):
+            application = cmd()
+            exit(0)
+
+        sys.exit(execute(cmd))
--- a/start-dev.sh
+++ b/start-dev.sh
@ -2,10 +2,18 @@ POD=mirrormanager2

 podman pod exists $POD || podman pod create -p 5000:5000 -n $POD

+podman container exists postgres || podman run --pod $POD \
+	--name postgres \
+	--replace -d \
+	-e POSTGRES_PASSWORD=mirrormanager \
+	-e POSTGRES_DB=mirrormanager \
+	-v pgdata:/var/lib/postgresql/data \
+	docker.io/library/postgres:15
+
 test -d tmp || mkdir tmp
 test -f client_secrets.json || (echo "missing client_secrets" && exit 2)

-podman build -t git.resf.org/infrastructure/mirrormanager2:dev -f Containerfile
+#podman build -t git.resf.org/infrastructure/mirrormanager2:dev -f Containerfile
 podman rm --force mm2 -t 1
 podman run \
    --pod $POD \
@ -14,14 +22,15 @@ podman run \
    -e 'MM2_CONFIG=/etc/mirrormanager/mirrormanager2.cfg' \
    -e "MM2_SECRET_KEY=$(openssl rand -hex 32)" \
    -e "MM2_PASSWORD_SEED=$(openssl rand -hex 32)" \
+    -e 'MM2_SQLALCHEMY_DATABASE_URI=postgresql://postgres:mirrormanager@postgres:5432/mirrormanager' \
    -e 'MM2_THEME_FOLDER=rocky' \
    -e 'FLASK_DEBUG=1' \
    -e 'MM2_DEBUG=1' \
    -v $PWD/mirrormanager2.cfg:/etc/mirrormanager/mirrormanager2.cfg:z,ro \
    -v $PWD/client_secrets.json:/etc/mirrormanager/client_secrets.json:z,ro \
    -v $PWD/tmp:/var/tmp:z,rw \
-    -v $PWD/static:/usr/lib/python3.12/site-packages/mirrormanager2/static:z,ro \
-    -v $PWD/templates:/usr/lib/python3.12/site-packages/mirrormanager2/templates:z,ro \
+    -v $PWD/static:/opt/app-root/lib/python3.12/site-packages/mirrormanager2/static:z,ro \
+    -v $PWD/templates:/opt/app-root/lib/python3.12/site-packages/mirrormanager2/templates:z,ro \
    -d git.resf.org/infrastructure/mirrormanager2:dev

 # Setup sqlite database
--- a/static/rocky/rocky.css
+++ b/static/rocky/rocky.css
@ -32,7 +32,8 @@
    --bs-danger: #dc3545;
    --bs-light: #f8f9fa;
    --bs-dark: #212529;
-    --bs-primary-rgb: 60, 151, 214;
+    /*--bs-primary-rgb: 60, 151, 214;*/
+    --bs-primary-rgb: 16, 185, 91;
    --bs-secondary-rgb: 108, 117, 125;
    --bs-success-rgb: 25, 135, 84;
    --bs-info-rgb: 13, 202, 240;
--- a/templates/rocky/_macros.html
+++ b/templates/rocky/_macros.html
@ -77,6 +77,7 @@
                {% endset %}
                <p>
                <a href="https://github.com/fedora-infra/mirrormanager2/">mirrormanager</a>
+                -- {{version}}
                -- <a href="http://mirrormanager.rtfd.org" rel="noopener noreferrer"
                        target="_blank">Documentation</a>
                -- <a href="http://mirrormanager.readthedocs.org/en/latest/contributors.html">Authors</a></p>
@ -277,3 +278,34 @@ href="{{ url_for('static', filename='mirrormanager2.css') }}"/>
    {{ form.csrf_token }}
  </form>
    {% endmacro %}
+
+
+  {% macro pagination_bar(result) %}
+    {% if result.total_pages > 1 %}
+      <nav aria-label="Pagination">
+        <ul class="pagination pagination-sm justify-content-center my-4">
+          {# Page list #}
+          {% for page_number in result.truncated_pages_list(margin=6) %}
+            {% if page_number == result.page_number %}
+              <li class="page-item active" aria-current="page">
+                <span class="page-link">
+                  {{ page_number }}
+                  <span class="sr-only">(current)</span>
+                </span>
+              </li>
+            {% elif page_number == None %}
+            <li class="page-item disabled">
+              <span class="page-link">...</span>
+            </li>
+            {% else %}
+              <li class="page-item">
+                <a class="page-link" href="{{ result.page_url(page_number) }}">
+                  {{ page_number }}
+                </a>
+              </li>
+            {% endif %}
+          {% endfor %}
+        </ul>
+      </nav>
+    {% endif %}
+  {% endmacro %}
--- a/templates/rocky/host.html
+++ b/templates/rocky/host.html
@ -231,7 +231,15 @@
          </div>
          {% endif %}
          {% if category.directories %}
-          <h3>Up-to-Date Directories this host carries</h3>
+          <div class="accordion accordion-flush" id="accordion-category-{{category.id}}">
+            <div class="accordion-item">
+              <h2 class="accordion-header fw-bold" id="heading-category-{{category.id}}">
+                <button class="accordion-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#collapse-category-{{category.id}}" aria-expanded="false" aria-controls="flush-collapseOne">
+                  <strong>Up-to-Date Directories this host carries <span class="badge bg-primary"> {{category.directories|length}}</span></strong>
+                </button>
+              </h2>
+              <div id="collapse-category-{{category.id}}" class="accordion-collapse collapse" aria-labelledby="#heading-category-{{category.id}}" data-bs-parent="#accordion-category-{{category.id}}">
+                <div class="accordion-body">
                  <ul>
                    {% for dir in category.directories %}
                      {% if dir.up2date %}
@ -239,6 +247,10 @@
                      {% endif %}
                    {% endfor %}
                    </ul>
+                </div>
+              </div>
+            </div>
+          </div>
          {% endif %}

          </li>
--- a/templates/rocky/index.html
+++ b/templates/rocky/index.html
@ -34,8 +34,14 @@ I2 means both Internet2 and its peer high speed research and development
 networks globally.
 </p>

+
 <table class="table table-sm mt-5">
-<tr id="matrixheadings">
+<tr id="matrixtitle">
+  <th colspan="{{ arches | length + 2 }}">
+    Mirror list filtering matrix
+  </th>
+</tr>
+<tr id="matrixheadings" >
  <th>Projects</th>
  <th>Versions</th>
  <th colspan="{{ arches | length}}">
--- a/templates/rocky/mirrors.html
+++ b/templates/rocky/mirrors.html
@ -1,5 +1,7 @@
 {% extends "master.html" %}

+{% from "_macros.html" import pagination_bar %}
+
 {% block title %}Mirrors{% endblock %}
 {%block tag %}mirrors{% endblock %}

@ -29,8 +31,8 @@
    <th>Internet2</th>
    <th>Comment</th>
  </tr>
-  {% for mirror in mirrors %}
-  <tr>
+  {% for mirror in mirrors.items %}
+  <tr class="mirror-row">
    <td>{{ mirror.country }}</td>
    <td>
      <a href="{{ mirror.site.org_url }}">
@ -58,6 +60,7 @@
  </tr>
  {% endfor %}
 </table>
+{{pagination_bar(mirrors)}}
 {% else %}
 <p>
  There are currently no active mirrors registered.
Author	SHA1	Message	Date
Neil Hanlon	049d8ea920	update run.py to enable a common entrypoint for the container	2024-12-20 13:32:11 -05:00
Neil Hanlon	363191a9b5	update theme for latest mirrormanager various fixes and updates for running in ECS, too	2024-12-20 12:30:23 -05:00