From 5e1fde102723cdac8260e729ef5d21a8902e5877 Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Sat, 4 Feb 2023 00:37:45 +0100 Subject: [PATCH] Add apollo_tree and fix pylint warnings --- .github/workflows/test.yaml | 18 + .vscode/settings.json | 3 + apollo/publishing_tools/apollo_tree.py | 346 ++++++++++++ apollo/rherrata/__init__.py | 6 +- apollo/rherrata/example.py | 1 - apollo/rpmworker/repomd.py | 2 - apollo/rpmworker/rh_matcher_activities.py | 14 +- apollo/server/routes/advisories.py | 2 +- apollo/server/routes/api_advisories.py | 2 +- apollo/server/routes/api_compat.py | 10 +- apollo/server/server.py | 2 +- apollo/tests/BUILD.bazel | 0 apollo/tests/publishing_tools/BUILD.bazel | 0 .../data/appstream__base__repomd__aarch64.xml | 88 ++++ .../data/appstream__base__repomd__x86_64.xml | 88 ++++ .../data/baseos__base__repomd__aarch64.xml | 80 +++ .../data/baseos__base__repomd__x86_64.xml | 80 +++ ...__base__repomd__x86_64_with_updateinfo.xml | 80 +++ .../data/updateinfo__test__1.xml | 111 ++++ .../publishing_tools/test_apollo_tree.py | 498 ++++++++++++++++++ build/scripts/pylint.bash | 15 + build/scripts/test.bash | 5 + common/testing.py | 13 + deploy/apollo/apollo-rpmworker/values.yaml | 2 +- deploy/apollo/apollo-server/values.yaml | 8 +- gazelle_python.yaml | 19 +- requirements.txt | 5 +- requirements_lock.txt | 33 ++ 28 files changed, 1503 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/test.yaml create mode 100644 apollo/publishing_tools/apollo_tree.py create mode 100644 apollo/tests/BUILD.bazel create mode 100644 apollo/tests/publishing_tools/BUILD.bazel create mode 100644 apollo/tests/publishing_tools/data/appstream__base__repomd__aarch64.xml create mode 100644 apollo/tests/publishing_tools/data/appstream__base__repomd__x86_64.xml create mode 100644 apollo/tests/publishing_tools/data/baseos__base__repomd__aarch64.xml create mode 100644 apollo/tests/publishing_tools/data/baseos__base__repomd__x86_64.xml create mode 100644 apollo/tests/publishing_tools/data/baseos__base__repomd__x86_64_with_updateinfo.xml create mode 100644 apollo/tests/publishing_tools/data/updateinfo__test__1.xml create mode 100644 apollo/tests/publishing_tools/test_apollo_tree.py create mode 100755 build/scripts/pylint.bash create mode 100755 build/scripts/test.bash create mode 100644 common/testing.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..8dbe626 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,18 @@ +name: Lint and test + +on: + push: + branches: [ "main" ] + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Lint + run: ./build/scripts/pylint.bash + - name: Test + run: ./build/scripts/test.bash diff --git a/.vscode/settings.json b/.vscode/settings.json index c51e800..43c4b3e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -14,5 +14,8 @@ "editor.formatOnSave": true, "[python]": { "editor.tabSize": 4 + }, + "[xml]": { + "editor.formatOnSave": false } } diff --git a/apollo/publishing_tools/apollo_tree.py b/apollo/publishing_tools/apollo_tree.py new file mode 100644 index 0000000..344a36d --- /dev/null +++ b/apollo/publishing_tools/apollo_tree.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 + +import os +import argparse +import asyncio +import logging +import hashlib +import gzip +from dataclasses import dataclass +import time +from urllib.parse import quote +from xml.etree import ElementTree as ET + +import aiohttp + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("apollo_tree") + +NS = { + "": "http://linux.duke.edu/metadata/repo", + "rpm": "http://linux.duke.edu/metadata/rpm" +} + + +@dataclass +class Repository: + base_repomd: str = None + source_repomd: str = None + debug_repomd: str = None + arch: str = None + + +async def scan_path( + base_path: str, + fmt: str, + ignore_dirs: list[str], +): + """ + Scan base path for repositories + The format string can contain $reponame and $arch + When we reach $reponame, that means we have found a repository + Follow the path further into the tree until $arch is found + That determines the architecture of the repository + """ + + repos = {} + + # First we need to find the root. + # Construct root by prepending base_path and all parts until $reponame + # Then we can walk the tree from there + root = base_path + parts = fmt.split("/") + repo_first = True + if "$reponame" in parts: + for part in parts: + parts.pop(0) + if part == "$reponame": + break + if part == "$arch": + repo_first = False + break + root = os.path.join(root, part) + + logger.info("Found root: %s", root) + + # Walk the base path + for directory in os.listdir(root): + if directory in ignore_dirs: + continue + current_parts = parts + if repo_first: + repo_name = directory + logger.info("Found repo: %s", repo_name) + else: + arch = directory + logger.info("Found arch: %s", arch) + repo_base = os.path.join(root, directory) + + if repo_first: + repos[repo_name] = [] + + # Construct repo base until we reach $arch + if "$arch" in current_parts: + for part in current_parts: + if (part == "$arch" and repo_first) or part == "$reponame": + break + repo_base = os.path.join(repo_base, part) + current_parts.pop(0) + + logger.warning("Searching for arches in %s", repo_base) + + # All dirs in repo_base is an architecture + for arch_ in os.listdir(repo_base): + if repo_first: + arch = arch_ + else: + repo_name = arch_ + # Now append each combination + rest of parts as repo_info + if repo_first: + logger.info("Found arch: %s", arch) + else: + logger.info("Found repo: %s", repo_name) + + if repo_first: + found_path = f"{repo_base}/{arch}/{'/'.join(current_parts[1:])}" + else: + found_path = f"{repo_base}/{repo_name}/{'/'.join(current_parts[1:])}" + + # Verify that the path exists + if not os.path.exists(found_path): + logger.warning("Path does not exist: %s, skipping", found_path) + continue + + repo = { + "name": repo_name, + "arch": arch, + "found_path": found_path, + } + if repo_name not in repos: + repos[repo_name] = [] + repos[repo_name].append(repo) + + return repos + + +async def fetch_updateinfo_from_apollo( + repo: dict, + product_name: str, + arch: str = None, + api_base: str = None, +) -> str: + if not api_base: + api_base = "https://apollo.build.resf.org/api/v3/updateinfo" + api_url = f"{api_base}/{quote(product_name)}/{quote(repo['name'])}/updateinfo.xml" + if arch: + api_url += f"?req_arch={arch}" + + logger.info("Fetching updateinfo from %s", api_url) + async with aiohttp.ClientSession() as session: + async with session.get(api_url) as resp: + if resp.status != 200: + logger.error("Failed to fetch updateinfo from %s", api_url) + return None + return await resp.text() + + +async def gzip_updateinfo(updateinfo: str) -> dict: + # Gzip updateinfo, get both open and closed size as + # well as the sha256sum for both + + # First get the sha256sum and size of the open updateinfo + sha256sum = hashlib.sha256(updateinfo.encode("utf-8")).hexdigest() + size = len(updateinfo) + + # Then gzip it and get hash and size + gzipped = gzip.compress(updateinfo.encode("utf-8"), mtime=0) + gzipped_sha256sum = hashlib.sha256(gzipped).hexdigest() + gzipped_size = len(gzipped) + + return { + "sha256sum": sha256sum, + "size": size, + "gzipped_sha256sum": gzipped_sha256sum, + "gzipped_size": gzipped_size, + "gzipped": gzipped, + } + + +async def write_updateinfo_to_file( + repomd_xml_path: str, updateinfo: dict +) -> str: + # Write updateinfo to file + repomd_dir = os.path.dirname(repomd_xml_path) + gzipped_sum = updateinfo["gzipped_sha256sum"] + updateinfo_path = os.path.join( + repomd_dir, f"{gzipped_sum}-updateinfo.xml.gz" + ) + with open(updateinfo_path, "wb") as f: + f.write(updateinfo["gzipped"]) + + return updateinfo_path + + +async def update_repomd_xml(repomd_xml_path: str, updateinfo: dict): + # Update repomd.xml with new updateinfo + gzipped_sum = updateinfo["gzipped_sha256sum"] + updateinfo_path = f"{gzipped_sum}-updateinfo.xml.gz" + + # Parse repomd.xml + ET.register_namespace("", NS[""]) + ET.register_namespace("rpm", NS["rpm"]) + repomd_xml = ET.parse(repomd_xml_path).getroot() + + # Iterate over data and find type="updateinfo" and delete it + existing_updateinfo_path = None + for data in repomd_xml.findall("data", NS): + data_type = data.attrib["type"] + if not data_type: + logger.warning("No type found in data, skipping") + continue + if data_type == "updateinfo": + # Delete the data element + repomd_xml.remove(data) + + # Get the location of the updateinfo file + location = data.find("location", NS) + if not location: + logger.warning("No location found in data, skipping") + continue + existing_updateinfo_path = location.attrib["href"] + break + + # Create new data element and set type to updateinfo + data = ET.Element("data") + data.set("type", "updateinfo") + + # Add checksum, open-checksum, location, timestamp, size and open-size + checksum = ET.SubElement(data, "checksum") + checksum.set("type", "sha256") + checksum.text = updateinfo["gzipped_sha256sum"] + + open_checksum = ET.SubElement(data, "open-checksum") + open_checksum.set("type", "sha256") + open_checksum.text = updateinfo["sha256sum"] + + location = ET.SubElement(data, "location") + location.set("href", f"repodata/{updateinfo_path}") + + timestamp = ET.SubElement(data, "timestamp") + timestamp.text = str(int(time.time())) + + size = ET.SubElement(data, "size") + size.text = str(updateinfo["gzipped_size"]) + + open_size = ET.SubElement(data, "open-size") + open_size.text = str(updateinfo["size"]) + + # Add data to repomd.xml + repomd_xml.append(data) + + # Create string + ET.indent(repomd_xml) + xml_str = ET.tostring( + repomd_xml, + xml_declaration=True, + encoding="utf-8", + short_empty_elements=True, + ) + + # Prepend declaration with double quotes + xml_str = xml_str.decode("utf-8") + xml_str = xml_str.replace("'", "\"") + xml_str = xml_str.replace("utf-8", "UTF-8") + + # "Fix" closing tags to not have a space + xml_str = xml_str.replace(" />", "/>") + + # Add xmlns:rpm + xml_str = xml_str.replace( + "repo\">", + f"repo\" xmlns:rpm=\"{NS['rpm']}\">", + ) + + # Write to repomd.xml + with open(repomd_xml_path, "w", encoding="utf-8") as f: + f.write(xml_str) + + # Delete old updateinfo file + if existing_updateinfo_path: + os.remove(existing_updateinfo_path) + + +async def main(args): + base_paths = await scan_path( + args.path, + args.base_format, + args.ignore, + ) + print(args) + pass + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="apollo_tree", + description="Apollo updateinfo.xml publisher (Local file tree)", + epilog="(C) 2023 Rocky Enterprise Software Foundation, Inc.", + ) + parser.add_argument( + "-b", + "--base-format", + default="$reponame/$arch/os/repodata/repomd.xml", + help="Format for main repo.xml file", + ) + parser.add_argument( + "-m", + "--manual", + action="store_true", + help="Manual mode", + ) + parser.add_argument( + "-r", + "--repos", + nargs="+", + action="append", + default=[], + help="Repositories to publish (manual mode), format: :", + ) + parser.add_argument( + "-a", + "--auto-scan", + default=True, + action="store_true", + help="Automatically scan for repos", + ) + parser.add_argument( + "-p", + "--path", + help="Default path to scan for repos", + ) + parser.add_argument( + "-i", + "--ignore", + nargs="+", + action="append", + default=[], + help="Directories in base path to ignore in auto-scan mode", + ) + parser.add_argument( + "-n", + "--product-name", + required=True, + help="Product name", + ) + + p_args = parser.parse_args() + if p_args.auto_scan and p_args.manual: + parser.error("Cannot use --auto-scan and --manual together") + + if p_args.manual and not p_args.repos: + parser.error("Must specify repos to publish in manual mode") + + if p_args.auto_scan and not p_args.path: + parser.error("Must specify path to scan for repos in auto-scan mode") + + asyncio.run(main(p_args)) diff --git a/apollo/rherrata/__init__.py b/apollo/rherrata/__init__.py index 4440f61..d30586b 100644 --- a/apollo/rherrata/__init__.py +++ b/apollo/rherrata/__init__.py @@ -126,7 +126,11 @@ class Advisory(JSONWizard): Returns whether this advisory affects the given RHEL version and architecture. """ for product in self.get_products(): - if product.variant == "Red Hat Enterprise Linux" and product.major_version == major_version and product.minor_version == minor_version and product.arch == arch.value: + is_variant = product.variant == "Red Hat Enterprise Linux" + is_major_version = product.major_version == major_version + is_minor_version = product.minor_version == minor_version + is_arch = product.arch == arch.value + if is_variant and is_major_version and is_minor_version and is_arch: return True return False diff --git a/apollo/rherrata/example.py b/apollo/rherrata/example.py index 9d97fa9..454c615 100644 --- a/apollo/rherrata/example.py +++ b/apollo/rherrata/example.py @@ -1,5 +1,4 @@ import asyncio -import datetime from __init__ import API, Architecture diff --git a/apollo/rpmworker/repomd.py b/apollo/rpmworker/repomd.py index 6724dd9..1795592 100644 --- a/apollo/rpmworker/repomd.py +++ b/apollo/rpmworker/repomd.py @@ -8,8 +8,6 @@ from os import path import aiohttp import yaml -from common.logger import Logger - NVRA_RE = re.compile( r"^(\S+)-([\w~%.+]+)-(\w+(?:\.[\w~%+]+)+?)(?:\.(\w+))?(?:\.rpm)?$" ) diff --git a/apollo/rpmworker/rh_matcher_activities.py b/apollo/rpmworker/rh_matcher_activities.py index eae8ff9..f36eaad 100644 --- a/apollo/rpmworker/rh_matcher_activities.py +++ b/apollo/rpmworker/rh_matcher_activities.py @@ -111,7 +111,7 @@ async def clone_advisory( logger = Logger() logger.info("Cloning advisory %s to %s", advisory.name, product.name) - acceptable_arches = list(set([x.match_arch for x in mirrors])) + acceptable_arches = list({x.match_arch for x in mirrors}) acceptable_arches.extend(["src", "noarch"]) for mirror in mirrors: if mirror.match_arch == "x86_64": @@ -377,14 +377,12 @@ async def clone_advisory( # Check if topic is empty, if so construct it if not new_advisory.topic: - package_names = list(set([p.package_name for p in new_pkgs])) + package_names = list({p.package_name for p in new_pkgs}) affected_products = list( - set( - [ - f"{product.name} {mirror.match_major_version}" - for mirror in mirrors - ] - ) + { + f"{product.name} {mirror.match_major_version}" + for mirror in mirrors + } ) topic = f"""An update is available for {', '.join(package_names)}. This update affects {', '.join(affected_products)}. diff --git a/apollo/server/routes/advisories.py b/apollo/server/routes/advisories.py index 5763983..d604ca8 100644 --- a/apollo/server/routes/advisories.py +++ b/apollo/server/routes/advisories.py @@ -125,4 +125,4 @@ async def get_advisory(request: Request, advisory_name: str): "advisory": advisory, "package_map": package_map, } - ) \ No newline at end of file + ) diff --git a/apollo/server/routes/api_advisories.py b/apollo/server/routes/api_advisories.py index c9bd34b..111f43d 100644 --- a/apollo/server/routes/api_advisories.py +++ b/apollo/server/routes/api_advisories.py @@ -1,6 +1,6 @@ from typing import TypeVar, Generic -from fastapi import APIRouter, Request +from fastapi import APIRouter from fastapi.exceptions import HTTPException from fastapi_pagination.links import Page from fastapi_pagination.ext.tortoise import paginate diff --git a/apollo/server/routes/api_compat.py b/apollo/server/routes/api_compat.py index 5841451..d4ca8b2 100644 --- a/apollo/server/routes/api_compat.py +++ b/apollo/server/routes/api_compat.py @@ -90,12 +90,10 @@ def v3_advisory_to_v2( kind = "TYPE_ENHANCEMENT" affected_products = list( - set( - [ - f"{ap.variant} {ap.major_version}" - for ap in advisory.affected_products - ] - ) + { + f"{ap.variant} {ap.major_version}" + for ap in advisory.affected_products + } ) cves = [] diff --git a/apollo/server/server.py b/apollo/server/server.py index 948eef9..f1e2e69 100644 --- a/apollo/server/server.py +++ b/apollo/server/server.py @@ -67,7 +67,7 @@ async def health(): @app.exception_handler(404) -async def not_found_handler(request, exc): +async def not_found_handler(request, exc): # pylint: disable=unused-argument if request.url.path.startswith("/api" ) or request.url.path.startswith("/v2"): return JSONResponse({"error": "Not found"}, status_code=404) diff --git a/apollo/tests/BUILD.bazel b/apollo/tests/BUILD.bazel new file mode 100644 index 0000000..e69de29 diff --git a/apollo/tests/publishing_tools/BUILD.bazel b/apollo/tests/publishing_tools/BUILD.bazel new file mode 100644 index 0000000..e69de29 diff --git a/apollo/tests/publishing_tools/data/appstream__base__repomd__aarch64.xml b/apollo/tests/publishing_tools/data/appstream__base__repomd__aarch64.xml new file mode 100644 index 0000000..1df3327 --- /dev/null +++ b/apollo/tests/publishing_tools/data/appstream__base__repomd__aarch64.xml @@ -0,0 +1,88 @@ + + + 8.7 + + Rocky Linux 8 + + + 602602cc03961da7245740aa32af042c28fa7a4bed01e9f51dcde8326c6240e3 + a69fcb753072006097c806f9f441553e5086b1369180a72ae719fad654246554 + + 1674782842 + 1648129 + 16701409 + + + 693e2deed8288074851942f71f525bfce865a71734214fa434010dd00a215283 + bc311897f94631363f555ef354525f28613df9effb7e4a0459acffea29bf3ba1 + + 1674782842 + 6162410 + 87526801 + + + fd97455be05a44ffe98685a63031d1ac10661ca0c0bafa747dfae6c479261e6d + 518c3a001a453adb34cf5a70ceecf11aa2d90f0c807c879d76a02f7123f5c22f + + 1674782842 + 1102970 + 12298845 + + + ae73143161fda8b62f6b1cc333ee3fcd4100daa0947a5b918e80b4a487ded360 + 9f9b5bb6f1772f3f742f66245041246021777015d1d30970d5410b988cdd233a + + 1674782850 + 2724240 + 17735680 + 10 + + + 75d9455e941cb3e8a5266980b78eccb26476198bb4a1f3142ee6cda05ed044f9 + 23c2249ccb4fe69474b41c1399aac94d4fd1a1d2c6c2cefc46cb23d4dd52e3b0 + + 1674782854 + 4556600 + 37732352 + 10 + + + f3968e32961df9482a5c265adaeae9a3435c03b81b2c76cfd7c8b1a74337b972 + e84a16aa1bec1a0d6af28ced81aa3a0708d2599b56abbedb858caced501576e9 + + 1674782846 + 1113696 + 10711040 + 10 + + + 341f7a33a85d237227c4b814e5ffd57cc7ead34dab6ca79db4feff8520803b22 + + 1674782725 + 428231 + + + c0562df3ad4d3d47bb8076482351eca478a02f4bd15e21b4683456268920154a + 341f7a33a85d237227c4b814e5ffd57cc7ead34dab6ca79db4feff8520803b22 + + 1674782842 + 73836 + 428231 + + + d53d510b653b6356691a5e9cf5adf2f803c525f0c9f908adf9ccbb6c88a13967 + 479622387203132ad97518c6ccdf2a41996f2fddc8a5c9bdf09fcb7407064c09 + + 1674783016 + 73124 + 712620 + + + be8915c2f7ed08fca20532364bd01b12ab51e826d45233f92832f318d1b78c3e + eec1c28a088aea1719b3ef2edacba8cdc78658fcf65009d2520f76e43ba90566 + + 1674284973 + 233607 + 2402781 + + diff --git a/apollo/tests/publishing_tools/data/appstream__base__repomd__x86_64.xml b/apollo/tests/publishing_tools/data/appstream__base__repomd__x86_64.xml new file mode 100644 index 0000000..6ff1695 --- /dev/null +++ b/apollo/tests/publishing_tools/data/appstream__base__repomd__x86_64.xml @@ -0,0 +1,88 @@ + + + 8.7 + + Rocky Linux 8 + + + 593ce13bf52aadd58d9da7c2b3fc2149bffd7bc4924c7032946b51cc340fc976 + c808007edc3392c44ac219825b4c73f4ba717721f4df2dc9aba023413d3dfab0 + + 1674781924 + 1962142 + 20488457 + + + bce97b0287c0f9f9d900b5c9681cfe73f69890c26aae9792fffe25429fc2186b + 4e7729a83d7917070277bc630cc8cea6a620fab59d584a12dc7252d33e8c5632 + + 1674781924 + 7066338 + 99500723 + + + 902ea6dea89caac9980c8599cd53ce154a1e8bff3a1d212d70c38d823e08c028 + c5f4414a2cacff80f6c038761e440f61ed4c6bdefeba66e0062f228be408afc9 + + 1674781924 + 1263392 + 15731326 + + + c1b32bc2fcef5863e3d0ab03263dbee5d645eb5b299d76a6edf23701504544a3 + ef04c60effc3269b15aa41d99713169e1fa0ea9196ce5167c6096d42e9ae8c0a + + 1674781934 + 3321344 + 21495808 + 10 + + + bcf3f665052787a1f84ce5de17cc05ab0aa526de282ef6dfcdd8c8a75bf2ac26 + 8bd4fd292c8b4041acfc0a113f8e457b55bc2ad75755e17c70e116d71f6b6019 + + 1674781938 + 5256956 + 44195840 + 10 + + + 5c1e7e3286c934c684bbe451356d9ac31968f4e786faa209f0bb3080fe02d67f + 88246653b636218807ef6f1174590614851df489be10ae8e462bd38e21c6ef2b + + 1674781929 + 1385640 + 13701120 + 10 + + + ec2fa1982439f4b0ca7dc86f882f18657457a2d42f06ac6718298c7f7528df43 + + 1674781761 + 486316 + + + 7ba43f88671d8107d6603ebff822fc071e73596a1e20779855af577b0e6e343a + ec2fa1982439f4b0ca7dc86f882f18657457a2d42f06ac6718298c7f7528df43 + + 1674781925 + 81840 + 486316 + + + 7facfc73e398abc34445310a258754aae5ea0add698c4ea039580a43897d0d02 + da557db5bfd41e8949f8c3904f963cd81fcc444aa4fcc9e99a47ed843948140b + + 1674782139 + 76500 + 742747 + + + 5c45121e44d7b58060fb9435733d3ac4e03f2e1965cacd5371eb1649e15d5c06 + dda8db3b243e33262f799fb4361bd21a16b1ff22ddd0224bd18070496942cf3d + + 1674284975 + 276699 + 2720880 + + diff --git a/apollo/tests/publishing_tools/data/baseos__base__repomd__aarch64.xml b/apollo/tests/publishing_tools/data/baseos__base__repomd__aarch64.xml new file mode 100644 index 0000000..4430a84 --- /dev/null +++ b/apollo/tests/publishing_tools/data/baseos__base__repomd__aarch64.xml @@ -0,0 +1,80 @@ + + + 8.7 + + Rocky Linux 8 + + + 9f55c0c1cf7cb1fe08bc208f0286d64f274339df4ba857ba978e544a9924fa92 + 8a5e972376710cb07fc5eb1f559d0c4456b1b58a36d219474eae66b4d10f7040 + + 1674782716 + 1346598 + 9900543 + + + 0457005390605cd3154a88584c7ef6a0e5e37a15c8035dfcde72f9d832ab14e2 + ae8651d8f568994088e3fc576538c6f4638ec29475d6d4c4739fd492028fc0fb + + 1674782716 + 1523844 + 19386269 + + + 30399987025d0024ccf2d1ee6e1b412361d547cc50813777e42730375a3925dd + 8ea81e1b6d4333cc8553fc4ddd95dc3119e763e7a0c81ea753531fb8f4d7d56a + + 1674782716 + 829148 + 5197359 + + + 5e78af661835050ef33470ed4c00b81ed2e9f8a76629fc7b8f04153ae273eaca + ced2ee6fd2db2e01b5ba521a2a7815c6ab84c1b1cc95dd41acf55faf90f819c6 + + 1674782721 + 1663124 + 10960896 + 10 + + + 513e605dd419c53c13f1aa3688ca53338e149dbe6de19b9c8453102bcb9ce882 + 3639a0d1ea827449d47564f4330fe8426d37c1cf96308ed13d40fb3460036104 + + 1674782720 + 1412020 + 10305536 + 10 + + + 33bddcd2c405ef925318e88662acd38f3a7246e78ebaf88639fe7c43c215095e + b8d0ffdf150621ecc96e0aa587a895f765e0401937b50ff98e5f2312e3965774 + + 1674782717 + 356456 + 5017600 + 10 + + + 5f2dec3cbb871be9fb6c456740458773b183987275a1d5c93920d4800323fdf9 + + 1674782695 + 289214 + + + 01f25f36dcb08b5b027ce222dec48e68bcaa6b9e96de863ce60f126897f76faf + 5f2dec3cbb871be9fb6c456740458773b183987275a1d5c93920d4800323fdf9 + + 1674782716 + 55624 + 289214 + + + cb9bb3ef856440c44a70f4f0aa9ee1b3621814c3457f922cdb02d7ed63a2701f + 596fc124c74802371ff879a1ecc3a7560704413bbfb4a5ab9299671baf12c2fb + + 1674284975 + 45141 + 433034 + + diff --git a/apollo/tests/publishing_tools/data/baseos__base__repomd__x86_64.xml b/apollo/tests/publishing_tools/data/baseos__base__repomd__x86_64.xml new file mode 100644 index 0000000..c8528e7 --- /dev/null +++ b/apollo/tests/publishing_tools/data/baseos__base__repomd__x86_64.xml @@ -0,0 +1,80 @@ + + + 8.7 + + Rocky Linux 8 + + + 712e4af06a8f9f13766274c3fa83bbdf5572a1014ec77f7cd30d56d492a40db2 + c3ec31964ce06e7dc14a8184ad11d6f1edce3a718bd63e4de8ed18349d27e85e + + 1674781748 + 1791032 + 13677600 + + + 052aea22e07e586ae9383f6da4056599417fc445431c9d0b1136f0316ef4843e + dd5952122ed4335b799b1853f096f05b9e9ea6ea1a48e344700b7b82f468d61f + + 1674781749 + 1800846 + 22806093 + + + 37421cd13a9594ff7dd3a0f2953aa19a01290e90be43db4ce279fef4394fdcd0 + 3be8a12fb7bcd600b178f6b00404d1e4e9196b9f0d5faea460c58b8d07771747 + + 1674781749 + 878178 + 6343206 + + + 2b1d2886dec67d493adcf7358e44b7f7fd4886c85bab7fa9854014ad9c404d7a + a2a2c52ce2b2ce80e936a9488c8799c8508cbfe1ff5dd03187e62f8cf659ca8b + + 1674781756 + 2230076 + 15314944 + 10 + + + 637abdee51be41d56603a2c031bfaf0dd4fc9da0127c826328d9a527aa3b5aa0 + 979021c34ae689ddfdd32f1a37e78da16fdb827e8bb5c0f8818ccfe880906b74 + + 1674781753 + 1689576 + 12529664 + 10 + + + 45ff5a2543dd16101778346f47e0d2362fe546698f137a05b7b72d638f03904f + e045ad4b00b3d20c2cd489a89297ed7c7d0efa1b279cd012b1e53dab06efef24 + + 1674781750 + 427448 + 6021120 + 10 + + + dae7e104812099a2f632ea4c5ef2769aca18ca1205abdd2c3ba6d171e319df3d + + 1674781726 + 298889 + + + 741d3c80487757df624285f4a107f925abfac16115210486760a3920d8724e38 + dae7e104812099a2f632ea4c5ef2769aca18ca1205abdd2c3ba6d171e319df3d + + 1674781749 + 57068 + 298889 + + + 568de83be47822b08ea890ce0f58fd263c61c9a5c5dc500789d25e7020827112 + 1be116b93938f8bf96bb27115a63be34b59b36fcb4b2bfd3fc22c4eb3e3ffce5 + + 1674284973 + 56390 + 508695 + + diff --git a/apollo/tests/publishing_tools/data/baseos__base__repomd__x86_64_with_updateinfo.xml b/apollo/tests/publishing_tools/data/baseos__base__repomd__x86_64_with_updateinfo.xml new file mode 100644 index 0000000..4cced6a --- /dev/null +++ b/apollo/tests/publishing_tools/data/baseos__base__repomd__x86_64_with_updateinfo.xml @@ -0,0 +1,80 @@ + + + 8.7 + + Rocky Linux 8 + + + 712e4af06a8f9f13766274c3fa83bbdf5572a1014ec77f7cd30d56d492a40db2 + c3ec31964ce06e7dc14a8184ad11d6f1edce3a718bd63e4de8ed18349d27e85e + + 1674781748 + 1791032 + 13677600 + + + 052aea22e07e586ae9383f6da4056599417fc445431c9d0b1136f0316ef4843e + dd5952122ed4335b799b1853f096f05b9e9ea6ea1a48e344700b7b82f468d61f + + 1674781749 + 1800846 + 22806093 + + + 37421cd13a9594ff7dd3a0f2953aa19a01290e90be43db4ce279fef4394fdcd0 + 3be8a12fb7bcd600b178f6b00404d1e4e9196b9f0d5faea460c58b8d07771747 + + 1674781749 + 878178 + 6343206 + + + 2b1d2886dec67d493adcf7358e44b7f7fd4886c85bab7fa9854014ad9c404d7a + a2a2c52ce2b2ce80e936a9488c8799c8508cbfe1ff5dd03187e62f8cf659ca8b + + 1674781756 + 2230076 + 15314944 + 10 + + + 637abdee51be41d56603a2c031bfaf0dd4fc9da0127c826328d9a527aa3b5aa0 + 979021c34ae689ddfdd32f1a37e78da16fdb827e8bb5c0f8818ccfe880906b74 + + 1674781753 + 1689576 + 12529664 + 10 + + + 45ff5a2543dd16101778346f47e0d2362fe546698f137a05b7b72d638f03904f + e045ad4b00b3d20c2cd489a89297ed7c7d0efa1b279cd012b1e53dab06efef24 + + 1674781750 + 427448 + 6021120 + 10 + + + dae7e104812099a2f632ea4c5ef2769aca18ca1205abdd2c3ba6d171e319df3d + + 1674781726 + 298889 + + + 741d3c80487757df624285f4a107f925abfac16115210486760a3920d8724e38 + dae7e104812099a2f632ea4c5ef2769aca18ca1205abdd2c3ba6d171e319df3d + + 1674781749 + 57068 + 298889 + + + 2242022f6b5935ee22d4ba78d65da0a37873e5a42f00de907795b809d3dadb59 + 8ad9b2cab0f009a09fc6ed6f6fb946ce734e1d90d0d2ff6faf878788393ba4d8 + + 1674284973 + 1706 + 8335 + + \ No newline at end of file diff --git a/apollo/tests/publishing_tools/data/updateinfo__test__1.xml b/apollo/tests/publishing_tools/data/updateinfo__test__1.xml new file mode 100644 index 0000000..25ad12e --- /dev/null +++ b/apollo/tests/publishing_tools/data/updateinfo__test__1.xml @@ -0,0 +1,111 @@ + + + RLSA-2022:1821 + Moderate: python27:2.7 security update + Python is an interpreted, interactive, object-oriented programming language that supports modules, classes, exceptions, high-level dynamic data types, and dynamic typing. The python27 packages provide a stable release of Python 2.7 with a number of additional utilities and database connectors for MySQL and PostgreSQL. + +Security Fix(es): + +* python: urllib: Regular expression DoS in AbstractBasicAuthHandler (CVE-2021-3733) + +* python: ftplib should not use the host from the PASV response (CVE-2021-4189) + +* python-lxml: HTML Cleaner allows crafted and SVG embedded scripts to pass through (CVE-2021-43818) + +* python: urllib.parse does not sanitize URLs containing ASCII newline and tabs (CVE-2022-0391) + +* python: urllib: HTTP client possible infinite loop on a 100 Continue response (CVE-2021-3737) + +For more details about the security issue(s), including the impact, a CVSS score, acknowledgments, and other related information, refer to the CVE page(s) listed in the References section. + +Additional Changes: + +For detailed information on changes in this release, see the Rocky Linux 8.6 Release Notes linked from the References section. + 2022-05-10 08:02:50 + 2023-02-02 13:41:28 + Copyright 2023 Rocky Enterprise Software Foundation + Rocky Linux 8 + 1 + Moderate + An update is available for python-pymongo, python2-rpm-macros, python-sqlalchemy, python-backports, python-docutils, pytest, python-psycopg2, python-lxml, python-PyMySQL, python-urllib3, PyYAML, python-pytest-mock, python-attrs, python-jinja2, python-docs, python-requests, python-mock, python-ipaddress, python-funcsigs, python2-six, python-py, python2, python2-pip, python-chardet, python-markupsafe, python-pluggy, python-pygments, python2-setuptools, Cython, python-virtualenv, babel, python-dns, python-wheel, python-pysocks, python-backports-ssl_match_hostname, python-coverage, python-setuptools_scm, pytz, python-nose, scipy, python-idna, numpy. +This update affects Rocky Linux 8. +A Common Vulnerability Scoring System (CVSS) base score, which gives a detailed severity rating, is available for each vulnerability from the CVE list + Python is an interpreted, interactive, object-oriented programming language that supports modules, classes, exceptions, high-level dynamic data types, and dynamic typing. The python27 packages provide a stable release of Python 2.7 with a number of additional utilities and database connectors for MySQL and PostgreSQL. + +Security Fix(es): + +* python: urllib: Regular expression DoS in AbstractBasicAuthHandler (CVE-2021-3733) + +* python: ftplib should not use the host from the PASV response (CVE-2021-4189) + +* python-lxml: HTML Cleaner allows crafted and SVG embedded scripts to pass through (CVE-2021-43818) + +* python: urllib.parse does not sanitize URLs containing ASCII newline and tabs (CVE-2022-0391) + +* python: urllib: HTTP client possible infinite loop on a 100 Continue response (CVE-2021-3737) + +For more details about the security issue(s), including the impact, a CVSS score, acknowledgments, and other related information, refer to the CVE page(s) listed in the References section. + +Additional Changes: + +For detailed information on changes in this release, see the Rocky Linux 8.6 Release Notes linked from the References section. + + + + + + + + + + + + + + + + + + + RLSA-2022:7593 + Moderate: python27:2.7 security update + Python is an interpreted, interactive, object-oriented programming language that supports modules, classes, exceptions, high-level dynamic data types, and dynamic typing. + +Security Fix(es): + +* python: mailcap: findmatch() function does not sanitize the second argument (CVE-2015-20107). + +For more details about the security issue(s), including the impact, a CVSS score, acknowledgments, and other related information, refer to the CVE page(s) listed in the References section. + +Additional Changes: + +For detailed information on changes in this release, see the Rocky Linux 8.7 Release Notes linked from the References section. + 2022-11-08 06:23:47 + 2023-02-02 13:52:34 + Copyright 2023 Rocky Enterprise Software Foundation + Rocky Linux 8 + 1 + Moderate + An update is available for python-pymongo, python2-rpm-macros, python-sqlalchemy, python-backports, python-docutils, pytest, python-psycopg2, python-lxml, python-PyMySQL, python-urllib3, PyYAML, python-pytest-mock, python-attrs, python-jinja2, python-docs, python-requests, python-mock, python-ipaddress, python-funcsigs, python2-six, python-py, python2, python2-pip, python-chardet, python-markupsafe, python-pluggy, python-pygments, python2-setuptools, Cython, python-virtualenv, babel, python-dns, python-wheel, python-pysocks, python-backports-ssl_match_hostname, python-coverage, python-setuptools_scm, pytz, python-nose, scipy, python-idna, numpy. +This update affects Rocky Linux 8. +A Common Vulnerability Scoring System (CVSS) base score, which gives a detailed severity rating, is available for each vulnerability from the CVE list + Python is an interpreted, interactive, object-oriented programming language that supports modules, classes, exceptions, high-level dynamic data types, and dynamic typing. + +Security Fix(es): + +* python: mailcap: findmatch() function does not sanitize the second argument (CVE-2015-20107). + +For more details about the security issue(s), including the impact, a CVSS score, acknowledgments, and other related information, refer to the CVE page(s) listed in the References section. + +Additional Changes: + +For detailed information on changes in this release, see the Rocky Linux 8.7 Release Notes linked from the References section. + + + + + + + + + \ No newline at end of file diff --git a/apollo/tests/publishing_tools/test_apollo_tree.py b/apollo/tests/publishing_tools/test_apollo_tree.py new file mode 100644 index 0000000..0843cac --- /dev/null +++ b/apollo/tests/publishing_tools/test_apollo_tree.py @@ -0,0 +1,498 @@ +import tempfile +import shutil +import pathlib +import hashlib +from os import path, environ + +import pytest + +from apollo.publishing_tools import apollo_tree + +from common.testing import MockResponse + +data = [ + "baseos__base__repomd__x86_64.xml", + "baseos__base__repomd__aarch64.xml", + "appstream__base__repomd__x86_64.xml", + "appstream__base__repomd__aarch64.xml", +] + + +async def _setup_test_baseos(directory: str): + file = data[0] + base_dir = path.join( + directory, + "BaseOS/x86_64/os/repodata", + ) + pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True) + shutil.copyfile( + path.join(path.dirname(__file__), "data", file), + path.join(base_dir, "repomd.xml"), + ) + + # Run scan_path + repos = await apollo_tree.scan_path( + directory, + "$reponame/$arch/os/repodata/repomd.xml", + [], + ) + + return repos + + +@pytest.mark.asyncio +async def test_scan_path_valid_structure(): + with tempfile.TemporaryDirectory() as directory: + # Copy test data to temp dir + for file in data: + fsplit = file.split("__") + base_dir = path.join( + directory, + fsplit[0], + fsplit[-1].removesuffix(".xml"), + "os/repodata", + ) + pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True) + shutil.copyfile( + path.join(path.dirname(__file__), "data", file), + path.join(base_dir, "repomd.xml"), + ) + + # Run scan_path + repos = await apollo_tree.scan_path( + directory, + "$reponame/$arch/os/repodata/repomd.xml", + [], + ) + + assert "baseos" in repos + assert "appstream" in repos + assert len(repos["baseos"]) == 2 + assert len(repos["appstream"]) == 2 + + for repo in repos["baseos"]: + assert repo["name"] == "baseos" + assert repo["arch"] in ["x86_64", "aarch64"] + assert repo["found_path"] == path.join( + directory, + "baseos", + repo["arch"], + "os/repodata/repomd.xml", + ) + + for repo in repos["appstream"]: + assert repo["name"] == "appstream" + assert repo["arch"] in ["x86_64", "aarch64"] + assert repo["found_path"] == path.join( + directory, + "appstream", + repo["arch"], + "os/repodata/repomd.xml", + ) + + +@pytest.mark.asyncio +async def test_scan_path_multiple_formats(): + with tempfile.TemporaryDirectory() as directory: + # Copy test data to temp dir + for file in data: + fsplit = file.split("__") + base_dir = path.join( + directory, + fsplit[0], + fsplit[-1].removesuffix(".xml"), + "os/repodata", + ) + pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True) + shutil.copyfile( + path.join(path.dirname(__file__), "data", file), + path.join(base_dir, "repomd.xml"), + ) + + file = data[0] + fsplit = file.split("__") + base_dir = path.join( + directory, + fsplit[0], + "source/tree/repodata", + ) + pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True) + shutil.copyfile( + path.join(path.dirname(__file__), "data", file), + path.join(base_dir, "repomd.xml"), + ) + + # Run scan_path + repos = await apollo_tree.scan_path( + directory, + "$reponame/$arch/os/repodata/repomd.xml", + [], + ) + + assert "baseos" in repos + assert "appstream" in repos + assert len(repos["baseos"]) == 2 + assert len(repos["appstream"]) == 2 + + for repo in repos["baseos"]: + assert repo["name"] == "baseos" + assert repo["arch"] in ["source", "x86_64", "aarch64"] + assert repo["found_path"] == path.join( + directory, + "baseos", + repo["arch"], + "os/repodata/repomd.xml", + ) + + for repo in repos["appstream"]: + assert repo["name"] == "appstream" + assert repo["arch"] in ["x86_64", "aarch64"] + assert repo["found_path"] == path.join( + directory, + "appstream", + repo["arch"], + "os/repodata/repomd.xml", + ) + + # Run scan_path for source + repos = await apollo_tree.scan_path( + directory, + "$reponame/source/tree/repodata/repomd.xml", + [], + ) + + assert "baseos" in repos + assert len(repos["baseos"]) == 1 + + for repo in repos["baseos"]: + assert repo["name"] == "baseos" + assert repo["arch"] == "source" + assert repo["found_path"] == path.join( + directory, + "baseos", + "source", + "tree/repodata/repomd.xml", + ) + + +@pytest.mark.asyncio +async def test_scan_path_valid_structure_arch_first(): + with tempfile.TemporaryDirectory() as directory: + # Copy test data to temp dir + for file in data: + fsplit = file.split("__") + base_dir = path.join( + directory, + fsplit[-1].removesuffix(".xml"), + fsplit[0], + "os/repodata", + ) + pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True) + shutil.copyfile( + path.join(path.dirname(__file__), "data", file), + path.join(base_dir, "repomd.xml"), + ) + + # Run scan_path + repos = await apollo_tree.scan_path( + directory, + "$arch/$reponame/os/repodata/repomd.xml", + [], + ) + + assert "baseos" in repos + assert "appstream" in repos + assert len(repos["baseos"]) == 2 + assert len(repos["appstream"]) == 2 + + for repo in repos["baseos"]: + assert repo["name"] == "baseos" + assert repo["arch"] in ["x86_64", "aarch64"] + assert repo["found_path"] == path.join( + directory, + repo["arch"], + "baseos", + "os/repodata/repomd.xml", + ) + + for repo in repos["appstream"]: + assert repo["name"] == "appstream" + assert repo["arch"] in ["x86_64", "aarch64"] + assert repo["found_path"] == path.join( + directory, + repo["arch"], + "appstream", + "os/repodata/repomd.xml", + ) + + +@pytest.mark.asyncio +async def test_fetch_updateinfo_from_apollo_live(): + # This test is only run if the environment variable + # TEST_WITH_SIDE_EFFECTS is set to 1 + if not environ.get("TEST_WITH_SIDE_EFFECTS"): + pytest.skip("Skipping test_fetch_updateinfo_from_apollo_live") + + with tempfile.TemporaryDirectory() as directory: + file = data[0] + base_dir = path.join( + directory, + "BaseOS/x86_64/os/repodata", + ) + pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True) + shutil.copyfile( + path.join(path.dirname(__file__), "data", file), + path.join(base_dir, "repomd.xml"), + ) + + # Run scan_path + repos = await apollo_tree.scan_path( + directory, + "$reponame/$arch/os/repodata/repomd.xml", + [], + ) + + assert "BaseOS" in repos + assert len(repos["BaseOS"]) == 1 + + # Run fetch_updateinfo_from_apollo + for _, repo_variants in repos.items(): + for repo in repo_variants: + updateinfo = await apollo_tree.fetch_updateinfo_from_apollo( + repo, + "Rocky Linux 8 x86_64", + ) + + assert updateinfo is not None + + +@pytest.mark.asyncio +async def test_fetch_updateinfo_from_apollo_live_no_updateinfo(): + # This test is only run if the environment variable + # TEST_WITH_SIDE_EFFECTS is set to 1 + if not environ.get("TEST_WITH_SIDE_EFFECTS"): + pytest.skip( + "Skipping test_fetch_updateinfo_from_apollo_live_no_updateinfo" + ) + + with tempfile.TemporaryDirectory() as directory: + file = data[0] + base_dir = path.join( + directory, + "BaseOS/x86_64/os/repodata", + ) + pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True) + shutil.copyfile( + path.join(path.dirname(__file__), "data", file), + path.join(base_dir, "repomd.xml"), + ) + + # Run scan_path + repos = await apollo_tree.scan_path( + directory, + "$reponame/$arch/os/repodata/repomd.xml", + [], + ) + + assert "BaseOS" in repos + assert len(repos["BaseOS"]) == 1 + + # Run fetch_updateinfo_from_apollo + for _, repo_variants in repos.items(): + for repo in repo_variants: + updateinfo = await apollo_tree.fetch_updateinfo_from_apollo( + repo, + "Rocky Linux 8 x86_64 NONEXISTENT", + ) + + assert updateinfo is None + + +@pytest.mark.asyncio +async def test_fetch_updateinfo_from_apollo_mock(mocker): + with tempfile.TemporaryDirectory() as directory: + repos = await _setup_test_baseos(directory) + + # Read data/updateinfo__test__1.xml + with open( + path.join( + path.dirname(__file__), "data", "updateinfo__test__1.xml" + ), + "r", + encoding="utf-8", + ) as f: + updateinfo_xml = f.read() + + resp = MockResponse(updateinfo_xml, 200) + mocker.patch("aiohttp.ClientSession.get", return_value=resp) + + # Run fetch_updateinfo_from_apollo + for _, repo_variants in repos.items(): + for repo in repo_variants: + updateinfo = await apollo_tree.fetch_updateinfo_from_apollo( + repo, + "Rocky Linux 8 x86_64", + True, + ) + + assert updateinfo == updateinfo_xml + + +@pytest.mark.asyncio +async def test_gzip_updateinfo(mocker): + with tempfile.TemporaryDirectory() as directory: + repos = await _setup_test_baseos(directory) + + # Read data/updateinfo__test__1.xml + with open( + path.join( + path.dirname(__file__), "data", "updateinfo__test__1.xml" + ), + "r", + encoding="utf-8", + ) as f: + updateinfo_xml = f.read() + + resp = MockResponse(updateinfo_xml, 200) + mocker.patch("aiohttp.ClientSession.get", return_value=resp) + + # Run fetch_updateinfo_from_apollo + updateinfo = None + for _, repo_variants in repos.items(): + for repo in repo_variants: + updateinfo = await apollo_tree.fetch_updateinfo_from_apollo( + repo, + "Rocky Linux 8 x86_64", + True, + ) + + assert updateinfo == updateinfo_xml + break + + # Run gzip_updateinfo + updateinfo_gz = await apollo_tree.gzip_updateinfo(updateinfo) + assert updateinfo_gz is not None + + +@pytest.mark.asyncio +async def test_write_updateinfo_to_file(mocker): + with tempfile.TemporaryDirectory() as directory: + repos = await _setup_test_baseos(directory) + + # Read data/updateinfo__test__1.xml + with open( + path.join( + path.dirname(__file__), "data", "updateinfo__test__1.xml" + ), + "r", + encoding="utf-8", + ) as f: + updateinfo_xml = f.read() + + resp = MockResponse(updateinfo_xml, 200) + mocker.patch("aiohttp.ClientSession.get", return_value=resp) + + # Run fetch_updateinfo_from_apollo + updateinfo = None + for _, repo_variants in repos.items(): + for repo in repo_variants: + updateinfo = await apollo_tree.fetch_updateinfo_from_apollo( + repo, + "Rocky Linux 8 x86_64", + True, + ) + + assert updateinfo == updateinfo_xml + break + + # Gzip first + gzipped = await apollo_tree.gzip_updateinfo(updateinfo) + + # Run write_updateinfo_to_file + updateinfo_file = await apollo_tree.write_updateinfo_to_file( + repos["BaseOS"][0]["found_path"], + gzipped, + ) + + assert updateinfo_file is not None + assert path.exists(updateinfo_file) + assert path.isfile(updateinfo_file) + + with open(updateinfo_file, "rb") as f: + updateinfo_file_contents = f.read() + + # Check sha256sum against written file + actual_hexdigest = hashlib.sha256(updateinfo_file_contents).hexdigest() + expected_hexdigest = gzipped["gzipped_sha256sum"] + assert actual_hexdigest == expected_hexdigest + + +@pytest.mark.asyncio +async def test_update_repomd_xml(mocker): + with tempfile.TemporaryDirectory() as directory: + repos = await _setup_test_baseos(directory) + + # Read data/updateinfo__test__1.xml + with open( + path.join( + path.dirname(__file__), "data", "updateinfo__test__1.xml" + ), + "r", + encoding="utf-8", + ) as f: + updateinfo_xml = f.read() + + resp = MockResponse(updateinfo_xml, 200) + mocker.patch("aiohttp.ClientSession.get", return_value=resp) + + # Run fetch_updateinfo_from_apollo + updateinfo = None + for _, repo_variants in repos.items(): + for repo in repo_variants: + updateinfo = await apollo_tree.fetch_updateinfo_from_apollo( + repo, + "Rocky Linux 8 x86_64", + True, + ) + + assert updateinfo == updateinfo_xml + break + + # Gzip first + gzipped = await apollo_tree.gzip_updateinfo(updateinfo) + + # Run write_updateinfo_to_file + updateinfo_file = await apollo_tree.write_updateinfo_to_file( + repos["BaseOS"][0]["found_path"], + gzipped, + ) + + assert updateinfo_file is not None + assert path.exists(updateinfo_file) + assert path.isfile(updateinfo_file) + + # Run update_repomd_xml + # This will replace the repomd.xml file with the new one + mocker.patch("time.time", return_value=1674284973) + repomd_xml_path = repos["BaseOS"][0]["found_path"] + await apollo_tree.update_repomd_xml( + repomd_xml_path, + gzipped, + ) + + # Check that the repomd.xml file matches baseos__base__repomd__x86_64_with_updateinfo.xml from data + with open( + path.join( + path.dirname(__file__), + "data", + "baseos__base__repomd__x86_64_with_updateinfo.xml", + ), + "r", + encoding="utf-8", + ) as f: + expected_repomd_xml = f.read() + + with open(repomd_xml_path, "r", encoding="utf-8") as f: + actual_repomd_xml = f.read() + + assert actual_repomd_xml == expected_repomd_xml diff --git a/build/scripts/pylint.bash b/build/scripts/pylint.bash new file mode 100755 index 0000000..60ea511 --- /dev/null +++ b/build/scripts/pylint.bash @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +shopt -s globstar + +python3 -m pylint --rcfile=.pylintrc --ignore-patterns "re.compile(r'bazel-.*|node-modules|.venv')" **/*.py -v | tee /tmp/pytest.txt +score=$(sed -n 's/^Your code has been rated at \([-0-9.]*\)\/.*/\1/p' /tmp/pytest.txt) + +echo "====================" +if (( $(echo "$score < 9.0" | bc -l) )); then + echo "Pylint score is too low: $score" + exit 1 +else + echo "Pylint score is good: $score" +fi +echo "====================" diff --git a/build/scripts/test.bash b/build/scripts/test.bash new file mode 100755 index 0000000..f8dad8f --- /dev/null +++ b/build/scripts/test.bash @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +shopt -s globstar + +python3 -m pytest --ignore node_modules --ignore .venv --ignore-glob "bazel-*" -v diff --git a/common/testing.py b/common/testing.py new file mode 100644 index 0000000..8ba10b3 --- /dev/null +++ b/common/testing.py @@ -0,0 +1,13 @@ +class MockResponse: + def __init__(self, text, status): + self._text = text + self.status = status + + async def text(self): + return self._text + + async def __aexit__(self, exc_type, exc, tb): + pass + + async def __aenter__(self): + return self diff --git a/deploy/apollo/apollo-rpmworker/values.yaml b/deploy/apollo/apollo-rpmworker/values.yaml index eac4bb8..be06a6b 100644 --- a/deploy/apollo/apollo-rpmworker/values.yaml +++ b/deploy/apollo/apollo-rpmworker/values.yaml @@ -8,7 +8,7 @@ image: repository: ghcr.io/resf/apollo-rpmworker pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: "0c2e297" + tag: "15c4ca5" imagePullSecrets: [] nameOverride: "" diff --git a/deploy/apollo/apollo-server/values.yaml b/deploy/apollo/apollo-server/values.yaml index a178abf..1464d7a 100644 --- a/deploy/apollo/apollo-server/values.yaml +++ b/deploy/apollo/apollo-server/values.yaml @@ -8,7 +8,7 @@ image: repository: ghcr.io/resf/apollo-server pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: "cc92597" + tag: "b498d3b" imagePullSecrets: [] nameOverride: "" @@ -44,7 +44,7 @@ istio: enabled: true gateway: istio-system/base-gateway-public externalDnsTarget: ingress.build.resf.org - host: apollo-v3-beta.build.resf.org + host: apollo.build.resf.org database: host: resf-peridot-dev.ctxqgglmfofx.us-east-2.rds.amazonaws.com @@ -84,9 +84,9 @@ resources: {} # memory: 128Mi autoscaling: - enabled: false + enabled: true minReplicas: 1 - maxReplicas: 100 + maxReplicas: 10 targetCPUUtilizationPercentage: 80 # targetMemoryUtilizationPercentage: 80 diff --git a/gazelle_python.yaml b/gazelle_python.yaml index ca3d678..8148c2d 100644 --- a/gazelle_python.yaml +++ b/gazelle_python.yaml @@ -362,6 +362,7 @@ manifest: dill.tests.test_source: dill dill.tests.test_temp: dill dill.tests.test_weakref: dill + exceptiongroup: exceptiongroup fastapi: fastapi fastapi.applications: fastapi fastapi.background: fastapi @@ -594,6 +595,8 @@ manifest: idna.intranges: idna idna.package_data: idna idna.uts46data: idna + iniconfig: iniconfig + iniconfig.exceptions: iniconfig iso8601: iso8601 iso8601.iso8601: iso8601 iso8601.test_iso8601: iso8601 @@ -731,6 +734,13 @@ manifest: openapi_python_client.schema.openapi_schema_pydantic.xml: openapi_python_client openapi_python_client.schema.parameter_location: openapi_python_client openapi_python_client.utils: openapi_python_client + packaging: packaging + packaging.markers: packaging + packaging.requirements: packaging + packaging.specifiers: packaging + packaging.tags: packaging + packaging.utils: packaging + packaging.version: packaging passlib: passlib passlib.apache: passlib passlib.apps: passlib @@ -830,8 +840,10 @@ manifest: platformdirs.unix: platformdirs platformdirs.version: platformdirs platformdirs.windows: platformdirs + pluggy: pluggy priority: priority priority.priority: priority + py: pytest pydantic: pydantic pydantic.annotated_types: pydantic pydantic.class_validators: pydantic @@ -1051,6 +1063,11 @@ manifest: pypika.queries: pypika_tortoise pypika.terms: pypika_tortoise pypika.utils: pypika_tortoise + pytest: pytest + pytest_asyncio: pytest_asyncio + pytest_asyncio.plugin: pytest_asyncio + pytest_mock: pytest_mock + pytest_mock.plugin: pytest_mock pytz: pytz pytz.exceptions: pytz pytz.lazy: pytz @@ -1507,4 +1524,4 @@ manifest: yarl: yarl pip_repository: name: pypi -integrity: 8d848d11c467949c981e296a23211f42a009566f0d2ab5fe11ceb59ade4cb74e +integrity: f886dabbf1a9923a165d6ef2ef657ca51f835c91992272bb8c12e9f78c128101 diff --git a/requirements.txt b/requirements.txt index 7c1edc3..1cb91e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,4 +18,7 @@ itsdangerous==2.1.2 PyYAML==6.0 beautifulsoup4==4.11.2 rssgen==0.9.0 -python-slugify==8.0.0 \ No newline at end of file +python-slugify==8.0.0 +pytest==7.2.1 +pytest-asyncio==0.20.3 +pytest-mock==3.10.0 \ No newline at end of file diff --git a/requirements_lock.txt b/requirements_lock.txt index 2e11256..5f415a4 100644 --- a/requirements_lock.txt +++ b/requirements_lock.txt @@ -159,6 +159,7 @@ attrs==22.2.0 \ # via # aiohttp # openapi-python-client + # pytest autoflake==2.0.0 \ --hash=sha256:7185b596e70d8970c6d4106c112ef41921e472bd26abf3613db99eca88cc8c2a \ --hash=sha256:d58ed4187c6b4f623a942b9a90c43ff84bf6a266f3682f407b42ca52073c9678 @@ -229,6 +230,10 @@ dill==0.3.6 \ --hash=sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0 \ --hash=sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373 # via pylint +exceptiongroup==1.1.0 \ + --hash=sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e \ + --hash=sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23 + # via pytest fastapi==0.89.1 \ --hash=sha256:15d9271ee52b572a015ca2ae5c72e1ce4241dd8532a534ad4f7ec70c376a580f \ --hash=sha256:f9773ea22290635b2f48b4275b2bf69a8fa721fda2e38228bed47139839dc877 @@ -355,6 +360,10 @@ idna==3.4 \ # anyio # rfc3986 # yarl +iniconfig==2.0.0 \ + --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ + --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 + # via pytest iso8601==1.1.0 \ --hash=sha256:32811e7b81deee2063ea6d2e94f8819a86d1f3811e49d23623a41fa832bef03f \ --hash=sha256:8400e90141bf792bce2634df533dc57e3bee19ea120a87bebcd3da89a58ad73f @@ -555,6 +564,10 @@ openapi-python-client==0.13.1 \ --hash=sha256:43bcd2e43e39dc31decba76ec09cbaeb6faad8709ce4684aec9b0228cd1bf3b5 \ --hash=sha256:adb5d886946cae2ff654f26396bd4d3f497234d5a9dafee805ee19587acbfdce # via -r ./requirements.txt +packaging==23.0 \ + --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ + --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 + # via pytest passlib[bcrypt]==1.7.4 \ --hash=sha256:aa6bca462b8d8bda89c70b382f0c298a20b5560af6cbfa2dce410c0a2fb669f1 \ --hash=sha256:defd50f72b65c5402ab2c573830a6978e5f202ad0d984793c8dde2c4152ebe04 @@ -569,6 +582,10 @@ platformdirs==2.6.2 \ # via # black # pylint +pluggy==1.0.0 \ + --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \ + --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 + # via pytest priority==2.0.0 \ --hash=sha256:6f8eefce5f3ad59baf2c080a664037bb4725cd0a790d53d59ab4059288faf6aa \ --hash=sha256:c965d54f1b8d0d0b19479db3924c7c36cf672dbf2aec92d43fbdaf4492ba18c0 @@ -643,6 +660,21 @@ pypika-tortoise==0.1.6 \ --hash=sha256:2d68bbb7e377673743cff42aa1059f3a80228d411fbcae591e4465e173109fd8 \ --hash=sha256:d802868f479a708e3263724c7b5719a26ad79399b2a70cea065f4a4cadbebf36 # via tortoise-orm +pytest==7.2.1 \ + --hash=sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5 \ + --hash=sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42 + # via + # -r ./requirements.txt + # pytest-asyncio + # pytest-mock +pytest-asyncio==0.20.3 \ + --hash=sha256:83cbf01169ce3e8eb71c6c278ccb0574d1a7a3bb8eaaf5e50e0ad342afb33b36 \ + --hash=sha256:f129998b209d04fcc65c96fc85c11e5316738358909a8399e93be553d7656442 + # via -r ./requirements.txt +pytest-mock==3.10.0 \ + --hash=sha256:f4c973eeae0282963eb293eb173ce91b091a79c1334455acfac9ddee8a1c784b \ + --hash=sha256:fbbdb085ef7c252a326fd8cdcac0aa3b1333d8811f131bdcc701002e1be7ed4f + # via -r ./requirements.txt python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 @@ -765,6 +797,7 @@ tomli==2.0.1 \ # autoflake # black # pylint + # pytest tomlkit==0.11.6 \ --hash=sha256:07de26b0d8cfc18f871aec595fda24d95b08fef89d147caa861939f37230bf4b \ --hash=sha256:71b952e5721688937fb02cf9d354dbcf0785066149d2855e44531ebdd2b65d73