Add apollo_tree and fix pylint warnings

This commit is contained in:
Mustafa Gezen 2023-02-04 00:37:45 +01:00
parent b498d3b889
commit 5e1fde1027
Signed by untrusted user who does not match committer: mustafa
GPG Key ID: DCDF010D946438C1
28 changed files with 1503 additions and 28 deletions

18
.github/workflows/test.yaml vendored Normal file
View File

@ -0,0 +1,18 @@
name: Lint and test
on:
push:
branches: [ "main" ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Lint
run: ./build/scripts/pylint.bash
- name: Test
run: ./build/scripts/test.bash

View File

@ -14,5 +14,8 @@
"editor.formatOnSave": true,
"[python]": {
"editor.tabSize": 4
},
"[xml]": {
"editor.formatOnSave": false
}
}

View File

@ -0,0 +1,346 @@
#!/usr/bin/env python3
import os
import argparse
import asyncio
import logging
import hashlib
import gzip
from dataclasses import dataclass
import time
from urllib.parse import quote
from xml.etree import ElementTree as ET
import aiohttp
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("apollo_tree")
NS = {
"": "http://linux.duke.edu/metadata/repo",
"rpm": "http://linux.duke.edu/metadata/rpm"
}
@dataclass
class Repository:
base_repomd: str = None
source_repomd: str = None
debug_repomd: str = None
arch: str = None
async def scan_path(
base_path: str,
fmt: str,
ignore_dirs: list[str],
):
"""
Scan base path for repositories
The format string can contain $reponame and $arch
When we reach $reponame, that means we have found a repository
Follow the path further into the tree until $arch is found
That determines the architecture of the repository
"""
repos = {}
# First we need to find the root.
# Construct root by prepending base_path and all parts until $reponame
# Then we can walk the tree from there
root = base_path
parts = fmt.split("/")
repo_first = True
if "$reponame" in parts:
for part in parts:
parts.pop(0)
if part == "$reponame":
break
if part == "$arch":
repo_first = False
break
root = os.path.join(root, part)
logger.info("Found root: %s", root)
# Walk the base path
for directory in os.listdir(root):
if directory in ignore_dirs:
continue
current_parts = parts
if repo_first:
repo_name = directory
logger.info("Found repo: %s", repo_name)
else:
arch = directory
logger.info("Found arch: %s", arch)
repo_base = os.path.join(root, directory)
if repo_first:
repos[repo_name] = []
# Construct repo base until we reach $arch
if "$arch" in current_parts:
for part in current_parts:
if (part == "$arch" and repo_first) or part == "$reponame":
break
repo_base = os.path.join(repo_base, part)
current_parts.pop(0)
logger.warning("Searching for arches in %s", repo_base)
# All dirs in repo_base is an architecture
for arch_ in os.listdir(repo_base):
if repo_first:
arch = arch_
else:
repo_name = arch_
# Now append each combination + rest of parts as repo_info
if repo_first:
logger.info("Found arch: %s", arch)
else:
logger.info("Found repo: %s", repo_name)
if repo_first:
found_path = f"{repo_base}/{arch}/{'/'.join(current_parts[1:])}"
else:
found_path = f"{repo_base}/{repo_name}/{'/'.join(current_parts[1:])}"
# Verify that the path exists
if not os.path.exists(found_path):
logger.warning("Path does not exist: %s, skipping", found_path)
continue
repo = {
"name": repo_name,
"arch": arch,
"found_path": found_path,
}
if repo_name not in repos:
repos[repo_name] = []
repos[repo_name].append(repo)
return repos
async def fetch_updateinfo_from_apollo(
repo: dict,
product_name: str,
arch: str = None,
api_base: str = None,
) -> str:
if not api_base:
api_base = "https://apollo.build.resf.org/api/v3/updateinfo"
api_url = f"{api_base}/{quote(product_name)}/{quote(repo['name'])}/updateinfo.xml"
if arch:
api_url += f"?req_arch={arch}"
logger.info("Fetching updateinfo from %s", api_url)
async with aiohttp.ClientSession() as session:
async with session.get(api_url) as resp:
if resp.status != 200:
logger.error("Failed to fetch updateinfo from %s", api_url)
return None
return await resp.text()
async def gzip_updateinfo(updateinfo: str) -> dict:
# Gzip updateinfo, get both open and closed size as
# well as the sha256sum for both
# First get the sha256sum and size of the open updateinfo
sha256sum = hashlib.sha256(updateinfo.encode("utf-8")).hexdigest()
size = len(updateinfo)
# Then gzip it and get hash and size
gzipped = gzip.compress(updateinfo.encode("utf-8"), mtime=0)
gzipped_sha256sum = hashlib.sha256(gzipped).hexdigest()
gzipped_size = len(gzipped)
return {
"sha256sum": sha256sum,
"size": size,
"gzipped_sha256sum": gzipped_sha256sum,
"gzipped_size": gzipped_size,
"gzipped": gzipped,
}
async def write_updateinfo_to_file(
repomd_xml_path: str, updateinfo: dict
) -> str:
# Write updateinfo to file
repomd_dir = os.path.dirname(repomd_xml_path)
gzipped_sum = updateinfo["gzipped_sha256sum"]
updateinfo_path = os.path.join(
repomd_dir, f"{gzipped_sum}-updateinfo.xml.gz"
)
with open(updateinfo_path, "wb") as f:
f.write(updateinfo["gzipped"])
return updateinfo_path
async def update_repomd_xml(repomd_xml_path: str, updateinfo: dict):
# Update repomd.xml with new updateinfo
gzipped_sum = updateinfo["gzipped_sha256sum"]
updateinfo_path = f"{gzipped_sum}-updateinfo.xml.gz"
# Parse repomd.xml
ET.register_namespace("", NS[""])
ET.register_namespace("rpm", NS["rpm"])
repomd_xml = ET.parse(repomd_xml_path).getroot()
# Iterate over data and find type="updateinfo" and delete it
existing_updateinfo_path = None
for data in repomd_xml.findall("data", NS):
data_type = data.attrib["type"]
if not data_type:
logger.warning("No type found in data, skipping")
continue
if data_type == "updateinfo":
# Delete the data element
repomd_xml.remove(data)
# Get the location of the updateinfo file
location = data.find("location", NS)
if not location:
logger.warning("No location found in data, skipping")
continue
existing_updateinfo_path = location.attrib["href"]
break
# Create new data element and set type to updateinfo
data = ET.Element("data")
data.set("type", "updateinfo")
# Add checksum, open-checksum, location, timestamp, size and open-size
checksum = ET.SubElement(data, "checksum")
checksum.set("type", "sha256")
checksum.text = updateinfo["gzipped_sha256sum"]
open_checksum = ET.SubElement(data, "open-checksum")
open_checksum.set("type", "sha256")
open_checksum.text = updateinfo["sha256sum"]
location = ET.SubElement(data, "location")
location.set("href", f"repodata/{updateinfo_path}")
timestamp = ET.SubElement(data, "timestamp")
timestamp.text = str(int(time.time()))
size = ET.SubElement(data, "size")
size.text = str(updateinfo["gzipped_size"])
open_size = ET.SubElement(data, "open-size")
open_size.text = str(updateinfo["size"])
# Add data to repomd.xml
repomd_xml.append(data)
# Create string
ET.indent(repomd_xml)
xml_str = ET.tostring(
repomd_xml,
xml_declaration=True,
encoding="utf-8",
short_empty_elements=True,
)
# Prepend declaration with double quotes
xml_str = xml_str.decode("utf-8")
xml_str = xml_str.replace("'", "\"")
xml_str = xml_str.replace("utf-8", "UTF-8")
# "Fix" closing tags to not have a space
xml_str = xml_str.replace(" />", "/>")
# Add xmlns:rpm
xml_str = xml_str.replace(
"repo\">",
f"repo\" xmlns:rpm=\"{NS['rpm']}\">",
)
# Write to repomd.xml
with open(repomd_xml_path, "w", encoding="utf-8") as f:
f.write(xml_str)
# Delete old updateinfo file
if existing_updateinfo_path:
os.remove(existing_updateinfo_path)
async def main(args):
base_paths = await scan_path(
args.path,
args.base_format,
args.ignore,
)
print(args)
pass
if __name__ == "__main__":
parser = argparse.ArgumentParser(
prog="apollo_tree",
description="Apollo updateinfo.xml publisher (Local file tree)",
epilog="(C) 2023 Rocky Enterprise Software Foundation, Inc.",
)
parser.add_argument(
"-b",
"--base-format",
default="$reponame/$arch/os/repodata/repomd.xml",
help="Format for main repo.xml file",
)
parser.add_argument(
"-m",
"--manual",
action="store_true",
help="Manual mode",
)
parser.add_argument(
"-r",
"--repos",
nargs="+",
action="append",
default=[],
help="Repositories to publish (manual mode), format: <arch>:<repomd>",
)
parser.add_argument(
"-a",
"--auto-scan",
default=True,
action="store_true",
help="Automatically scan for repos",
)
parser.add_argument(
"-p",
"--path",
help="Default path to scan for repos",
)
parser.add_argument(
"-i",
"--ignore",
nargs="+",
action="append",
default=[],
help="Directories in base path to ignore in auto-scan mode",
)
parser.add_argument(
"-n",
"--product-name",
required=True,
help="Product name",
)
p_args = parser.parse_args()
if p_args.auto_scan and p_args.manual:
parser.error("Cannot use --auto-scan and --manual together")
if p_args.manual and not p_args.repos:
parser.error("Must specify repos to publish in manual mode")
if p_args.auto_scan and not p_args.path:
parser.error("Must specify path to scan for repos in auto-scan mode")
asyncio.run(main(p_args))

View File

@ -126,7 +126,11 @@ class Advisory(JSONWizard):
Returns whether this advisory affects the given RHEL version and architecture.
"""
for product in self.get_products():
if product.variant == "Red Hat Enterprise Linux" and product.major_version == major_version and product.minor_version == minor_version and product.arch == arch.value:
is_variant = product.variant == "Red Hat Enterprise Linux"
is_major_version = product.major_version == major_version
is_minor_version = product.minor_version == minor_version
is_arch = product.arch == arch.value
if is_variant and is_major_version and is_minor_version and is_arch:
return True
return False

View File

@ -1,5 +1,4 @@
import asyncio
import datetime
from __init__ import API, Architecture

View File

@ -8,8 +8,6 @@ from os import path
import aiohttp
import yaml
from common.logger import Logger
NVRA_RE = re.compile(
r"^(\S+)-([\w~%.+]+)-(\w+(?:\.[\w~%+]+)+?)(?:\.(\w+))?(?:\.rpm)?$"
)

View File

@ -111,7 +111,7 @@ async def clone_advisory(
logger = Logger()
logger.info("Cloning advisory %s to %s", advisory.name, product.name)
acceptable_arches = list(set([x.match_arch for x in mirrors]))
acceptable_arches = list({x.match_arch for x in mirrors})
acceptable_arches.extend(["src", "noarch"])
for mirror in mirrors:
if mirror.match_arch == "x86_64":
@ -377,14 +377,12 @@ async def clone_advisory(
# Check if topic is empty, if so construct it
if not new_advisory.topic:
package_names = list(set([p.package_name for p in new_pkgs]))
package_names = list({p.package_name for p in new_pkgs})
affected_products = list(
set(
[
f"{product.name} {mirror.match_major_version}"
for mirror in mirrors
]
)
{
f"{product.name} {mirror.match_major_version}"
for mirror in mirrors
}
)
topic = f"""An update is available for {', '.join(package_names)}.
This update affects {', '.join(affected_products)}.

View File

@ -125,4 +125,4 @@ async def get_advisory(request: Request, advisory_name: str):
"advisory": advisory,
"package_map": package_map,
}
)
)

View File

@ -1,6 +1,6 @@
from typing import TypeVar, Generic
from fastapi import APIRouter, Request
from fastapi import APIRouter
from fastapi.exceptions import HTTPException
from fastapi_pagination.links import Page
from fastapi_pagination.ext.tortoise import paginate

View File

@ -90,12 +90,10 @@ def v3_advisory_to_v2(
kind = "TYPE_ENHANCEMENT"
affected_products = list(
set(
[
f"{ap.variant} {ap.major_version}"
for ap in advisory.affected_products
]
)
{
f"{ap.variant} {ap.major_version}"
for ap in advisory.affected_products
}
)
cves = []

View File

@ -67,7 +67,7 @@ async def health():
@app.exception_handler(404)
async def not_found_handler(request, exc):
async def not_found_handler(request, exc): # pylint: disable=unused-argument
if request.url.path.startswith("/api"
) or request.url.path.startswith("/v2"):
return JSONResponse({"error": "Not found"}, status_code=404)

0
apollo/tests/BUILD.bazel Normal file
View File

View File

@ -0,0 +1,88 @@
<?xml version="1.0" encoding="UTF-8"?>
<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
<revision>8.7</revision>
<tags>
<distro cpeid="cpe:/o:rocky:rocky:8">Rocky Linux 8</distro>
</tags>
<data type="primary">
<checksum type="sha256">602602cc03961da7245740aa32af042c28fa7a4bed01e9f51dcde8326c6240e3</checksum>
<open-checksum type="sha256">a69fcb753072006097c806f9f441553e5086b1369180a72ae719fad654246554</open-checksum>
<location href="repodata/602602cc03961da7245740aa32af042c28fa7a4bed01e9f51dcde8326c6240e3-primary.xml.gz"/>
<timestamp>1674782842</timestamp>
<size>1648129</size>
<open-size>16701409</open-size>
</data>
<data type="filelists">
<checksum type="sha256">693e2deed8288074851942f71f525bfce865a71734214fa434010dd00a215283</checksum>
<open-checksum type="sha256">bc311897f94631363f555ef354525f28613df9effb7e4a0459acffea29bf3ba1</open-checksum>
<location href="repodata/693e2deed8288074851942f71f525bfce865a71734214fa434010dd00a215283-filelists.xml.gz"/>
<timestamp>1674782842</timestamp>
<size>6162410</size>
<open-size>87526801</open-size>
</data>
<data type="other">
<checksum type="sha256">fd97455be05a44ffe98685a63031d1ac10661ca0c0bafa747dfae6c479261e6d</checksum>
<open-checksum type="sha256">518c3a001a453adb34cf5a70ceecf11aa2d90f0c807c879d76a02f7123f5c22f</open-checksum>
<location href="repodata/fd97455be05a44ffe98685a63031d1ac10661ca0c0bafa747dfae6c479261e6d-other.xml.gz"/>
<timestamp>1674782842</timestamp>
<size>1102970</size>
<open-size>12298845</open-size>
</data>
<data type="primary_db">
<checksum type="sha256">ae73143161fda8b62f6b1cc333ee3fcd4100daa0947a5b918e80b4a487ded360</checksum>
<open-checksum type="sha256">9f9b5bb6f1772f3f742f66245041246021777015d1d30970d5410b988cdd233a</open-checksum>
<location href="repodata/ae73143161fda8b62f6b1cc333ee3fcd4100daa0947a5b918e80b4a487ded360-primary.sqlite.xz"/>
<timestamp>1674782850</timestamp>
<size>2724240</size>
<open-size>17735680</open-size>
<database_version>10</database_version>
</data>
<data type="filelists_db">
<checksum type="sha256">75d9455e941cb3e8a5266980b78eccb26476198bb4a1f3142ee6cda05ed044f9</checksum>
<open-checksum type="sha256">23c2249ccb4fe69474b41c1399aac94d4fd1a1d2c6c2cefc46cb23d4dd52e3b0</open-checksum>
<location href="repodata/75d9455e941cb3e8a5266980b78eccb26476198bb4a1f3142ee6cda05ed044f9-filelists.sqlite.xz"/>
<timestamp>1674782854</timestamp>
<size>4556600</size>
<open-size>37732352</open-size>
<database_version>10</database_version>
</data>
<data type="other_db">
<checksum type="sha256">f3968e32961df9482a5c265adaeae9a3435c03b81b2c76cfd7c8b1a74337b972</checksum>
<open-checksum type="sha256">e84a16aa1bec1a0d6af28ced81aa3a0708d2599b56abbedb858caced501576e9</open-checksum>
<location href="repodata/f3968e32961df9482a5c265adaeae9a3435c03b81b2c76cfd7c8b1a74337b972-other.sqlite.xz"/>
<timestamp>1674782846</timestamp>
<size>1113696</size>
<open-size>10711040</open-size>
<database_version>10</database_version>
</data>
<data type="group">
<checksum type="sha256">341f7a33a85d237227c4b814e5ffd57cc7ead34dab6ca79db4feff8520803b22</checksum>
<location href="repodata/341f7a33a85d237227c4b814e5ffd57cc7ead34dab6ca79db4feff8520803b22-comps-AppStream.aarch64.xml"/>
<timestamp>1674782725</timestamp>
<size>428231</size>
</data>
<data type="group_xz">
<checksum type="sha256">c0562df3ad4d3d47bb8076482351eca478a02f4bd15e21b4683456268920154a</checksum>
<open-checksum type="sha256">341f7a33a85d237227c4b814e5ffd57cc7ead34dab6ca79db4feff8520803b22</open-checksum>
<location href="repodata/c0562df3ad4d3d47bb8076482351eca478a02f4bd15e21b4683456268920154a-comps-AppStream.aarch64.xml.xz"/>
<timestamp>1674782842</timestamp>
<size>73836</size>
<open-size>428231</open-size>
</data>
<data type="modules">
<checksum type="sha256">d53d510b653b6356691a5e9cf5adf2f803c525f0c9f908adf9ccbb6c88a13967</checksum>
<open-checksum type="sha256">479622387203132ad97518c6ccdf2a41996f2fddc8a5c9bdf09fcb7407064c09</open-checksum>
<location href="repodata/d53d510b653b6356691a5e9cf5adf2f803c525f0c9f908adf9ccbb6c88a13967-modules.yaml.xz"/>
<timestamp>1674783016</timestamp>
<size>73124</size>
<open-size>712620</open-size>
</data>
<data type="updateinfo">
<checksum type="sha256">be8915c2f7ed08fca20532364bd01b12ab51e826d45233f92832f318d1b78c3e</checksum>
<open-checksum type="sha256">eec1c28a088aea1719b3ef2edacba8cdc78658fcf65009d2520f76e43ba90566</open-checksum>
<location href="repodata/be8915c2f7ed08fca20532364bd01b12ab51e826d45233f92832f318d1b78c3e-updateinfo.xml.gz"/>
<timestamp>1674284973</timestamp>
<size>233607</size>
<open-size>2402781</open-size>
</data>
</repomd>

View File

@ -0,0 +1,88 @@
<?xml version="1.0" encoding="UTF-8"?>
<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
<revision>8.7</revision>
<tags>
<distro cpeid="cpe:/o:rocky:rocky:8">Rocky Linux 8</distro>
</tags>
<data type="primary">
<checksum type="sha256">593ce13bf52aadd58d9da7c2b3fc2149bffd7bc4924c7032946b51cc340fc976</checksum>
<open-checksum type="sha256">c808007edc3392c44ac219825b4c73f4ba717721f4df2dc9aba023413d3dfab0</open-checksum>
<location href="repodata/593ce13bf52aadd58d9da7c2b3fc2149bffd7bc4924c7032946b51cc340fc976-primary.xml.gz"/>
<timestamp>1674781924</timestamp>
<size>1962142</size>
<open-size>20488457</open-size>
</data>
<data type="filelists">
<checksum type="sha256">bce97b0287c0f9f9d900b5c9681cfe73f69890c26aae9792fffe25429fc2186b</checksum>
<open-checksum type="sha256">4e7729a83d7917070277bc630cc8cea6a620fab59d584a12dc7252d33e8c5632</open-checksum>
<location href="repodata/bce97b0287c0f9f9d900b5c9681cfe73f69890c26aae9792fffe25429fc2186b-filelists.xml.gz"/>
<timestamp>1674781924</timestamp>
<size>7066338</size>
<open-size>99500723</open-size>
</data>
<data type="other">
<checksum type="sha256">902ea6dea89caac9980c8599cd53ce154a1e8bff3a1d212d70c38d823e08c028</checksum>
<open-checksum type="sha256">c5f4414a2cacff80f6c038761e440f61ed4c6bdefeba66e0062f228be408afc9</open-checksum>
<location href="repodata/902ea6dea89caac9980c8599cd53ce154a1e8bff3a1d212d70c38d823e08c028-other.xml.gz"/>
<timestamp>1674781924</timestamp>
<size>1263392</size>
<open-size>15731326</open-size>
</data>
<data type="primary_db">
<checksum type="sha256">c1b32bc2fcef5863e3d0ab03263dbee5d645eb5b299d76a6edf23701504544a3</checksum>
<open-checksum type="sha256">ef04c60effc3269b15aa41d99713169e1fa0ea9196ce5167c6096d42e9ae8c0a</open-checksum>
<location href="repodata/c1b32bc2fcef5863e3d0ab03263dbee5d645eb5b299d76a6edf23701504544a3-primary.sqlite.xz"/>
<timestamp>1674781934</timestamp>
<size>3321344</size>
<open-size>21495808</open-size>
<database_version>10</database_version>
</data>
<data type="filelists_db">
<checksum type="sha256">bcf3f665052787a1f84ce5de17cc05ab0aa526de282ef6dfcdd8c8a75bf2ac26</checksum>
<open-checksum type="sha256">8bd4fd292c8b4041acfc0a113f8e457b55bc2ad75755e17c70e116d71f6b6019</open-checksum>
<location href="repodata/bcf3f665052787a1f84ce5de17cc05ab0aa526de282ef6dfcdd8c8a75bf2ac26-filelists.sqlite.xz"/>
<timestamp>1674781938</timestamp>
<size>5256956</size>
<open-size>44195840</open-size>
<database_version>10</database_version>
</data>
<data type="other_db">
<checksum type="sha256">5c1e7e3286c934c684bbe451356d9ac31968f4e786faa209f0bb3080fe02d67f</checksum>
<open-checksum type="sha256">88246653b636218807ef6f1174590614851df489be10ae8e462bd38e21c6ef2b</open-checksum>
<location href="repodata/5c1e7e3286c934c684bbe451356d9ac31968f4e786faa209f0bb3080fe02d67f-other.sqlite.xz"/>
<timestamp>1674781929</timestamp>
<size>1385640</size>
<open-size>13701120</open-size>
<database_version>10</database_version>
</data>
<data type="group">
<checksum type="sha256">ec2fa1982439f4b0ca7dc86f882f18657457a2d42f06ac6718298c7f7528df43</checksum>
<location href="repodata/ec2fa1982439f4b0ca7dc86f882f18657457a2d42f06ac6718298c7f7528df43-comps-AppStream.x86_64.xml"/>
<timestamp>1674781761</timestamp>
<size>486316</size>
</data>
<data type="group_xz">
<checksum type="sha256">7ba43f88671d8107d6603ebff822fc071e73596a1e20779855af577b0e6e343a</checksum>
<open-checksum type="sha256">ec2fa1982439f4b0ca7dc86f882f18657457a2d42f06ac6718298c7f7528df43</open-checksum>
<location href="repodata/7ba43f88671d8107d6603ebff822fc071e73596a1e20779855af577b0e6e343a-comps-AppStream.x86_64.xml.xz"/>
<timestamp>1674781925</timestamp>
<size>81840</size>
<open-size>486316</open-size>
</data>
<data type="modules">
<checksum type="sha256">7facfc73e398abc34445310a258754aae5ea0add698c4ea039580a43897d0d02</checksum>
<open-checksum type="sha256">da557db5bfd41e8949f8c3904f963cd81fcc444aa4fcc9e99a47ed843948140b</open-checksum>
<location href="repodata/7facfc73e398abc34445310a258754aae5ea0add698c4ea039580a43897d0d02-modules.yaml.xz"/>
<timestamp>1674782139</timestamp>
<size>76500</size>
<open-size>742747</open-size>
</data>
<data type="updateinfo">
<checksum type="sha256">5c45121e44d7b58060fb9435733d3ac4e03f2e1965cacd5371eb1649e15d5c06</checksum>
<open-checksum type="sha256">dda8db3b243e33262f799fb4361bd21a16b1ff22ddd0224bd18070496942cf3d</open-checksum>
<location href="repodata/5c45121e44d7b58060fb9435733d3ac4e03f2e1965cacd5371eb1649e15d5c06-updateinfo.xml.gz"/>
<timestamp>1674284975</timestamp>
<size>276699</size>
<open-size>2720880</open-size>
</data>
</repomd>

View File

@ -0,0 +1,80 @@
<?xml version="1.0" encoding="UTF-8"?>
<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
<revision>8.7</revision>
<tags>
<distro cpeid="cpe:/o:rocky:rocky:8">Rocky Linux 8</distro>
</tags>
<data type="primary">
<checksum type="sha256">9f55c0c1cf7cb1fe08bc208f0286d64f274339df4ba857ba978e544a9924fa92</checksum>
<open-checksum type="sha256">8a5e972376710cb07fc5eb1f559d0c4456b1b58a36d219474eae66b4d10f7040</open-checksum>
<location href="repodata/9f55c0c1cf7cb1fe08bc208f0286d64f274339df4ba857ba978e544a9924fa92-primary.xml.gz"/>
<timestamp>1674782716</timestamp>
<size>1346598</size>
<open-size>9900543</open-size>
</data>
<data type="filelists">
<checksum type="sha256">0457005390605cd3154a88584c7ef6a0e5e37a15c8035dfcde72f9d832ab14e2</checksum>
<open-checksum type="sha256">ae8651d8f568994088e3fc576538c6f4638ec29475d6d4c4739fd492028fc0fb</open-checksum>
<location href="repodata/0457005390605cd3154a88584c7ef6a0e5e37a15c8035dfcde72f9d832ab14e2-filelists.xml.gz"/>
<timestamp>1674782716</timestamp>
<size>1523844</size>
<open-size>19386269</open-size>
</data>
<data type="other">
<checksum type="sha256">30399987025d0024ccf2d1ee6e1b412361d547cc50813777e42730375a3925dd</checksum>
<open-checksum type="sha256">8ea81e1b6d4333cc8553fc4ddd95dc3119e763e7a0c81ea753531fb8f4d7d56a</open-checksum>
<location href="repodata/30399987025d0024ccf2d1ee6e1b412361d547cc50813777e42730375a3925dd-other.xml.gz"/>
<timestamp>1674782716</timestamp>
<size>829148</size>
<open-size>5197359</open-size>
</data>
<data type="primary_db">
<checksum type="sha256">5e78af661835050ef33470ed4c00b81ed2e9f8a76629fc7b8f04153ae273eaca</checksum>
<open-checksum type="sha256">ced2ee6fd2db2e01b5ba521a2a7815c6ab84c1b1cc95dd41acf55faf90f819c6</open-checksum>
<location href="repodata/5e78af661835050ef33470ed4c00b81ed2e9f8a76629fc7b8f04153ae273eaca-primary.sqlite.xz"/>
<timestamp>1674782721</timestamp>
<size>1663124</size>
<open-size>10960896</open-size>
<database_version>10</database_version>
</data>
<data type="filelists_db">
<checksum type="sha256">513e605dd419c53c13f1aa3688ca53338e149dbe6de19b9c8453102bcb9ce882</checksum>
<open-checksum type="sha256">3639a0d1ea827449d47564f4330fe8426d37c1cf96308ed13d40fb3460036104</open-checksum>
<location href="repodata/513e605dd419c53c13f1aa3688ca53338e149dbe6de19b9c8453102bcb9ce882-filelists.sqlite.xz"/>
<timestamp>1674782720</timestamp>
<size>1412020</size>
<open-size>10305536</open-size>
<database_version>10</database_version>
</data>
<data type="other_db">
<checksum type="sha256">33bddcd2c405ef925318e88662acd38f3a7246e78ebaf88639fe7c43c215095e</checksum>
<open-checksum type="sha256">b8d0ffdf150621ecc96e0aa587a895f765e0401937b50ff98e5f2312e3965774</open-checksum>
<location href="repodata/33bddcd2c405ef925318e88662acd38f3a7246e78ebaf88639fe7c43c215095e-other.sqlite.xz"/>
<timestamp>1674782717</timestamp>
<size>356456</size>
<open-size>5017600</open-size>
<database_version>10</database_version>
</data>
<data type="group">
<checksum type="sha256">5f2dec3cbb871be9fb6c456740458773b183987275a1d5c93920d4800323fdf9</checksum>
<location href="repodata/5f2dec3cbb871be9fb6c456740458773b183987275a1d5c93920d4800323fdf9-comps-BaseOS.aarch64.xml"/>
<timestamp>1674782695</timestamp>
<size>289214</size>
</data>
<data type="group_xz">
<checksum type="sha256">01f25f36dcb08b5b027ce222dec48e68bcaa6b9e96de863ce60f126897f76faf</checksum>
<open-checksum type="sha256">5f2dec3cbb871be9fb6c456740458773b183987275a1d5c93920d4800323fdf9</open-checksum>
<location href="repodata/01f25f36dcb08b5b027ce222dec48e68bcaa6b9e96de863ce60f126897f76faf-comps-BaseOS.aarch64.xml.xz"/>
<timestamp>1674782716</timestamp>
<size>55624</size>
<open-size>289214</open-size>
</data>
<data type="updateinfo">
<checksum type="sha256">cb9bb3ef856440c44a70f4f0aa9ee1b3621814c3457f922cdb02d7ed63a2701f</checksum>
<open-checksum type="sha256">596fc124c74802371ff879a1ecc3a7560704413bbfb4a5ab9299671baf12c2fb</open-checksum>
<location href="repodata/cb9bb3ef856440c44a70f4f0aa9ee1b3621814c3457f922cdb02d7ed63a2701f-updateinfo.xml.gz"/>
<timestamp>1674284975</timestamp>
<size>45141</size>
<open-size>433034</open-size>
</data>
</repomd>

View File

@ -0,0 +1,80 @@
<?xml version="1.0" encoding="UTF-8"?>
<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
<revision>8.7</revision>
<tags>
<distro cpeid="cpe:/o:rocky:rocky:8">Rocky Linux 8</distro>
</tags>
<data type="primary">
<checksum type="sha256">712e4af06a8f9f13766274c3fa83bbdf5572a1014ec77f7cd30d56d492a40db2</checksum>
<open-checksum type="sha256">c3ec31964ce06e7dc14a8184ad11d6f1edce3a718bd63e4de8ed18349d27e85e</open-checksum>
<location href="repodata/712e4af06a8f9f13766274c3fa83bbdf5572a1014ec77f7cd30d56d492a40db2-primary.xml.gz"/>
<timestamp>1674781748</timestamp>
<size>1791032</size>
<open-size>13677600</open-size>
</data>
<data type="filelists">
<checksum type="sha256">052aea22e07e586ae9383f6da4056599417fc445431c9d0b1136f0316ef4843e</checksum>
<open-checksum type="sha256">dd5952122ed4335b799b1853f096f05b9e9ea6ea1a48e344700b7b82f468d61f</open-checksum>
<location href="repodata/052aea22e07e586ae9383f6da4056599417fc445431c9d0b1136f0316ef4843e-filelists.xml.gz"/>
<timestamp>1674781749</timestamp>
<size>1800846</size>
<open-size>22806093</open-size>
</data>
<data type="other">
<checksum type="sha256">37421cd13a9594ff7dd3a0f2953aa19a01290e90be43db4ce279fef4394fdcd0</checksum>
<open-checksum type="sha256">3be8a12fb7bcd600b178f6b00404d1e4e9196b9f0d5faea460c58b8d07771747</open-checksum>
<location href="repodata/37421cd13a9594ff7dd3a0f2953aa19a01290e90be43db4ce279fef4394fdcd0-other.xml.gz"/>
<timestamp>1674781749</timestamp>
<size>878178</size>
<open-size>6343206</open-size>
</data>
<data type="primary_db">
<checksum type="sha256">2b1d2886dec67d493adcf7358e44b7f7fd4886c85bab7fa9854014ad9c404d7a</checksum>
<open-checksum type="sha256">a2a2c52ce2b2ce80e936a9488c8799c8508cbfe1ff5dd03187e62f8cf659ca8b</open-checksum>
<location href="repodata/2b1d2886dec67d493adcf7358e44b7f7fd4886c85bab7fa9854014ad9c404d7a-primary.sqlite.xz"/>
<timestamp>1674781756</timestamp>
<size>2230076</size>
<open-size>15314944</open-size>
<database_version>10</database_version>
</data>
<data type="filelists_db">
<checksum type="sha256">637abdee51be41d56603a2c031bfaf0dd4fc9da0127c826328d9a527aa3b5aa0</checksum>
<open-checksum type="sha256">979021c34ae689ddfdd32f1a37e78da16fdb827e8bb5c0f8818ccfe880906b74</open-checksum>
<location href="repodata/637abdee51be41d56603a2c031bfaf0dd4fc9da0127c826328d9a527aa3b5aa0-filelists.sqlite.xz"/>
<timestamp>1674781753</timestamp>
<size>1689576</size>
<open-size>12529664</open-size>
<database_version>10</database_version>
</data>
<data type="other_db">
<checksum type="sha256">45ff5a2543dd16101778346f47e0d2362fe546698f137a05b7b72d638f03904f</checksum>
<open-checksum type="sha256">e045ad4b00b3d20c2cd489a89297ed7c7d0efa1b279cd012b1e53dab06efef24</open-checksum>
<location href="repodata/45ff5a2543dd16101778346f47e0d2362fe546698f137a05b7b72d638f03904f-other.sqlite.xz"/>
<timestamp>1674781750</timestamp>
<size>427448</size>
<open-size>6021120</open-size>
<database_version>10</database_version>
</data>
<data type="group">
<checksum type="sha256">dae7e104812099a2f632ea4c5ef2769aca18ca1205abdd2c3ba6d171e319df3d</checksum>
<location href="repodata/dae7e104812099a2f632ea4c5ef2769aca18ca1205abdd2c3ba6d171e319df3d-comps-BaseOS.x86_64.xml"/>
<timestamp>1674781726</timestamp>
<size>298889</size>
</data>
<data type="group_xz">
<checksum type="sha256">741d3c80487757df624285f4a107f925abfac16115210486760a3920d8724e38</checksum>
<open-checksum type="sha256">dae7e104812099a2f632ea4c5ef2769aca18ca1205abdd2c3ba6d171e319df3d</open-checksum>
<location href="repodata/741d3c80487757df624285f4a107f925abfac16115210486760a3920d8724e38-comps-BaseOS.x86_64.xml.xz"/>
<timestamp>1674781749</timestamp>
<size>57068</size>
<open-size>298889</open-size>
</data>
<data type="updateinfo">
<checksum type="sha256">568de83be47822b08ea890ce0f58fd263c61c9a5c5dc500789d25e7020827112</checksum>
<open-checksum type="sha256">1be116b93938f8bf96bb27115a63be34b59b36fcb4b2bfd3fc22c4eb3e3ffce5</open-checksum>
<location href="repodata/568de83be47822b08ea890ce0f58fd263c61c9a5c5dc500789d25e7020827112-updateinfo.xml.gz"/>
<timestamp>1674284973</timestamp>
<size>56390</size>
<open-size>508695</open-size>
</data>
</repomd>

View File

@ -0,0 +1,80 @@
<?xml version="1.0" encoding="UTF-8"?>
<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
<revision>8.7</revision>
<tags>
<distro cpeid="cpe:/o:rocky:rocky:8">Rocky Linux 8</distro>
</tags>
<data type="primary">
<checksum type="sha256">712e4af06a8f9f13766274c3fa83bbdf5572a1014ec77f7cd30d56d492a40db2</checksum>
<open-checksum type="sha256">c3ec31964ce06e7dc14a8184ad11d6f1edce3a718bd63e4de8ed18349d27e85e</open-checksum>
<location href="repodata/712e4af06a8f9f13766274c3fa83bbdf5572a1014ec77f7cd30d56d492a40db2-primary.xml.gz"/>
<timestamp>1674781748</timestamp>
<size>1791032</size>
<open-size>13677600</open-size>
</data>
<data type="filelists">
<checksum type="sha256">052aea22e07e586ae9383f6da4056599417fc445431c9d0b1136f0316ef4843e</checksum>
<open-checksum type="sha256">dd5952122ed4335b799b1853f096f05b9e9ea6ea1a48e344700b7b82f468d61f</open-checksum>
<location href="repodata/052aea22e07e586ae9383f6da4056599417fc445431c9d0b1136f0316ef4843e-filelists.xml.gz"/>
<timestamp>1674781749</timestamp>
<size>1800846</size>
<open-size>22806093</open-size>
</data>
<data type="other">
<checksum type="sha256">37421cd13a9594ff7dd3a0f2953aa19a01290e90be43db4ce279fef4394fdcd0</checksum>
<open-checksum type="sha256">3be8a12fb7bcd600b178f6b00404d1e4e9196b9f0d5faea460c58b8d07771747</open-checksum>
<location href="repodata/37421cd13a9594ff7dd3a0f2953aa19a01290e90be43db4ce279fef4394fdcd0-other.xml.gz"/>
<timestamp>1674781749</timestamp>
<size>878178</size>
<open-size>6343206</open-size>
</data>
<data type="primary_db">
<checksum type="sha256">2b1d2886dec67d493adcf7358e44b7f7fd4886c85bab7fa9854014ad9c404d7a</checksum>
<open-checksum type="sha256">a2a2c52ce2b2ce80e936a9488c8799c8508cbfe1ff5dd03187e62f8cf659ca8b</open-checksum>
<location href="repodata/2b1d2886dec67d493adcf7358e44b7f7fd4886c85bab7fa9854014ad9c404d7a-primary.sqlite.xz"/>
<timestamp>1674781756</timestamp>
<size>2230076</size>
<open-size>15314944</open-size>
<database_version>10</database_version>
</data>
<data type="filelists_db">
<checksum type="sha256">637abdee51be41d56603a2c031bfaf0dd4fc9da0127c826328d9a527aa3b5aa0</checksum>
<open-checksum type="sha256">979021c34ae689ddfdd32f1a37e78da16fdb827e8bb5c0f8818ccfe880906b74</open-checksum>
<location href="repodata/637abdee51be41d56603a2c031bfaf0dd4fc9da0127c826328d9a527aa3b5aa0-filelists.sqlite.xz"/>
<timestamp>1674781753</timestamp>
<size>1689576</size>
<open-size>12529664</open-size>
<database_version>10</database_version>
</data>
<data type="other_db">
<checksum type="sha256">45ff5a2543dd16101778346f47e0d2362fe546698f137a05b7b72d638f03904f</checksum>
<open-checksum type="sha256">e045ad4b00b3d20c2cd489a89297ed7c7d0efa1b279cd012b1e53dab06efef24</open-checksum>
<location href="repodata/45ff5a2543dd16101778346f47e0d2362fe546698f137a05b7b72d638f03904f-other.sqlite.xz"/>
<timestamp>1674781750</timestamp>
<size>427448</size>
<open-size>6021120</open-size>
<database_version>10</database_version>
</data>
<data type="group">
<checksum type="sha256">dae7e104812099a2f632ea4c5ef2769aca18ca1205abdd2c3ba6d171e319df3d</checksum>
<location href="repodata/dae7e104812099a2f632ea4c5ef2769aca18ca1205abdd2c3ba6d171e319df3d-comps-BaseOS.x86_64.xml"/>
<timestamp>1674781726</timestamp>
<size>298889</size>
</data>
<data type="group_xz">
<checksum type="sha256">741d3c80487757df624285f4a107f925abfac16115210486760a3920d8724e38</checksum>
<open-checksum type="sha256">dae7e104812099a2f632ea4c5ef2769aca18ca1205abdd2c3ba6d171e319df3d</open-checksum>
<location href="repodata/741d3c80487757df624285f4a107f925abfac16115210486760a3920d8724e38-comps-BaseOS.x86_64.xml.xz"/>
<timestamp>1674781749</timestamp>
<size>57068</size>
<open-size>298889</open-size>
</data>
<data type="updateinfo">
<checksum type="sha256">2242022f6b5935ee22d4ba78d65da0a37873e5a42f00de907795b809d3dadb59</checksum>
<open-checksum type="sha256">8ad9b2cab0f009a09fc6ed6f6fb946ce734e1d90d0d2ff6faf878788393ba4d8</open-checksum>
<location href="repodata/2242022f6b5935ee22d4ba78d65da0a37873e5a42f00de907795b809d3dadb59-updateinfo.xml.gz"/>
<timestamp>1674284973</timestamp>
<size>1706</size>
<open-size>8335</open-size>
</data>
</repomd>

View File

@ -0,0 +1,111 @@
<updates>
<update from="releng@rockylinux.org" status="final" type="security" version="2">
<id>RLSA-2022:1821</id>
<title>Moderate: python27:2.7 security update</title>
<description>Python is an interpreted, interactive, object-oriented programming language that supports modules, classes, exceptions, high-level dynamic data types, and dynamic typing. The python27 packages provide a stable release of Python 2.7 with a number of additional utilities and database connectors for MySQL and PostgreSQL.
Security Fix(es):
* python: urllib: Regular expression DoS in AbstractBasicAuthHandler (CVE-2021-3733)
* python: ftplib should not use the host from the PASV response (CVE-2021-4189)
* python-lxml: HTML Cleaner allows crafted and SVG embedded scripts to pass through (CVE-2021-43818)
* python: urllib.parse does not sanitize URLs containing ASCII newline and tabs (CVE-2022-0391)
* python: urllib: HTTP client possible infinite loop on a 100 Continue response (CVE-2021-3737)
For more details about the security issue(s), including the impact, a CVSS score, acknowledgments, and other related information, refer to the CVE page(s) listed in the References section.
Additional Changes:
For detailed information on changes in this release, see the Rocky Linux 8.6 Release Notes linked from the References section.</description>
<issued>2022-05-10 08:02:50</issued>
<updated>2023-02-02 13:41:28</updated>
<rights>Copyright 2023 Rocky Enterprise Software Foundation</rights>
<release>Rocky Linux 8</release>
<pushcount>1</pushcount>
<severity>Moderate</severity>
<summary>An update is available for python-pymongo, python2-rpm-macros, python-sqlalchemy, python-backports, python-docutils, pytest, python-psycopg2, python-lxml, python-PyMySQL, python-urllib3, PyYAML, python-pytest-mock, python-attrs, python-jinja2, python-docs, python-requests, python-mock, python-ipaddress, python-funcsigs, python2-six, python-py, python2, python2-pip, python-chardet, python-markupsafe, python-pluggy, python-pygments, python2-setuptools, Cython, python-virtualenv, babel, python-dns, python-wheel, python-pysocks, python-backports-ssl_match_hostname, python-coverage, python-setuptools_scm, pytz, python-nose, scipy, python-idna, numpy.
This update affects Rocky Linux 8.
A Common Vulnerability Scoring System (CVSS) base score, which gives a detailed severity rating, is available for each vulnerability from the CVE list</summary>
<description>Python is an interpreted, interactive, object-oriented programming language that supports modules, classes, exceptions, high-level dynamic data types, and dynamic typing. The python27 packages provide a stable release of Python 2.7 with a number of additional utilities and database connectors for MySQL and PostgreSQL.
Security Fix(es):
* python: urllib: Regular expression DoS in AbstractBasicAuthHandler (CVE-2021-3733)
* python: ftplib should not use the host from the PASV response (CVE-2021-4189)
* python-lxml: HTML Cleaner allows crafted and SVG embedded scripts to pass through (CVE-2021-43818)
* python: urllib.parse does not sanitize URLs containing ASCII newline and tabs (CVE-2022-0391)
* python: urllib: HTTP client possible infinite loop on a 100 Continue response (CVE-2021-3737)
For more details about the security issue(s), including the impact, a CVSS score, acknowledgments, and other related information, refer to the CVE page(s) listed in the References section.
Additional Changes:
For detailed information on changes in this release, see the Rocky Linux 8.6 Release Notes linked from the References section.</description>
<solution />
<references>
<reference href="https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-3733" id="CVE-2021-3733" type="cve" title="CVE-2021-3733" />
<reference href="https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-3737" id="CVE-2021-3737" type="cve" title="CVE-2021-3737" />
<reference href="https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-4189" id="CVE-2021-4189" type="cve" title="CVE-2021-4189" />
<reference href="https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-43818" id="CVE-2021-43818" type="cve" title="CVE-2021-43818" />
<reference href="https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-0391" id="CVE-2022-0391" type="cve" title="CVE-2022-0391" />
<reference href="https://bugzilla.redhat.com/show_bug.cgi?id=1995162" id="1995162" type="bugzilla" title="" />
<reference href="https://bugzilla.redhat.com/show_bug.cgi?id=1995234" id="1995234" type="bugzilla" title="" />
<reference href="https://bugzilla.redhat.com/show_bug.cgi?id=2006792" id="2006792" type="bugzilla" title="" />
<reference href="https://bugzilla.redhat.com/show_bug.cgi?id=2032569" id="2032569" type="bugzilla" title="" />
<reference href="https://bugzilla.redhat.com/show_bug.cgi?id=2036020" id="2036020" type="bugzilla" title="" />
<reference href="https://bugzilla.redhat.com/show_bug.cgi?id=2047376" id="2047376" type="bugzilla" title="" />
<reference href="https://errata.rockylinux.org/RLSA-2022:1821" id="RLSA-2022:1821" type="self" title="RLSA-2022:1821" />
</references>
<pkglist />
</update>
<update from="releng@rockylinux.org" status="final" type="security" version="2">
<id>RLSA-2022:7593</id>
<title>Moderate: python27:2.7 security update</title>
<description>Python is an interpreted, interactive, object-oriented programming language that supports modules, classes, exceptions, high-level dynamic data types, and dynamic typing.
Security Fix(es):
* python: mailcap: findmatch() function does not sanitize the second argument (CVE-2015-20107).
For more details about the security issue(s), including the impact, a CVSS score, acknowledgments, and other related information, refer to the CVE page(s) listed in the References section.
Additional Changes:
For detailed information on changes in this release, see the Rocky Linux 8.7 Release Notes linked from the References section.</description>
<issued>2022-11-08 06:23:47</issued>
<updated>2023-02-02 13:52:34</updated>
<rights>Copyright 2023 Rocky Enterprise Software Foundation</rights>
<release>Rocky Linux 8</release>
<pushcount>1</pushcount>
<severity>Moderate</severity>
<summary>An update is available for python-pymongo, python2-rpm-macros, python-sqlalchemy, python-backports, python-docutils, pytest, python-psycopg2, python-lxml, python-PyMySQL, python-urllib3, PyYAML, python-pytest-mock, python-attrs, python-jinja2, python-docs, python-requests, python-mock, python-ipaddress, python-funcsigs, python2-six, python-py, python2, python2-pip, python-chardet, python-markupsafe, python-pluggy, python-pygments, python2-setuptools, Cython, python-virtualenv, babel, python-dns, python-wheel, python-pysocks, python-backports-ssl_match_hostname, python-coverage, python-setuptools_scm, pytz, python-nose, scipy, python-idna, numpy.
This update affects Rocky Linux 8.
A Common Vulnerability Scoring System (CVSS) base score, which gives a detailed severity rating, is available for each vulnerability from the CVE list</summary>
<description>Python is an interpreted, interactive, object-oriented programming language that supports modules, classes, exceptions, high-level dynamic data types, and dynamic typing.
Security Fix(es):
* python: mailcap: findmatch() function does not sanitize the second argument (CVE-2015-20107).
For more details about the security issue(s), including the impact, a CVSS score, acknowledgments, and other related information, refer to the CVE page(s) listed in the References section.
Additional Changes:
For detailed information on changes in this release, see the Rocky Linux 8.7 Release Notes linked from the References section.</description>
<solution />
<references>
<reference href="https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-20107" id="CVE-2015-20107" type="cve" title="CVE-2015-20107" />
<reference href="https://bugzilla.redhat.com/show_bug.cgi?id=2075390" id="2075390" type="bugzilla" title="" />
<reference href="https://errata.rockylinux.org/RLSA-2022:7593" id="RLSA-2022:7593" type="self" title="RLSA-2022:7593" />
</references>
<pkglist />
</update>
</updates>

View File

@ -0,0 +1,498 @@
import tempfile
import shutil
import pathlib
import hashlib
from os import path, environ
import pytest
from apollo.publishing_tools import apollo_tree
from common.testing import MockResponse
data = [
"baseos__base__repomd__x86_64.xml",
"baseos__base__repomd__aarch64.xml",
"appstream__base__repomd__x86_64.xml",
"appstream__base__repomd__aarch64.xml",
]
async def _setup_test_baseos(directory: str):
file = data[0]
base_dir = path.join(
directory,
"BaseOS/x86_64/os/repodata",
)
pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True)
shutil.copyfile(
path.join(path.dirname(__file__), "data", file),
path.join(base_dir, "repomd.xml"),
)
# Run scan_path
repos = await apollo_tree.scan_path(
directory,
"$reponame/$arch/os/repodata/repomd.xml",
[],
)
return repos
@pytest.mark.asyncio
async def test_scan_path_valid_structure():
with tempfile.TemporaryDirectory() as directory:
# Copy test data to temp dir
for file in data:
fsplit = file.split("__")
base_dir = path.join(
directory,
fsplit[0],
fsplit[-1].removesuffix(".xml"),
"os/repodata",
)
pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True)
shutil.copyfile(
path.join(path.dirname(__file__), "data", file),
path.join(base_dir, "repomd.xml"),
)
# Run scan_path
repos = await apollo_tree.scan_path(
directory,
"$reponame/$arch/os/repodata/repomd.xml",
[],
)
assert "baseos" in repos
assert "appstream" in repos
assert len(repos["baseos"]) == 2
assert len(repos["appstream"]) == 2
for repo in repos["baseos"]:
assert repo["name"] == "baseos"
assert repo["arch"] in ["x86_64", "aarch64"]
assert repo["found_path"] == path.join(
directory,
"baseos",
repo["arch"],
"os/repodata/repomd.xml",
)
for repo in repos["appstream"]:
assert repo["name"] == "appstream"
assert repo["arch"] in ["x86_64", "aarch64"]
assert repo["found_path"] == path.join(
directory,
"appstream",
repo["arch"],
"os/repodata/repomd.xml",
)
@pytest.mark.asyncio
async def test_scan_path_multiple_formats():
with tempfile.TemporaryDirectory() as directory:
# Copy test data to temp dir
for file in data:
fsplit = file.split("__")
base_dir = path.join(
directory,
fsplit[0],
fsplit[-1].removesuffix(".xml"),
"os/repodata",
)
pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True)
shutil.copyfile(
path.join(path.dirname(__file__), "data", file),
path.join(base_dir, "repomd.xml"),
)
file = data[0]
fsplit = file.split("__")
base_dir = path.join(
directory,
fsplit[0],
"source/tree/repodata",
)
pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True)
shutil.copyfile(
path.join(path.dirname(__file__), "data", file),
path.join(base_dir, "repomd.xml"),
)
# Run scan_path
repos = await apollo_tree.scan_path(
directory,
"$reponame/$arch/os/repodata/repomd.xml",
[],
)
assert "baseos" in repos
assert "appstream" in repos
assert len(repos["baseos"]) == 2
assert len(repos["appstream"]) == 2
for repo in repos["baseos"]:
assert repo["name"] == "baseos"
assert repo["arch"] in ["source", "x86_64", "aarch64"]
assert repo["found_path"] == path.join(
directory,
"baseos",
repo["arch"],
"os/repodata/repomd.xml",
)
for repo in repos["appstream"]:
assert repo["name"] == "appstream"
assert repo["arch"] in ["x86_64", "aarch64"]
assert repo["found_path"] == path.join(
directory,
"appstream",
repo["arch"],
"os/repodata/repomd.xml",
)
# Run scan_path for source
repos = await apollo_tree.scan_path(
directory,
"$reponame/source/tree/repodata/repomd.xml",
[],
)
assert "baseos" in repos
assert len(repos["baseos"]) == 1
for repo in repos["baseos"]:
assert repo["name"] == "baseos"
assert repo["arch"] == "source"
assert repo["found_path"] == path.join(
directory,
"baseos",
"source",
"tree/repodata/repomd.xml",
)
@pytest.mark.asyncio
async def test_scan_path_valid_structure_arch_first():
with tempfile.TemporaryDirectory() as directory:
# Copy test data to temp dir
for file in data:
fsplit = file.split("__")
base_dir = path.join(
directory,
fsplit[-1].removesuffix(".xml"),
fsplit[0],
"os/repodata",
)
pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True)
shutil.copyfile(
path.join(path.dirname(__file__), "data", file),
path.join(base_dir, "repomd.xml"),
)
# Run scan_path
repos = await apollo_tree.scan_path(
directory,
"$arch/$reponame/os/repodata/repomd.xml",
[],
)
assert "baseos" in repos
assert "appstream" in repos
assert len(repos["baseos"]) == 2
assert len(repos["appstream"]) == 2
for repo in repos["baseos"]:
assert repo["name"] == "baseos"
assert repo["arch"] in ["x86_64", "aarch64"]
assert repo["found_path"] == path.join(
directory,
repo["arch"],
"baseos",
"os/repodata/repomd.xml",
)
for repo in repos["appstream"]:
assert repo["name"] == "appstream"
assert repo["arch"] in ["x86_64", "aarch64"]
assert repo["found_path"] == path.join(
directory,
repo["arch"],
"appstream",
"os/repodata/repomd.xml",
)
@pytest.mark.asyncio
async def test_fetch_updateinfo_from_apollo_live():
# This test is only run if the environment variable
# TEST_WITH_SIDE_EFFECTS is set to 1
if not environ.get("TEST_WITH_SIDE_EFFECTS"):
pytest.skip("Skipping test_fetch_updateinfo_from_apollo_live")
with tempfile.TemporaryDirectory() as directory:
file = data[0]
base_dir = path.join(
directory,
"BaseOS/x86_64/os/repodata",
)
pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True)
shutil.copyfile(
path.join(path.dirname(__file__), "data", file),
path.join(base_dir, "repomd.xml"),
)
# Run scan_path
repos = await apollo_tree.scan_path(
directory,
"$reponame/$arch/os/repodata/repomd.xml",
[],
)
assert "BaseOS" in repos
assert len(repos["BaseOS"]) == 1
# Run fetch_updateinfo_from_apollo
for _, repo_variants in repos.items():
for repo in repo_variants:
updateinfo = await apollo_tree.fetch_updateinfo_from_apollo(
repo,
"Rocky Linux 8 x86_64",
)
assert updateinfo is not None
@pytest.mark.asyncio
async def test_fetch_updateinfo_from_apollo_live_no_updateinfo():
# This test is only run if the environment variable
# TEST_WITH_SIDE_EFFECTS is set to 1
if not environ.get("TEST_WITH_SIDE_EFFECTS"):
pytest.skip(
"Skipping test_fetch_updateinfo_from_apollo_live_no_updateinfo"
)
with tempfile.TemporaryDirectory() as directory:
file = data[0]
base_dir = path.join(
directory,
"BaseOS/x86_64/os/repodata",
)
pathlib.Path(base_dir).mkdir(parents=True, exist_ok=True)
shutil.copyfile(
path.join(path.dirname(__file__), "data", file),
path.join(base_dir, "repomd.xml"),
)
# Run scan_path
repos = await apollo_tree.scan_path(
directory,
"$reponame/$arch/os/repodata/repomd.xml",
[],
)
assert "BaseOS" in repos
assert len(repos["BaseOS"]) == 1
# Run fetch_updateinfo_from_apollo
for _, repo_variants in repos.items():
for repo in repo_variants:
updateinfo = await apollo_tree.fetch_updateinfo_from_apollo(
repo,
"Rocky Linux 8 x86_64 NONEXISTENT",
)
assert updateinfo is None
@pytest.mark.asyncio
async def test_fetch_updateinfo_from_apollo_mock(mocker):
with tempfile.TemporaryDirectory() as directory:
repos = await _setup_test_baseos(directory)
# Read data/updateinfo__test__1.xml
with open(
path.join(
path.dirname(__file__), "data", "updateinfo__test__1.xml"
),
"r",
encoding="utf-8",
) as f:
updateinfo_xml = f.read()
resp = MockResponse(updateinfo_xml, 200)
mocker.patch("aiohttp.ClientSession.get", return_value=resp)
# Run fetch_updateinfo_from_apollo
for _, repo_variants in repos.items():
for repo in repo_variants:
updateinfo = await apollo_tree.fetch_updateinfo_from_apollo(
repo,
"Rocky Linux 8 x86_64",
True,
)
assert updateinfo == updateinfo_xml
@pytest.mark.asyncio
async def test_gzip_updateinfo(mocker):
with tempfile.TemporaryDirectory() as directory:
repos = await _setup_test_baseos(directory)
# Read data/updateinfo__test__1.xml
with open(
path.join(
path.dirname(__file__), "data", "updateinfo__test__1.xml"
),
"r",
encoding="utf-8",
) as f:
updateinfo_xml = f.read()
resp = MockResponse(updateinfo_xml, 200)
mocker.patch("aiohttp.ClientSession.get", return_value=resp)
# Run fetch_updateinfo_from_apollo
updateinfo = None
for _, repo_variants in repos.items():
for repo in repo_variants:
updateinfo = await apollo_tree.fetch_updateinfo_from_apollo(
repo,
"Rocky Linux 8 x86_64",
True,
)
assert updateinfo == updateinfo_xml
break
# Run gzip_updateinfo
updateinfo_gz = await apollo_tree.gzip_updateinfo(updateinfo)
assert updateinfo_gz is not None
@pytest.mark.asyncio
async def test_write_updateinfo_to_file(mocker):
with tempfile.TemporaryDirectory() as directory:
repos = await _setup_test_baseos(directory)
# Read data/updateinfo__test__1.xml
with open(
path.join(
path.dirname(__file__), "data", "updateinfo__test__1.xml"
),
"r",
encoding="utf-8",
) as f:
updateinfo_xml = f.read()
resp = MockResponse(updateinfo_xml, 200)
mocker.patch("aiohttp.ClientSession.get", return_value=resp)
# Run fetch_updateinfo_from_apollo
updateinfo = None
for _, repo_variants in repos.items():
for repo in repo_variants:
updateinfo = await apollo_tree.fetch_updateinfo_from_apollo(
repo,
"Rocky Linux 8 x86_64",
True,
)
assert updateinfo == updateinfo_xml
break
# Gzip first
gzipped = await apollo_tree.gzip_updateinfo(updateinfo)
# Run write_updateinfo_to_file
updateinfo_file = await apollo_tree.write_updateinfo_to_file(
repos["BaseOS"][0]["found_path"],
gzipped,
)
assert updateinfo_file is not None
assert path.exists(updateinfo_file)
assert path.isfile(updateinfo_file)
with open(updateinfo_file, "rb") as f:
updateinfo_file_contents = f.read()
# Check sha256sum against written file
actual_hexdigest = hashlib.sha256(updateinfo_file_contents).hexdigest()
expected_hexdigest = gzipped["gzipped_sha256sum"]
assert actual_hexdigest == expected_hexdigest
@pytest.mark.asyncio
async def test_update_repomd_xml(mocker):
with tempfile.TemporaryDirectory() as directory:
repos = await _setup_test_baseos(directory)
# Read data/updateinfo__test__1.xml
with open(
path.join(
path.dirname(__file__), "data", "updateinfo__test__1.xml"
),
"r",
encoding="utf-8",
) as f:
updateinfo_xml = f.read()
resp = MockResponse(updateinfo_xml, 200)
mocker.patch("aiohttp.ClientSession.get", return_value=resp)
# Run fetch_updateinfo_from_apollo
updateinfo = None
for _, repo_variants in repos.items():
for repo in repo_variants:
updateinfo = await apollo_tree.fetch_updateinfo_from_apollo(
repo,
"Rocky Linux 8 x86_64",
True,
)
assert updateinfo == updateinfo_xml
break
# Gzip first
gzipped = await apollo_tree.gzip_updateinfo(updateinfo)
# Run write_updateinfo_to_file
updateinfo_file = await apollo_tree.write_updateinfo_to_file(
repos["BaseOS"][0]["found_path"],
gzipped,
)
assert updateinfo_file is not None
assert path.exists(updateinfo_file)
assert path.isfile(updateinfo_file)
# Run update_repomd_xml
# This will replace the repomd.xml file with the new one
mocker.patch("time.time", return_value=1674284973)
repomd_xml_path = repos["BaseOS"][0]["found_path"]
await apollo_tree.update_repomd_xml(
repomd_xml_path,
gzipped,
)
# Check that the repomd.xml file matches baseos__base__repomd__x86_64_with_updateinfo.xml from data
with open(
path.join(
path.dirname(__file__),
"data",
"baseos__base__repomd__x86_64_with_updateinfo.xml",
),
"r",
encoding="utf-8",
) as f:
expected_repomd_xml = f.read()
with open(repomd_xml_path, "r", encoding="utf-8") as f:
actual_repomd_xml = f.read()
assert actual_repomd_xml == expected_repomd_xml

15
build/scripts/pylint.bash Executable file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env bash
shopt -s globstar
python3 -m pylint --rcfile=.pylintrc --ignore-patterns "re.compile(r'bazel-.*|node-modules|.venv')" **/*.py -v | tee /tmp/pytest.txt
score=$(sed -n 's/^Your code has been rated at \([-0-9.]*\)\/.*/\1/p' /tmp/pytest.txt)
echo "===================="
if (( $(echo "$score < 9.0" | bc -l) )); then
echo "Pylint score is too low: $score"
exit 1
else
echo "Pylint score is good: $score"
fi
echo "===================="

5
build/scripts/test.bash Executable file
View File

@ -0,0 +1,5 @@
#!/usr/bin/env bash
shopt -s globstar
python3 -m pytest --ignore node_modules --ignore .venv --ignore-glob "bazel-*" -v

13
common/testing.py Normal file
View File

@ -0,0 +1,13 @@
class MockResponse:
def __init__(self, text, status):
self._text = text
self.status = status
async def text(self):
return self._text
async def __aexit__(self, exc_type, exc, tb):
pass
async def __aenter__(self):
return self

View File

@ -8,7 +8,7 @@ image:
repository: ghcr.io/resf/apollo-rpmworker
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: "0c2e297"
tag: "15c4ca5"
imagePullSecrets: []
nameOverride: ""

View File

@ -8,7 +8,7 @@ image:
repository: ghcr.io/resf/apollo-server
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: "cc92597"
tag: "b498d3b"
imagePullSecrets: []
nameOverride: ""
@ -44,7 +44,7 @@ istio:
enabled: true
gateway: istio-system/base-gateway-public
externalDnsTarget: ingress.build.resf.org
host: apollo-v3-beta.build.resf.org
host: apollo.build.resf.org
database:
host: resf-peridot-dev.ctxqgglmfofx.us-east-2.rds.amazonaws.com
@ -84,9 +84,9 @@ resources: {}
# memory: 128Mi
autoscaling:
enabled: false
enabled: true
minReplicas: 1
maxReplicas: 100
maxReplicas: 10
targetCPUUtilizationPercentage: 80
# targetMemoryUtilizationPercentage: 80

View File

@ -362,6 +362,7 @@ manifest:
dill.tests.test_source: dill
dill.tests.test_temp: dill
dill.tests.test_weakref: dill
exceptiongroup: exceptiongroup
fastapi: fastapi
fastapi.applications: fastapi
fastapi.background: fastapi
@ -594,6 +595,8 @@ manifest:
idna.intranges: idna
idna.package_data: idna
idna.uts46data: idna
iniconfig: iniconfig
iniconfig.exceptions: iniconfig
iso8601: iso8601
iso8601.iso8601: iso8601
iso8601.test_iso8601: iso8601
@ -731,6 +734,13 @@ manifest:
openapi_python_client.schema.openapi_schema_pydantic.xml: openapi_python_client
openapi_python_client.schema.parameter_location: openapi_python_client
openapi_python_client.utils: openapi_python_client
packaging: packaging
packaging.markers: packaging
packaging.requirements: packaging
packaging.specifiers: packaging
packaging.tags: packaging
packaging.utils: packaging
packaging.version: packaging
passlib: passlib
passlib.apache: passlib
passlib.apps: passlib
@ -830,8 +840,10 @@ manifest:
platformdirs.unix: platformdirs
platformdirs.version: platformdirs
platformdirs.windows: platformdirs
pluggy: pluggy
priority: priority
priority.priority: priority
py: pytest
pydantic: pydantic
pydantic.annotated_types: pydantic
pydantic.class_validators: pydantic
@ -1051,6 +1063,11 @@ manifest:
pypika.queries: pypika_tortoise
pypika.terms: pypika_tortoise
pypika.utils: pypika_tortoise
pytest: pytest
pytest_asyncio: pytest_asyncio
pytest_asyncio.plugin: pytest_asyncio
pytest_mock: pytest_mock
pytest_mock.plugin: pytest_mock
pytz: pytz
pytz.exceptions: pytz
pytz.lazy: pytz
@ -1507,4 +1524,4 @@ manifest:
yarl: yarl
pip_repository:
name: pypi
integrity: 8d848d11c467949c981e296a23211f42a009566f0d2ab5fe11ceb59ade4cb74e
integrity: f886dabbf1a9923a165d6ef2ef657ca51f835c91992272bb8c12e9f78c128101

View File

@ -18,4 +18,7 @@ itsdangerous==2.1.2
PyYAML==6.0
beautifulsoup4==4.11.2
rssgen==0.9.0
python-slugify==8.0.0
python-slugify==8.0.0
pytest==7.2.1
pytest-asyncio==0.20.3
pytest-mock==3.10.0

View File

@ -159,6 +159,7 @@ attrs==22.2.0 \
# via
# aiohttp
# openapi-python-client
# pytest
autoflake==2.0.0 \
--hash=sha256:7185b596e70d8970c6d4106c112ef41921e472bd26abf3613db99eca88cc8c2a \
--hash=sha256:d58ed4187c6b4f623a942b9a90c43ff84bf6a266f3682f407b42ca52073c9678
@ -229,6 +230,10 @@ dill==0.3.6 \
--hash=sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0 \
--hash=sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373
# via pylint
exceptiongroup==1.1.0 \
--hash=sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e \
--hash=sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23
# via pytest
fastapi==0.89.1 \
--hash=sha256:15d9271ee52b572a015ca2ae5c72e1ce4241dd8532a534ad4f7ec70c376a580f \
--hash=sha256:f9773ea22290635b2f48b4275b2bf69a8fa721fda2e38228bed47139839dc877
@ -355,6 +360,10 @@ idna==3.4 \
# anyio
# rfc3986
# yarl
iniconfig==2.0.0 \
--hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \
--hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374
# via pytest
iso8601==1.1.0 \
--hash=sha256:32811e7b81deee2063ea6d2e94f8819a86d1f3811e49d23623a41fa832bef03f \
--hash=sha256:8400e90141bf792bce2634df533dc57e3bee19ea120a87bebcd3da89a58ad73f
@ -555,6 +564,10 @@ openapi-python-client==0.13.1 \
--hash=sha256:43bcd2e43e39dc31decba76ec09cbaeb6faad8709ce4684aec9b0228cd1bf3b5 \
--hash=sha256:adb5d886946cae2ff654f26396bd4d3f497234d5a9dafee805ee19587acbfdce
# via -r ./requirements.txt
packaging==23.0 \
--hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \
--hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97
# via pytest
passlib[bcrypt]==1.7.4 \
--hash=sha256:aa6bca462b8d8bda89c70b382f0c298a20b5560af6cbfa2dce410c0a2fb669f1 \
--hash=sha256:defd50f72b65c5402ab2c573830a6978e5f202ad0d984793c8dde2c4152ebe04
@ -569,6 +582,10 @@ platformdirs==2.6.2 \
# via
# black
# pylint
pluggy==1.0.0 \
--hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \
--hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3
# via pytest
priority==2.0.0 \
--hash=sha256:6f8eefce5f3ad59baf2c080a664037bb4725cd0a790d53d59ab4059288faf6aa \
--hash=sha256:c965d54f1b8d0d0b19479db3924c7c36cf672dbf2aec92d43fbdaf4492ba18c0
@ -643,6 +660,21 @@ pypika-tortoise==0.1.6 \
--hash=sha256:2d68bbb7e377673743cff42aa1059f3a80228d411fbcae591e4465e173109fd8 \
--hash=sha256:d802868f479a708e3263724c7b5719a26ad79399b2a70cea065f4a4cadbebf36
# via tortoise-orm
pytest==7.2.1 \
--hash=sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5 \
--hash=sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42
# via
# -r ./requirements.txt
# pytest-asyncio
# pytest-mock
pytest-asyncio==0.20.3 \
--hash=sha256:83cbf01169ce3e8eb71c6c278ccb0574d1a7a3bb8eaaf5e50e0ad342afb33b36 \
--hash=sha256:f129998b209d04fcc65c96fc85c11e5316738358909a8399e93be553d7656442
# via -r ./requirements.txt
pytest-mock==3.10.0 \
--hash=sha256:f4c973eeae0282963eb293eb173ce91b091a79c1334455acfac9ddee8a1c784b \
--hash=sha256:fbbdb085ef7c252a326fd8cdcac0aa3b1333d8811f131bdcc701002e1be7ed4f
# via -r ./requirements.txt
python-dateutil==2.8.2 \
--hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \
--hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
@ -765,6 +797,7 @@ tomli==2.0.1 \
# autoflake
# black
# pylint
# pytest
tomlkit==0.11.6 \
--hash=sha256:07de26b0d8cfc18f871aec595fda24d95b08fef89d147caa861939f37230bf4b \
--hash=sha256:71b952e5721688937fb02cf9d354dbcf0785066149d2855e44531ebdd2b65d73