From 988ac4d04292a4ec72781a529a516736699dbcf1 Mon Sep 17 00:00:00 2001 From: Mustafa Gezen Date: Thu, 2 Feb 2023 14:53:35 +0100 Subject: [PATCH] Add compat RSS endpoint --- apollo/rhworker/BUILD.bazel | 5 +- apollo/rpmworker/BUILD.bazel | 6 +- apollo/server/BUILD.bazel | 1 + apollo/server/routes/api_compat.py | 151 ++++++++++++++++++++++++----- apollo/server/settings.py | 3 + build/macros/pyimage.bzl | 6 +- gazelle_python.yaml | 8 +- requirements.txt | 3 +- requirements_lock.txt | 5 + 9 files changed, 152 insertions(+), 36 deletions(-) diff --git a/apollo/rhworker/BUILD.bazel b/apollo/rhworker/BUILD.bazel index 457de32..2f7d426 100644 --- a/apollo/rhworker/BUILD.bazel +++ b/apollo/rhworker/BUILD.bazel @@ -1,7 +1,7 @@ load("@aspect_rules_py//py:defs.bzl", "py_library") -load("//build/macros:pyimage.bzl", "pyimage") +load("//build/macros:pyimage.bzl", "py_binary") -pyimage( +py_binary( name = "rhworker", srcs = ["__main__.py"], image_name = "apollo-rhworker", @@ -28,7 +28,6 @@ py_library( deps = [ "//apollo/db:db_lib", "//apollo/rherrata:rherrata_lib", - "//apollo/rhwebscraper:rhwebscraper_lib", "//common:common_lib", "@pypi_aiohttp//:pkg", "@pypi_temporalio//:pkg", diff --git a/apollo/rpmworker/BUILD.bazel b/apollo/rpmworker/BUILD.bazel index ba95992..a75e545 100644 --- a/apollo/rpmworker/BUILD.bazel +++ b/apollo/rpmworker/BUILD.bazel @@ -1,5 +1,5 @@ load("@aspect_rules_py//py:defs.bzl", "py_library") -load("//build/macros:pyimage.bzl", "pyimage") +load("//build/macros:pyimage.bzl", "py_binary") py_library( name = "rpmworker_lib", @@ -21,10 +21,10 @@ py_library( ], ) -pyimage( +py_binary( name = "rpmworker", - image_name = "apollo-rpmworker", srcs = ["__main__.py"], + image_name = "apollo-rpmworker", imports = ["../.."], main = "__main__.py", visibility = ["//:__subpackages__"], diff --git a/apollo/server/BUILD.bazel b/apollo/server/BUILD.bazel index 345fe67..27e056a 100644 --- a/apollo/server/BUILD.bazel +++ b/apollo/server/BUILD.bazel @@ -35,6 +35,7 @@ py_library( "@pypi_jinja2//:pkg", "@pypi_passlib//:pkg", "@pypi_python_multipart//:pkg", + "@pypi_rssgen//:pkg", "@pypi_starlette//:pkg", "@pypi_tortoise_orm//:pkg", ], diff --git a/apollo/server/routes/api_compat.py b/apollo/server/routes/api_compat.py index a7b2dd6..d8cc29a 100644 --- a/apollo/server/routes/api_compat.py +++ b/apollo/server/routes/api_compat.py @@ -7,14 +7,17 @@ from typing import TypeVar, Generic, Optional from tortoise import connections -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Depends, Query, Response from fastapi.exceptions import HTTPException from fastapi_pagination.links import Page from fastapi_pagination import Params from fastapi_pagination.ext.tortoise import create_page +from rssgen.feed import RssGenerator + from apollo.db import Advisory, RedHatIndexState from apollo.db.serialize import Advisory_Pydantic_V2, Advisory_Pydantic_V2_CVE, Advisory_Pydantic_V2_Fix +from apollo.server.settings import UI_URL, COMPANY_NAME, MANAGING_EDITOR, get_setting from common.fastapi import RenderErrorTemplateException @@ -31,6 +34,13 @@ class Pagination(Page[T], Generic[T]): fields = {"items": {"alias": "advisories"}} +class CompatParams(Params): + limit: int = Query(50, ge=1, le=100, description="Page size") + + def get_size(self) -> int: + return self.limit if self.limit else self.size + + def v3_advisory_to_v2( advisory: Advisory, include_rpms=True, @@ -104,20 +114,16 @@ def v3_advisory_to_v2( ) -@router.get( - "/", - response_model=Pagination[Advisory_Pydantic_V2], -) -async def list_advisories_compat_v2( - params: Params = Depends(), - product: str = Query(default=None, alias="filters.product"), - before_raw: str = Query(default=None, alias="filters.before"), - after_raw: str = Query(default=None, alias="filters.after"), - cve: str = Query(default=None, alias="filters.cve"), - synopsis: str = Query(default=None, alias="filters.synopsis"), - keyword: str = Query(default=None, alias="filters.keyword"), - severity: str = Query(default=None, alias="filters.severity"), - kind: str = Query(default=None, alias="filters.type"), +async def fetch_advisories_compat( + params: CompatParams, + product: str, + before_raw: str, + after_raw: str, + cve: str, + synopsis: str, + keyword: str, + severity: str, + kind: str, ): before = None after = None @@ -128,15 +134,13 @@ async def list_advisories_compat_v2( before_raw.removesuffix("Z") ) except: - raise RenderErrorTemplateException("Invalid before date", 400) + raise RenderErrorTemplateException("Invalid before date", 400) # noqa # pylint: disable=raise-missing-from try: if after_raw: after = datetime.datetime.fromisoformat(after_raw.removesuffix("Z")) except: - raise RenderErrorTemplateException("Invalid after date", 400) - - state = await RedHatIndexState.first() + raise RenderErrorTemplateException("Invalid after date", 400) # noqa # pylint: disable=raise-missing-from a = """ with vars (search, size, page_offset, product, before, after, cve, synopsis, severity, kind) as ( @@ -157,11 +161,10 @@ async def list_advisories_compat_v2( count(a.*) over () as total from advisories a - left outer join advisory_affected_products ap on ap.advisory_id = a.id left outer join advisory_cves c on c.advisory_id = a.id left outer join advisory_fixes f on f.advisory_id = a.id where - ((select product from vars) is null or ap.name ilike '%' || (select product from vars) || '%') + ((select product from vars) is null or exists (select name from advisory_affected_products where advisory_id = a.id and name like '%' || (select product from vars) || '%')) and ((select before from vars) is null or a.published_at < (select before from vars)) and ((select after from vars) is null or a.published_at > (select after from vars)) and (a.published_at is not null) @@ -170,7 +173,7 @@ async def list_advisories_compat_v2( and ((select severity from vars) is null or a.severity = (select severity from vars)) and ((select kind from vars) is null or a.kind = (select kind from vars)) and ((select search from vars) is null or - ap.name ilike '%' || (select search from vars) || '%' or + exists (select name from advisory_affected_products where advisory_id = a.id and name like '%' || (select product from vars) || '%') or a.synopsis ilike '%' || (select search from vars) || '%' or a.description ilike '%' || (select search from vars) || '%' or exists (select cve from advisory_cves where advisory_id = a.id and cve ilike '%' || (select search from vars) || '%') or @@ -184,8 +187,16 @@ async def list_advisories_compat_v2( connection = connections.get("default") results = await connection.execute_query( a, [ - keyword, params.size, params.size * (params.page - 1), product, - before, after, cve, synopsis, severity, kind + keyword, + params.get_size(), + params.get_size() * (params.page - 1), + product, + before, + after, + cve, + synopsis, + severity, + kind, ] ) @@ -194,8 +205,42 @@ async def list_advisories_compat_v2( if results[1]: count = results[1][0]["total"] + return (count, results[1]) + + +@router.get( + "/", + response_model=Pagination[Advisory_Pydantic_V2], +) +async def list_advisories_compat_v2( + params: CompatParams = Depends(), + product: str = Query(default=None, alias="filters.product"), + before_raw: str = Query(default=None, alias="filters.before"), + after_raw: str = Query(default=None, alias="filters.after"), + cve: str = Query(default=None, alias="filters.cve"), + synopsis: str = Query(default=None, alias="filters.synopsis"), + keyword: str = Query(default=None, alias="filters.keyword"), + severity: str = Query(default=None, alias="filters.severity"), + kind: str = Query(default=None, alias="filters.type"), +): + + state = await RedHatIndexState.first() + + fetch_adv = await fetch_advisories_compat( + params, + product, + before_raw, + after_raw, + cve, + synopsis, + keyword, + severity, + kind, + ) + count = fetch_adv[0] + advisories = [] - for adv in results[1]: + for adv in fetch_adv[1]: advisory = Advisory(**adv) await advisory.fetch_related( "packages", @@ -220,6 +265,62 @@ async def list_advisories_compat_v2( return page +@router.get(":rss") +async def list_advisories_compat_v2_rss( + params: CompatParams = Depends(), + product: str = Query(default=None, alias="filters.product"), + before_raw: str = Query(default=None, alias="filters.before"), + after_raw: str = Query(default=None, alias="filters.after"), + cve: str = Query(default=None, alias="filters.cve"), + synopsis: str = Query(default=None, alias="filters.synopsis"), + keyword: str = Query(default=None, alias="filters.keyword"), + severity: str = Query(default=None, alias="filters.severity"), + kind: str = Query(default=None, alias="filters.type"), +): + fetch_adv = await fetch_advisories_compat( + params, + product, + before_raw, + after_raw, + cve, + synopsis, + keyword, + severity, + kind, + ) + count = fetch_adv[0] + advisories = fetch_adv[1] + + ui_url = await get_setting(UI_URL) + company_name = await get_setting(COMPANY_NAME) + managing_editor = await get_setting(MANAGING_EDITOR) + + fg = RssGenerator() + fg.title(f"{company_name} Errata Feed") + fg.link(href=ui_url, rel="alternate") + fg.language("en") + fg.description(f"Advisories issued by {company_name}") + fg.copyright( + f"(C) {company_name} {datetime.datetime.now().year}. All rights reserved. CVE sources are copyright of their respective owners." + ) + fg.managingEditor(f"{managing_editor} ({company_name})") + + if count != 0: + fg.pubDate(advisories[0]["published_at"]) + fg.lastBuildDate(advisories[0]["published_at"]) + + for adv in advisories: + advisory = Advisory(**adv) + fe = fg.add_entry() + fe.title(f"{advisory.name}: {advisory.synopsis}") + fe.link(href=f"{ui_url}/{advisory.name}", rel="alternate") + fe.description(advisory.topic) + fe.id(str(advisory.id)) + fe.pubDate(advisory.published_at) + + return Response(content=fg.rss_str(), media_type="application/xml") + + @router.get( "/{advisory_name}", response_model=Advisory_Pydantic_V2, diff --git a/apollo/server/settings.py b/apollo/server/settings.py index f324192..d6f0cc2 100644 --- a/apollo/server/settings.py +++ b/apollo/server/settings.py @@ -16,6 +16,9 @@ OIDC_ADMIN_ROLE = "oidc-admin-role" OIDC_ELEVATED_ROLE = "oidc-elevated-role" RH_MATCH_STALE = "rh-match-stale" DISABLE_SERVING_RH_ADVISORIES = "disable-serving-rh-advisories" +UI_URL = "ui-url" +COMPANY_NAME = "company-name" +MANAGING_EDITOR = "managing-editor" async def get_setting(name: str) -> Optional[str]: diff --git a/build/macros/pyimage.bzl b/build/macros/pyimage.bzl index 6511f8b..70cd83a 100644 --- a/build/macros/pyimage.bzl +++ b/build/macros/pyimage.bzl @@ -1,9 +1,9 @@ -load("@aspect_rules_py//py:defs.bzl", "py_binary") +load("@aspect_rules_py//py:defs.bzl", _py_binary = "py_binary") load("@io_bazel_rules_docker//python3:image.bzl", "py3_image") load("@io_bazel_rules_docker//container:container.bzl", "container_push") -def pyimage(name, image_name, **kwargs): - py_binary( +def py_binary(name, image_name, **kwargs): + _py_binary( name = name, **kwargs ) diff --git a/gazelle_python.yaml b/gazelle_python.yaml index f794c71..bcebd8a 100644 --- a/gazelle_python.yaml +++ b/gazelle_python.yaml @@ -1069,6 +1069,12 @@ manifest: rfc3986.parseresult: rfc3986 rfc3986.uri: rfc3986 rfc3986.validators: rfc3986 + rssgen: rssgen + rssgen.compat: rssgen + rssgen.entry: rssgen + rssgen.feed: rssgen + rssgen.util: rssgen + rssgen.version: rssgen setuptools: setuptools setuptools.archive_util: setuptools setuptools.build_meta: setuptools @@ -1497,4 +1503,4 @@ manifest: yarl: yarl pip_repository: name: pypi -integrity: 6adcf30189668fa8123faef9b05219d91bf45a1950bc648c64a3c84e829b117d +integrity: 98955591e0f143193fb26aa58a3c5c9120c83f3270459709acd849f9613119ac diff --git a/requirements.txt b/requirements.txt index 6cc0ed4..698a649 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,5 @@ passlib[bcrypt]==1.7.4 python-multipart==0.0.5 itsdangerous==2.1.2 PyYAML==6.0 -beautifulsoup4==4.11.2 \ No newline at end of file +beautifulsoup4==4.11.2 +rssgen==0.9.0 \ No newline at end of file diff --git a/requirements_lock.txt b/requirements_lock.txt index d12064c..ab76dbd 100644 --- a/requirements_lock.txt +++ b/requirements_lock.txt @@ -648,6 +648,7 @@ python-dateutil==2.8.2 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via # openapi-python-client + # rssgen # temporalio python-multipart==0.0.5 \ --hash=sha256:f7bb5f611fc600d15fa47b3974c8aa16e93724513b49b5f95c81e6624c83fa43 @@ -704,6 +705,10 @@ rfc3986[idna2008]==1.5.0 \ --hash=sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835 \ --hash=sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97 # via httpx +rssgen==0.9.0 \ + --hash=sha256:26b6ec12d59a55b9962b83d4044e3bf9cd2d11c4942d8e93692eac70b9f3f6fa \ + --hash=sha256:c8c19bc1540a5789221e2df0be591d5864e619c49ab6517a8f4a524f8b7be868 + # via -r ./requirements.txt setuptools==58.2.0 \ --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11 \ --hash=sha256:2c55bdb85d5bb460bd2e3b12052b677879cffcf46c0c688f2e5bf51d36001145