Add compat RSS endpoint

This commit is contained in:
Mustafa Gezen 2023-02-02 14:53:35 +01:00
parent 6eccd0968c
commit 988ac4d042
Signed by untrusted user who does not match committer: mustafa
GPG Key ID: DCDF010D946438C1
9 changed files with 152 additions and 36 deletions

View File

@ -1,7 +1,7 @@
load("@aspect_rules_py//py:defs.bzl", "py_library") load("@aspect_rules_py//py:defs.bzl", "py_library")
load("//build/macros:pyimage.bzl", "pyimage") load("//build/macros:pyimage.bzl", "py_binary")
pyimage( py_binary(
name = "rhworker", name = "rhworker",
srcs = ["__main__.py"], srcs = ["__main__.py"],
image_name = "apollo-rhworker", image_name = "apollo-rhworker",
@ -28,7 +28,6 @@ py_library(
deps = [ deps = [
"//apollo/db:db_lib", "//apollo/db:db_lib",
"//apollo/rherrata:rherrata_lib", "//apollo/rherrata:rherrata_lib",
"//apollo/rhwebscraper:rhwebscraper_lib",
"//common:common_lib", "//common:common_lib",
"@pypi_aiohttp//:pkg", "@pypi_aiohttp//:pkg",
"@pypi_temporalio//:pkg", "@pypi_temporalio//:pkg",

View File

@ -1,5 +1,5 @@
load("@aspect_rules_py//py:defs.bzl", "py_library") load("@aspect_rules_py//py:defs.bzl", "py_library")
load("//build/macros:pyimage.bzl", "pyimage") load("//build/macros:pyimage.bzl", "py_binary")
py_library( py_library(
name = "rpmworker_lib", name = "rpmworker_lib",
@ -21,10 +21,10 @@ py_library(
], ],
) )
pyimage( py_binary(
name = "rpmworker", name = "rpmworker",
image_name = "apollo-rpmworker",
srcs = ["__main__.py"], srcs = ["__main__.py"],
image_name = "apollo-rpmworker",
imports = ["../.."], imports = ["../.."],
main = "__main__.py", main = "__main__.py",
visibility = ["//:__subpackages__"], visibility = ["//:__subpackages__"],

View File

@ -35,6 +35,7 @@ py_library(
"@pypi_jinja2//:pkg", "@pypi_jinja2//:pkg",
"@pypi_passlib//:pkg", "@pypi_passlib//:pkg",
"@pypi_python_multipart//:pkg", "@pypi_python_multipart//:pkg",
"@pypi_rssgen//:pkg",
"@pypi_starlette//:pkg", "@pypi_starlette//:pkg",
"@pypi_tortoise_orm//:pkg", "@pypi_tortoise_orm//:pkg",
], ],

View File

@ -7,14 +7,17 @@ from typing import TypeVar, Generic, Optional
from tortoise import connections from tortoise import connections
from fastapi import APIRouter, Depends, Query from fastapi import APIRouter, Depends, Query, Response
from fastapi.exceptions import HTTPException from fastapi.exceptions import HTTPException
from fastapi_pagination.links import Page from fastapi_pagination.links import Page
from fastapi_pagination import Params from fastapi_pagination import Params
from fastapi_pagination.ext.tortoise import create_page from fastapi_pagination.ext.tortoise import create_page
from rssgen.feed import RssGenerator
from apollo.db import Advisory, RedHatIndexState from apollo.db import Advisory, RedHatIndexState
from apollo.db.serialize import Advisory_Pydantic_V2, Advisory_Pydantic_V2_CVE, Advisory_Pydantic_V2_Fix from apollo.db.serialize import Advisory_Pydantic_V2, Advisory_Pydantic_V2_CVE, Advisory_Pydantic_V2_Fix
from apollo.server.settings import UI_URL, COMPANY_NAME, MANAGING_EDITOR, get_setting
from common.fastapi import RenderErrorTemplateException from common.fastapi import RenderErrorTemplateException
@ -31,6 +34,13 @@ class Pagination(Page[T], Generic[T]):
fields = {"items": {"alias": "advisories"}} fields = {"items": {"alias": "advisories"}}
class CompatParams(Params):
limit: int = Query(50, ge=1, le=100, description="Page size")
def get_size(self) -> int:
return self.limit if self.limit else self.size
def v3_advisory_to_v2( def v3_advisory_to_v2(
advisory: Advisory, advisory: Advisory,
include_rpms=True, include_rpms=True,
@ -104,20 +114,16 @@ def v3_advisory_to_v2(
) )
@router.get( async def fetch_advisories_compat(
"/", params: CompatParams,
response_model=Pagination[Advisory_Pydantic_V2], product: str,
) before_raw: str,
async def list_advisories_compat_v2( after_raw: str,
params: Params = Depends(), cve: str,
product: str = Query(default=None, alias="filters.product"), synopsis: str,
before_raw: str = Query(default=None, alias="filters.before"), keyword: str,
after_raw: str = Query(default=None, alias="filters.after"), severity: str,
cve: str = Query(default=None, alias="filters.cve"), kind: str,
synopsis: str = Query(default=None, alias="filters.synopsis"),
keyword: str = Query(default=None, alias="filters.keyword"),
severity: str = Query(default=None, alias="filters.severity"),
kind: str = Query(default=None, alias="filters.type"),
): ):
before = None before = None
after = None after = None
@ -128,15 +134,13 @@ async def list_advisories_compat_v2(
before_raw.removesuffix("Z") before_raw.removesuffix("Z")
) )
except: except:
raise RenderErrorTemplateException("Invalid before date", 400) raise RenderErrorTemplateException("Invalid before date", 400) # noqa # pylint: disable=raise-missing-from
try: try:
if after_raw: if after_raw:
after = datetime.datetime.fromisoformat(after_raw.removesuffix("Z")) after = datetime.datetime.fromisoformat(after_raw.removesuffix("Z"))
except: except:
raise RenderErrorTemplateException("Invalid after date", 400) raise RenderErrorTemplateException("Invalid after date", 400) # noqa # pylint: disable=raise-missing-from
state = await RedHatIndexState.first()
a = """ a = """
with vars (search, size, page_offset, product, before, after, cve, synopsis, severity, kind) as ( with vars (search, size, page_offset, product, before, after, cve, synopsis, severity, kind) as (
@ -157,11 +161,10 @@ async def list_advisories_compat_v2(
count(a.*) over () as total count(a.*) over () as total
from from
advisories a advisories a
left outer join advisory_affected_products ap on ap.advisory_id = a.id
left outer join advisory_cves c on c.advisory_id = a.id left outer join advisory_cves c on c.advisory_id = a.id
left outer join advisory_fixes f on f.advisory_id = a.id left outer join advisory_fixes f on f.advisory_id = a.id
where where
((select product from vars) is null or ap.name ilike '%' || (select product from vars) || '%') ((select product from vars) is null or exists (select name from advisory_affected_products where advisory_id = a.id and name like '%' || (select product from vars) || '%'))
and ((select before from vars) is null or a.published_at < (select before from vars)) and ((select before from vars) is null or a.published_at < (select before from vars))
and ((select after from vars) is null or a.published_at > (select after from vars)) and ((select after from vars) is null or a.published_at > (select after from vars))
and (a.published_at is not null) and (a.published_at is not null)
@ -170,7 +173,7 @@ async def list_advisories_compat_v2(
and ((select severity from vars) is null or a.severity = (select severity from vars)) and ((select severity from vars) is null or a.severity = (select severity from vars))
and ((select kind from vars) is null or a.kind = (select kind from vars)) and ((select kind from vars) is null or a.kind = (select kind from vars))
and ((select search from vars) is null or and ((select search from vars) is null or
ap.name ilike '%' || (select search from vars) || '%' or exists (select name from advisory_affected_products where advisory_id = a.id and name like '%' || (select product from vars) || '%') or
a.synopsis ilike '%' || (select search from vars) || '%' or a.synopsis ilike '%' || (select search from vars) || '%' or
a.description ilike '%' || (select search from vars) || '%' or a.description ilike '%' || (select search from vars) || '%' or
exists (select cve from advisory_cves where advisory_id = a.id and cve ilike '%' || (select search from vars) || '%') or exists (select cve from advisory_cves where advisory_id = a.id and cve ilike '%' || (select search from vars) || '%') or
@ -184,8 +187,16 @@ async def list_advisories_compat_v2(
connection = connections.get("default") connection = connections.get("default")
results = await connection.execute_query( results = await connection.execute_query(
a, [ a, [
keyword, params.size, params.size * (params.page - 1), product, keyword,
before, after, cve, synopsis, severity, kind params.get_size(),
params.get_size() * (params.page - 1),
product,
before,
after,
cve,
synopsis,
severity,
kind,
] ]
) )
@ -194,8 +205,42 @@ async def list_advisories_compat_v2(
if results[1]: if results[1]:
count = results[1][0]["total"] count = results[1][0]["total"]
return (count, results[1])
@router.get(
"/",
response_model=Pagination[Advisory_Pydantic_V2],
)
async def list_advisories_compat_v2(
params: CompatParams = Depends(),
product: str = Query(default=None, alias="filters.product"),
before_raw: str = Query(default=None, alias="filters.before"),
after_raw: str = Query(default=None, alias="filters.after"),
cve: str = Query(default=None, alias="filters.cve"),
synopsis: str = Query(default=None, alias="filters.synopsis"),
keyword: str = Query(default=None, alias="filters.keyword"),
severity: str = Query(default=None, alias="filters.severity"),
kind: str = Query(default=None, alias="filters.type"),
):
state = await RedHatIndexState.first()
fetch_adv = await fetch_advisories_compat(
params,
product,
before_raw,
after_raw,
cve,
synopsis,
keyword,
severity,
kind,
)
count = fetch_adv[0]
advisories = [] advisories = []
for adv in results[1]: for adv in fetch_adv[1]:
advisory = Advisory(**adv) advisory = Advisory(**adv)
await advisory.fetch_related( await advisory.fetch_related(
"packages", "packages",
@ -220,6 +265,62 @@ async def list_advisories_compat_v2(
return page return page
@router.get(":rss")
async def list_advisories_compat_v2_rss(
params: CompatParams = Depends(),
product: str = Query(default=None, alias="filters.product"),
before_raw: str = Query(default=None, alias="filters.before"),
after_raw: str = Query(default=None, alias="filters.after"),
cve: str = Query(default=None, alias="filters.cve"),
synopsis: str = Query(default=None, alias="filters.synopsis"),
keyword: str = Query(default=None, alias="filters.keyword"),
severity: str = Query(default=None, alias="filters.severity"),
kind: str = Query(default=None, alias="filters.type"),
):
fetch_adv = await fetch_advisories_compat(
params,
product,
before_raw,
after_raw,
cve,
synopsis,
keyword,
severity,
kind,
)
count = fetch_adv[0]
advisories = fetch_adv[1]
ui_url = await get_setting(UI_URL)
company_name = await get_setting(COMPANY_NAME)
managing_editor = await get_setting(MANAGING_EDITOR)
fg = RssGenerator()
fg.title(f"{company_name} Errata Feed")
fg.link(href=ui_url, rel="alternate")
fg.language("en")
fg.description(f"Advisories issued by {company_name}")
fg.copyright(
f"(C) {company_name} {datetime.datetime.now().year}. All rights reserved. CVE sources are copyright of their respective owners."
)
fg.managingEditor(f"{managing_editor} ({company_name})")
if count != 0:
fg.pubDate(advisories[0]["published_at"])
fg.lastBuildDate(advisories[0]["published_at"])
for adv in advisories:
advisory = Advisory(**adv)
fe = fg.add_entry()
fe.title(f"{advisory.name}: {advisory.synopsis}")
fe.link(href=f"{ui_url}/{advisory.name}", rel="alternate")
fe.description(advisory.topic)
fe.id(str(advisory.id))
fe.pubDate(advisory.published_at)
return Response(content=fg.rss_str(), media_type="application/xml")
@router.get( @router.get(
"/{advisory_name}", "/{advisory_name}",
response_model=Advisory_Pydantic_V2, response_model=Advisory_Pydantic_V2,

View File

@ -16,6 +16,9 @@ OIDC_ADMIN_ROLE = "oidc-admin-role"
OIDC_ELEVATED_ROLE = "oidc-elevated-role" OIDC_ELEVATED_ROLE = "oidc-elevated-role"
RH_MATCH_STALE = "rh-match-stale" RH_MATCH_STALE = "rh-match-stale"
DISABLE_SERVING_RH_ADVISORIES = "disable-serving-rh-advisories" DISABLE_SERVING_RH_ADVISORIES = "disable-serving-rh-advisories"
UI_URL = "ui-url"
COMPANY_NAME = "company-name"
MANAGING_EDITOR = "managing-editor"
async def get_setting(name: str) -> Optional[str]: async def get_setting(name: str) -> Optional[str]:

View File

@ -1,9 +1,9 @@
load("@aspect_rules_py//py:defs.bzl", "py_binary") load("@aspect_rules_py//py:defs.bzl", _py_binary = "py_binary")
load("@io_bazel_rules_docker//python3:image.bzl", "py3_image") load("@io_bazel_rules_docker//python3:image.bzl", "py3_image")
load("@io_bazel_rules_docker//container:container.bzl", "container_push") load("@io_bazel_rules_docker//container:container.bzl", "container_push")
def pyimage(name, image_name, **kwargs): def py_binary(name, image_name, **kwargs):
py_binary( _py_binary(
name = name, name = name,
**kwargs **kwargs
) )

View File

@ -1069,6 +1069,12 @@ manifest:
rfc3986.parseresult: rfc3986 rfc3986.parseresult: rfc3986
rfc3986.uri: rfc3986 rfc3986.uri: rfc3986
rfc3986.validators: rfc3986 rfc3986.validators: rfc3986
rssgen: rssgen
rssgen.compat: rssgen
rssgen.entry: rssgen
rssgen.feed: rssgen
rssgen.util: rssgen
rssgen.version: rssgen
setuptools: setuptools setuptools: setuptools
setuptools.archive_util: setuptools setuptools.archive_util: setuptools
setuptools.build_meta: setuptools setuptools.build_meta: setuptools
@ -1497,4 +1503,4 @@ manifest:
yarl: yarl yarl: yarl
pip_repository: pip_repository:
name: pypi name: pypi
integrity: 6adcf30189668fa8123faef9b05219d91bf45a1950bc648c64a3c84e829b117d integrity: 98955591e0f143193fb26aa58a3c5c9120c83f3270459709acd849f9613119ac

View File

@ -17,3 +17,4 @@ python-multipart==0.0.5
itsdangerous==2.1.2 itsdangerous==2.1.2
PyYAML==6.0 PyYAML==6.0
beautifulsoup4==4.11.2 beautifulsoup4==4.11.2
rssgen==0.9.0

View File

@ -648,6 +648,7 @@ python-dateutil==2.8.2 \
--hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
# via # via
# openapi-python-client # openapi-python-client
# rssgen
# temporalio # temporalio
python-multipart==0.0.5 \ python-multipart==0.0.5 \
--hash=sha256:f7bb5f611fc600d15fa47b3974c8aa16e93724513b49b5f95c81e6624c83fa43 --hash=sha256:f7bb5f611fc600d15fa47b3974c8aa16e93724513b49b5f95c81e6624c83fa43
@ -704,6 +705,10 @@ rfc3986[idna2008]==1.5.0 \
--hash=sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835 \ --hash=sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835 \
--hash=sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97 --hash=sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97
# via httpx # via httpx
rssgen==0.9.0 \
--hash=sha256:26b6ec12d59a55b9962b83d4044e3bf9cd2d11c4942d8e93692eac70b9f3f6fa \
--hash=sha256:c8c19bc1540a5789221e2df0be591d5864e619c49ab6517a8f4a524f8b7be868
# via -r ./requirements.txt
setuptools==58.2.0 \ setuptools==58.2.0 \
--hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11 \ --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11 \
--hash=sha256:2c55bdb85d5bb460bd2e3b12052b677879cffcf46c0c688f2e5bf51d36001145 --hash=sha256:2c55bdb85d5bb460bd2e3b12052b677879cffcf46c0c688f2e5bf51d36001145