Add compat RSS endpoint

This commit is contained in:
Mustafa Gezen 2023-02-02 14:53:35 +01:00
parent 6eccd0968c
commit 988ac4d042
Signed by untrusted user who does not match committer: mustafa
GPG Key ID: DCDF010D946438C1
9 changed files with 152 additions and 36 deletions

View File

@ -1,7 +1,7 @@
load("@aspect_rules_py//py:defs.bzl", "py_library")
load("//build/macros:pyimage.bzl", "pyimage")
load("//build/macros:pyimage.bzl", "py_binary")
pyimage(
py_binary(
name = "rhworker",
srcs = ["__main__.py"],
image_name = "apollo-rhworker",
@ -28,7 +28,6 @@ py_library(
deps = [
"//apollo/db:db_lib",
"//apollo/rherrata:rherrata_lib",
"//apollo/rhwebscraper:rhwebscraper_lib",
"//common:common_lib",
"@pypi_aiohttp//:pkg",
"@pypi_temporalio//:pkg",

View File

@ -1,5 +1,5 @@
load("@aspect_rules_py//py:defs.bzl", "py_library")
load("//build/macros:pyimage.bzl", "pyimage")
load("//build/macros:pyimage.bzl", "py_binary")
py_library(
name = "rpmworker_lib",
@ -21,10 +21,10 @@ py_library(
],
)
pyimage(
py_binary(
name = "rpmworker",
image_name = "apollo-rpmworker",
srcs = ["__main__.py"],
image_name = "apollo-rpmworker",
imports = ["../.."],
main = "__main__.py",
visibility = ["//:__subpackages__"],

View File

@ -35,6 +35,7 @@ py_library(
"@pypi_jinja2//:pkg",
"@pypi_passlib//:pkg",
"@pypi_python_multipart//:pkg",
"@pypi_rssgen//:pkg",
"@pypi_starlette//:pkg",
"@pypi_tortoise_orm//:pkg",
],

View File

@ -7,14 +7,17 @@ from typing import TypeVar, Generic, Optional
from tortoise import connections
from fastapi import APIRouter, Depends, Query
from fastapi import APIRouter, Depends, Query, Response
from fastapi.exceptions import HTTPException
from fastapi_pagination.links import Page
from fastapi_pagination import Params
from fastapi_pagination.ext.tortoise import create_page
from rssgen.feed import RssGenerator
from apollo.db import Advisory, RedHatIndexState
from apollo.db.serialize import Advisory_Pydantic_V2, Advisory_Pydantic_V2_CVE, Advisory_Pydantic_V2_Fix
from apollo.server.settings import UI_URL, COMPANY_NAME, MANAGING_EDITOR, get_setting
from common.fastapi import RenderErrorTemplateException
@ -31,6 +34,13 @@ class Pagination(Page[T], Generic[T]):
fields = {"items": {"alias": "advisories"}}
class CompatParams(Params):
limit: int = Query(50, ge=1, le=100, description="Page size")
def get_size(self) -> int:
return self.limit if self.limit else self.size
def v3_advisory_to_v2(
advisory: Advisory,
include_rpms=True,
@ -104,20 +114,16 @@ def v3_advisory_to_v2(
)
@router.get(
"/",
response_model=Pagination[Advisory_Pydantic_V2],
)
async def list_advisories_compat_v2(
params: Params = Depends(),
product: str = Query(default=None, alias="filters.product"),
before_raw: str = Query(default=None, alias="filters.before"),
after_raw: str = Query(default=None, alias="filters.after"),
cve: str = Query(default=None, alias="filters.cve"),
synopsis: str = Query(default=None, alias="filters.synopsis"),
keyword: str = Query(default=None, alias="filters.keyword"),
severity: str = Query(default=None, alias="filters.severity"),
kind: str = Query(default=None, alias="filters.type"),
async def fetch_advisories_compat(
params: CompatParams,
product: str,
before_raw: str,
after_raw: str,
cve: str,
synopsis: str,
keyword: str,
severity: str,
kind: str,
):
before = None
after = None
@ -128,15 +134,13 @@ async def list_advisories_compat_v2(
before_raw.removesuffix("Z")
)
except:
raise RenderErrorTemplateException("Invalid before date", 400)
raise RenderErrorTemplateException("Invalid before date", 400) # noqa # pylint: disable=raise-missing-from
try:
if after_raw:
after = datetime.datetime.fromisoformat(after_raw.removesuffix("Z"))
except:
raise RenderErrorTemplateException("Invalid after date", 400)
state = await RedHatIndexState.first()
raise RenderErrorTemplateException("Invalid after date", 400) # noqa # pylint: disable=raise-missing-from
a = """
with vars (search, size, page_offset, product, before, after, cve, synopsis, severity, kind) as (
@ -157,11 +161,10 @@ async def list_advisories_compat_v2(
count(a.*) over () as total
from
advisories a
left outer join advisory_affected_products ap on ap.advisory_id = a.id
left outer join advisory_cves c on c.advisory_id = a.id
left outer join advisory_fixes f on f.advisory_id = a.id
where
((select product from vars) is null or ap.name ilike '%' || (select product from vars) || '%')
((select product from vars) is null or exists (select name from advisory_affected_products where advisory_id = a.id and name like '%' || (select product from vars) || '%'))
and ((select before from vars) is null or a.published_at < (select before from vars))
and ((select after from vars) is null or a.published_at > (select after from vars))
and (a.published_at is not null)
@ -170,7 +173,7 @@ async def list_advisories_compat_v2(
and ((select severity from vars) is null or a.severity = (select severity from vars))
and ((select kind from vars) is null or a.kind = (select kind from vars))
and ((select search from vars) is null or
ap.name ilike '%' || (select search from vars) || '%' or
exists (select name from advisory_affected_products where advisory_id = a.id and name like '%' || (select product from vars) || '%') or
a.synopsis ilike '%' || (select search from vars) || '%' or
a.description ilike '%' || (select search from vars) || '%' or
exists (select cve from advisory_cves where advisory_id = a.id and cve ilike '%' || (select search from vars) || '%') or
@ -184,8 +187,16 @@ async def list_advisories_compat_v2(
connection = connections.get("default")
results = await connection.execute_query(
a, [
keyword, params.size, params.size * (params.page - 1), product,
before, after, cve, synopsis, severity, kind
keyword,
params.get_size(),
params.get_size() * (params.page - 1),
product,
before,
after,
cve,
synopsis,
severity,
kind,
]
)
@ -194,8 +205,42 @@ async def list_advisories_compat_v2(
if results[1]:
count = results[1][0]["total"]
return (count, results[1])
@router.get(
"/",
response_model=Pagination[Advisory_Pydantic_V2],
)
async def list_advisories_compat_v2(
params: CompatParams = Depends(),
product: str = Query(default=None, alias="filters.product"),
before_raw: str = Query(default=None, alias="filters.before"),
after_raw: str = Query(default=None, alias="filters.after"),
cve: str = Query(default=None, alias="filters.cve"),
synopsis: str = Query(default=None, alias="filters.synopsis"),
keyword: str = Query(default=None, alias="filters.keyword"),
severity: str = Query(default=None, alias="filters.severity"),
kind: str = Query(default=None, alias="filters.type"),
):
state = await RedHatIndexState.first()
fetch_adv = await fetch_advisories_compat(
params,
product,
before_raw,
after_raw,
cve,
synopsis,
keyword,
severity,
kind,
)
count = fetch_adv[0]
advisories = []
for adv in results[1]:
for adv in fetch_adv[1]:
advisory = Advisory(**adv)
await advisory.fetch_related(
"packages",
@ -220,6 +265,62 @@ async def list_advisories_compat_v2(
return page
@router.get(":rss")
async def list_advisories_compat_v2_rss(
params: CompatParams = Depends(),
product: str = Query(default=None, alias="filters.product"),
before_raw: str = Query(default=None, alias="filters.before"),
after_raw: str = Query(default=None, alias="filters.after"),
cve: str = Query(default=None, alias="filters.cve"),
synopsis: str = Query(default=None, alias="filters.synopsis"),
keyword: str = Query(default=None, alias="filters.keyword"),
severity: str = Query(default=None, alias="filters.severity"),
kind: str = Query(default=None, alias="filters.type"),
):
fetch_adv = await fetch_advisories_compat(
params,
product,
before_raw,
after_raw,
cve,
synopsis,
keyword,
severity,
kind,
)
count = fetch_adv[0]
advisories = fetch_adv[1]
ui_url = await get_setting(UI_URL)
company_name = await get_setting(COMPANY_NAME)
managing_editor = await get_setting(MANAGING_EDITOR)
fg = RssGenerator()
fg.title(f"{company_name} Errata Feed")
fg.link(href=ui_url, rel="alternate")
fg.language("en")
fg.description(f"Advisories issued by {company_name}")
fg.copyright(
f"(C) {company_name} {datetime.datetime.now().year}. All rights reserved. CVE sources are copyright of their respective owners."
)
fg.managingEditor(f"{managing_editor} ({company_name})")
if count != 0:
fg.pubDate(advisories[0]["published_at"])
fg.lastBuildDate(advisories[0]["published_at"])
for adv in advisories:
advisory = Advisory(**adv)
fe = fg.add_entry()
fe.title(f"{advisory.name}: {advisory.synopsis}")
fe.link(href=f"{ui_url}/{advisory.name}", rel="alternate")
fe.description(advisory.topic)
fe.id(str(advisory.id))
fe.pubDate(advisory.published_at)
return Response(content=fg.rss_str(), media_type="application/xml")
@router.get(
"/{advisory_name}",
response_model=Advisory_Pydantic_V2,

View File

@ -16,6 +16,9 @@ OIDC_ADMIN_ROLE = "oidc-admin-role"
OIDC_ELEVATED_ROLE = "oidc-elevated-role"
RH_MATCH_STALE = "rh-match-stale"
DISABLE_SERVING_RH_ADVISORIES = "disable-serving-rh-advisories"
UI_URL = "ui-url"
COMPANY_NAME = "company-name"
MANAGING_EDITOR = "managing-editor"
async def get_setting(name: str) -> Optional[str]:

View File

@ -1,9 +1,9 @@
load("@aspect_rules_py//py:defs.bzl", "py_binary")
load("@aspect_rules_py//py:defs.bzl", _py_binary = "py_binary")
load("@io_bazel_rules_docker//python3:image.bzl", "py3_image")
load("@io_bazel_rules_docker//container:container.bzl", "container_push")
def pyimage(name, image_name, **kwargs):
py_binary(
def py_binary(name, image_name, **kwargs):
_py_binary(
name = name,
**kwargs
)

View File

@ -1069,6 +1069,12 @@ manifest:
rfc3986.parseresult: rfc3986
rfc3986.uri: rfc3986
rfc3986.validators: rfc3986
rssgen: rssgen
rssgen.compat: rssgen
rssgen.entry: rssgen
rssgen.feed: rssgen
rssgen.util: rssgen
rssgen.version: rssgen
setuptools: setuptools
setuptools.archive_util: setuptools
setuptools.build_meta: setuptools
@ -1497,4 +1503,4 @@ manifest:
yarl: yarl
pip_repository:
name: pypi
integrity: 6adcf30189668fa8123faef9b05219d91bf45a1950bc648c64a3c84e829b117d
integrity: 98955591e0f143193fb26aa58a3c5c9120c83f3270459709acd849f9613119ac

View File

@ -17,3 +17,4 @@ python-multipart==0.0.5
itsdangerous==2.1.2
PyYAML==6.0
beautifulsoup4==4.11.2
rssgen==0.9.0

View File

@ -648,6 +648,7 @@ python-dateutil==2.8.2 \
--hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
# via
# openapi-python-client
# rssgen
# temporalio
python-multipart==0.0.5 \
--hash=sha256:f7bb5f611fc600d15fa47b3974c8aa16e93724513b49b5f95c81e6624c83fa43
@ -704,6 +705,10 @@ rfc3986[idna2008]==1.5.0 \
--hash=sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835 \
--hash=sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97
# via httpx
rssgen==0.9.0 \
--hash=sha256:26b6ec12d59a55b9962b83d4044e3bf9cd2d11c4942d8e93692eac70b9f3f6fa \
--hash=sha256:c8c19bc1540a5789221e2df0be591d5864e619c49ab6517a8f4a524f8b7be868
# via -r ./requirements.txt
setuptools==58.2.0 \
--hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11 \
--hash=sha256:2c55bdb85d5bb460bd2e3b12052b677879cffcf46c0c688f2e5bf51d36001145