resf/distro-tools
1
0
Fork 0
mirror of https://github.com/resf/distro-tools.git synced 2024-11-21 12:41:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Hydra API can return list[str] for boostVersion (#12)

Browse Source 
* Hydra API can return list[str] for boostVersion

* Max row count is now 999
This commit is contained in:
Mustafa Gezen 2023-06-24 20:35:45 +02:00 committed by GitHub
parent 9648bfba93
commit f89f71f3b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 47 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apollo/rherrata/__init__.py
View File

@ -87,7 +87,7 @@ class Advisory(JSONWizard):
portal_child_ids: list[str] = None
portal_product_filter: list[str] = None
boostProduct: str = None
boostVersion: int = None
boostVersion: int | list[str] = None
detectedProducts: list[str] = None
caseCount: int = None
caseCount_365: int = None
@ -208,4 +208,5 @@ class API:
raise Exception((await response.text()))
elif body.get("response", {}).get("numFound", 0) == 0:
return []
return Advisory.from_list(list(body["response"]["docs"]))
advisory_list = list(body["response"]["docs"])
return Advisory.from_list(advisory_list)

4
apollo/rherrata/example.py
View File

@ -6,9 +6,9 @@ async def main():
api = API()
res = await api.search(
detected_product="rhel",
rows=1000,
rows=999,
from_date="2019-05-05T22:00:00Z",
sort_asc=True
sort_asc=True,
)
contains_9 = 0
contains_8 = 0

82
apollo/rhworker/poll_rh_activities.py
View File

@ -9,7 +9,11 @@ from temporalio import activity
from tortoise.transactions import in_transaction
from apollo.db import RedHatIndexState, RedHatAdvisory, RedHatAdvisoryPackage
from apollo.db import RedHatAdvisoryBugzillaBug, RedHatAdvisoryAffectedProduct, RedHatAdvisoryCVE
from apollo.db import (
RedHatAdvisoryBugzillaBug,
RedHatAdvisoryAffectedProduct,
RedHatAdvisoryCVE,
)
from apollo.rherrata import API
from common.logger import Logger
@ -25,16 +29,22 @@ def parse_red_hat_date(rhdate: str) -> datetime.datetime:
@activity.defn
async def get_last_indexed_date() -> Optional[str]:
state = await RedHatIndexState.get_or_none()
return re.sub(
r"\+\d\d:\d\d",
"",
state.last_indexed_at.isoformat("T") + "Z",
) if state else None
return (
re.sub(
r"\+\d\d:\d\d",
"",
state.last_indexed_at.isoformat("T") + "Z",
)
if state
else None
)
async def fetch_mapped_oval() -> dict[str, ET.ElementTree]:
# Download the oval_url using aiohttp, decompress using bzip and parse
oval_url = "https://access.redhat.com/security/data/oval/com.redhat.rhsa-all.xml.bz2"
oval_url = (
"https://access.redhat.com/security/data/oval/com.redhat.rhsa-all.xml.bz2"
)
async with aiohttp.ClientSession() as session:
async with session.get(oval_url) as response:
if response.status == 200:
@ -58,9 +68,7 @@ async def fetch_mapped_oval() -> dict[str, ET.ElementTree]:
@activity.defn
async def get_rh_advisories(from_timestamp: str = None) -> None:
logger = Logger()
advisories = await API().search(
from_date=from_timestamp, rows=10000, sort_asc=True
)
advisories = await API().search(from_date=from_timestamp, rows=999, sort_asc=True)
oval = await fetch_mapped_oval()
for advisory in advisories:
@ -73,14 +81,13 @@ async def get_rh_advisories(from_timestamp: str = None) -> None:
state.last_indexed_at = advisory_last_indexed_at
await state.save()
else:
await RedHatIndexState().create(
last_index_at=advisory_last_indexed_at
)
await RedHatIndexState().create(last_index_at=advisory_last_indexed_at)
logger.info("Processing advisory %s", advisory.id)
existing_advisory = await RedHatAdvisory.filter(name=advisory.id
).get_or_none()
existing_advisory = await RedHatAdvisory.filter(
name=advisory.id
).get_or_none()
if existing_advisory:
logger.info("Advisory %s already exists, skipping", advisory.id)
continue
@ -110,13 +117,11 @@ async def get_rh_advisories(from_timestamp: str = None) -> None:
await RedHatAdvisoryPackage.bulk_create(
[
RedHatAdvisoryPackage(
**{
"red_hat_advisory_id": ra.id,
"nevra": nevra
}
) for nevra in advisory.portal_package
**{"red_hat_advisory_id": ra.id, "nevra": nevra}
)
for nevra in advisory.portal_package
],
ignore_conflicts=True
ignore_conflicts=True,
)
if advisory.portal_CVE:
@ -147,10 +152,10 @@ async def get_rh_advisories(from_timestamp: str = None) -> None:
cvss3 = cve.attrib.get("cvss3")
if cvss3:
cvss3_raw = cvss3.split("/", 1)
cvss3_scoring_vector = cvss3_raw[
1] if cvss3_raw else "UNKNOWN"
cvss3_base_score = cvss3_raw[
0] if cvss3_raw else "UNKNOWN"
cvss3_scoring_vector = (
cvss3_raw[1] if cvss3_raw else "UNKNOWN"
)
cvss3_base_score = cvss3_raw[0] if cvss3_raw else "UNKNOWN"
cwe = cve.attrib.get("cwe")
if not cwe:
@ -159,16 +164,11 @@ async def get_rh_advisories(from_timestamp: str = None) -> None:
cves_to_save.append(
RedHatAdvisoryCVE(
**{
"red_hat_advisory_id":
ra.id,
"cve":
cve.text,
"cvss3_scoring_vector":
cvss3_scoring_vector,
"cvss3_base_score":
cvss3_base_score,
"cwe":
cwe,
"red_hat_advisory_id": ra.id,
"cve": cve.text,
"cvss3_scoring_vector": cvss3_scoring_vector,
"cvss3_base_score": cvss3_base_score,
"cwe": cwe,
}
)
)
@ -203,9 +203,10 @@ async def get_rh_advisories(from_timestamp: str = None) -> None:
"bugzilla_bug_id": bugzilla_bug_id,
"description": bz_map.get(bugzilla_bug_id, ""),
}
) for bugzilla_bug_id in advisory.portal_BZ
)
for bugzilla_bug_id in advisory.portal_BZ
],
ignore_conflicts=True
ignore_conflicts=True,
)
affected_products = advisory.get_products()
@ -219,11 +220,12 @@ async def get_rh_advisories(from_timestamp: str = None) -> None:
"name": product.name,
"major_version": product.major_version,
"minor_version": product.minor_version,
"arch": product.arch
"arch": product.arch,
}
) for product in affected_products
)
for product in affected_products
],
ignore_conflicts=True
ignore_conflicts=True,
)
logger.info("Processed advisory %s", advisory.id)