From 57c78a7afeb764b2fa21d9ea40b9643b31fd164a Mon Sep 17 00:00:00 2001 From: Louis Abel Date: Mon, 26 Sep 2022 12:36:49 -0700 Subject: [PATCH] add version parser --- mangle/generators/common | 1 + .../generators/generate_versions_from_stream | 52 ++++++++++++++ mangle/generators/version_parser.py | 72 +++++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100755 mangle/generators/generate_versions_from_stream create mode 100755 mangle/generators/version_parser.py diff --git a/mangle/generators/common b/mangle/generators/common index c8247d1..9d102cc 100644 --- a/mangle/generators/common +++ b/mangle/generators/common @@ -6,6 +6,7 @@ if [ -z "$RLVER" ]; then fi PREPOPDROP="/tmp/prepopulate.json" +VERSDROP="/tmp/versions.list" # Source Major common # Override: Not Allowed diff --git a/mangle/generators/generate_versions_from_stream b/mangle/generators/generate_versions_from_stream new file mode 100755 index 0000000..bca3846 --- /dev/null +++ b/mangle/generators/generate_versions_from_stream @@ -0,0 +1,52 @@ +#!/bin/bash +# Parses a CentOS Stream compose's repos +#set -x + +if [ -n "$1" ] && [ -n "$2" ]; then + MAJOR=$1 + DATE=$2 +else + echo "Major version not specified" + exit 1 +fi + +# Verify the date format +echo "${DATE}" | grep -Eq '[0-9]+\.[0-9]' +grep_val=$? + +if [ "$grep_val" -ne 0 ]; then + echo "Date format incorrect. You must use: YYYYMMDD.X" +fi + +export RLVER=$MAJOR +source common + +drop="${VERSDROP}" +current=$(pwd) +tmpdir=$(mktemp -d) +stream_compose_url="${STREAM_COMPOSE_BASEURL}/CentOS-Stream-${MAJOR}-${DATE}/compose" + +pushd "${tmpdir}" || { echo "Could not change directory"; exit 1; } +for x in "${REPO[@]}"; do + echo "Working on ${x}" + for y in "${ARCH[@]}"; do + repodatas=( $(dnf reposync --repofrompath ${x},${stream_compose_url}/${x}/${y}/os --download-metadata --repoid=${x} -p ${x}/${y} --forcearch ${y} --norepopath --remote-time --assumeyes -u | grep repodata) ) + mkdir -p "${x}/${y}/repodata" + pushd "${x}/${y}/repodata" || { echo "Could not change directory"; exit 1; } + for z in "${repodatas[@]}"; do + wget -q -nc "${z}" + done + wget -q -nc "${stream_compose_url}/${x}/${y}/os/repodata/repomd.xml" + popd || { echo "Could not change back..."; exit 1; } + done +done +/usr/bin/python3 "${current}/version_parser.py" +ret_val=$? +popd || { echo "Could not change back..."; exit 1; } + +if [ "$ret_val" -ne "0" ]; then + echo "There was an error running through the parser." + exit 1 +fi + +echo "File located at: $drop" diff --git a/mangle/generators/version_parser.py b/mangle/generators/version_parser.py new file mode 100755 index 0000000..9491c52 --- /dev/null +++ b/mangle/generators/version_parser.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +import os +import os.path +import json +import dnf +import createrepo_c as cr +from common import * + +REPOS = switcher.rlver((os.environ['RLVER'])) + +# Source packages we do not ship or are rocky branded +IGNORES = [ + 'insights-client', + 'rhc', + 'centos-indexhtml', + 'centos-logos', + 'centos-stream-release', + 'redhat-indexhtml', + 'redhat-logos', + 'redhat-release' +] + +def warningcb(warning_type, message): + print("WARNING: %s" % message) + return True + +repo_prepop = {} +with open('/tmp/versions.list', 'w+') as fp: + for k in REPOS: + repo_prepop[k] = {} + for arch in REPOS[k]: + PRIMARY_XML_PATH = None + FILELISTS_XML_PATH = None + OTHER_XML_PATH = None + REPO_PATH = k + '/' + arch + repomd = cr.Repomd() + cr.xml_parse_repomd(os.path.join(REPO_PATH, "repodata/repomd.xml"), repomd, warningcb) + for record in repomd.records: + if record.type == "primary": + PRIMARY_XML_PATH = os.path.join(REPO_PATH, record.location_href) + elif record.type == "filelists": + FILELISTS_XML_PATH = os.path.join(REPO_PATH, record.location_href) + elif record.type == "other": + OTHER_XML_PATH = os.path.join(REPO_PATH, record.location_href) + + package_iterator = cr.PackageIterator(primary_path=PRIMARY_XML_PATH, filelists_path=FILELISTS_XML_PATH, other_path=OTHER_XML_PATH, warningcb=warningcb) + repo_prepop[k][arch] = {} + for pkg in package_iterator: + subject = dnf.subject.Subject(pkg.rpm_sourcerpm) + possible_nevra = subject.get_nevra_possibilities() + srcname = possible_nevra[0].name + srcvers = possible_nevra[0].version + srcrele = possible_nevra[0].release + full = srcname + '-' + srcvers + '-' + srcrele + # Ignore packages (by source) that we do not ship + if srcname in IGNORES: + continue + + # Create the initial list if the package (by source) does not exist + if srcname not in repo_prepop[k][arch]: + repo_prepop[k][arch][srcname] = {} + + # Avoids duplicate entries - This is especially helpful for modules + repo_prepop[k][arch][srcname]['version'] = srcvers + repo_prepop[k][arch][srcname]['release'] = srcrele + + fp.write(full + '\n') + fp.close() + +entry_point = open('/tmp/versions.json', 'w+') +json.dump(repo_prepop, entry_point, indent=2, sort_keys=True) +entry_point.close()