add version parser

mangle/generators/common

@@ -6,6 +6,7 @@ if [ -z "$RLVER" ]; then
 fi
 
 PREPOPDROP="/tmp/prepopulate.json"
+VERSDROP="/tmp/versions.list"
 
 # Source Major common
 # Override: Not Allowed

mangle/generators/generate_versions_from_stream

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Parses a CentOS Stream compose's repos
+#set -x
+
+if [ -n "$1" ] && [ -n "$2" ]; then
+  MAJOR=$1
+  DATE=$2
+else
+  echo "Major version not specified"
+  exit 1
+fi
+
+# Verify the date format
+echo "${DATE}" | grep -Eq '[0-9]+\.[0-9]'
+grep_val=$?
+
+if [ "$grep_val" -ne 0 ]; then
+  echo "Date format incorrect. You must use: YYYYMMDD.X"
+fi
+
+export RLVER=$MAJOR
+source common
+
+drop="${VERSDROP}"
+current=$(pwd)
+tmpdir=$(mktemp -d)
+stream_compose_url="${STREAM_COMPOSE_BASEURL}/CentOS-Stream-${MAJOR}-${DATE}/compose"
+
+pushd "${tmpdir}" || { echo "Could not change directory"; exit 1; }
+for x in "${REPO[@]}"; do
+  echo "Working on ${x}"
+  for y in "${ARCH[@]}"; do
+    repodatas=( $(dnf reposync --repofrompath ${x},${stream_compose_url}/${x}/${y}/os --download-metadata --repoid=${x} -p ${x}/${y} --forcearch ${y} --norepopath --remote-time --assumeyes -u | grep repodata) )
+    mkdir -p "${x}/${y}/repodata"
+    pushd "${x}/${y}/repodata" || { echo "Could not change directory"; exit 1; }
+    for z in "${repodatas[@]}"; do
+      wget -q -nc "${z}"
+    done
+    wget -q -nc "${stream_compose_url}/${x}/${y}/os/repodata/repomd.xml"
+    popd || { echo "Could not change back..."; exit 1; }
+  done
+done
+/usr/bin/python3 "${current}/version_parser.py"
+ret_val=$?
+popd || { echo "Could not change back..."; exit 1; }
+
+if [ "$ret_val" -ne "0" ]; then
+  echo "There was an error running through the parser."
+  exit 1
+fi
+
+echo "File located at: $drop"

mangle/generators/version_parser.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+import os
+import os.path
+import json
+import dnf
+import createrepo_c as cr
+from common import *
+
+REPOS = switcher.rlver((os.environ['RLVER']))
+
+# Source packages we do not ship or are rocky branded
+IGNORES = [
+        'insights-client',
+        'rhc',
+        'centos-indexhtml',
+        'centos-logos',
+        'centos-stream-release',
+        'redhat-indexhtml',
+        'redhat-logos',
+        'redhat-release'
+]
+
+def warningcb(warning_type, message):
+    print("WARNING: %s" % message)
+    return True
+
+repo_prepop = {}
+with open('/tmp/versions.list', 'w+') as fp:
+    for k in REPOS:
+        repo_prepop[k] = {}
+        for arch in REPOS[k]:
+            PRIMARY_XML_PATH   = None
+            FILELISTS_XML_PATH = None
+            OTHER_XML_PATH     = None
+            REPO_PATH          = k + '/' + arch
+            repomd = cr.Repomd()
+            cr.xml_parse_repomd(os.path.join(REPO_PATH, "repodata/repomd.xml"), repomd, warningcb)
+            for record in repomd.records:
+                if record.type == "primary":
+                    PRIMARY_XML_PATH = os.path.join(REPO_PATH, record.location_href)
+                elif record.type == "filelists":
+                    FILELISTS_XML_PATH = os.path.join(REPO_PATH, record.location_href)
+                elif record.type == "other":
+                    OTHER_XML_PATH = os.path.join(REPO_PATH, record.location_href)
+
+            package_iterator = cr.PackageIterator(primary_path=PRIMARY_XML_PATH, filelists_path=FILELISTS_XML_PATH, other_path=OTHER_XML_PATH, warningcb=warningcb)
+            repo_prepop[k][arch] = {}
+            for pkg in package_iterator:
+                subject = dnf.subject.Subject(pkg.rpm_sourcerpm)
+                possible_nevra = subject.get_nevra_possibilities()
+                srcname = possible_nevra[0].name
+                srcvers = possible_nevra[0].version
+                srcrele = possible_nevra[0].release
+                full = srcname + '-' + srcvers + '-' + srcrele
+                # Ignore packages (by source) that we do not ship
+                if srcname in IGNORES:
+                    continue
+
+                # Create the initial list if the package (by source) does not exist
+                if srcname not in repo_prepop[k][arch]:
+                    repo_prepop[k][arch][srcname] = {}
+
+                # Avoids duplicate entries - This is especially helpful for modules
+                repo_prepop[k][arch][srcname]['version'] = srcvers
+                repo_prepop[k][arch][srcname]['release'] = srcrele
+
+                fp.write(full + '\n')
+    fp.close()
+
+entry_point = open('/tmp/versions.json', 'w+')
+json.dump(repo_prepop, entry_point, indent=2, sort_keys=True)
+entry_point.close()