modify manglers for version parsing

mangle/generators/common

@@ -17,4 +17,6 @@ if [ "$?" -ne 0 ]; then
 fi
 
 STREAM_COMPOSE_BASEURL="https://composes.stream.centos.org/production"
+STREAM_KOJI_REPO="https://kojihub.stream.centos.org/kojifiles/repos"
+PERIDOT_REPO="https://yumrepofs.build.resf.org/v1/projects"
 COMPOSE_BASEDIR="/mnt/compose"

mangle/generators/common.py

@@ -1,6 +1,18 @@
+import sys
+
 class common:
-    def rlver(self, rlver):
+    def rlver(self, rlver, stream=False, all_repo=False):
         default = "Not Supported"
+        if stream and all_repo:
+            print("incompatible options used")
+            sys.exit(1)
+
+        if stream:
+            return getattr(self, 'c' + str(rlver) + 's', lambda: default)()
+
+        if all_repo:
+            return getattr(self, 'rl' + str(rlver) + 'all', lambda: default)()
+
         return getattr(self, 'rl' + str(rlver), lambda: default)()
 
     def rl8(self):
@@ -28,4 +40,23 @@ class common:
         }
         return REPOS
 
+    def rl9all(self):
+        REPOS = {
+                'all': ['aarch64', 'ppc64le', 's390x', 'x86_64'],
+        }
+        return REPOS
+
+    # Parse tags of koji
+    def c8s(self):
+        REPOS = {
+                'c8s-build': ['aarch64', 'ppc64le', 'x86_64'],
+        }
+        return REPOS
+
+    def c9s(self):
+        REPOS = {
+                'c9s-build': ['aarch64', 'ppc64le', 's390x', 'x86_64'],
+        }
+        return REPOS
+
 switcher = common()

mangle/generators/common_9

@@ -2,3 +2,16 @@
 
 REPO=("BaseOS" "AppStream" "CRB" "HighAvailability" "ResilientStorage" "NFV" "RT" "SAP" "SAPHANA")
 ARCH=("aarch64" "ppc64le" "s390x" "x86_64")
+
+VER="r${MAJOR}${LH}"
+
+case "$VER" in
+  r9)
+    export PERIDOT_PROJECT_ID=0048077b-1573-4cb7-8ba7-cce823857ba5 ;;
+  r9lh)
+    export PERIDOT_PROJECT_ID=6794b5a8-290b-4d0d-ad5a-47164329cbb0 ;;
+  *)
+    echo "Not supported"
+    exit 1
+    ;;
+esac

mangle/generators/generate_versions_from_kpr

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Parses a CentOS Stream Koji Tag
+#set -x
+
+if [ -n "$1" ]; then
+  MAJOR="$1"
+else
+  echo "Major version not specified"
+  exit 1
+fi
+
+export RLVER="${MAJOR}"
+source common
+
+drop="${VERSDROP}"
+current=$(pwd)
+tmpdir=$(mktemp -d)
+tag_template="c${MAJOR}s-build"
+stream_repo_url="${STREAM_KOJI_REPO}/${tag_template}/latest"
+
+pushd "${tmpdir}" || { echo "Could not change directory"; exit 1; }
+for y in "${ARCH[@]}"; do
+  repodatas=( $(dnf reposync --repofrompath ${tag_template},${stream_repo_url}/${y} --download-metadata --repoid=${tag_template} -p ${tag_template}/${y} --forcearch ${y} --norepopath --remote-time --assumeyes -u | grep repodata) )
+  mkdir -p "${tag_template}/${y}/repodata"
+  pushd "${tag_template}/${y}/repodata" || { echo "Could not change directory"; exit 1; }
+  for z in "${repodatas[@]}"; do
+    wget -q -nc "${z}"
+  done
+  wget -q -nc "${stream_repo_url}/${y}/repodata/repomd.xml"
+  popd || { echo "Could not change back..."; exit 1; }
+done
+/usr/bin/python3 "${current}/version_parser.py" --version "${MAJOR}" --stream
+ret_val=$?
+popd || { echo "Could not change back..."; exit 1; }
+
+if [ "$ret_val" -ne "0" ]; then
+  echo "There was an error running through the parser."
+  exit 1
+fi
+
+echo "File located at: $drop"

mangle/generators/generate_versions_from_ppr

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Parses a peridot project
+#set -x
+
+if [ -n "$1" ]; then
+  MAJOR="$1"
+else
+  echo "Major version not specified"
+  exit 1
+fi
+
+if [ -n "$2" ] && [[ "$2" == "lh" ]]; then
+  export LH="lh"
+fi
+
+export RLVER="${MAJOR}"
+source common
+
+drop="${VERSDROP}"
+current=$(pwd)
+tmpdir=$(mktemp -d)
+tag_template="all"
+peridot_repo_url="${PERIDOT_REPO}/${PERIDOT_PROJECT_ID}/repo/${tag_template}"
+
+pushd "${tmpdir}" || { echo "Could not change directory"; exit 1; }
+for y in "${ARCH[@]}"; do
+  repodatas=( $(dnf reposync --repofrompath ${tag_template},${peridot_repo_url}/${y} --download-metadata --repoid=${tag_template} -p ${tag_template}/${y} --forcearch ${y} --norepopath --remote-time --assumeyes -u | grep repodata) )
+  mkdir -p "${tag_template}/${y}/repodata"
+  pushd "${tag_template}/${y}/repodata" || { echo "Could not change directory"; exit 1; }
+  for z in "${repodatas[@]}"; do
+    wget -q -nc "${z}"
+  done
+  wget -q -nc "${peridot_repo_url}/${y}/repodata/repomd.xml"
+  popd || { echo "Could not change back..."; exit 1; }
+done
+/usr/bin/python3 "${current}/version_parser.py" --version "${MAJOR}" --all-repo
+ret_val=$?
+popd || { echo "Could not change back..."; exit 1; }
+
+if [ "$ret_val" -ne "0" ]; then
+  echo "There was an error running through the parser."
+  exit 1
+fi
+
+echo "File located at: $drop"

mangle/generators/generate_versions_from_stream

@@ -3,8 +3,8 @@
 #set -x
 
 if [ -n "$1" ] && [ -n "$2" ]; then
-  MAJOR=$1
+  MAJOR="$1"
-  DATE=$2
+  DATE="$2"
 else
   echo "Major version not specified"
   exit 1
@@ -18,7 +18,7 @@ if [ "$grep_val" -ne 0 ]; then
   echo "Date format incorrect. You must use: YYYYMMDD.X"
 fi
 
-export RLVER=$MAJOR
+export RLVER="${MAJOR}"
 source common
 
 drop="${VERSDROP}"
@@ -40,7 +40,7 @@ for x in "${REPO[@]}"; do
     popd || { echo "Could not change back..."; exit 1; }
   done
 done
-/usr/bin/python3 "${current}/version_parser.py"
+/usr/bin/python3 "${current}/version_parser.py" --version "${MAJOR}"
 ret_val=$?
 popd || { echo "Could not change back..."; exit 1; }
 

mangle/generators/version_parser.py

@@ -2,11 +2,20 @@
 import os
 import os.path
 import json
+import argparse
 import dnf
 import createrepo_c as cr
 from common import *
 
-REPOS = switcher.rlver((os.environ['RLVER']))
+parser = argparse.ArgumentParser(description="Version Parser")
+parser.add_argument('--version', type=str, required=True)
+parser.add_argument('--stream', action='store_true', help="Stream koji only")
+parser.add_argument('--all-repo', action='store_true', help="Looks at the all repo for peridot")
+results = parser.parse_args()
+
+REPOS = switcher.rlver(results.version,
+                       stream=results.stream,
+                       all_repo=results.all_repo)
 
 # Source packages we do not ship or are rocky branded
 IGNORES = [