Create generators

mangle/README.md

@@ -1 +1 @@
-# Mirrormanager Mangling tools
+# Mirrormanager Mangling tools and other Accessories

mangle/generators/README.md

@@ -0,0 +1,3 @@
+# Generators
+
+These help generate comps or other data we need for peridot or pungi

mangle/generators/common

@@ -0,0 +1,19 @@
+# To be sourced by scripts to use
+
+if [ -z "$RLVER" ]; then
+  echo "You must set RLVER."
+  exit 1
+fi
+
+PREPOPDROP="/tmp/prepopulate.json"
+
+# Source Major common
+# Override: Not Allowed
+test -f "$(dirname "${BASH_SOURCE[0]}")/common_${RLVER}" && source "$(dirname "${BASH_SOURCE[0]}")/common_${RLVER}"
+if [ "$?" -ne 0 ]; then
+  echo "Could not source common_${RLVER}"
+  exit 1
+fi
+
+STREAM_COMPOSE_BASEURL="https://composes.stream.centos.org/production"
+COMPOSE_BASEDIR="/mnt/compose"

mangle/generators/common.py

@@ -0,0 +1,31 @@
+class common:
+    def rlver(self, rlver):
+        default = "Not Supported"
+        return getattr(self, 'rl' + str(rlver), lambda: default)()
+
+    def rl8(self):
+        REPOS = {
+                'AppStream': ['aarch64', 'x86_64'],
+                'BaseOS': ['aarch64', 'x86_64'],
+                'HighAvailability': ['aarch64', 'x86_64'],
+                'PowerTools': ['aarch64', 'x86_64'],
+                'ResilientStorage': ['aarch64', 'x86_64'],
+                'RT': ['x86_64'],
+        }
+        return REPOS
+
+    def rl9(self):
+        REPOS = {
+                'AppStream': ['aarch64', 'ppc64le', 's390x', 'x86_64'],
+                'BaseOS': ['aarch64', 'ppc64le', 's390x', 'x86_64'],
+                'CRB': ['aarch64', 'ppc64le', 's390x', 'x86_64'],
+                'HighAvailability': ['aarch64', 'ppc64le', 's390x', 'x86_64'],
+                'NFV': ['x86_64'],
+                'ResilientStorage': ['ppc64le', 's390x', 'x86_64'],
+                'RT': ['x86_64'],
+                'SAP': ['ppc64le', 's390x', 'x86_64'],
+                'SAPHANA': ['ppc64le', 'x86_64']
+        }
+        return REPOS
+
+switcher = common()

mangle/generators/common_8

@@ -0,0 +1,4 @@
+# To be sourced by scripts to use
+
+REPO=("BaseOS" "AppStream" "CRB" "HighAvailability" "ResilientStorage" "NFV" "RT")
+ARCH=("aarch64" "x86_64")

mangle/generators/common_9

@@ -0,0 +1,4 @@
+# To be sourced by scripts to use
+
+REPO=("BaseOS" "AppStream" "CRB" "HighAvailability" "ResilientStorage" "NFV" "RT" "SAP" "SAPHANA")
+ARCH=("aarch64" "ppc64le" "s390x" "x86_64")

mangle/generators/generate_comps

@@ -6,12 +6,7 @@ if [ ! -f "/usr/bin/pungi-koji" ]; then
   exit 1
 fi
 
-if [[ $# -eq 0  ]]; then
+VER="${RLVER}"
-  echo "You must provide a major version."
-  exit 1
-fi
-
-VER="${1}"
 BRANCH="r${VER}"
 REPO="${GIT_URL}/rocky/pungi-rocky.git"
 

mangle/generators/generate_prepopulate_from_pungi

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Parses a local compose's repos
+#set -x
+
+if [ -n "$1" ] && [ -n "$2" ]; then
+  MAJOR=$1
+  DATE=$2
+else
+  echo "Major version not specified"
+  exit 1
+fi
+
+# Verify the date format
+echo "${DATE}" | grep -Eq '[0-9]+\.[0-9]'
+grep_val=$?
+
+if [ "$grep_val" -ne 0 ]; then
+  echo "Date format incorrect. You must use: YYYYMMDD.X"
+fi
+
+export RLVER=$MAJOR
+source common
+
+drop="${PREPOPDROP}"
+current=$(pwd)
+compose_dir="${COMPOSE_BASEDIR}/Rocky-${MAJOR}-${DATE}/compose"
+
+pushd "${compose_dir}" || { echo "Could not change directory"; exit 1; }
+/usr/bin/python3 "${current}/prepopulate_parser.py"
+ret_val=$?
+popd || { echo "Could not change back..."; exit 1; }
+
+if [ "$ret_val" -ne "0" ]; then
+  echo "There was an error running through the parser."
+  exit 1
+fi
+
+echo "File located at: $drop"

mangle/generators/generate_prepopulate_from_stream

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Parses a CentOS Stream compose's repos
+#set -x
+
+if [ -n "$1" ] && [ -n "$2" ]; then
+  MAJOR=$1
+  DATE=$2
+else
+  echo "Major version not specified"
+  exit 1
+fi
+
+# Verify the date format
+echo "${DATE}" | grep -Eq '[0-9]+\.[0-9]'
+grep_val=$?
+
+if [ "$grep_val" -ne 0 ]; then
+  echo "Date format incorrect. You must use: YYYYMMDD.X"
+fi
+
+export RLVER=$MAJOR
+source common
+
+drop="${PREPOPDROP}"
+current=$(pwd)
+tmpdir=$(mktemp -d)
+stream_compose_url="${STREAM_COMPOSE_BASEURL}/CentOS-Stream-${MAJOR}-${DATE}/compose"
+
+pushd "${tmpdir}" || { echo "Could not change directory"; exit 1; }
+for x in "${REPO[@]}"; do
+  echo "Working on ${x}"
+  for y in "${ARCH[@]}"; do
+    repodatas=( $(dnf reposync --repofrompath ${x},${stream_compose_url}/${x}/${y}/os --download-metadata --repoid=${x} -p ${x}/${y} --forcearch ${y} --norepopath --remote-time --assumeyes -u | grep repodata) )
+    mkdir -p "${x}/${y}/repodata"
+    pushd "${x}/${y}/repodata" || { echo "Could not change directory"; exit 1; }
+    for z in "${repodatas[@]}"; do
+      wget -q -nc "${z}"
+    done
+    wget -q -nc "${stream_compose_url}/${x}/${y}/os/repodata/repomd.xml"
+    popd || { echo "Could not change back..."; exit 1; }
+  done
+done
+/usr/bin/python3 "${current}/prepopulate_parser.py"
+ret_val=$?
+popd || { echo "Could not change back..."; exit 1; }
+
+if [ "$ret_val" -ne "0" ]; then
+  echo "There was an error running through the parser."
+  exit 1
+fi
+
+echo "File located at: $drop"

mangle/generators/prepopulate_parser.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+import os
+import os.path
+import json
+import dnf
+import createrepo_c as cr
+from common import *
+
+REPOS = switcher.rlver((os.environ['RLVER']))
+
+# Source packages we do not ship or are rocky branded
+IGNORES = [
+        'insights-client',
+        'rhc',
+        'centos-indexhtml',
+        'centos-logos',
+        'centos-stream-release',
+        'redhat-indexhtml',
+        'redhat-logos',
+        'redhat-release'
+]
+
+def warningcb(warning_type, message):
+    print("WARNING: %s" % message)
+    return True
+
+repo_prepop = {}
+for k in REPOS:
+    repo_prepop[k] = {}
+    for arch in REPOS[k]:
+        PRIMARY_XML_PATH   = None
+        FILELISTS_XML_PATH = None
+        OTHER_XML_PATH     = None
+        REPO_PATH          = k + '/' + arch
+        repomd = cr.Repomd()
+        cr.xml_parse_repomd(os.path.join(REPO_PATH, "repodata/repomd.xml"), repomd, warningcb)
+        for record in repomd.records:
+            if record.type == "primary":
+                PRIMARY_XML_PATH = os.path.join(REPO_PATH, record.location_href)
+            elif record.type == "filelists":
+                FILELISTS_XML_PATH = os.path.join(REPO_PATH, record.location_href)
+            elif record.type == "other":
+                OTHER_XML_PATH = os.path.join(REPO_PATH, record.location_href)
+
+        package_iterator = cr.PackageIterator(primary_path=PRIMARY_XML_PATH, filelists_path=FILELISTS_XML_PATH, other_path=OTHER_XML_PATH, warningcb=warningcb)
+        repo_prepop[k][arch] = {}
+        for pkg in package_iterator:
+            name = pkg.name + '.' + pkg.arch
+            subject = dnf.subject.Subject(pkg.rpm_sourcerpm)
+            possible_nevra = subject.get_nevra_possibilities()
+            srcname = possible_nevra[0].name
+            # Ignore packages (by source) that we do not ship
+            if srcname in IGNORES:
+                continue
+
+            # Create the initial list if the package (by source) does not exist
+            if srcname not in repo_prepop[k][arch]:
+                repo_prepop[k][arch][srcname] = []
+
+            # Avoids duplicate entries - This is especially helpful for modules
+            if name not in repo_prepop[k][arch][srcname]:
+                repo_prepop[k][arch][srcname].append(name)
+
+            # Sorts the list items
+            repo_prepop[k][arch][srcname].sort()
+
+entry_point = open('/tmp/prepopulate.json', 'w+')
+json.dump(repo_prepop, entry_point, indent=2, sort_keys=True)
+entry_point.close()