From f0115de699032b5b7242394e92a5ab81519c4ac2 Mon Sep 17 00:00:00 2001 From: Louis Abel Date: Sun, 25 Sep 2022 19:31:40 -0700 Subject: [PATCH] Create generators --- mangle/README.md | 2 +- mangle/generators/README.md | 3 + mangle/generators/common | 19 +++++ mangle/generators/common.py | 31 +++++++++ mangle/generators/common_8 | 4 ++ mangle/generators/common_9 | 4 ++ mangle/{ => generators}/generate_comps | 7 +- .../generate_prepopulate_from_pungi | 38 ++++++++++ .../generate_prepopulate_from_stream | 52 ++++++++++++++ mangle/generators/prepopulate_parser.py | 69 +++++++++++++++++++ 10 files changed, 222 insertions(+), 7 deletions(-) create mode 100644 mangle/generators/README.md create mode 100644 mangle/generators/common create mode 100644 mangle/generators/common.py create mode 100644 mangle/generators/common_8 create mode 100644 mangle/generators/common_9 rename mangle/{ => generators}/generate_comps (81%) create mode 100755 mangle/generators/generate_prepopulate_from_pungi create mode 100755 mangle/generators/generate_prepopulate_from_stream create mode 100755 mangle/generators/prepopulate_parser.py diff --git a/mangle/README.md b/mangle/README.md index 6711deb..3aba9a4 100644 --- a/mangle/README.md +++ b/mangle/README.md @@ -1 +1 @@ -# Mirrormanager Mangling tools +# Mirrormanager Mangling tools and other Accessories diff --git a/mangle/generators/README.md b/mangle/generators/README.md new file mode 100644 index 0000000..da5e026 --- /dev/null +++ b/mangle/generators/README.md @@ -0,0 +1,3 @@ +# Generators + +These help generate comps or other data we need for peridot or pungi diff --git a/mangle/generators/common b/mangle/generators/common new file mode 100644 index 0000000..c8247d1 --- /dev/null +++ b/mangle/generators/common @@ -0,0 +1,19 @@ +# To be sourced by scripts to use + +if [ -z "$RLVER" ]; then + echo "You must set RLVER." + exit 1 +fi + +PREPOPDROP="/tmp/prepopulate.json" + +# Source Major common +# Override: Not Allowed +test -f "$(dirname "${BASH_SOURCE[0]}")/common_${RLVER}" && source "$(dirname "${BASH_SOURCE[0]}")/common_${RLVER}" +if [ "$?" -ne 0 ]; then + echo "Could not source common_${RLVER}" + exit 1 +fi + +STREAM_COMPOSE_BASEURL="https://composes.stream.centos.org/production" +COMPOSE_BASEDIR="/mnt/compose" diff --git a/mangle/generators/common.py b/mangle/generators/common.py new file mode 100644 index 0000000..602e1f4 --- /dev/null +++ b/mangle/generators/common.py @@ -0,0 +1,31 @@ +class common: + def rlver(self, rlver): + default = "Not Supported" + return getattr(self, 'rl' + str(rlver), lambda: default)() + + def rl8(self): + REPOS = { + 'AppStream': ['aarch64', 'x86_64'], + 'BaseOS': ['aarch64', 'x86_64'], + 'HighAvailability': ['aarch64', 'x86_64'], + 'PowerTools': ['aarch64', 'x86_64'], + 'ResilientStorage': ['aarch64', 'x86_64'], + 'RT': ['x86_64'], + } + return REPOS + + def rl9(self): + REPOS = { + 'AppStream': ['aarch64', 'ppc64le', 's390x', 'x86_64'], + 'BaseOS': ['aarch64', 'ppc64le', 's390x', 'x86_64'], + 'CRB': ['aarch64', 'ppc64le', 's390x', 'x86_64'], + 'HighAvailability': ['aarch64', 'ppc64le', 's390x', 'x86_64'], + 'NFV': ['x86_64'], + 'ResilientStorage': ['ppc64le', 's390x', 'x86_64'], + 'RT': ['x86_64'], + 'SAP': ['ppc64le', 's390x', 'x86_64'], + 'SAPHANA': ['ppc64le', 'x86_64'] + } + return REPOS + +switcher = common() diff --git a/mangle/generators/common_8 b/mangle/generators/common_8 new file mode 100644 index 0000000..3a979c6 --- /dev/null +++ b/mangle/generators/common_8 @@ -0,0 +1,4 @@ +# To be sourced by scripts to use + +REPO=("BaseOS" "AppStream" "CRB" "HighAvailability" "ResilientStorage" "NFV" "RT") +ARCH=("aarch64" "x86_64") diff --git a/mangle/generators/common_9 b/mangle/generators/common_9 new file mode 100644 index 0000000..d48ac58 --- /dev/null +++ b/mangle/generators/common_9 @@ -0,0 +1,4 @@ +# To be sourced by scripts to use + +REPO=("BaseOS" "AppStream" "CRB" "HighAvailability" "ResilientStorage" "NFV" "RT" "SAP" "SAPHANA") +ARCH=("aarch64" "ppc64le" "s390x" "x86_64") diff --git a/mangle/generate_comps b/mangle/generators/generate_comps similarity index 81% rename from mangle/generate_comps rename to mangle/generators/generate_comps index 41e9f8a..10d8f80 100755 --- a/mangle/generate_comps +++ b/mangle/generators/generate_comps @@ -6,12 +6,7 @@ if [ ! -f "/usr/bin/pungi-koji" ]; then exit 1 fi -if [[ $# -eq 0 ]]; then - echo "You must provide a major version." - exit 1 -fi - -VER="${1}" +VER="${RLVER}" BRANCH="r${VER}" REPO="${GIT_URL}/rocky/pungi-rocky.git" diff --git a/mangle/generators/generate_prepopulate_from_pungi b/mangle/generators/generate_prepopulate_from_pungi new file mode 100755 index 0000000..25b75b0 --- /dev/null +++ b/mangle/generators/generate_prepopulate_from_pungi @@ -0,0 +1,38 @@ +#!/bin/bash +# Parses a local compose's repos +#set -x + +if [ -n "$1" ] && [ -n "$2" ]; then + MAJOR=$1 + DATE=$2 +else + echo "Major version not specified" + exit 1 +fi + +# Verify the date format +echo "${DATE}" | grep -Eq '[0-9]+\.[0-9]' +grep_val=$? + +if [ "$grep_val" -ne 0 ]; then + echo "Date format incorrect. You must use: YYYYMMDD.X" +fi + +export RLVER=$MAJOR +source common + +drop="${PREPOPDROP}" +current=$(pwd) +compose_dir="${COMPOSE_BASEDIR}/Rocky-${MAJOR}-${DATE}/compose" + +pushd "${compose_dir}" || { echo "Could not change directory"; exit 1; } +/usr/bin/python3 "${current}/prepopulate_parser.py" +ret_val=$? +popd || { echo "Could not change back..."; exit 1; } + +if [ "$ret_val" -ne "0" ]; then + echo "There was an error running through the parser." + exit 1 +fi + +echo "File located at: $drop" diff --git a/mangle/generators/generate_prepopulate_from_stream b/mangle/generators/generate_prepopulate_from_stream new file mode 100755 index 0000000..a4f5ede --- /dev/null +++ b/mangle/generators/generate_prepopulate_from_stream @@ -0,0 +1,52 @@ +#!/bin/bash +# Parses a CentOS Stream compose's repos +#set -x + +if [ -n "$1" ] && [ -n "$2" ]; then + MAJOR=$1 + DATE=$2 +else + echo "Major version not specified" + exit 1 +fi + +# Verify the date format +echo "${DATE}" | grep -Eq '[0-9]+\.[0-9]' +grep_val=$? + +if [ "$grep_val" -ne 0 ]; then + echo "Date format incorrect. You must use: YYYYMMDD.X" +fi + +export RLVER=$MAJOR +source common + +drop="${PREPOPDROP}" +current=$(pwd) +tmpdir=$(mktemp -d) +stream_compose_url="${STREAM_COMPOSE_BASEURL}/CentOS-Stream-${MAJOR}-${DATE}/compose" + +pushd "${tmpdir}" || { echo "Could not change directory"; exit 1; } +for x in "${REPO[@]}"; do + echo "Working on ${x}" + for y in "${ARCH[@]}"; do + repodatas=( $(dnf reposync --repofrompath ${x},${stream_compose_url}/${x}/${y}/os --download-metadata --repoid=${x} -p ${x}/${y} --forcearch ${y} --norepopath --remote-time --assumeyes -u | grep repodata) ) + mkdir -p "${x}/${y}/repodata" + pushd "${x}/${y}/repodata" || { echo "Could not change directory"; exit 1; } + for z in "${repodatas[@]}"; do + wget -q -nc "${z}" + done + wget -q -nc "${stream_compose_url}/${x}/${y}/os/repodata/repomd.xml" + popd || { echo "Could not change back..."; exit 1; } + done +done +/usr/bin/python3 "${current}/prepopulate_parser.py" +ret_val=$? +popd || { echo "Could not change back..."; exit 1; } + +if [ "$ret_val" -ne "0" ]; then + echo "There was an error running through the parser." + exit 1 +fi + +echo "File located at: $drop" diff --git a/mangle/generators/prepopulate_parser.py b/mangle/generators/prepopulate_parser.py new file mode 100755 index 0000000..43f9787 --- /dev/null +++ b/mangle/generators/prepopulate_parser.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +import os +import os.path +import json +import dnf +import createrepo_c as cr +from common import * + +REPOS = switcher.rlver((os.environ['RLVER'])) + +# Source packages we do not ship or are rocky branded +IGNORES = [ + 'insights-client', + 'rhc', + 'centos-indexhtml', + 'centos-logos', + 'centos-stream-release', + 'redhat-indexhtml', + 'redhat-logos', + 'redhat-release' +] + +def warningcb(warning_type, message): + print("WARNING: %s" % message) + return True + +repo_prepop = {} +for k in REPOS: + repo_prepop[k] = {} + for arch in REPOS[k]: + PRIMARY_XML_PATH = None + FILELISTS_XML_PATH = None + OTHER_XML_PATH = None + REPO_PATH = k + '/' + arch + repomd = cr.Repomd() + cr.xml_parse_repomd(os.path.join(REPO_PATH, "repodata/repomd.xml"), repomd, warningcb) + for record in repomd.records: + if record.type == "primary": + PRIMARY_XML_PATH = os.path.join(REPO_PATH, record.location_href) + elif record.type == "filelists": + FILELISTS_XML_PATH = os.path.join(REPO_PATH, record.location_href) + elif record.type == "other": + OTHER_XML_PATH = os.path.join(REPO_PATH, record.location_href) + + package_iterator = cr.PackageIterator(primary_path=PRIMARY_XML_PATH, filelists_path=FILELISTS_XML_PATH, other_path=OTHER_XML_PATH, warningcb=warningcb) + repo_prepop[k][arch] = {} + for pkg in package_iterator: + name = pkg.name + '.' + pkg.arch + subject = dnf.subject.Subject(pkg.rpm_sourcerpm) + possible_nevra = subject.get_nevra_possibilities() + srcname = possible_nevra[0].name + # Ignore packages (by source) that we do not ship + if srcname in IGNORES: + continue + + # Create the initial list if the package (by source) does not exist + if srcname not in repo_prepop[k][arch]: + repo_prepop[k][arch][srcname] = [] + + # Avoids duplicate entries - This is especially helpful for modules + if name not in repo_prepop[k][arch][srcname]: + repo_prepop[k][arch][srcname].append(name) + + # Sorts the list items + repo_prepop[k][arch][srcname].sort() + +entry_point = open('/tmp/prepopulate.json', 'w+') +json.dump(repo_prepop, entry_point, indent=2, sort_keys=True) +entry_point.close()