forked from sig_core/toolkit
Create generators
This commit is contained in:
parent
f97546ca4a
commit
f0115de699
@ -1 +1 @@
|
||||
# Mirrormanager Mangling tools
|
||||
# Mirrormanager Mangling tools and other Accessories
|
||||
|
3
mangle/generators/README.md
Normal file
3
mangle/generators/README.md
Normal file
@ -0,0 +1,3 @@
|
||||
# Generators
|
||||
|
||||
These help generate comps or other data we need for peridot or pungi
|
19
mangle/generators/common
Normal file
19
mangle/generators/common
Normal file
@ -0,0 +1,19 @@
|
||||
# To be sourced by scripts to use
|
||||
|
||||
if [ -z "$RLVER" ]; then
|
||||
echo "You must set RLVER."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PREPOPDROP="/tmp/prepopulate.json"
|
||||
|
||||
# Source Major common
|
||||
# Override: Not Allowed
|
||||
test -f "$(dirname "${BASH_SOURCE[0]}")/common_${RLVER}" && source "$(dirname "${BASH_SOURCE[0]}")/common_${RLVER}"
|
||||
if [ "$?" -ne 0 ]; then
|
||||
echo "Could not source common_${RLVER}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STREAM_COMPOSE_BASEURL="https://composes.stream.centos.org/production"
|
||||
COMPOSE_BASEDIR="/mnt/compose"
|
31
mangle/generators/common.py
Normal file
31
mangle/generators/common.py
Normal file
@ -0,0 +1,31 @@
|
||||
class common:
|
||||
def rlver(self, rlver):
|
||||
default = "Not Supported"
|
||||
return getattr(self, 'rl' + str(rlver), lambda: default)()
|
||||
|
||||
def rl8(self):
|
||||
REPOS = {
|
||||
'AppStream': ['aarch64', 'x86_64'],
|
||||
'BaseOS': ['aarch64', 'x86_64'],
|
||||
'HighAvailability': ['aarch64', 'x86_64'],
|
||||
'PowerTools': ['aarch64', 'x86_64'],
|
||||
'ResilientStorage': ['aarch64', 'x86_64'],
|
||||
'RT': ['x86_64'],
|
||||
}
|
||||
return REPOS
|
||||
|
||||
def rl9(self):
|
||||
REPOS = {
|
||||
'AppStream': ['aarch64', 'ppc64le', 's390x', 'x86_64'],
|
||||
'BaseOS': ['aarch64', 'ppc64le', 's390x', 'x86_64'],
|
||||
'CRB': ['aarch64', 'ppc64le', 's390x', 'x86_64'],
|
||||
'HighAvailability': ['aarch64', 'ppc64le', 's390x', 'x86_64'],
|
||||
'NFV': ['x86_64'],
|
||||
'ResilientStorage': ['ppc64le', 's390x', 'x86_64'],
|
||||
'RT': ['x86_64'],
|
||||
'SAP': ['ppc64le', 's390x', 'x86_64'],
|
||||
'SAPHANA': ['ppc64le', 'x86_64']
|
||||
}
|
||||
return REPOS
|
||||
|
||||
switcher = common()
|
4
mangle/generators/common_8
Normal file
4
mangle/generators/common_8
Normal file
@ -0,0 +1,4 @@
|
||||
# To be sourced by scripts to use
|
||||
|
||||
REPO=("BaseOS" "AppStream" "CRB" "HighAvailability" "ResilientStorage" "NFV" "RT")
|
||||
ARCH=("aarch64" "x86_64")
|
4
mangle/generators/common_9
Normal file
4
mangle/generators/common_9
Normal file
@ -0,0 +1,4 @@
|
||||
# To be sourced by scripts to use
|
||||
|
||||
REPO=("BaseOS" "AppStream" "CRB" "HighAvailability" "ResilientStorage" "NFV" "RT" "SAP" "SAPHANA")
|
||||
ARCH=("aarch64" "ppc64le" "s390x" "x86_64")
|
@ -6,12 +6,7 @@ if [ ! -f "/usr/bin/pungi-koji" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ $# -eq 0 ]]; then
|
||||
echo "You must provide a major version."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
VER="${1}"
|
||||
VER="${RLVER}"
|
||||
BRANCH="r${VER}"
|
||||
REPO="${GIT_URL}/rocky/pungi-rocky.git"
|
||||
|
38
mangle/generators/generate_prepopulate_from_pungi
Executable file
38
mangle/generators/generate_prepopulate_from_pungi
Executable file
@ -0,0 +1,38 @@
|
||||
#!/bin/bash
|
||||
# Parses a local compose's repos
|
||||
#set -x
|
||||
|
||||
if [ -n "$1" ] && [ -n "$2" ]; then
|
||||
MAJOR=$1
|
||||
DATE=$2
|
||||
else
|
||||
echo "Major version not specified"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify the date format
|
||||
echo "${DATE}" | grep -Eq '[0-9]+\.[0-9]'
|
||||
grep_val=$?
|
||||
|
||||
if [ "$grep_val" -ne 0 ]; then
|
||||
echo "Date format incorrect. You must use: YYYYMMDD.X"
|
||||
fi
|
||||
|
||||
export RLVER=$MAJOR
|
||||
source common
|
||||
|
||||
drop="${PREPOPDROP}"
|
||||
current=$(pwd)
|
||||
compose_dir="${COMPOSE_BASEDIR}/Rocky-${MAJOR}-${DATE}/compose"
|
||||
|
||||
pushd "${compose_dir}" || { echo "Could not change directory"; exit 1; }
|
||||
/usr/bin/python3 "${current}/prepopulate_parser.py"
|
||||
ret_val=$?
|
||||
popd || { echo "Could not change back..."; exit 1; }
|
||||
|
||||
if [ "$ret_val" -ne "0" ]; then
|
||||
echo "There was an error running through the parser."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "File located at: $drop"
|
52
mangle/generators/generate_prepopulate_from_stream
Executable file
52
mangle/generators/generate_prepopulate_from_stream
Executable file
@ -0,0 +1,52 @@
|
||||
#!/bin/bash
|
||||
# Parses a CentOS Stream compose's repos
|
||||
#set -x
|
||||
|
||||
if [ -n "$1" ] && [ -n "$2" ]; then
|
||||
MAJOR=$1
|
||||
DATE=$2
|
||||
else
|
||||
echo "Major version not specified"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify the date format
|
||||
echo "${DATE}" | grep -Eq '[0-9]+\.[0-9]'
|
||||
grep_val=$?
|
||||
|
||||
if [ "$grep_val" -ne 0 ]; then
|
||||
echo "Date format incorrect. You must use: YYYYMMDD.X"
|
||||
fi
|
||||
|
||||
export RLVER=$MAJOR
|
||||
source common
|
||||
|
||||
drop="${PREPOPDROP}"
|
||||
current=$(pwd)
|
||||
tmpdir=$(mktemp -d)
|
||||
stream_compose_url="${STREAM_COMPOSE_BASEURL}/CentOS-Stream-${MAJOR}-${DATE}/compose"
|
||||
|
||||
pushd "${tmpdir}" || { echo "Could not change directory"; exit 1; }
|
||||
for x in "${REPO[@]}"; do
|
||||
echo "Working on ${x}"
|
||||
for y in "${ARCH[@]}"; do
|
||||
repodatas=( $(dnf reposync --repofrompath ${x},${stream_compose_url}/${x}/${y}/os --download-metadata --repoid=${x} -p ${x}/${y} --forcearch ${y} --norepopath --remote-time --assumeyes -u | grep repodata) )
|
||||
mkdir -p "${x}/${y}/repodata"
|
||||
pushd "${x}/${y}/repodata" || { echo "Could not change directory"; exit 1; }
|
||||
for z in "${repodatas[@]}"; do
|
||||
wget -q -nc "${z}"
|
||||
done
|
||||
wget -q -nc "${stream_compose_url}/${x}/${y}/os/repodata/repomd.xml"
|
||||
popd || { echo "Could not change back..."; exit 1; }
|
||||
done
|
||||
done
|
||||
/usr/bin/python3 "${current}/prepopulate_parser.py"
|
||||
ret_val=$?
|
||||
popd || { echo "Could not change back..."; exit 1; }
|
||||
|
||||
if [ "$ret_val" -ne "0" ]; then
|
||||
echo "There was an error running through the parser."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "File located at: $drop"
|
69
mangle/generators/prepopulate_parser.py
Executable file
69
mangle/generators/prepopulate_parser.py
Executable file
@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import os.path
|
||||
import json
|
||||
import dnf
|
||||
import createrepo_c as cr
|
||||
from common import *
|
||||
|
||||
REPOS = switcher.rlver((os.environ['RLVER']))
|
||||
|
||||
# Source packages we do not ship or are rocky branded
|
||||
IGNORES = [
|
||||
'insights-client',
|
||||
'rhc',
|
||||
'centos-indexhtml',
|
||||
'centos-logos',
|
||||
'centos-stream-release',
|
||||
'redhat-indexhtml',
|
||||
'redhat-logos',
|
||||
'redhat-release'
|
||||
]
|
||||
|
||||
def warningcb(warning_type, message):
|
||||
print("WARNING: %s" % message)
|
||||
return True
|
||||
|
||||
repo_prepop = {}
|
||||
for k in REPOS:
|
||||
repo_prepop[k] = {}
|
||||
for arch in REPOS[k]:
|
||||
PRIMARY_XML_PATH = None
|
||||
FILELISTS_XML_PATH = None
|
||||
OTHER_XML_PATH = None
|
||||
REPO_PATH = k + '/' + arch
|
||||
repomd = cr.Repomd()
|
||||
cr.xml_parse_repomd(os.path.join(REPO_PATH, "repodata/repomd.xml"), repomd, warningcb)
|
||||
for record in repomd.records:
|
||||
if record.type == "primary":
|
||||
PRIMARY_XML_PATH = os.path.join(REPO_PATH, record.location_href)
|
||||
elif record.type == "filelists":
|
||||
FILELISTS_XML_PATH = os.path.join(REPO_PATH, record.location_href)
|
||||
elif record.type == "other":
|
||||
OTHER_XML_PATH = os.path.join(REPO_PATH, record.location_href)
|
||||
|
||||
package_iterator = cr.PackageIterator(primary_path=PRIMARY_XML_PATH, filelists_path=FILELISTS_XML_PATH, other_path=OTHER_XML_PATH, warningcb=warningcb)
|
||||
repo_prepop[k][arch] = {}
|
||||
for pkg in package_iterator:
|
||||
name = pkg.name + '.' + pkg.arch
|
||||
subject = dnf.subject.Subject(pkg.rpm_sourcerpm)
|
||||
possible_nevra = subject.get_nevra_possibilities()
|
||||
srcname = possible_nevra[0].name
|
||||
# Ignore packages (by source) that we do not ship
|
||||
if srcname in IGNORES:
|
||||
continue
|
||||
|
||||
# Create the initial list if the package (by source) does not exist
|
||||
if srcname not in repo_prepop[k][arch]:
|
||||
repo_prepop[k][arch][srcname] = []
|
||||
|
||||
# Avoids duplicate entries - This is especially helpful for modules
|
||||
if name not in repo_prepop[k][arch][srcname]:
|
||||
repo_prepop[k][arch][srcname].append(name)
|
||||
|
||||
# Sorts the list items
|
||||
repo_prepop[k][arch][srcname].sort()
|
||||
|
||||
entry_point = open('/tmp/prepopulate.json', 'w+')
|
||||
json.dump(repo_prepop, entry_point, indent=2, sort_keys=True)
|
||||
entry_point.close()
|
Loading…
Reference in New Issue
Block a user