Final Commit for 5/21

Add better syncing for current bash scripts (thanks pgreco)
Continued python scripts, creating classes for local module imports,
with some inspiration coming from cobbler
This commit is contained in:
Louis Abel 2022-05-21 00:10:37 -07:00
parent b0bd9a8367
commit 4f9bfd44a0
Signed by untrusted user: label
GPG Key ID: B37E62D143879B36
14 changed files with 208 additions and 53 deletions

View File

@ -1,8 +1,6 @@
iso
===
# iso
scripts
-------
## scripts
* sync-variant-pungi
* sync-variant-peridot
@ -12,8 +10,41 @@ scripts
* build-all-iso
* sign-repos-only
wrappers
--------
## wrappers
* lorax-generators
* sync-generators
## rules
### imports
When making a script, you *must* import common. This is insanely bad practice,
but we would prefer if we started out this way:
```
from common import *
import argparse
```
Whatever is imported in common will effectively be imported in your scripts as
well, but there is nothing stopping you from defining them again, even out of
habit. `argparse` is there because you better have a very, *very* good reason
to not be writing scripts that are major version specific.
If you are writing something that could be arch specific based on the major
version (which is likely), make sure to import the util module and use it arch
checker appropriately. Small (but weak) example.
```
from util import Checks
rlvars = rldict['9']
r = Checks(rlvars, arch)
r.check_valid_arch()
```
### script names and permissions
* Callable scripts should *not* end in `.py`
* They should have at least `775` or `+x` permissions

View File

@ -1,9 +1,11 @@
# All imports are here
import os
import platform
import time
import glob
import rpm
import yaml
import logging
# These are a bunch of colors we may use in terminal output
class Color:
@ -19,10 +21,17 @@ class Color:
END = '\033[0m'
# vars and additional checks
#RLVER = rpm.expandMacro('%rhel')
RLVER = '9'
rldict = {}
arch = platform.machine()
config = {
"rlmacro": rpm.expandMacro('%rhel'),
"arch": platform.machine(),
"date_stamp": time.strftime("%Y%m%d", time.localtime()),
"staging_root": "/mnt/repos-staging",
"production_root": "/mnt/repos-production",
"category_stub": "/mirror/pub/rocky",
"sig_category_stub": "/mirror/pub/sig",
"repo_base_url": "https://yumrepofs.build.resf.org/v1/projects/"
}
# Importing the config from yaml
for conf in glob.iglob('configs/*.yaml'):
@ -30,28 +39,17 @@ for conf in glob.iglob('configs/*.yaml'):
rldict.update(yaml.safe_load(file))
# The system needs to be a RHEL-like system. It cannot be Fedora or SuSE.
#if "%rhel" in RLVER:
#if "%rhel" in config['RLMACRO']:
# raise SystemExit(Color.BOLD + 'This is not a RHEL-like system.' + Color.END
# + '\n\nPlease verify you are running on a RHEL-like system that is '
# 'not Fedora nor SuSE. This means that the %rhel macro will be '
# 'defined with a value equal to the version you are targetting. RHEL'
# ' and its derivatives have this set.')
# Generic rlvars for the particular EL release we're on. This does not have to
# be used. A different one can be assigned based on script need.
rlvars = rldict[RLVER]
# Is our arch allowed for this particular release? Some previous releases do
# not support ppc or s390x
if arch not in rlvars['allowed_arches']:
raise SystemExit(Color.BOLD + 'This architecture is not supported.'
+ Color.END + '\n\nEnsure that the architecture you are building '
'for is supported for this compose process.')
date_stamp = time.strftime("%Y%m%d", time.localtime())
COMPOSE_ROOT = "/mnt/compose/" + RLVER
COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp
STAGING_ROOT = "/mnt/repos-staging"
PRODUCTION_ROOT = "/mnt/repos-production"
CATEGORY_STUB = "/mirror/pub/rocky"
REVISION = rlvars['revision'] + '-' + rlvars['rclvl']
# These will be set in their respective var files
#REVISION = rlvars['revision'] + '-' + rlvars['rclvl']
#rlvars = rldict[RLVER]
#rlvars = rldict[RLMACRO]
#COMPOSE_ROOT = "/mnt/compose/" + RLVER
#COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp

View File

@ -6,14 +6,15 @@
- x86_64
- aarch64
provide_multilib: False
repo_url_list: []
project_id: ''
required_packages:
- 'lorax'
- 'genisoimage'
- 'isomd5sum'
symlink_refs:
repo_symlinks:
devel: 'Devel'
NFV: 'nfv'
renames: {}
all_repos:
- 'BaseOS'
- 'AppStream'

View File

@ -8,31 +8,18 @@
- ppc64le
- s390x
provide_multilib: True
repo_url_list:
- 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/all'
- 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/BaseOS'
- 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/AppStream'
- 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/CRB'
- 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/HighAvailability'
- 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/ResilientStorage'
- 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/RT'
- 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/NFV'
- 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/SAP'
- 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/SAPHANA'
- 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/extras'
- 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/devel'
- 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/plus'
project_id: '55b17281-bc54-4929-8aca-a8a11d628738'
required_packages:
- 'lorax'
- 'genisoimage'
- 'isomd5sum'
symlink_refs:
repo_symlinks:
devel: 'Devel'
NFV: 'nfv'
renames:
all: 'nplb'
all_repos:
- 'nplb'
- 'all'
- 'BaseOS'
- 'AppStream'
- 'HighAvailability'
@ -46,7 +33,7 @@
- 'devel'
- 'plus'
no_comps_or_groups:
- 'nplb'
- 'all'
- 'extras'
- 'devel'
- 'plus'

10
iso/py/sig/cloud.yaml Normal file
View File

@ -0,0 +1,10 @@
---
'8':
cloud-kernel:
project_id: 'f91da90d-5bdb-4cf2-80ea-e07f8dae5a5c'
cloud-common:
project_id: ''
'9':
cloud-common:
project_id: ''
...

12
iso/py/sig/core.yaml Normal file
View File

@ -0,0 +1,12 @@
---
'8':
core-common:
project_id: ''
core-infra:
project_id: ''
'9':
core-common:
project_id: ''
core-infra:
project_id: ''
...

13
iso/py/sync-all-peridot Executable file
View File

@ -0,0 +1,13 @@
#!/usr/bin/env python3
from common import *
import argparse
from util import Checks
from util import RepoSync
rlvars = rldict['9']
r = Checks(rlvars, config['arch'])
r.check_valid_arch()
a = RepoSync(rlvars, config, repo="BaseOS")
print(a.sync())

View File

@ -1,6 +1,9 @@
#!/usr/bin/env python3
from common import *
import argparse
from util import Checks
print(platform.machine())
print(arch)
rlvars = rldict['9']
r = Checks(rlvars, arch)
r.check_valid_arch()

12
iso/py/util/__init__.py Normal file
View File

@ -0,0 +1,12 @@
from .check import (
Checks,
)
from .dnf_utils import (
RepoSync,
)
__all__ = [
'Checks',
'RepoSync'
]

14
iso/py/util/check.py Normal file
View File

@ -0,0 +1,14 @@
# Is our arch allowed for this particular release? Some previous releases do
# not support ppc or s390x
from common import Color
class Checks:
"""This class helps check some things"""
def __init__(self, rlvars, arch):
self.arches = rlvars['allowed_arches']
self.arch = arch
def check_valid_arch(self):
if self.arch not in self.arches:
raise SystemExit(Color.BOLD + 'This architecture is not supported.'
+ Color.END + '\n\nEnsure that the architecture you are '
'building for is supported for this compose process.')

69
iso/py/util/dnf_utils.py Normal file
View File

@ -0,0 +1,69 @@
HAS_LIBREPO = True
import os
import os.path
import pipes
import shutil
try:
import librepo
except:
HAS_LIBREPO = False
class RepoSync:
"""
This helps us do reposync operations for the base system. SIG syncs are a
different class entirely. This is on purpose. Please use the SigRepoSync
class for SIG syncs.
"""
def __init__(self, rlvars, config, repo=None, nofail: bool = False):
self.nofail = nofail
# Relevant config items
self.major_version = config['rlmacro']
self.date_stamp = config['date_stamp']
self.staging_dir = config['staging_root'] + config['category_stub'] + self.major_version
self.repo_base_url = config['repo_base_url']
# Relevant major version items
self.revision = rlvars['revision'] + "-" + rlvars['rclvl']
self.arches = rlvars['allowed_arches']
self.project_id = rlvars['project_id']
self.repo_renames = rlvars['renames']
self.repos = rlvars['all_repos']
self.repo = repo
def run(self):
pass
def sync(self):
cmd = self.reposync_cmd()
print(self.revision)
def generate_conf(self, dest_path: str, repo):
"""
Generates the necessary repo conf file for the operation. This repo
file should be temporary in nature. This will generate a repo file
with all repos by default. If a repo is chosen for sync, that will be
the only one synced.
:param dest_path: The destination where the temporary conf goes
:param repo: The repo object to create a file for
"""
pass
def reposync_cmd(self) -> str:
"""
This generates the reposync command. We don't support reposync by
itself and will raise an error.
:return: The path to the reposync command. If dnf exists, we'll use
that.
"""
cmd = None
if os.path.exists("/usr/bin/dnf"):
cmd = "/usr/bin/dnf reposync"
else:
raise SystemExit("/usr/bin/dnf was not found. /usr/bin/reposync is "
"is not sufficient and you are likely running on an el7 "
"system, which tells us that you made changes to these "
"tools.")
return cmd

View File

@ -30,6 +30,7 @@ NONMODS_REPOS=(
declare -A LINK_REPOS
LINK_REPOS=(
[NFV]="nfv"
[Devel]="devel"
)
# These repos have comps/groups, except for debuginfo and sources

View File

@ -56,11 +56,12 @@ for ARCH in "${ARCHES[@]}"; do
# Hardcoding this for now
SOURCE="/mnt/compose/${MAJ}/latest-Rocky-${MAJ}/compose/${x}/${ARCH}/iso"
TARGET_ARCH="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${x}/${ARCH}/iso"
mkdir -p "${SOURCE}" "${TARGET}" "${TARGET_ARCH}"
mkdir -p "${TARGET}"
#mkdir -p "${SOURCE}" "${TARGET}" "${TARGET_ARCH}"
# Copy the ISO and manifests into their target architecture
cp -n "${SOURCE}"/*.iso "${TARGET_ARCH}/"
cp -n "${SOURCE}"/*.iso.manifest "${TARGET_ARCH}/"
cp -n "${SOURCE}/CHECKSUM" "${TARGET_ARCH}/"
#cp -n "${SOURCE}"/*.iso "${TARGET_ARCH}/"
#cp -n "${SOURCE}"/*.iso.manifest "${TARGET_ARCH}/"
#cp -n "${SOURCE}/CHECKSUM" "${TARGET_ARCH}/"
# Copy the ISO and manifests into the main isos target
cp "${SOURCE}"/*.iso "${TARGET}/"
cp "${SOURCE}"/*.iso.manifest "${TARGET}/"

View File

@ -16,7 +16,10 @@ if [ $ret_val -eq "0" ]; then
# disabling because none of our files should be starting with dashes. If they
# are something is *seriously* wrong here.
# shellcheck disable=SC2035
sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
sudo -l && find ./ -mindepth 1 -maxdepth 1 -type d -exec find {}/ -mindepth 1 -maxdepth 1 -type d \;|sed 's/^..//g' | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
{} "${TARGET}"
# shellcheck disable=SC2035
sudo -l && find ** -maxdepth 0 -type l | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
{} "${TARGET}"
# Full file list update