Final Commit for 5/21

Add better syncing for current bash scripts (thanks pgreco)
Continued python scripts, creating classes for local module imports,
with some inspiration coming from cobbler

iso/py/README.md

@@ -1,8 +1,6 @@
-iso
+# iso
-===
 
-scripts
+## scripts
--------
 
 * sync-variant-pungi
 * sync-variant-peridot
@@ -12,8 +10,41 @@ scripts
 * build-all-iso
 * sign-repos-only
 
-wrappers
+## wrappers
---------
 
 * lorax-generators
 * sync-generators
+
+## rules
+
+### imports
+
+When making a script, you *must* import common. This is insanely bad practice,
+but we would prefer if we started out this way:
+
+```
+from common import *
+import argparse
+```
+
+Whatever is imported in common will effectively be imported in your scripts as
+well, but there is nothing stopping you from defining them again, even out of
+habit. `argparse` is there because you better have a very, *very* good reason
+to not be writing scripts that are major version specific.
+
+If you are writing something that could be arch specific based on the major
+version (which is likely), make sure to import the util module and use it arch
+checker appropriately. Small (but weak) example.
+
+```
+from util import Checks
+
+rlvars = rldict['9']
+r = Checks(rlvars, arch)
+r.check_valid_arch()
+```
+
+### script names and permissions
+
+* Callable scripts should *not* end in `.py`
+* They should have at least `775` or `+x` permissions

iso/py/common.py

@@ -1,9 +1,11 @@
 # All imports are here
+import os
 import platform
 import time
 import glob
 import rpm
 import yaml
+import logging
 
 # These are a bunch of colors we may use in terminal output
 class Color:
@@ -19,10 +21,17 @@ class Color:
     END = '\033[0m'
 
 # vars and additional checks
-#RLVER = rpm.expandMacro('%rhel')
-RLVER = '9'
 rldict = {}
-arch = platform.machine()
+config = {
+    "rlmacro": rpm.expandMacro('%rhel'),
+    "arch": platform.machine(),
+    "date_stamp": time.strftime("%Y%m%d", time.localtime()),
+    "staging_root": "/mnt/repos-staging",
+    "production_root": "/mnt/repos-production",
+    "category_stub": "/mirror/pub/rocky",
+    "sig_category_stub": "/mirror/pub/sig",
+    "repo_base_url": "https://yumrepofs.build.resf.org/v1/projects/"
+}
 
 # Importing the config from yaml
 for conf in glob.iglob('configs/*.yaml'):
@@ -30,28 +39,17 @@ for conf in glob.iglob('configs/*.yaml'):
         rldict.update(yaml.safe_load(file))
 
 # The system needs to be a RHEL-like system. It cannot be Fedora or SuSE.
-#if "%rhel" in RLVER:
+#if "%rhel" in config['RLMACRO']:
 #    raise SystemExit(Color.BOLD + 'This is not a RHEL-like system.' + Color.END
 #            + '\n\nPlease verify you are running on a RHEL-like system that is '
 #            'not Fedora nor SuSE. This means that the %rhel macro will be '
 #            'defined with a value equal to the version you are targetting. RHEL'
 #            ' and its derivatives have this set.')
 
-# Generic rlvars for the particular EL release we're on. This does not have to
-# be used. A different one can be assigned based on script need.
-rlvars = rldict[RLVER]
 
-# Is our arch allowed for this particular release? Some previous releases do
+# These will be set in their respective var files
-# not support ppc or s390x
+#REVISION = rlvars['revision'] + '-' + rlvars['rclvl']
-if arch not in rlvars['allowed_arches']:
+#rlvars = rldict[RLVER]
-    raise SystemExit(Color.BOLD + 'This architecture is not supported.'
+#rlvars = rldict[RLMACRO]
-            + Color.END + '\n\nEnsure that the architecture you are building '
+#COMPOSE_ROOT = "/mnt/compose/" + RLVER
-            'for is supported for this compose process.')
+#COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp
-
-date_stamp = time.strftime("%Y%m%d", time.localtime())
-COMPOSE_ROOT = "/mnt/compose/" + RLVER
-COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp
-STAGING_ROOT = "/mnt/repos-staging"
-PRODUCTION_ROOT = "/mnt/repos-production"
-CATEGORY_STUB = "/mirror/pub/rocky"
-REVISION = rlvars['revision'] + '-' + rlvars['rclvl']

iso/py/configs/el8.yaml

@@ -6,14 +6,15 @@
     - x86_64
     - aarch64
   provide_multilib: False
-  repo_url_list: []
+  project_id: ''
   required_packages:
     - 'lorax'
     - 'genisoimage'
     - 'isomd5sum'
-  symlink_refs:
+  repo_symlinks:
     devel: 'Devel'
     NFV: 'nfv'
+  renames: {}
   all_repos:
     - 'BaseOS'
     - 'AppStream'

iso/py/configs/el9.yaml

@@ -8,31 +8,18 @@
     - ppc64le
     - s390x
   provide_multilib: True
-  repo_url_list:
+  project_id: '55b17281-bc54-4929-8aca-a8a11d628738'
-    - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/all'
-    - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/BaseOS'
-    - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/AppStream'
-    - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/CRB'
-    - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/HighAvailability'
-    - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/ResilientStorage'
-    - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/RT'
-    - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/NFV'
-    - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/SAP'
-    - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/SAPHANA'
-    - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/extras'
-    - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/devel'
-    - 'https://yumrepofs.build.resf.org/v1/projects/55b17281-bc54-4929-8aca-a8a11d628738/repo/plus'
   required_packages:
     - 'lorax'
     - 'genisoimage'
     - 'isomd5sum'
-  symlink_refs:
+  repo_symlinks:
     devel: 'Devel'
     NFV: 'nfv'
   renames:
     all: 'nplb'
   all_repos:
-    - 'nplb'
+    - 'all'
     - 'BaseOS'
     - 'AppStream'
     - 'HighAvailability'
@@ -46,7 +33,7 @@
     - 'devel'
     - 'plus'
   no_comps_or_groups:
-    - 'nplb'
+    - 'all'
     - 'extras'
     - 'devel'
     - 'plus'

iso/py/sig/cloud.yaml

@@ -0,0 +1,10 @@
+---
+'8':
+  cloud-kernel:
+    project_id: 'f91da90d-5bdb-4cf2-80ea-e07f8dae5a5c'
+  cloud-common:
+    project_id: ''
+'9':
+  cloud-common:
+    project_id: ''
+...

iso/py/sig/core.yaml

@@ -0,0 +1,12 @@
+---
+'8':
+  core-common:
+    project_id: ''
+  core-infra:
+    project_id: ''
+'9':
+  core-common:
+    project_id: ''
+  core-infra:
+    project_id: ''
+...

iso/py/sync-all-peridot

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+
+from common import *
+import argparse
+from util import Checks
+from util import RepoSync
+
+rlvars = rldict['9']
+r = Checks(rlvars, config['arch'])
+r.check_valid_arch()
+
+a = RepoSync(rlvars, config, repo="BaseOS")
+print(a.sync())

iso/py/test.py

@@ -1,6 +1,9 @@
 #!/usr/bin/env python3
 
 from common import *
+import argparse
+from util import Checks
 
-print(platform.machine())
+rlvars = rldict['9']
-print(arch)
+r = Checks(rlvars, arch)
+r.check_valid_arch()

iso/py/util/__init__.py

@@ -0,0 +1,12 @@
+from .check import (
+        Checks,
+)
+
+from .dnf_utils import (
+        RepoSync,
+)
+
+__all__ = [
+        'Checks',
+        'RepoSync'
+]

iso/py/util/check.py

@@ -0,0 +1,14 @@
+# Is our arch allowed for this particular release? Some previous releases do
+# not support ppc or s390x
+from common import Color
+class Checks:
+    """This class helps check some things"""
+    def __init__(self, rlvars, arch):
+        self.arches = rlvars['allowed_arches']
+        self.arch = arch
+
+    def check_valid_arch(self):
+        if self.arch not in self.arches:
+            raise SystemExit(Color.BOLD + 'This architecture is not supported.'
+                    + Color.END + '\n\nEnsure that the architecture you are '
+                    'building for is supported for this compose process.')

iso/py/util/dnf_utils.py

@@ -0,0 +1,69 @@
+HAS_LIBREPO = True
+import os
+import os.path
+import pipes
+import shutil
+
+try:
+    import librepo
+except:
+    HAS_LIBREPO = False
+
+class RepoSync:
+    """
+    This helps us do reposync operations for the base system. SIG syncs are a
+    different class entirely. This is on purpose. Please use the SigRepoSync
+    class for SIG syncs.
+    """
+    def __init__(self, rlvars, config, repo=None, nofail: bool = False):
+        self.nofail = nofail
+        # Relevant config items
+        self.major_version = config['rlmacro']
+        self.date_stamp = config['date_stamp']
+        self.staging_dir = config['staging_root'] + config['category_stub'] + self.major_version
+        self.repo_base_url = config['repo_base_url']
+
+        # Relevant major version items
+        self.revision = rlvars['revision'] + "-" + rlvars['rclvl']
+        self.arches = rlvars['allowed_arches']
+        self.project_id = rlvars['project_id']
+        self.repo_renames = rlvars['renames']
+        self.repos = rlvars['all_repos']
+        self.repo = repo
+
+    def run(self):
+        pass
+
+    def sync(self):
+        cmd = self.reposync_cmd()
+        print(self.revision)
+
+    def generate_conf(self, dest_path: str, repo):
+        """
+        Generates the necessary repo conf file for the operation. This repo
+        file should be temporary in nature. This will generate a repo file
+        with all repos by default. If a repo is chosen for sync, that will be
+        the only one synced.
+
+        :param dest_path: The destination where the temporary conf goes
+        :param repo: The repo object to create a file for
+        """
+        pass
+
+    def reposync_cmd(self) -> str:
+        """
+        This generates the reposync command. We don't support reposync by
+        itself and will raise an error.
+
+        :return: The path to the reposync command. If dnf exists, we'll use
+        that.
+        """
+        cmd = None
+        if os.path.exists("/usr/bin/dnf"):
+            cmd = "/usr/bin/dnf reposync"
+        else:
+            raise SystemExit("/usr/bin/dnf was not found. /usr/bin/reposync is "
+                    "is not sufficient and you are likely running on an el7 "
+                    "system, which tells us that you made changes to these "
+                    "tools.")
+        return cmd

sync/common_8

@@ -30,6 +30,7 @@ NONMODS_REPOS=(
 declare -A LINK_REPOS
 LINK_REPOS=(
   [NFV]="nfv"
+  [Devel]="devel"
 )
 
 # These repos have comps/groups, except for debuginfo and sources

sync/minor-release-sync-to-staging.sh

@@ -56,11 +56,12 @@ for ARCH in "${ARCHES[@]}"; do
     # Hardcoding this for now
     SOURCE="/mnt/compose/${MAJ}/latest-Rocky-${MAJ}/compose/${x}/${ARCH}/iso"
     TARGET_ARCH="${STAGING_ROOT}/${CATEGORY_STUB}/${REV}/${x}/${ARCH}/iso"
-    mkdir -p "${SOURCE}" "${TARGET}" "${TARGET_ARCH}"
+    mkdir -p "${TARGET}"
+    #mkdir -p "${SOURCE}" "${TARGET}" "${TARGET_ARCH}"
     # Copy the ISO and manifests into their target architecture
-    cp -n "${SOURCE}"/*.iso "${TARGET_ARCH}/"
+    #cp -n "${SOURCE}"/*.iso "${TARGET_ARCH}/"
-    cp -n "${SOURCE}"/*.iso.manifest "${TARGET_ARCH}/"
+    #cp -n "${SOURCE}"/*.iso.manifest "${TARGET_ARCH}/"
-    cp -n "${SOURCE}/CHECKSUM" "${TARGET_ARCH}/"
+    #cp -n "${SOURCE}/CHECKSUM" "${TARGET_ARCH}/"
     # Copy the ISO and manifests into the main isos target
     cp "${SOURCE}"/*.iso "${TARGET}/"
     cp "${SOURCE}"/*.iso.manifest "${TARGET}/"

sync/sync-to-prod.sh

@@ -16,7 +16,10 @@ if [ $ret_val -eq "0" ]; then
   # disabling because none of our files should be starting with dashes. If they
   # are something is *seriously* wrong here.
   # shellcheck disable=SC2035
-  sudo -l && find **/* -maxdepth 0 -type d | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
+  sudo -l && find ./ -mindepth 1 -maxdepth 1 -type d -exec find {}/ -mindepth 1 -maxdepth 1 -type d \;|sed 's/^..//g' | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
+      {} "${TARGET}"
+  # shellcheck disable=SC2035
+  sudo -l && find ** -maxdepth 0 -type l | parallel --will-cite -j 18 sudo rsync -av --chown=10004:10005 --progress --relative --human-readable \
       {} "${TARGET}"
 
   # Full file list update