Final commit for 5/21

iso/py/README.md

@@ -4,8 +4,8 @@
 
 * sync-variant-pungi
 * sync-variant-peridot
-* sync-all-pungi
+* sync-from-pungi
-* sync-all-peridot
+* sync-from-peridot
 * sync-sig
 * build-all-iso
 * sign-repos-only

iso/py/common.py

@@ -25,12 +25,13 @@ rldict = {}
 config = {
     "rlmacro": rpm.expandMacro('%rhel'),
     "arch": platform.machine(),
-    "date_stamp": time.strftime("%Y%m%d", time.localtime()),
+    "date_stamp": time.strftime("%Y%m%d.%H%M%S", time.localtime()),
+    "compose_root": "/mnt/compose",
     "staging_root": "/mnt/repos-staging",
     "production_root": "/mnt/repos-production",
-    "category_stub": "/mirror/pub/rocky",
+    "category_stub": "mirror/pub/rocky",
-    "sig_category_stub": "/mirror/pub/sig",
+    "sig_category_stub": "mirror/pub/sig",
-    "repo_base_url": "https://yumrepofs.build.resf.org/v1/projects/"
+    "repo_base_url": "https://yumrepofs.build.resf.org/v1/projects"
 }
 
 # Importing the config from yaml
@@ -39,7 +40,7 @@ for conf in glob.iglob('configs/*.yaml'):
         rldict.update(yaml.safe_load(file))
 
 # The system needs to be a RHEL-like system. It cannot be Fedora or SuSE.
-#if "%rhel" in config['RLMACRO']:
+#if "%rhel" in config['rlmacro']:
 #    raise SystemExit(Color.BOLD + 'This is not a RHEL-like system.' + Color.END
 #            + '\n\nPlease verify you are running on a RHEL-like system that is '
 #            'not Fedora nor SuSE. This means that the %rhel macro will be '
@@ -49,7 +50,6 @@ for conf in glob.iglob('configs/*.yaml'):
 
 # These will be set in their respective var files
 #REVISION = rlvars['revision'] + '-' + rlvars['rclvl']
-#rlvars = rldict[RLVER]
+#rlvars = rldict[rlver]
-#rlvars = rldict[RLMACRO]
+#rlvars = rldict[rlmacro]
-#COMPOSE_ROOT = "/mnt/compose/" + RLVER
 #COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp

iso/py/configs/el8.yaml

@@ -18,9 +18,9 @@
   all_repos:
     - 'BaseOS'
     - 'AppStream'
+    - 'PowerTools'
     - 'HighAvailability'
     - 'ResilientStorage'
-    - 'PowerTools'
     - 'RT'
     - 'NFV'
     - 'extras'
@@ -35,9 +35,9 @@
   comps_or_groups:
     - 'BaseOS'
     - 'AppStream'
+    - 'PowerTools'
     - 'HighAvailability'
     - 'ResilientStorage'
-    - 'PowerTools'
     - 'RT'
     - 'NFV'
   has_modules:

iso/py/configs/el9.yaml

@@ -22,9 +22,9 @@
     - 'all'
     - 'BaseOS'
     - 'AppStream'
+    - 'CRB'
     - 'HighAvailability'
     - 'ResilientStorage'
-    - 'CRB'
     - 'RT'
     - 'NFV'
     - 'SAP'
@@ -40,9 +40,9 @@
   comps_or_groups:
     - 'BaseOS'
     - 'AppStream'
+    - 'CRB'
     - 'HighAvailability'
     - 'ResilientStorage'
-    - 'CRB'
     - 'RT'
     - 'NFV'
     - 'SAP'

iso/py/sync-all-peridot

@@ -1,13 +0,0 @@
-#!/usr/bin/env python3
-
-from common import *
-import argparse
-from util import Checks
-from util import RepoSync
-
-rlvars = rldict['9']
-r = Checks(rlvars, config['arch'])
-r.check_valid_arch()
-
-a = RepoSync(rlvars, config, repo="BaseOS")
-print(a.sync())

iso/py/sync-from-peridot

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+# This is intended for doing "full" syncs, not periodic update syncs.
+
+from common import *
+import argparse
+from util import Checks
+from util import RepoSync
+
+rlvars = rldict['9']
+r = Checks(rlvars, config['arch'])
+r.check_valid_arch()
+
+a = RepoSync(rlvars, config, major="9", repo="BaseOS")
+a.run()
+#a.generate_conf()
+#somedir = a.generate_compose_dirs()
+#print(a.sync())

iso/py/util/dnf_utils.py

@@ -1,13 +1,21 @@
-HAS_LIBREPO = True
+"""
+Syncs yum repos for mirroring and composing.
+
+Louis Abel <label AT rockylinux.org>
+"""
+#import shutil
+import logging
+import sys
 import os
 import os.path
-import pipes
+#import pipes
-import shutil
+from common import Color
 
-try:
+#HAS_LIBREPO = True
-    import librepo
+#try:
-except:
+#    import librepo
-    HAS_LIBREPO = False
+#except:
+#    HAS_LIBREPO = False
 
 class RepoSync:
     """
@@ -15,13 +23,31 @@ class RepoSync:
     different class entirely. This is on purpose. Please use the SigRepoSync
     class for SIG syncs.
     """
-    def __init__(self, rlvars, config, repo=None, nofail: bool = False):
+    def __init__(
+            self,
+            rlvars,
+            config,
+            major,
+            repo=None,
+            arch=None,
+            ignore_debug=False,
+            ignore_source=False,
+            dryrun: bool = False,
+            fullrun: bool = False,
+            nofail: bool = False,
+            logger=None
+        ):
         self.nofail = nofail
+        self.dryrun = dryrun
+        self.fullrun = fullrun
+        self.arch = arch
+        self.ignore_debug = ignore_debug
+        self.ignore_source = ignore_source
         # Relevant config items
-        self.major_version = config['rlmacro']
+        self.major_version = major
         self.date_stamp = config['date_stamp']
-        self.staging_dir = config['staging_root'] + config['category_stub'] + self.major_version
         self.repo_base_url = config['repo_base_url']
+        self.compose_base = config['compose_root'] + "/" + major
 
         # Relevant major version items
         self.revision = rlvars['revision'] + "-" + rlvars['rclvl']
@@ -31,14 +57,106 @@ class RepoSync:
         self.repos = rlvars['all_repos']
         self.repo = repo
 
+        self.staging_dir = os.path.join(
+                    config['staging_root'],
+                    config['category_stub'],
+                    self.revision
+        )
+
+        self.compose_latest_dir = os.path.join(
+                config['compose_root'],
+                major,
+                "latest-Rocky-{}".format(major)
+        )
+
+        self.compose_latest_sync = os.path.join(
+                self.compose_latest_dir,
+                "compose"
+        )
+
+        # This is temporary for now.
+        if logger is None:
+            self.log = logging.getLogger("reposync")
+            self.log.setLevel(logging.INFO)
+            handler = logging.StreamHandler(sys.stdout)
+            handler.setLevel(logging.INFO)
+            formatter = logging.Formatter(
+                    '%(asctime)s :: %(name)s :: %(message)s',
+                    '%Y-%m-%d %H:%M:%S'
+            )
+            handler.setFormatter(formatter)
+            self.log.addHandler(handler)
+
+        self.log.info('reposync init')
+        self.log.info(self.revision)
+
     def run(self):
+        """
+        This must be called to perform the sync. This will run through, create
+        the configuration file as required, and try to do a sync of every repo
+        applicable or the repo actually specified. If self.repo is None, it
+        will be assumed all repos are synced as dictated by rlvars.
+
+         * Dry runs only create initial directories and structure
+         * Full runs sync everything from the top and setup structure,
+           including creating a symlink to latest-Rocky-X
+         * self.repo is ignored during full runs (noted in stdout)
+         * self.arch being set will force only that arch to sync
+        """
+        if self.fullrun and self.repo:
+            self.log.error('WARNING: repo ignored when doing a full sync')
+        if self.fullrun and self.dryrun:
+            self.log.error('A full and dry run is currently not supported.')
+            raise SystemExit('\nA full and dry run is currently not supported.')
+
+        self.generate_conf()
+
+        if self.fullrun:
+            sync_root = os.path.join(
+                    self.generate_compose_dirs(),
+                    'compose'
+            )
+        else:
+            sync_root = self.compose_latest_sync
+
+    def sync(self, repo, sync_root, arch=None):
+        """
+        Does the actual syncing of the repo. We generally sync each component
+        of a repo:
+            * each architecture
+            * each architecture debug
+            * each source
+        """
+        # dnf reposync --download-metadata \
+        #       --repoid fedora -p /tmp/test \
+        #       --forcearch aarch64 --norepopath
+        cmd = self.reposync_cmd()
+
+    def generate_compose_dirs(self) -> str:
+        """
+        Generate compose dirs for full runs
+        """
+        compose_base_dir = os.path.join(
+                self.compose_base,
+                "Rocky-{}-{}".format(self.major_version, self.date_stamp)
+        )
+        self.log.info('Creating compose directory %s' % compose_base_dir)
+        if not os.path.exists(compose_base_dir):
+            os.makedirs(compose_base_dir)
+
+        return compose_base_dir
+
+    def symlink_to_latest(self):
+        """
+        Emulates pungi and symlinks latest-Rocky-X
+
+        This link will be what is updated in full runs. Whatever is in this
+        'latest' directory is what is rsynced on to staging after completion.
+        This link should not change often.
+        """
         pass
 
-    def sync(self):
+    def generate_conf(self, dest_path='/var/tmp'):
-        cmd = self.reposync_cmd()
-        print(self.revision)
-
-    def generate_conf(self, dest_path: str, repo):
         """
         Generates the necessary repo conf file for the operation. This repo
         file should be temporary in nature. This will generate a repo file
@@ -48,7 +166,62 @@ class RepoSync:
         :param dest_path: The destination where the temporary conf goes
         :param repo: The repo object to create a file for
         """
-        pass
+        fname = os.path.join(
+                dest_path,
+                "{}-config.repo".format(self.major_version)
+        )
+        self.log.info('Generating the repo configuration: %s' % fname)
+
+        if self.repo_base_url.startswith("/"):
+            self.log.error("Local file syncs are not supported.")
+            raise SystemExit(Color.BOLD + "Local file syncs are not "
+                "supported." + Color.END)
+
+        # create dest_path
+        if not os.path.exists(dest_path):
+            os.makedirs(dest_path, exist_ok=True)
+        config_file = open(fname, "w+")
+        for repo in self.repos:
+            constructed_url = '{}/{}/repo/{}/$basearch'.format(
+                    self.repo_base_url,
+                    self.project_id,
+                    repo,
+            )
+
+            constructed_url_debug = '{}/{}/repo/{}/$basearch-debug'.format(
+                    self.repo_base_url,
+                    self.project_id,
+                    repo,
+            )
+
+            constructed_url_src = '{}/{}/repo/{}/src'.format(
+                    self.repo_base_url,
+                    self.project_id,
+                    repo,
+            )
+
+            # normal
+            config_file.write('[%s]\n' % repo)
+            config_file.write('name=%s\n' % repo)
+            config_file.write('baseurl=%s\n' % constructed_url)
+            config_file.write("enabled=1\n")
+            config_file.write("gpgcheck=0\n\n")
+
+            # debug
+            config_file.write('[%s-debug]\n' % repo)
+            config_file.write('name=%s debug\n' % repo)
+            config_file.write('baseurl=%s\n' % constructed_url_debug)
+            config_file.write("enabled=1\n")
+            config_file.write("gpgcheck=0\n\n")
+
+            # src
+            config_file.write('[%s-source]\n' % repo)
+            config_file.write('name=%s source\n' % repo)
+            config_file.write('baseurl=%s\n' % constructed_url_src)
+            config_file.write("enabled=1\n")
+            config_file.write("gpgcheck=0\n\n")
+
+
 
     def reposync_cmd(self) -> str:
         """
@@ -56,14 +229,22 @@ class RepoSync:
         itself and will raise an error.
 
         :return: The path to the reposync command. If dnf exists, we'll use
-        that.
+        that. Otherwise, fail immediately.
         """
         cmd = None
         if os.path.exists("/usr/bin/dnf"):
             cmd = "/usr/bin/dnf reposync"
         else:
-            raise SystemExit("/usr/bin/dnf was not found. /usr/bin/reposync is "
+            self.log.error('/usr/bin/dnf was not found. Good bye.')
+            raise SystemExit("/usr/bin/dnf was not found. \n\n/usr/bin/reposync "
                     "is not sufficient and you are likely running on an el7 "
-                    "system, which tells us that you made changes to these "
+                    "system or a grossly modified EL8+ system, " + Color.BOLD +
-                    "tools.")
+                    "which tells us that you probably made changes to these tools "
+                    "expecting them to work and got to this point." + Color.END)
         return cmd
+
+class SigRepoSync:
+    """
+    This helps us do reposync operations for SIG's. Do not use this for the
+    base system. Use RepoSync for that.
+    """