add justcopyit for live images

iso/empanadas/README.md

@@ -41,6 +41,8 @@ Changes to the poetry.lock should be commited if dependencies are added or updat
 * finalize_compose     -> Finalizes a compose with metadata and checksums, as well as copies images
 * launch-builds        -> Creates a kube config to run build-iso
 * build-image          -> Runs build-iso
+* generate_compose     -> Creates a compose directory right away and optionally links it as latest
+                          (You should only use this if you are running into errors with images)
 ```
 
 ## wrappers

iso/empanadas/empanadas/scripts/build_iso_live.py

@@ -15,6 +15,7 @@ parser.add_argument('--image', type=str, help="Granular choice in which live ima
 parser.add_argument('--logger', type=str)
 parser.add_argument('--live-iso-mode', type=str, default='local')
 parser.add_argument('--hashed', action='store_true')
+parser.add_argument('--just-copy-it', action='store_true', help="Just copy the images to the compose dir")
 results = parser.parse_args()
 rlvars = rldict[results.release]
 major = rlvars['major']
@@ -28,6 +29,7 @@ a = LiveBuild(
         image=results.image,
         compose_dir_is_here=results.local_compose,
         hashed=results.hashed,
+        justcopyit=results.just_copy_it,
         logger=results.logger
 )
 

iso/empanadas/empanadas/scripts/generate_compose.py

@@ -0,0 +1,72 @@
+# This script can be called to do single syncs or full on syncs.
+
+import argparse
+import logging
+import sys
+
+from empanadas.common import *
+from empanadas.util import Checks
+from empanadas.util import RepoSync
+from empanadas.util import Shared
+
+# Start up the parser baby
+parser = argparse.ArgumentParser(description="Peridot Sync and Compose")
+
+# All of our options
+parser.add_argument('--release', type=str, help="Major Release Version or major-type (eg 9-beta)", required=True)
+parser.add_argument('--logger', type=str)
+
+# Parse them
+results = parser.parse_args()
+rlvars = rldict[results.release]
+major = rlvars['major']
+
+r = Checks(rlvars, config['arch'])
+r.check_valid_arch()
+
+# Send them and do whatever I guess
+def run():
+    if results.logger is None:
+        log = logging.getLogger("generate")
+        log.setLevel(logging.INFO)
+        handler = logging.StreamHandler(sys.stdout)
+        handler.setLevel(logging.INFO)
+        formatter = logging.Formatter(
+            '%(asctime)s :: %(name)s :: %(message)s',
+            '%Y-%m-%d %H:%M:%S'
+        )
+        handler.setFormatter(formatter)
+        log.addHandler(handler)
+    else:
+        log = results.logger
+
+    compose_base = config['compose_root'] + "/" + major
+    shortname = config['shortname']
+    version = rlvars['revision']
+    date_stamp = config['date_stamp']
+    profile = rlvars['profile']
+    logger = log
+
+    generated_dir = Shared.generate_compose_dirs(
+            compose_base,
+            shortname,
+            version,
+            date_stamp,
+            logger
+    )
+
+    if results.symlink:
+        compose_latest_dir = os.path.join(
+                config['compose_root'],
+                major,
+                "latest-{}-{}".format(
+                    shortname,
+                    profile,
+                )
+        )
+        if os.path.exists(compose_latest_dir):
+            os.remove(compose_latest_dir)
+
+        os.symlink(generated_dir, compose_latest_dir)
+
+    log.info('Generated compose dirs.')

iso/empanadas/empanadas/util/iso_utils.py

@@ -1039,12 +1039,12 @@ class IsoBuild:
             self.log.info(Color.INFO + 'Building ' + i + ' completed')
 
             if len(bad_exit_list) == 0:
-                self.log.info(Color.INFO + 'Copying ISOs over to compose directory...')
+                self.log.info(Color.INFO + 'Images built successfully.')
             else:
                 self.log.error(
                         Color.FAIL +
                         'There were issues with the work done. As a result, ' +
-                        'the ISOs will not be copied.'
+                        'some/all ISOs may not exist.'
                 )
 
 
@@ -1425,10 +1425,12 @@ class LiveBuild:
             compose_dir_is_here: bool = False,
             hashed: bool = False,
             image=None,
+            justcopyit: bool = False,
             logger=None
     ):
 
         self.image = image
+        self.justcopyit = justcopyit
         self.fullname = rlvars['fullname']
         self.distname = config['distname']
         self.shortname = config['shortname']
@@ -1523,6 +1525,14 @@ class LiveBuild:
         )
         self.log.info(self.revision)
 
+        if not os.path.exists(self.compose_latest_dir):
+            self.log.warn(Color.WARN + 'A compose directory was not found ' +
+                    'here. If there is a failure, it may be due to it ' +
+                    'missing. You may want to generate a fake compose if ' +
+                    'you are simply making your own live images and you run ' +
+                    'into any errors beyond this point.'
+            )
+
     def run_build_live_iso(self):
         """
         Builds DVD images based on the data created from the initial lorax on
@@ -1575,7 +1585,10 @@ class LiveBuild:
                 raise SystemExit()
 
         if self.live_iso_mode == 'podman':
-            self._live_iso_podman_run(self.current_arch, images_to_build, work_root)
+            #self._live_iso_podman_run(self.current_arch, images_to_build, work_root)
+            self.log.error(Color.FAIL + 'At this time, live images cannot be ' +
+                    'built in podman.')
+            raise SystemExit()
 
     def _live_iso_local_config(self, image, work_root):
         """
@@ -1723,6 +1736,7 @@ class LiveBuild:
         bad_exit_list = []
         checksum_list = []
         entry_name_list = []
+        self.log.warn(Color.WARN + 'This mode does not work properly. It will fail.')
         for i in images:
             image_name = i
             entry_name = 'buildLiveImage-{}-{}.sh'.format(arch, i)
@@ -1821,12 +1835,12 @@ class LiveBuild:
         self.log.info(Color.INFO + 'Building live images completed')
 
         if len(bad_exit_list) == 0:
-            self.log.info(Color.INFO + 'Copying ISOs over to compose directory...')
+            self.log.info(Color.INFO + 'Live images completed successfully.')
         else:
             self.log.error(
                     Color.FAIL +
                     'There were issues with the work done. As a result, ' +
-                    'the ISOs will not be copied.'
+                    'some or all ISOs may not be copied later.'
             )
 
     def _live_iso_local_run(self, arch, image, work_root):
@@ -1836,6 +1850,19 @@ class LiveBuild:
         have mock available.
         """
         entries_dir = os.path.join(work_root, "entries")
+        live_dir_arch = os.path.join(self.live_work_dir, arch)
+        isoname = '{}-{}-{}-{}-{}.iso'.format(
+                self.shortname,
+                image,
+                self.release,
+                arch,
+                self.date
+        )
+        live_res_dir = '/var/lib/mock/{}-{}-{}/result'.format(
+                self.shortname,
+                self.major_version,
+                arch
+        )
         live_iso_cmd = '/bin/bash {}/liveisobuild-{}-{}.sh'.format(entries_dir, arch, image)
         self.log.info('Starting mock build...')
         p = subprocess.call(shlex.split(live_iso_cmd))
@@ -1850,6 +1877,21 @@ class LiveBuild:
         )
         self.log.warn(
                 Color.WARN +
-                'If you are looping images, your built image WILL get ' +
+                'If you are looping images, your built image may get ' +
-                'overwritten.'
+                'overwritten. Ensure you have justcopyit enabled to avoid this.'
         )
+
+        if self.justcopyit:
+            self.log.info(Color.INFO + 'Copying image to work directory')
+            source_path = os.path.join(live_res_dir, isoname)
+            dest_path = os.path.join(live_dir_arch, isoname)
+            os.makedirs(live_dir_arch, exist_ok=True)
+            shutil.copy2(source_path, dest_path)
+            self.log.info(Color.INFO + 'Generating checksum')
+            checksum = Shared.get_checksum(dest_path, self.checksum, self.log)
+            if not checksum:
+                self.log.error(Color.FAIL + dest_path + ' not found. Did we copy it?')
+                return
+            with open(dest_path + '.CHECKSUM', "w+") as c:
+                c.write(checksum)
+                c.close()

iso/empanadas/empanadas/util/shared.py

@@ -319,6 +319,10 @@ class Shared:
         logger.info('Creating compose directory %s' % compose_base_dir)
         if not os.path.exists(compose_base_dir):
             os.makedirs(compose_base_dir)
+            os.makedirs(compose_base_dir + '/work')
+            os.makedirs(compose_base_dir + '/work/entries')
+            os.makedirs(compose_base_dir + '/work/logs')
+            os.makedirs(compose_base_dir + '/compose')
 
         return compose_base_dir
 

iso/empanadas/pyproject.toml

@@ -33,6 +33,7 @@ launch-builds = "empanadas.scripts.launch_builds:run"
 build-image = "empanadas.scripts.build_image:run"
 finalize_compose = "empanadas.scripts.finalize_compose:run"
 pull-cloud-image = "empanadas.scripts.pull_cloud_image:run"
+generate_compose = "empanadas.scripts.generate_compose:run"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]