Merge pull request 'feature/variant-images-download' (#3) from feature/variant-images-download into devel

Reviewed-on: sig_core/toolkit#3

iso/empanadas/empanadas/common.py

@@ -103,10 +103,10 @@ for conf in glob.iglob(f"{_rootdir}/sig/*.yaml"):
 
 
 ALLOWED_TYPE_VARIANTS = {
-        "Azure": None,
+        "Azure": ["Base", "LVM"],
         "Container": ["Base", "Minimal", "UBI"],
-        "EC2": None,
+        "EC2": ["Base", "LVM"],
-        "GenericCloud": None,
+        "GenericCloud": ["Base", "LVM"],
         "Vagrant": ["Libvirt", "Vbox"],
         "OCP": None
 }

iso/empanadas/empanadas/configs/el8-beta.yaml

@@ -74,10 +74,23 @@
         - 'xorriso'
   cloudimages:
     images:
+      Azure:
+        format: vhd
+        variants: [Base, LVM]
+        primary_variant: 'Base'
       EC2:
         format: raw
+        variants: [Base, LVM]
+        primary_variant: 'Base'
       GenericCloud:
         format: qcow2
+        variants: [Base, LVM]
+        primary_variant: 'Base'
+      Container:
+        format: tar.xz
+        variants: [Base, Minimal, UBI]
+      OCP:
+        format: qcow2
   livemap:
     git_repo: 'https://git.resf.org/sig_core/kickstarts.git'
     branch: 'r8'

iso/empanadas/empanadas/configs/el8.yaml

@@ -74,10 +74,23 @@
         - 'xorriso'
   cloudimages:
     images:
+      Azure:
+        format: vhd
+        variants: [Base, LVM]
+        primary_variant: 'Base'
       EC2:
         format: raw
+        variants: [Base, LVM]
+        primary_variant: 'Base'
       GenericCloud:
         format: qcow2
+        variants: [Base, LVM]
+        primary_variant: 'Base'
+      Container:
+        format: tar.xz
+        variants: [Base, Minimal, UBI]
+      OCP:
+        format: qcow2
   livemap:
     git_repo: 'https://git.resf.org/sig_core/kickstarts.git'
     branch: 'r8'

iso/empanadas/empanadas/configs/el8lh.yaml

@@ -74,10 +74,23 @@
         - 'xorriso'
   cloudimages:
     images:
+      Azure:
+        format: vhd
+        variants: [Base, LVM]
+        primary_variant: 'Base'
       EC2:
         format: raw
+        variants: [Base, LVM]
+        primary_variant: 'Base'
       GenericCloud:
         format: qcow2
+        variants: [Base, LVM]
+        primary_variant: 'Base'
+      Container:
+        format: tar.xz
+        variants: [Base, Minimal, UBI]
+      OCP:
+        format: qcow2
   livemap:
     git_repo: 'https://git.resf.org/sig_core/kickstarts.git'
     branch: 'r8'

iso/empanadas/empanadas/configs/el9-beta.yaml

@@ -77,10 +77,23 @@
         - 'xorriso'
   cloudimages:
     images:
+      Azure:
+        format: vhd
+        variants: [Base, LVM]
+        primary_variant: 'Base'
       EC2:
         format: raw
+        variants: [Base, LVM]
+        primary_variant: 'Base'
       GenericCloud:
         format: qcow2
+        variants: [Base, LVM]
+        primary_variant: 'Base'
+      Container:
+        format: tar.xz
+        variants: [Base, Minimal, UBI]
+      OCP:
+        format: qcow2
   livemap:
     git_repo: 'https://git.resf.org/sig_core/kickstarts.git'
     branch: 'r9-beta'

iso/empanadas/empanadas/configs/el9.yaml

@@ -78,10 +78,23 @@
         - 'xorriso'
   cloudimages:
     images:
+      Azure:
+        format: vhd
+        variants: [Base, LVM]
+        primary_variant: 'Base'
       EC2:
         format: raw
+        variants: [Base, LVM]
+        primary_variant: 'Base'
       GenericCloud:
         format: qcow2
+        variants: [Base, LVM]
+        primary_variant: 'Base'
+      Container:
+        format: tar.xz
+        variants: [Base, Minimal, UBI]
+      OCP:
+        format: qcow2
   livemap:
     git_repo: 'https://git.resf.org/sig_core/kickstarts.git'
     branch: 'r9'

iso/empanadas/empanadas/configs/el9lh.yaml

@@ -77,10 +77,23 @@
         - 'xorriso'
   cloudimages:
     images:
+      Azure:
+        format: vhd
+        variants: [Base, LVM]
+        primary_variant: 'Base'
       EC2:
         format: raw
+        variants: [Base, LVM]
+        primary_variant: 'Base'
       GenericCloud:
         format: qcow2
+        variants: [Base, LVM]
+        primary_variant: 'Base'
+      Container:
+        format: tar.xz
+        variants: [Base, Minimal, UBI]
+      OCP:
+        format: qcow2
   livemap:
     git_repo: 'https://git.resf.org/sig_core/kickstarts.git'
     branch: 'r9lh'

iso/empanadas/empanadas/configs/rln.yaml

@@ -74,10 +74,23 @@
         - 'xorriso'
   cloudimages:
     images:
+      Azure:
+        format: vhd
+        variants: [Base, LVM]
+        primary_variant: 'Base'
       EC2:
         format: raw
+        variants: [Base, LVM]
+        primary_variant: 'Base'
       GenericCloud:
         format: qcow2
+        variants: [Base, LVM]
+        primary_variant: 'Base'
+      Container:
+        format: tar.xz
+        variants: [Base, Minimal, UBI]
+      OCP:
+        format: qcow2
   livemap:
     git_repo: 'https://git.resf.org/sig_core/kickstarts.git'
     branch: 'rln'

iso/empanadas/empanadas/util/iso_utils.py

@@ -1357,52 +1357,84 @@ class IsoBuild:
         """
         unpack_single_arch = False
         arches_to_unpack = self.arches
+        latest_artifacts = {}
         if self.arch:
             unpack_single_arch = True
             arches_to_unpack = [self.arch]
 
-        for imagename in self.cloudimages['images']:
+        for name, extra in self.cloudimages['images'].items():
-            self.log.info(Color.INFO + 'Determining the latest images for ' + imagename + ' ...')
+            self.log.info(Color.INFO + 'Determining the latest images for ' + name + ' ...')
-            formattype = self.cloudimages['images'][imagename]['format']
+            formattype = extra['format']
+            latest_artifacts[name] = {}
+            primary_variant = extra['primary_variant'] if 'primary_variant' in extra else None
+            latest_artifacts[name]['primary'] = primary_variant
 
+            variants = extra['variants'] if 'variants' in extra.keys() else [None] # need to loop once
+            imagename = name
+            variantname = name
+
+            for variant in variants:
+                if variant:
+                    variantname = f"{name}-{variant}"
+                    self.log.info(Color.INFO + 'Getting latest for variant ' + variant + ' ...')
                 if self.s3:
-                latest_artifacts = Shared.s3_determine_latest(
+                    latest_artifacts[name][variantname] = Shared.s3_determine_latest(
                             self.s3_bucket,
                             self.release,
                             arches_to_unpack,
                             formattype,
-                        imagename,
+                            variantname,
                             self.log
                     )
 
                 else:
-                latest_artifacts = Shared.reqs_determine_latest(
+                    latest_artifacts[name][variantname] = Shared.reqs_determine_latest(
                             self.s3_bucket_url,
                             self.release,
                             arches_to_unpack,
                             formattype,
-                        imagename,
+                            variantname,
                             self.log
                     )
 
-            if not len(latest_artifacts) > 0:
+                # latest_artifacts should have at least 1 result if has_variants, else == 1
-                self.log.warn(Color.WARN + 'No images found.')
+                if not len(latest_artifacts[name][variantname]) > 0:
+                    self.log.warn(Color.WARN + 'No images found for ' + variantname +
+                            '. This means it will be skipped.')
+
+            del imagename
+            del variantname
+            del variants
+
+        #print(latest_artifacts)
+        for keyname in latest_artifacts.keys():
+            primary = latest_artifacts[keyname]['primary']
+            for imgname in latest_artifacts[keyname]:
+                keysect = latest_artifacts[keyname][imgname]
+                if imgname == 'primary':
                     continue
 
-            self.log.info(Color.INFO + 'Attempting to download requested artifacts')
+                if not keysect:
+                    continue
+
+                self.log.info(Color.INFO + 'Attempting to download requested ' +
+                              'artifacts (' + keyname + ')')
+
                 for arch in arches_to_unpack:
                     image_arch_dir = os.path.join(
                             self.image_work_dir,
                             arch
                     )
 
-                if arch not in latest_artifacts.keys():
+                    source_path = keysect[arch]
-                    self.log.warn(Color.WARN + 'Artifact for ' + imagename +
-                            ' ' + arch + ' (' + formattype + ') does not exist.')
-                    continue
-
-                source_path = latest_artifacts[arch]
                     drop_name = source_path.split('/')[-1]
+
+                    # Docker containers get a "layer" name, this hack gets
+                    # around it. I didn't feel like adding another config opt.
+                    if 'layer' in drop_name:
+                        fsuffix = drop_name.replace('layer', '')
+                        drop_name = source_path.split('/')[-3] + fsuffix
+
                     checksum_name = drop_name + '.CHECKSUM'
                     full_drop = '{}/{}'.format(
                             image_arch_dir,
@@ -1449,7 +1481,7 @@ class IsoBuild:
                             image_arch_dir,
                             self.shortname,
                             self.major_version,
-                        imagename,
+                            imgname,
                             arch,
                             formattype
                     )
@@ -1458,7 +1490,7 @@ class IsoBuild:
                             image_arch_dir,
                             self.shortname,
                             self.major_version,
-                        imagename,
+                            imgname,
                             arch,
                             formattype
                     )
@@ -1480,6 +1512,40 @@ class IsoBuild:
                         link.write(checkdata)
                         link.close()
 
+                    # If this is the primary image, set the appropriate symlink
+                    # and checksum
+                    if primary and primary in drop_name:
+                        # If an image is the primary, we set this.
+                        latest_primary_name = '{}/{}-{}-{}.latest.{}.{}'.format(
+                                image_arch_dir,
+                                self.shortname,
+                                self.major_version,
+                                keyname,
+                                arch,
+                                formattype
+                        )
+                        latest_primary_checksum = '{}/{}-{}-{}.latest.{}.{}.CHECKSUM'.format(
+                                image_arch_dir,
+                                self.shortname,
+                                self.major_version,
+                                keyname,
+                                arch,
+                                formattype
+                        )
+                        latest_primary_path = latest_name.split('/')[-1]
+
+                        self.log.info('This is the primary image, setting link and checksum')
+                        if os.path.exists(latest_primary_name):
+                            os.remove(latest_primary_name)
+                        os.symlink(drop_name, latest_primary_name)
+                        shutil.copy2(checksum_drop, latest_primary_checksum)
+                        with open(latest_primary_checksum) as link:
+                            checkpdata = link.read()
+                        checkpdata = checkpdata.replace(drop_name, latest_primary_path)
+                        with open(latest_primary_checksum, 'w+') as link:
+                            link.write(checkpdata)
+                            link.close()
+
         self.log.info(Color.INFO + 'Image download phase completed')