Merge branch 'feature/iso-kube' into feature/imagebuild

* use a flag to determine if we want an RC or not
* Convert rldict and sigdict to an AttributeDict to allow access via
  __getattr__
* add fedora_release variable to configs for controlling icicle
  templates
* build_image.py script to generate per-architecture XML files used by
  imagefactory
* refactor time to call utcnow() once
* add jinja types to development dependencies until we move past jinja
  2.x
* Generate TDL templates per architecture for each image variant on
  demand
* Generate imagefactory and copy commands to execute image build
* Refactor Kubernetes job template to be generic for all current jobs

iso/empanadas/empanadas/common.py

@@ -7,6 +7,24 @@ import rpm
 import yaml
 import logging
 
+
+from collections import defaultdict
+from typing import Tuple
+
+# An implementation from the Fabric python library
+class AttributeDict(defaultdict):
+    def __init__(self):
+        super(AttributeDict, self).__init__(AttributeDict)
+
+    def __getattr__(self, key):
+        try:
+            return self[key]
+        except KeyError:
+            raise AttributeError(key)
+
+    def __setattr__(self, key, value):
+        self[key] = value
+
 # These are a bunch of colors we may use in terminal output
 class Color:
     RED = '\033[91m'
@@ -21,8 +39,8 @@ class Color:
     END = '\033[0m'
 
 # vars and additional checks
-rldict = {}
+rldict = AttributeDict()
-sigdict = {}
+sigdict = AttributeDict()
 config = {
     "rlmacro": rpm.expandMacro('%rhel'),
     "dist": 'el' + rpm.expandMacro('%rhel'),
@@ -76,3 +94,26 @@ for conf in glob.iglob(f"{_rootdir}/sig/*.yaml"):
 #rlvars = rldict[rlver]
 #rlvars = rldict[rlmacro]
 #COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp
+
+
+def valid_type_variant(_type: str, variant: str="") -> Tuple[bool, str]:
+    ALLOWED_TYPE_VARIANTS = {
+            "Container": ["Base", "Minimal"],
+            "GenericCloud": [],
+    }
+
+    if _type not in ALLOWED_TYPE_VARIANTS:
+        return False, f"Type is invalid: ({_type}, {variant})"
+    elif variant not in ALLOWED_TYPE_VARIANTS[_type]:
+        if variant.capitalize() in ALLOWED_TYPE_VARIANTS[_type]:
+            return False, f"Capitalization mismatch. Found: ({_type}, {variant}). Expected: ({_type}, {variant.capitalize()})"
+        return False, f"Type/Variant Combination is not allowed: ({_type}, {variant})"
+    return True, ""
+
+class Architecture(str):
+    @staticmethod
+    def New(architecture: str, version: int):
+        if architecture not in rldict[version]["allowed_arches"]:
+            print("Invalid architecture/version combo, skipping")
+            exit()
+        return Architecture(architecture)

iso/empanadas/empanadas/configs/el8.yaml

@@ -5,6 +5,7 @@
   rclvl: 'RC2'
   major: '8'
   minor: '6'
+  fedora_release: 28
   allowed_arches:
     - x86_64
     - aarch64

iso/empanadas/empanadas/configs/el9.yaml

@@ -5,6 +5,7 @@
   rclvl: 'RC1'
   major: '9'
   minor: '0'
+  fedora_release: 34
   bugurl: 'https://bugs.rockylinux.org'
   allowed_arches:
     - x86_64

iso/empanadas/empanadas/scripts/build_image.py

@@ -0,0 +1,144 @@
+# Builds an image given a version, type, variant, and architecture
+# Defaults to the running host's architecture
+
+import argparse
+import datetime
+import os
+import tempfile
+import pathlib
+
+from jinja2 import Environment, FileSystemLoader, Template
+from typing import List, Tuple
+
+from empanadas.common import Architecture, rldict, valid_type_variant
+from empanadas.common import _rootdir
+
+parser = argparse.ArgumentParser(description="ISO Compose")
+
+parser.add_argument('--version', type=str, help="Release Version (8.6, 9.1)", required=True)
+parser.add_argument('--rc', action='store_true', help="Release Candidate")
+parser.add_argument('--kickstartdir', action='store_true', help="Use the kickstart dir instead of the os dir for repositories")
+parser.add_argument('--debug', action='store_true', help="debug?")
+parser.add_argument('--type', type=str, help="Image type (container, genclo, azure, aws, vagrant)", required=True)
+parser.add_argument('--variant', type=str, help="", required=False)
+parser.add_argument('--release', type=str, help="Image release for subsequent builds with the same date stamp (rarely needed)", required=False)
+
+results = parser.parse_args()
+rlvars = rldict[results.version]
+major = rlvars["major"]
+
+STORAGE_DIR = pathlib.Path("/var/lib/imagefactory/storage")
+KICKSTART_PATH = pathlib.Path(os.environ.get("KICKSTART_PATH", "/kickstarts"))
+BUILDTIME = datetime.datetime.utcnow()
+
+
+def render_icicle_template(template: Template, architecture: Architecture) -> str:
+    handle, output = tempfile.mkstemp()
+    if not handle:
+        exit(3)
+    with os.fdopen(handle, "wb") as tmp:
+        _template = template.render(
+            architecture=architecture,
+            fedora_version=rlvars["fedora_release"],
+            iso8601date=BUILDTIME.strftime("%Y%m%d"),
+            installdir="kickstart" if results.kickstartdir else "os",
+            major=major,
+            release=results.release if results.release else 0,
+            size="10G",
+            type=results.type.capitalize(),
+            utcnow=BUILDTIME,
+            version_variant=rlvars["revision"] if not results.variant else f"{rlvars['revision']}-{results.variant.capitalize()}",
+        )
+        tmp.write(_template.encode())
+    return output
+
+
+def generate_kickstart_imagefactory_args(debug: bool = False) -> str:
+    type_variant = results.type if not results.variant else f"{results.type}-{results.variant}" # todo -cleanup
+    kickstart_path = pathlib.Path(f"{KICKSTART_PATH}/Rocky-{major}-{type_variant}.ks")
+
+    if not kickstart_path.is_file():
+        print(f"Kickstart file is not available: {kickstart_path}")
+        if not debug:
+            exit(2)
+
+    return f"--file-parameter install_script {kickstart_path}"
+
+def get_image_format(_type: str) -> str:
+    mapping = {
+            "Container": "docker"
+    }
+    return mapping[_type] if _type in mapping.keys() else ''
+
+def generate_imagefactory_commands(tdl_template: Template, architecture: Architecture) -> List[List[str]]:
+    template_path = render_icicle_template(tdl_template, architecture)
+    if not template_path:
+        exit(2)
+
+    args_mapping = {
+        "debug": "--debug" 
+    }
+
+    # only supports boolean flags right now?
+    args = [param for name, param in args_mapping.items() if getattr(results,name)]
+    package_args = []
+
+    kickstart_arg = generate_kickstart_imagefactory_args(True) # REMOVE DEBUG ARG
+
+    if results.type == "Container":
+        args += ["--parameter", "offline_icicle", "true"]
+        package_args += ["--parameter", "compress", "xz"]
+        tar_command = ["tar", "-Oxf", f"{STORAGE_DIR}/*.body" "./layer.tar"]
+
+    type_variant = results.type if not results.variant else f"{results.type}-{results.variant}" # todo -cleanup
+    outname = f"Rocky-{rlvars['major']}-{type_variant}.{BUILDTIME.strftime('%Y%m%d')}.{results.release if results.release else 0}.{architecture}"
+
+    outdir = pathlib.Path(f"/tmp/{outname}")
+
+    build_command = (f"imagefactory base_image {kickstart_arg} {' '.join(args)} {template_path}"
+                        f" | tee -a {outdir}/logs/base_image-{outname}.out"
+                        f" | tail -n4 > {outdir}/base.meta || exit 2" 
+            )
+
+
+    out_type = get_image_format(results.type)
+    package_command = ["imagefactory", "target_image", *args, template_path,
+                        "--id", "$(awk '$1==\"UUID:\"{print $NF}'"+f" /tmp/{outname}/base.meta)",
+                        *package_args, 
+                        "--parameter", "repository", outname, out_type,
+                        "|", "tee", "-a", f"{outdir}/base_image-{outname}.out",
+                        "|", "tail", "-n4", ">", f"{outdir}/target.meta", "||", "exit", "3" 
+            ]
+
+    copy_command = (f"aws s3 cp --recursive {outdir}/* s3://resf-empanadas/buildimage-{ outname }/{ BUILDTIME.strftime('%s') }/")
+    commands = [build_command, package_command, copy_command]
+    return commands
+    
+def run():
+    result, error = valid_type_variant(results.type, results.variant)
+    if not result:
+        print(error)
+        exit(2)
+
+    file_loader = FileSystemLoader(f"{_rootdir}/templates")
+    tmplenv = Environment(loader=file_loader)
+    tdl_template = tmplenv.get_template('icicle/tdl.xml.tmpl')
+    job_template = tmplenv.get_template('kube/Job.tmpl')
+
+    for architecture in rlvars["allowed_arches"]:
+        architecture = Architecture.New(architecture, major)
+
+        commands = generate_imagefactory_commands(tdl_template, architecture)
+
+        print(job_template.render(
+            architecture=architecture,
+            backoffLimit=4,
+            buildTime=datetime.datetime.utcnow().strftime("%s"),
+            command=commands,
+            imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest",
+            jobname="buildimage",
+            namespace="empanadas",
+            major=major,
+            restartPolicy="Never",
+        ))
+

iso/empanadas/empanadas/scripts/launch_builds.py

@@ -12,6 +12,7 @@ parser = argparse.ArgumentParser(description="ISO Compose")
 
 parser.add_argument('--release', type=str, help="Major Release Version", required=True)
 parser.add_argument('--env', type=str, help="environment", required=True)
+parser.add_argument('--rc', action='store_true', help="Release Candidate")
 results = parser.parse_args()
 rlvars = rldict[results.release]
 major = rlvars['major']
@@ -30,16 +31,25 @@ def run():
     elif results.env == "all":
         arches = EKSARCH+EXTARCH
 
-    command = ["build-iso", "--release", f"{results.release}", "--rc", "--isolation", "simple"]
+    command = ["build-iso", "--release", f"{results.release}", "--isolation", "simple"]
+    if results.rc:
+        command += ["--rc"]
+
+    buildstamp = datetime.datetime.utcnow()
 
     out = ""
-    for arch in arches:
+    for architecture in arches:
+        copy_command = (f"aws s3 cp --recursive --exclude=* --include=lorax* "
+                            f"/var/lib/mock/rocky-{ major }-$(uname -m)/root/builddir/ "
+                            f"s3://resf-empanadas/buildiso-{ major }-{ architecture }/{ buildstamp.strftime('%s') }/"
+        )
         out += job_template.render(
-            architecture=arch,
+            architecture=architecture,
             backoffLimit=4,
-            buildTime=datetime.datetime.utcnow().strftime("%s"),
+            buildTime=buildstamp.strftime("%s"),
-            command=command,
+            command=[command, copy_command],
             imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest",
+            jobname="buildiso",
             namespace="empanadas",
             major=major,
             restartPolicy="Never",

iso/empanadas/empanadas/templates/icicle/tdl.xml.tmpl

@@ -0,0 +1,21 @@
+<template>
+  <name>Rocky-{{major}}-{{type}}-{{version_variant}}.{{iso8601date}}.{{release}}.{{architecture}}</name>
+        <os>
+            <name>Fedora</name>
+            <version>{{fedora_version}}</version>
+            <arch>{{architecture}}</arch>
+            <install type='url'>
+              <url>https://dl.rockylinux.org/stg/rocky/{{major}}/BaseOS/{{architecture}}/{{installdir}}/</url>
+            </install>
+        <icicle>
+                <extra_command>rpm -qa --qf '%{NAME},%{VERSION},%{RELEASE},%{ARCH},%{EPOCH},%{SIZE},%{SIGMD5},%{BUILDTIME}
+'</extra_command>
+            </icicle>
+        </os>
+        <description>Rocky-{{major}}-{{type}}-{{version_variant}}.{{iso8601date}}.{{release}}.{{architecture}} Generated on {{utcnow}}</description>
+    <disk>
+      <size>{{size}}</size>
+    </disk>
+</template>
+
+

iso/empanadas/empanadas/templates/kube/Job.tmpl

@@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: build-iso-{{ major }}-{{ architecture }}
+  name: {{ jobname }}-{{ major }}-{{ architecture }}
   namespace: {{ namespace }}
 spec:
   template:
@@ -11,15 +11,18 @@ spec:
         peridot.rockylinux.org/workflow-tolerates-arch: {{ architecture }}
     spec:
       containers:
-      - name: buildiso-{{ major }}-{{ architecture }}
+      - name: {{ jobname }}-{{ major }}-{{ architecture }}
         image: {{ imageName }}
         command: ["/bin/bash", "-c"]
         args:
           - |
-            {{ command | join(' ') }}
+{%- for c in command -%}
-            aws s3 cp --recursive --exclude=* --include=lorax* \
+{%- if c is string %}
-            /var/lib/mock/rocky-{{ major }}-$(uname -m)/root/builddir/ \
+            {{ c }}
-            "s3://resf-empanadas/buildiso-{{ major }}-{{ architecture }}/{{ buildTime }}/"
+{%- else %}
+            {{ ' '.join(c) }}
+{%- endif %}
+{%- endfor %}
         securityContext:
           runAsUser: 0
           runAsGroup: 0

iso/empanadas/poetry.lock

@@ -302,6 +302,25 @@ category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
 
+[[package]]
+name = "types-jinja2"
+version = "2.11.9"
+description = "Typing stubs for Jinja2"
+category = "dev"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+types-MarkupSafe = "*"
+
+[[package]]
+name = "types-markupsafe"
+version = "1.1.10"
+description = "Typing stubs for MarkupSafe"
+category = "dev"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "typing-extensions"
 version = "4.2.0"
@@ -354,7 +373,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
 [metadata]
 lock-version = "1.1"
 python-versions = ">=3.7,<4"
-content-hash = "d011f4622c248f6aa107fd679616eaa19a897147398c6f52dd0dea0ab1d74486"
+content-hash = "76b6a6434217e66d6388431460ff32288198b91647e16f236c41164afc445b32"
 
 [metadata.files]
 atomicwrites = [
@@ -558,6 +577,14 @@ six = [
     {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
     {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
 ]
+types-jinja2 = [
+    {file = "types-Jinja2-2.11.9.tar.gz", hash = "sha256:dbdc74a40aba7aed520b7e4d89e8f0fe4286518494208b35123bcf084d4b8c81"},
+    {file = "types_Jinja2-2.11.9-py3-none-any.whl", hash = "sha256:60a1e21e8296979db32f9374d8a239af4cb541ff66447bb915d8ad398f9c63b2"},
+]
+types-markupsafe = [
+    {file = "types-MarkupSafe-1.1.10.tar.gz", hash = "sha256:85b3a872683d02aea3a5ac2a8ef590193c344092032f58457287fbf8e06711b1"},
+    {file = "types_MarkupSafe-1.1.10-py3-none-any.whl", hash = "sha256:ca2bee0f4faafc45250602567ef38d533e877d2ddca13003b319c551ff5b3cc5"},
+]
 typing-extensions = [
     {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"},
     {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"},

iso/empanadas/pyproject.toml

@@ -18,6 +18,7 @@ requests = "^2.28.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "~5"
+types-Jinja2 = "^2.11.9" # Remove when upgrading past Jinja 2.x as type annotations are in-tree
 
 [tool.poetry.scripts]
 sync_from_peridot = "empanadas.scripts.sync_from_peridot:run"
@@ -26,6 +27,7 @@ sync_sig = "empanadas.scripts.sync_sig:run"
 build-iso = "empanadas.scripts.build_iso:run"
 pull-unpack-tree = "empanadas.scripts.pull_unpack_tree:run"
 launch-builds = "empanadas.scripts.launch_builds:run"
+build-image = "empanadas.scripts.build_image:run"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]