Merge branch 'feature/imagebuild' into 'devel'

Run builds in a container

See merge request release-engineering/public/toolkit!46

iso/empanadas/Containerfile

@@ -56,7 +56,7 @@ RUN rm -rf /etc/yum.repos.d/*.repo
 RUN useradd -o -d /var/peridot -u 1002 peridotbuilder && usermod -a -G mock peridotbuilder
 RUN chown peridotbuilder:mock /etc/yum.conf && chown -R peridotbuilder:mock /etc/dnf && chown -R peridotbuilder:mock /etc/rpm && chown -R peridotbuilder:mock /etc/yum.repos.d
 
-RUN pip install 'git+https://git.rockylinux.org/release-engineering/public/toolkit.git@feature/iso-kube#egg=empanadas&subdirectory=iso/empanadas'
+RUN pip install 'git+https://git.rockylinux.org/release-engineering/public/toolkit.git@devel#egg=empanadas&subdirectory=iso/empanadas'
 
 RUN pip install awscli
 

iso/empanadas/empanadas/common.py

@@ -8,6 +8,24 @@ import yaml
 import logging
 import hashlib
 
+
+from collections import defaultdict
+from typing import Tuple
+
+# An implementation from the Fabric python library
+class AttributeDict(defaultdict):
+    def __init__(self):
+        super(AttributeDict, self).__init__(AttributeDict)
+
+    def __getattr__(self, key):
+        try:
+            return self[key]
+        except KeyError:
+            raise AttributeError(key)
+
+    def __setattr__(self, key, value):
+        self[key] = value
+
 # These are a bunch of colors we may use in terminal output
 class Color:
     RED = '\033[91m'
@@ -22,8 +40,8 @@ class Color:
     END = '\033[0m'
 
 # vars and additional checks
-rldict = {}
-sigdict = {}
+rldict = AttributeDict()
+sigdict = AttributeDict()
 config = {
     "rlmacro": rpm.expandMacro('%rhel'),
     "dist": 'el' + rpm.expandMacro('%rhel'),
@@ -77,3 +95,26 @@ for conf in glob.iglob(f"{_rootdir}/sig/*.yaml"):
 #rlvars = rldict[rlver]
 #rlvars = rldict[rlmacro]
 #COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp
+
+
+def valid_type_variant(_type: str, variant: str="") -> Tuple[bool, str]:
+    ALLOWED_TYPE_VARIANTS = {
+            "Container": ["Base", "Minimal"],
+            "GenericCloud": [],
+    }
+
+    if _type not in ALLOWED_TYPE_VARIANTS:
+        return False, f"Type is invalid: ({_type}, {variant})"
+    elif variant not in ALLOWED_TYPE_VARIANTS[_type]:
+        if variant.capitalize() in ALLOWED_TYPE_VARIANTS[_type]:
+            return False, f"Capitalization mismatch. Found: ({_type}, {variant}). Expected: ({_type}, {variant.capitalize()})"
+        return False, f"Type/Variant Combination is not allowed: ({_type}, {variant})"
+    return True, ""
+
+class Architecture(str):
+    @staticmethod
+    def New(architecture: str, version: int):
+        if architecture not in rldict[version]["allowed_arches"]:
+            print("Invalid architecture/version combo, skipping")
+            exit()
+        return Architecture(architecture)

iso/empanadas/empanadas/configs/el8.yaml

@@ -7,6 +7,7 @@
   minor: '6'
   profile: '8'
   bugurl: 'https://bugs.rockylinux.org'
+  fedora_release: 28
   allowed_arches:
     - x86_64
     - aarch64

iso/empanadas/empanadas/configs/el9-beta.yaml

@@ -8,6 +8,7 @@
   profile: '9-beta'
   bugurl: 'https://bugs.rockylinux.org'
   checksum: 'sha256'
+  fedora_release: 34
   allowed_arches:
     - x86_64
     - aarch64

iso/empanadas/empanadas/configs/el9.yaml

@@ -6,6 +6,7 @@
   major: '9'
   minor: '0'
   profile: '9'
+  fedora_release: 34
   bugurl: 'https://bugs.rockylinux.org'
   checksum: 'sha256'
   allowed_arches:

iso/empanadas/empanadas/configs/el9lh.yaml

@@ -8,6 +8,7 @@
   profile: '9-lookahead'
   bugurl: 'https://bugs.rockylinux.org'
   checksum: 'sha256'
+  fedora_release: 34
   allowed_arches:
     - x86_64
     - aarch64

iso/empanadas/empanadas/scripts/build_image.py

@@ -0,0 +1,144 @@
+# Builds an image given a version, type, variant, and architecture
+# Defaults to the running host's architecture
+
+import argparse
+import datetime
+import os
+import tempfile
+import pathlib
+
+from jinja2 import Environment, FileSystemLoader, Template
+from typing import List, Tuple
+
+from empanadas.common import Architecture, rldict, valid_type_variant
+from empanadas.common import _rootdir
+
+parser = argparse.ArgumentParser(description="ISO Compose")
+
+parser.add_argument('--version', type=str, help="Release Version (8.6, 9.1)", required=True)
+parser.add_argument('--rc', action='store_true', help="Release Candidate")
+parser.add_argument('--kickstartdir', action='store_true', help="Use the kickstart dir instead of the os dir for repositories")
+parser.add_argument('--debug', action='store_true', help="debug?")
+parser.add_argument('--type', type=str, help="Image type (container, genclo, azure, aws, vagrant)", required=True)
+parser.add_argument('--variant', type=str, help="", required=False)
+parser.add_argument('--release', type=str, help="Image release for subsequent builds with the same date stamp (rarely needed)", required=False)
+
+results = parser.parse_args()
+rlvars = rldict[results.version]
+major = rlvars["major"]
+
+STORAGE_DIR = pathlib.Path("/var/lib/imagefactory/storage")
+KICKSTART_PATH = pathlib.Path(os.environ.get("KICKSTART_PATH", "/kickstarts"))
+BUILDTIME = datetime.datetime.utcnow()
+
+
+def render_icicle_template(template: Template, architecture: Architecture) -> str:
+    handle, output = tempfile.mkstemp()
+    if not handle:
+        exit(3)
+    with os.fdopen(handle, "wb") as tmp:
+        _template = template.render(
+            architecture=architecture,
+            fedora_version=rlvars["fedora_release"],
+            iso8601date=BUILDTIME.strftime("%Y%m%d"),
+            installdir="kickstart" if results.kickstartdir else "os",
+            major=major,
+            release=results.release if results.release else 0,
+            size="10G",
+            type=results.type.capitalize(),
+            utcnow=BUILDTIME,
+            version_variant=rlvars["revision"] if not results.variant else f"{rlvars['revision']}-{results.variant.capitalize()}",
+        )
+        tmp.write(_template.encode())
+    return output
+
+
+def generate_kickstart_imagefactory_args(debug: bool = False) -> str:
+    type_variant = results.type if not results.variant else f"{results.type}-{results.variant}" # todo -cleanup
+    kickstart_path = pathlib.Path(f"{KICKSTART_PATH}/Rocky-{major}-{type_variant}.ks")
+
+    if not kickstart_path.is_file():
+        print(f"Kickstart file is not available: {kickstart_path}")
+        if not debug:
+            exit(2)
+
+    return f"--file-parameter install_script {kickstart_path}"
+
+def get_image_format(_type: str) -> str:
+    mapping = {
+            "Container": "docker"
+    }
+    return mapping[_type] if _type in mapping.keys() else ''
+
+def generate_imagefactory_commands(tdl_template: Template, architecture: Architecture) -> List[List[str]]:
+    template_path = render_icicle_template(tdl_template, architecture)
+    if not template_path:
+        exit(2)
+
+    args_mapping = {
+        "debug": "--debug" 
+    }
+
+    # only supports boolean flags right now?
+    args = [param for name, param in args_mapping.items() if getattr(results,name)]
+    package_args = []
+
+    kickstart_arg = generate_kickstart_imagefactory_args(True) # REMOVE DEBUG ARG
+
+    if results.type == "Container":
+        args += ["--parameter", "offline_icicle", "true"]
+        package_args += ["--parameter", "compress", "xz"]
+        tar_command = ["tar", "-Oxf", f"{STORAGE_DIR}/*.body" "./layer.tar"]
+
+    type_variant = results.type if not results.variant else f"{results.type}-{results.variant}" # todo -cleanup
+    outname = f"Rocky-{rlvars['major']}-{type_variant}.{BUILDTIME.strftime('%Y%m%d')}.{results.release if results.release else 0}.{architecture}"
+
+    outdir = pathlib.Path(f"/tmp/{outname}")
+
+    build_command = (f"imagefactory base_image {kickstart_arg} {' '.join(args)} {template_path}"
+                        f" | tee -a {outdir}/logs/base_image-{outname}.out"
+                        f" | tail -n4 > {outdir}/base.meta || exit 2" 
+            )
+
+
+    out_type = get_image_format(results.type)
+    package_command = ["imagefactory", "target_image", *args, template_path,
+                        "--id", "$(awk '$1==\"UUID:\"{print $NF}'"+f" /tmp/{outname}/base.meta)",
+                        *package_args, 
+                        "--parameter", "repository", outname, out_type,
+                        "|", "tee", "-a", f"{outdir}/base_image-{outname}.out",
+                        "|", "tail", "-n4", ">", f"{outdir}/target.meta", "||", "exit", "3" 
+            ]
+
+    copy_command = (f"aws s3 cp --recursive {outdir}/* s3://resf-empanadas/buildimage-{ outname }/{ BUILDTIME.strftime('%s') }/")
+    commands = [build_command, package_command, copy_command]
+    return commands
+    
+def run():
+    result, error = valid_type_variant(results.type, results.variant)
+    if not result:
+        print(error)
+        exit(2)
+
+    file_loader = FileSystemLoader(f"{_rootdir}/templates")
+    tmplenv = Environment(loader=file_loader)
+    tdl_template = tmplenv.get_template('icicle/tdl.xml.tmpl')
+    job_template = tmplenv.get_template('kube/Job.tmpl')
+
+    for architecture in rlvars["allowed_arches"]:
+        architecture = Architecture.New(architecture, major)
+
+        commands = generate_imagefactory_commands(tdl_template, architecture)
+
+        print(job_template.render(
+            architecture=architecture,
+            backoffLimit=4,
+            buildTime=datetime.datetime.utcnow().strftime("%s"),
+            command=commands,
+            imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest",
+            jobname="buildimage",
+            namespace="empanadas",
+            major=major,
+            restartPolicy="Never",
+        ))
+

iso/empanadas/empanadas/scripts/launch_builds.py

@@ -8,10 +8,11 @@ from empanadas.common import _rootdir
 
 from jinja2 import Environment, FileSystemLoader
 
-parser = argparse.ArgumentParser(description="ISO Compose")
+parser = argparse.ArgumentParser(description="Generate Kubernetes Jobs to run lorax in mock and upload the result. Pipe into kubectl for the appropriate cluster")
 
-parser.add_argument('--release', type=str, help="Major Release Version", required=True)
-parser.add_argument('--env', type=str, help="environment", required=True)
+parser.add_argument('--release', type=str, help="Major Release Version: (8|9)", required=True)
+parser.add_argument('--env', type=str, help="environment: one of (eks|ext|all). presently jobs are scheduled on different kubernetes clusters", required=True)
+parser.add_argument('--rc', action='store_true', help="Release Candidate, Beta, RLN")
 results = parser.parse_args()
 rlvars = rldict[results.release]
 major = rlvars['major']
@@ -30,16 +31,25 @@ def run():
     elif results.env == "all":
         arches = EKSARCH+EXTARCH
 
-    command = ["build-iso", "--release", f"{results.release}", "--rc", "--isolation", "simple"]
+    command = ["build-iso", "--release", f"{results.release}", "--isolation", "simple"]
+    if results.rc:
+        command += ["--rc"]
+
+    buildstamp = datetime.datetime.utcnow()
 
     out = ""
-    for arch in arches:
+    for architecture in arches:
+        copy_command = (f"aws s3 cp --recursive --exclude=* --include=lorax* "
+                            f"/var/lib/mock/rocky-{ major }-$(uname -m)/root/builddir/ "
+                            f"s3://resf-empanadas/buildiso-{ major }-{ architecture }/{ buildstamp.strftime('%s') }/"
+        )
         out += job_template.render(
-            architecture=arch,
+            architecture=architecture,
             backoffLimit=4,
-            buildTime=datetime.datetime.utcnow().strftime("%s"),
-            command=command,
+            buildTime=buildstamp.strftime("%s"),
+            command=[command, copy_command],
             imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest",
+            jobname="buildiso",
             namespace="empanadas",
             major=major,
             restartPolicy="Never",

iso/empanadas/empanadas/templates/icicle/tdl.xml.tmpl

@@ -0,0 +1,21 @@
+<template>
+  <name>Rocky-{{major}}-{{type}}-{{version_variant}}.{{iso8601date}}.{{release}}.{{architecture}}</name>
+        <os>
+            <name>Fedora</name>
+            <version>{{fedora_version}}</version>
+            <arch>{{architecture}}</arch>
+            <install type='url'>
+              <url>https://dl.rockylinux.org/stg/rocky/{{major}}/BaseOS/{{architecture}}/{{installdir}}/</url>
+            </install>
+        <icicle>
+                <extra_command>rpm -qa --qf '%{NAME},%{VERSION},%{RELEASE},%{ARCH},%{EPOCH},%{SIZE},%{SIGMD5},%{BUILDTIME}
+'</extra_command>
+            </icicle>
+        </os>
+        <description>Rocky-{{major}}-{{type}}-{{version_variant}}.{{iso8601date}}.{{release}}.{{architecture}} Generated on {{utcnow}}</description>
+    <disk>
+      <size>{{size}}</size>
+    </disk>
+</template>
+
+

iso/empanadas/empanadas/templates/kube/Job.tmpl

@@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: build-iso-{{ major }}-{{ architecture }}
+  name: {{ jobname }}-{{ major }}-{{ architecture }}
   namespace: {{ namespace }}
 spec:
   template:
@@ -11,15 +11,18 @@ spec:
         peridot.rockylinux.org/workflow-tolerates-arch: {{ architecture }}
     spec:
       containers:
-      - name: buildiso-{{ major }}-{{ architecture }}
+      - name: {{ jobname }}-{{ major }}-{{ architecture }}
         image: {{ imageName }}
         command: ["/bin/bash", "-c"]
         args:
           - |
-            {{ command | join(' ') }}
-            aws s3 cp --recursive --exclude=* --include=lorax* \
-            /var/lib/mock/rocky-{{ major }}-$(uname -m)/root/builddir/ \
-            "s3://resf-empanadas/buildiso-{{ major }}-{{ architecture }}/{{ buildTime }}/"
+{%- for c in command -%}
+{%- if c is string %}
+            {{ c }}
+{%- else %}
+            {{ ' '.join(c) }}
+{%- endif %}
+{%- endfor %}
         securityContext:
           runAsUser: 0
           runAsGroup: 0

iso/empanadas/pyproject.toml

@@ -19,6 +19,7 @@ kobo = "^0.24.1"
 
 [tool.poetry.dev-dependencies]
 pytest = "~5"
+types-Jinja2 = "^2.11.9" # Remove when upgrading past Jinja 2.x as type annotations are in-tree
 
 [tool.poetry.scripts]
 sync_from_peridot = "empanadas.scripts.sync_from_peridot:run"
@@ -28,6 +29,7 @@ build-iso = "empanadas.scripts.build_iso:run"
 build-iso-extra = "empanadas.scripts.build_iso_extra:run"
 pull-unpack-tree = "empanadas.scripts.pull_unpack_tree:run"
 launch-builds = "empanadas.scripts.launch_builds:run"
+build-image = "empanadas.scripts.build_image:run"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]