From 4324e977d1883676e43efd0df9078606bb3609b0 Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Mon, 20 Jun 2022 20:12:20 -0400 Subject: [PATCH] Implement a feature to assist in generating various images * use a flag to determine if we want an RC or not * Convert rldict and sigdict to an AttributeDict to allow access via __getattr__ * add fedora_release variable to configs for controlling icicle templates * build_image.py script to generate per-architecture XML files used by imagefactory * refactor time to call utcnow() once * add jinja types to development dependencies until we move past jinja 2.x * Generate TDL templates per architecture for each image variant on demand * Generate imagefactory and copy commands to execute image build * Refactor Kubernetes job template to be generic for all current jobs --- iso/empanadas/empanadas/common.py | 45 +++++- iso/empanadas/empanadas/configs/el8.yaml | 1 + iso/empanadas/empanadas/configs/el9.yaml | 1 + .../empanadas/scripts/build_image.py | 145 ++++++++++++++++++ .../empanadas/scripts/launch_builds.py | 20 ++- .../empanadas/templates/icicle/tdl.xml.tmpl | 21 +++ .../empanadas/templates/kube/Job.tmpl | 15 +- iso/empanadas/poetry.lock | 29 +++- iso/empanadas/pyproject.toml | 2 + 9 files changed, 265 insertions(+), 14 deletions(-) create mode 100644 iso/empanadas/empanadas/scripts/build_image.py create mode 100644 iso/empanadas/empanadas/templates/icicle/tdl.xml.tmpl diff --git a/iso/empanadas/empanadas/common.py b/iso/empanadas/empanadas/common.py index bf081ab..e0fafb0 100644 --- a/iso/empanadas/empanadas/common.py +++ b/iso/empanadas/empanadas/common.py @@ -7,6 +7,24 @@ import rpm import yaml import logging + +from collections import defaultdict +from typing import Tuple + +# An implementation from the Fabric python library +class AttributeDict(defaultdict): + def __init__(self): + super(AttributeDict, self).__init__(AttributeDict) + + def __getattr__(self, key): + try: + return self[key] + except KeyError: + raise AttributeError(key) + + def __setattr__(self, key, value): + self[key] = value + # These are a bunch of colors we may use in terminal output class Color: RED = '\033[91m' @@ -21,8 +39,8 @@ class Color: END = '\033[0m' # vars and additional checks -rldict = {} -sigdict = {} +rldict = AttributeDict() +sigdict = AttributeDict() config = { "rlmacro": rpm.expandMacro('%rhel'), "dist": 'el' + rpm.expandMacro('%rhel'), @@ -76,3 +94,26 @@ for conf in glob.iglob(f"{_rootdir}/sig/*.yaml"): #rlvars = rldict[rlver] #rlvars = rldict[rlmacro] #COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp + + +def valid_type_variant(_type: str, variant: str="") -> Tuple[bool, str]: + ALLOWED_TYPE_VARIANTS = { + "Container": ["Base", "Minimal"], + "GenericCloud": [], + } + + if _type not in ALLOWED_TYPE_VARIANTS: + return False, f"Type is invalid: ({_type}, {variant})" + elif variant not in ALLOWED_TYPE_VARIANTS[_type]: + if variant.capitalize() in ALLOWED_TYPE_VARIANTS[_type]: + return False, f"Capitalization mismatch. Found: ({_type}, {variant}). Expected: ({_type}, {variant.capitalize()})" + return False, f"Type/Variant Combination is not allowed: ({_type}, {variant})" + return True, "" + +class Architecture(str): + @staticmethod + def New(architecture: str, version: int): + if architecture not in rldict[version]["allowed_arches"]: + print("Invalid architecture/version combo, skipping") + exit() + return Architecture(architecture) diff --git a/iso/empanadas/empanadas/configs/el8.yaml b/iso/empanadas/empanadas/configs/el8.yaml index 032a9de..ea5ef9a 100644 --- a/iso/empanadas/empanadas/configs/el8.yaml +++ b/iso/empanadas/empanadas/configs/el8.yaml @@ -5,6 +5,7 @@ rclvl: 'RC2' major: '8' minor: '6' + fedora_release: 28 allowed_arches: - x86_64 - aarch64 diff --git a/iso/empanadas/empanadas/configs/el9.yaml b/iso/empanadas/empanadas/configs/el9.yaml index 34eef95..ae3d18a 100644 --- a/iso/empanadas/empanadas/configs/el9.yaml +++ b/iso/empanadas/empanadas/configs/el9.yaml @@ -5,6 +5,7 @@ rclvl: 'RC1' major: '9' minor: '0' + fedora_release: 34 bugurl: 'https://bugs.rockylinux.org' allowed_arches: - x86_64 diff --git a/iso/empanadas/empanadas/scripts/build_image.py b/iso/empanadas/empanadas/scripts/build_image.py new file mode 100644 index 0000000..c9f7782 --- /dev/null +++ b/iso/empanadas/empanadas/scripts/build_image.py @@ -0,0 +1,145 @@ +# Builds an image given a version, type, variant, and architecture +# Defaults to the running host's architecture + +import argparse +import datetime +import os +import tempfile +import pathlib + +from jinja2 import Environment, FileSystemLoader, Template +from typing import List, Tuple + +from empanadas.common import Architecture, rldict, valid_type_variant +from empanadas.common import _rootdir + +parser = argparse.ArgumentParser(description="ISO Compose") + +parser.add_argument('--version', type=str, help="Release Version (8.6, 9.1)", required=True) +parser.add_argument('--rc', action='store_true', help="Release Candidate") +parser.add_argument('--kickstartdir', action='store_true', help="Use the kickstart dir instead of the os dir for repositories") +parser.add_argument('--debug', action='store_true', help="debug?") +parser.add_argument('--type', type=str, help="Image type (container, genclo, azure, aws, vagrant)", required=True) +parser.add_argument('--variant', type=str, help="", required=False) +parser.add_argument('--release', type=str, help="Image release for subsequent builds with the same date stamp (rarely needed)", required=False) + +results = parser.parse_args() +rlvars = rldict[results.version] +major = rlvars["major"] + +STORAGE_DIR = pathlib.Path("/var/lib/imagefactory/storage") +KICKSTART_PATH = pathlib.Path(os.environ.get("KICKSTART_PATH", "/kickstarts")) +BUILDTIME = datetime.datetime.utcnow() + + +def render_icicle_template(template: Template, architecture: Architecture) -> str: + handle, output = tempfile.mkstemp() + if not handle: + exit(3) + with os.fdopen(handle, "wb") as tmp: + _template = template.render( + architecture=architecture, + fedora_version=rlvars["fedora_release"], + iso8601date=BUILDTIME.strftime("%Y%m%d"), + installdir="kickstart" if results.kickstartdir else "os", + major=major, + release=results.release if results.release else 0, + size="10G", + type=results.type.capitalize(), + utcnow=BUILDTIME, + version_variant=rlvars["revision"] if not results.variant else f"{rlvars['revision']}-{results.variant.capitalize()}", + ) + tmp.write(_template.encode()) + return output + + +def generate_kickstart_imagefactory_args(debug: bool = False) -> str: + type_variant = results.type if not results.variant else f"{results.type}-{results.variant}" # todo -cleanup + kickstart_path = pathlib.Path(f"{KICKSTART_PATH}/Rocky-{major}-{type_variant}.ks") + + if not kickstart_path.is_file(): + print(f"Kickstart file is not available: {kickstart_path}") + if not debug: + exit(2) + + return f"--file-parameter install_script {kickstart_path}" + +def get_image_format(_type: str) -> str: + mapping = { + "Container": "docker" + } + return mapping[_type] if _type in mapping.keys() else '' + +def generate_imagefactory_commands(tdl_template: Template, architecture: Architecture) -> List[List[str]]: + template_path = render_icicle_template(tdl_template, architecture) + if not template_path: + exit(2) + + args_mapping = { + "debug": "--debug" + } + + # only supports boolean flags right now? + args = [param for name, param in args_mapping.items() if getattr(results,name)] + package_args = [] + + kickstart_arg = generate_kickstart_imagefactory_args(True) # REMOVE DEBUG ARG + + if results.type == "Container": + args += ["--parameter", "offline_icicle", "true"] + package_args += ["--parameter", "compress", "xz"] + tar_command = ["tar", "-Oxf", f"{STORAGE_DIR}/*.body" "./layer.tar"] + + type_variant = results.type if not results.variant else f"{results.type}-{results.variant}" # todo -cleanup + outname = f"Rocky-{rlvars['major']}-{type_variant}.{BUILDTIME.strftime('%Y%m%d')}.{results.release if results.release else 0}.{architecture}" + + outdir = pathlib.Path(f"/tmp/{outname}") + + build_command = (f"imagefactory base_image {kickstart_arg} {' '.join(args)} {template_path}" + f" | tee -a {outdir}/logs/base_image-{outname}.out" + f" | tail -n4 > {outdir}/base.meta || exit 2" + ) + + + out_type = get_image_format(results.type) + package_command = ["imagefactory", "target_image", *args, template_path, + "--id", "$(awk '$1==\"UUID\":{print $NF}'"+f" /tmp/{outname}/base.meta)", + *package_args, + "--parameter", "repository", outname, out_type, + "|", "tee", "-a", f"{outdir}/base_image-{outname}.out", + "|", "tail", "-n4", ">", f"{outdir}/target.meta", "||", "exit", "3" + ] + + copy_command = (f"aws s3 cp --recursive {outdir}/ s3://resf-empanadas/buildimage-{ outname }/{ BUILDTIME.strftime('%s') }/" + ) + commands = [build_command, package_command, copy_command] + return commands + +def run(): + result, error = valid_type_variant(results.type, results.variant) + if not result: + print(error) + exit(2) + + file_loader = FileSystemLoader(f"{_rootdir}/templates") + tmplenv = Environment(loader=file_loader) + tdl_template = tmplenv.get_template('icicle/tdl.xml.tmpl') + job_template = tmplenv.get_template('kube/Job.tmpl') + + for architecture in rlvars["allowed_arches"]: + architecture = Architecture.New(architecture, major) + + commands = generate_imagefactory_commands(tdl_template, architecture) + + print(job_template.render( + architecture=architecture, + backoffLimit=4, + buildTime=datetime.datetime.utcnow().strftime("%s"), + command=commands, + imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest", + jobname="buildimage", + namespace="empanadas", + major=major, + restartPolicy="Never", + )) + diff --git a/iso/empanadas/empanadas/scripts/launch_builds.py b/iso/empanadas/empanadas/scripts/launch_builds.py index f0f82f7..abd01a4 100755 --- a/iso/empanadas/empanadas/scripts/launch_builds.py +++ b/iso/empanadas/empanadas/scripts/launch_builds.py @@ -12,6 +12,7 @@ parser = argparse.ArgumentParser(description="ISO Compose") parser.add_argument('--release', type=str, help="Major Release Version", required=True) parser.add_argument('--env', type=str, help="environment", required=True) +parser.add_argument('--rc', action='store_true', help="Release Candidate") results = parser.parse_args() rlvars = rldict[results.release] major = rlvars['major'] @@ -30,16 +31,25 @@ def run(): elif results.env == "all": arches = EKSARCH+EXTARCH - command = ["build-iso", "--release", f"{results.release}", "--rc", "--isolation", "simple"] + command = ["build-iso", "--release", f"{results.release}", "--isolation", "simple"] + if results.rc: + command += ["--rc"] + + buildstamp = datetime.datetime.utcnow() out = "" - for arch in arches: + for architecture in arches: + copy_command = (f"aws s3 cp --recursive --exclude=* --include=lorax* " + f"/var/lib/mock/rocky-{ major }-$(uname -m)/root/builddir/ " + f"s3://resf-empanadas/buildiso-{ major }-{ architecture }/{ buildstamp.strftime('%s') }/" + ) out += job_template.render( - architecture=arch, + architecture=architecture, backoffLimit=4, - buildTime=datetime.datetime.utcnow().strftime("%s"), - command=command, + buildTime=buildstamp.strftime("%s"), + command=[command, copy_command], imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest", + jobname="buildiso", namespace="empanadas", major=major, restartPolicy="Never", diff --git a/iso/empanadas/empanadas/templates/icicle/tdl.xml.tmpl b/iso/empanadas/empanadas/templates/icicle/tdl.xml.tmpl new file mode 100644 index 0000000..14e8dd8 --- /dev/null +++ b/iso/empanadas/empanadas/templates/icicle/tdl.xml.tmpl @@ -0,0 +1,21 @@ + + + diff --git a/iso/empanadas/empanadas/templates/kube/Job.tmpl b/iso/empanadas/empanadas/templates/kube/Job.tmpl index bfcc20a..1ddf1f2 100644 --- a/iso/empanadas/empanadas/templates/kube/Job.tmpl +++ b/iso/empanadas/empanadas/templates/kube/Job.tmpl @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: build-iso-{{ major }}-{{ architecture }} + name: {{ jobname }}-{{ major }}-{{ architecture }} namespace: {{ namespace }} spec: template: @@ -11,15 +11,18 @@ spec: peridot.rockylinux.org/workflow-tolerates-arch: {{ architecture }} spec: containers: - - name: buildiso-{{ major }}-{{ architecture }} + - name: {{ jobname }}-{{ major }}-{{ architecture }} image: {{ imageName }} command: ["/bin/bash", "-c"] args: - | - {{ command | join(' ') }} - aws s3 cp --recursive --exclude=* --include=lorax* \ - /var/lib/mock/rocky-{{ major }}-$(uname -m)/root/builddir/ \ - "s3://resf-empanadas/buildiso-{{ major }}-{{ architecture }}/{{ buildTime }}/" +{%- for c in command -%} +{%- if c is string %} + {{ c }} +{%- else %} + {{ ' '.join(c) }} +{%- endif %} +{%- endfor %} securityContext: runAsUser: 0 runAsGroup: 0 diff --git a/iso/empanadas/poetry.lock b/iso/empanadas/poetry.lock index 716b15a..9ffaf92 100644 --- a/iso/empanadas/poetry.lock +++ b/iso/empanadas/poetry.lock @@ -302,6 +302,25 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "types-jinja2" +version = "2.11.9" +description = "Typing stubs for Jinja2" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +types-MarkupSafe = "*" + +[[package]] +name = "types-markupsafe" +version = "1.1.10" +description = "Typing stubs for MarkupSafe" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "typing-extensions" version = "4.2.0" @@ -354,7 +373,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = ">=3.7,<4" -content-hash = "d011f4622c248f6aa107fd679616eaa19a897147398c6f52dd0dea0ab1d74486" +content-hash = "76b6a6434217e66d6388431460ff32288198b91647e16f236c41164afc445b32" [metadata.files] atomicwrites = [ @@ -558,6 +577,14 @@ six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +types-jinja2 = [ + {file = "types-Jinja2-2.11.9.tar.gz", hash = "sha256:dbdc74a40aba7aed520b7e4d89e8f0fe4286518494208b35123bcf084d4b8c81"}, + {file = "types_Jinja2-2.11.9-py3-none-any.whl", hash = "sha256:60a1e21e8296979db32f9374d8a239af4cb541ff66447bb915d8ad398f9c63b2"}, +] +types-markupsafe = [ + {file = "types-MarkupSafe-1.1.10.tar.gz", hash = "sha256:85b3a872683d02aea3a5ac2a8ef590193c344092032f58457287fbf8e06711b1"}, + {file = "types_MarkupSafe-1.1.10-py3-none-any.whl", hash = "sha256:ca2bee0f4faafc45250602567ef38d533e877d2ddca13003b319c551ff5b3cc5"}, +] typing-extensions = [ {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"}, {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"}, diff --git a/iso/empanadas/pyproject.toml b/iso/empanadas/pyproject.toml index fb44274..154461e 100644 --- a/iso/empanadas/pyproject.toml +++ b/iso/empanadas/pyproject.toml @@ -18,6 +18,7 @@ requests = "^2.28.0" [tool.poetry.dev-dependencies] pytest = "~5" +types-Jinja2 = "^2.11.9" # Remove when upgrading past Jinja 2.x as type annotations are in-tree [tool.poetry.scripts] sync_from_peridot = "empanadas.scripts.sync_from_peridot:run" @@ -26,6 +27,7 @@ sync_sig = "empanadas.scripts.sync_sig:run" build-iso = "empanadas.scripts.build_iso:run" pull-unpack-tree = "empanadas.scripts.pull_unpack_tree:run" launch-builds = "empanadas.scripts.launch_builds:run" +build-image = "empanadas.scripts.build_image:run" [build-system] requires = ["poetry-core>=1.0.0"]