From 4bf6fb66185dc42cc9ca15b905efe25637fb5416 Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Mon, 20 Jun 2022 20:12:20 -0400 Subject: [PATCH 1/3] Implement a feature to assist in generating various images * use a flag to determine if we want an RC or not * Convert rldict and sigdict to an AttributeDict to allow access via __getattr__ * add fedora_release variable to configs for controlling icicle templates * build_image.py script to generate per-architecture XML files used by imagefactory * refactor time to call utcnow() once * add jinja types to development dependencies until we move past jinja 2.x * Generate TDL templates per architecture for each image variant on demand * Generate imagefactory and copy commands to execute image build * Refactor Kubernetes job template to be generic for all current jobs --- iso/empanadas/empanadas/common.py | 45 +++++- iso/empanadas/empanadas/configs/el8.yaml | 1 + iso/empanadas/empanadas/configs/el9-beta.yaml | 1 + iso/empanadas/empanadas/configs/el9.yaml | 1 + iso/empanadas/empanadas/configs/el9lh.yaml | 1 + .../empanadas/scripts/build_image.py | 145 ++++++++++++++++++ .../empanadas/scripts/launch_builds.py | 20 ++- .../empanadas/templates/icicle/tdl.xml.tmpl | 21 +++ .../empanadas/templates/kube/Job.tmpl | 15 +- iso/empanadas/pyproject.toml | 2 + 10 files changed, 239 insertions(+), 13 deletions(-) create mode 100644 iso/empanadas/empanadas/scripts/build_image.py create mode 100644 iso/empanadas/empanadas/templates/icicle/tdl.xml.tmpl diff --git a/iso/empanadas/empanadas/common.py b/iso/empanadas/empanadas/common.py index c3619ce..edb2533 100644 --- a/iso/empanadas/empanadas/common.py +++ b/iso/empanadas/empanadas/common.py @@ -8,6 +8,24 @@ import yaml import logging import hashlib + +from collections import defaultdict +from typing import Tuple + +# An implementation from the Fabric python library +class AttributeDict(defaultdict): + def __init__(self): + super(AttributeDict, self).__init__(AttributeDict) + + def __getattr__(self, key): + try: + return self[key] + except KeyError: + raise AttributeError(key) + + def __setattr__(self, key, value): + self[key] = value + # These are a bunch of colors we may use in terminal output class Color: RED = '\033[91m' @@ -22,8 +40,8 @@ class Color: END = '\033[0m' # vars and additional checks -rldict = {} -sigdict = {} +rldict = AttributeDict() +sigdict = AttributeDict() config = { "rlmacro": rpm.expandMacro('%rhel'), "dist": 'el' + rpm.expandMacro('%rhel'), @@ -77,3 +95,26 @@ for conf in glob.iglob(f"{_rootdir}/sig/*.yaml"): #rlvars = rldict[rlver] #rlvars = rldict[rlmacro] #COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp + + +def valid_type_variant(_type: str, variant: str="") -> Tuple[bool, str]: + ALLOWED_TYPE_VARIANTS = { + "Container": ["Base", "Minimal"], + "GenericCloud": [], + } + + if _type not in ALLOWED_TYPE_VARIANTS: + return False, f"Type is invalid: ({_type}, {variant})" + elif variant not in ALLOWED_TYPE_VARIANTS[_type]: + if variant.capitalize() in ALLOWED_TYPE_VARIANTS[_type]: + return False, f"Capitalization mismatch. Found: ({_type}, {variant}). Expected: ({_type}, {variant.capitalize()})" + return False, f"Type/Variant Combination is not allowed: ({_type}, {variant})" + return True, "" + +class Architecture(str): + @staticmethod + def New(architecture: str, version: int): + if architecture not in rldict[version]["allowed_arches"]: + print("Invalid architecture/version combo, skipping") + exit() + return Architecture(architecture) diff --git a/iso/empanadas/empanadas/configs/el8.yaml b/iso/empanadas/empanadas/configs/el8.yaml index eb80aff..8ce71c7 100644 --- a/iso/empanadas/empanadas/configs/el8.yaml +++ b/iso/empanadas/empanadas/configs/el8.yaml @@ -7,6 +7,7 @@ minor: '6' profile: '8' bugurl: 'https://bugs.rockylinux.org' + fedora_release: 28 allowed_arches: - x86_64 - aarch64 diff --git a/iso/empanadas/empanadas/configs/el9-beta.yaml b/iso/empanadas/empanadas/configs/el9-beta.yaml index 19d6cd5..116cc45 100644 --- a/iso/empanadas/empanadas/configs/el9-beta.yaml +++ b/iso/empanadas/empanadas/configs/el9-beta.yaml @@ -8,6 +8,7 @@ profile: '9-beta' bugurl: 'https://bugs.rockylinux.org' checksum: 'sha256' + fedora_release: 34 allowed_arches: - x86_64 - aarch64 diff --git a/iso/empanadas/empanadas/configs/el9.yaml b/iso/empanadas/empanadas/configs/el9.yaml index 88a978b..786c46a 100644 --- a/iso/empanadas/empanadas/configs/el9.yaml +++ b/iso/empanadas/empanadas/configs/el9.yaml @@ -6,6 +6,7 @@ major: '9' minor: '0' profile: '9' + fedora_release: 34 bugurl: 'https://bugs.rockylinux.org' checksum: 'sha256' allowed_arches: diff --git a/iso/empanadas/empanadas/configs/el9lh.yaml b/iso/empanadas/empanadas/configs/el9lh.yaml index 4176f66..ea833f7 100644 --- a/iso/empanadas/empanadas/configs/el9lh.yaml +++ b/iso/empanadas/empanadas/configs/el9lh.yaml @@ -8,6 +8,7 @@ profile: '9-lookahead' bugurl: 'https://bugs.rockylinux.org' checksum: 'sha256' + fedora_release: 34 allowed_arches: - x86_64 - aarch64 diff --git a/iso/empanadas/empanadas/scripts/build_image.py b/iso/empanadas/empanadas/scripts/build_image.py new file mode 100644 index 0000000..c9f7782 --- /dev/null +++ b/iso/empanadas/empanadas/scripts/build_image.py @@ -0,0 +1,145 @@ +# Builds an image given a version, type, variant, and architecture +# Defaults to the running host's architecture + +import argparse +import datetime +import os +import tempfile +import pathlib + +from jinja2 import Environment, FileSystemLoader, Template +from typing import List, Tuple + +from empanadas.common import Architecture, rldict, valid_type_variant +from empanadas.common import _rootdir + +parser = argparse.ArgumentParser(description="ISO Compose") + +parser.add_argument('--version', type=str, help="Release Version (8.6, 9.1)", required=True) +parser.add_argument('--rc', action='store_true', help="Release Candidate") +parser.add_argument('--kickstartdir', action='store_true', help="Use the kickstart dir instead of the os dir for repositories") +parser.add_argument('--debug', action='store_true', help="debug?") +parser.add_argument('--type', type=str, help="Image type (container, genclo, azure, aws, vagrant)", required=True) +parser.add_argument('--variant', type=str, help="", required=False) +parser.add_argument('--release', type=str, help="Image release for subsequent builds with the same date stamp (rarely needed)", required=False) + +results = parser.parse_args() +rlvars = rldict[results.version] +major = rlvars["major"] + +STORAGE_DIR = pathlib.Path("/var/lib/imagefactory/storage") +KICKSTART_PATH = pathlib.Path(os.environ.get("KICKSTART_PATH", "/kickstarts")) +BUILDTIME = datetime.datetime.utcnow() + + +def render_icicle_template(template: Template, architecture: Architecture) -> str: + handle, output = tempfile.mkstemp() + if not handle: + exit(3) + with os.fdopen(handle, "wb") as tmp: + _template = template.render( + architecture=architecture, + fedora_version=rlvars["fedora_release"], + iso8601date=BUILDTIME.strftime("%Y%m%d"), + installdir="kickstart" if results.kickstartdir else "os", + major=major, + release=results.release if results.release else 0, + size="10G", + type=results.type.capitalize(), + utcnow=BUILDTIME, + version_variant=rlvars["revision"] if not results.variant else f"{rlvars['revision']}-{results.variant.capitalize()}", + ) + tmp.write(_template.encode()) + return output + + +def generate_kickstart_imagefactory_args(debug: bool = False) -> str: + type_variant = results.type if not results.variant else f"{results.type}-{results.variant}" # todo -cleanup + kickstart_path = pathlib.Path(f"{KICKSTART_PATH}/Rocky-{major}-{type_variant}.ks") + + if not kickstart_path.is_file(): + print(f"Kickstart file is not available: {kickstart_path}") + if not debug: + exit(2) + + return f"--file-parameter install_script {kickstart_path}" + +def get_image_format(_type: str) -> str: + mapping = { + "Container": "docker" + } + return mapping[_type] if _type in mapping.keys() else '' + +def generate_imagefactory_commands(tdl_template: Template, architecture: Architecture) -> List[List[str]]: + template_path = render_icicle_template(tdl_template, architecture) + if not template_path: + exit(2) + + args_mapping = { + "debug": "--debug" + } + + # only supports boolean flags right now? + args = [param for name, param in args_mapping.items() if getattr(results,name)] + package_args = [] + + kickstart_arg = generate_kickstart_imagefactory_args(True) # REMOVE DEBUG ARG + + if results.type == "Container": + args += ["--parameter", "offline_icicle", "true"] + package_args += ["--parameter", "compress", "xz"] + tar_command = ["tar", "-Oxf", f"{STORAGE_DIR}/*.body" "./layer.tar"] + + type_variant = results.type if not results.variant else f"{results.type}-{results.variant}" # todo -cleanup + outname = f"Rocky-{rlvars['major']}-{type_variant}.{BUILDTIME.strftime('%Y%m%d')}.{results.release if results.release else 0}.{architecture}" + + outdir = pathlib.Path(f"/tmp/{outname}") + + build_command = (f"imagefactory base_image {kickstart_arg} {' '.join(args)} {template_path}" + f" | tee -a {outdir}/logs/base_image-{outname}.out" + f" | tail -n4 > {outdir}/base.meta || exit 2" + ) + + + out_type = get_image_format(results.type) + package_command = ["imagefactory", "target_image", *args, template_path, + "--id", "$(awk '$1==\"UUID\":{print $NF}'"+f" /tmp/{outname}/base.meta)", + *package_args, + "--parameter", "repository", outname, out_type, + "|", "tee", "-a", f"{outdir}/base_image-{outname}.out", + "|", "tail", "-n4", ">", f"{outdir}/target.meta", "||", "exit", "3" + ] + + copy_command = (f"aws s3 cp --recursive {outdir}/ s3://resf-empanadas/buildimage-{ outname }/{ BUILDTIME.strftime('%s') }/" + ) + commands = [build_command, package_command, copy_command] + return commands + +def run(): + result, error = valid_type_variant(results.type, results.variant) + if not result: + print(error) + exit(2) + + file_loader = FileSystemLoader(f"{_rootdir}/templates") + tmplenv = Environment(loader=file_loader) + tdl_template = tmplenv.get_template('icicle/tdl.xml.tmpl') + job_template = tmplenv.get_template('kube/Job.tmpl') + + for architecture in rlvars["allowed_arches"]: + architecture = Architecture.New(architecture, major) + + commands = generate_imagefactory_commands(tdl_template, architecture) + + print(job_template.render( + architecture=architecture, + backoffLimit=4, + buildTime=datetime.datetime.utcnow().strftime("%s"), + command=commands, + imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest", + jobname="buildimage", + namespace="empanadas", + major=major, + restartPolicy="Never", + )) + diff --git a/iso/empanadas/empanadas/scripts/launch_builds.py b/iso/empanadas/empanadas/scripts/launch_builds.py index f0f82f7..abd01a4 100755 --- a/iso/empanadas/empanadas/scripts/launch_builds.py +++ b/iso/empanadas/empanadas/scripts/launch_builds.py @@ -12,6 +12,7 @@ parser = argparse.ArgumentParser(description="ISO Compose") parser.add_argument('--release', type=str, help="Major Release Version", required=True) parser.add_argument('--env', type=str, help="environment", required=True) +parser.add_argument('--rc', action='store_true', help="Release Candidate") results = parser.parse_args() rlvars = rldict[results.release] major = rlvars['major'] @@ -30,16 +31,25 @@ def run(): elif results.env == "all": arches = EKSARCH+EXTARCH - command = ["build-iso", "--release", f"{results.release}", "--rc", "--isolation", "simple"] + command = ["build-iso", "--release", f"{results.release}", "--isolation", "simple"] + if results.rc: + command += ["--rc"] + + buildstamp = datetime.datetime.utcnow() out = "" - for arch in arches: + for architecture in arches: + copy_command = (f"aws s3 cp --recursive --exclude=* --include=lorax* " + f"/var/lib/mock/rocky-{ major }-$(uname -m)/root/builddir/ " + f"s3://resf-empanadas/buildiso-{ major }-{ architecture }/{ buildstamp.strftime('%s') }/" + ) out += job_template.render( - architecture=arch, + architecture=architecture, backoffLimit=4, - buildTime=datetime.datetime.utcnow().strftime("%s"), - command=command, + buildTime=buildstamp.strftime("%s"), + command=[command, copy_command], imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest", + jobname="buildiso", namespace="empanadas", major=major, restartPolicy="Never", diff --git a/iso/empanadas/empanadas/templates/icicle/tdl.xml.tmpl b/iso/empanadas/empanadas/templates/icicle/tdl.xml.tmpl new file mode 100644 index 0000000..14e8dd8 --- /dev/null +++ b/iso/empanadas/empanadas/templates/icicle/tdl.xml.tmpl @@ -0,0 +1,21 @@ + + + diff --git a/iso/empanadas/empanadas/templates/kube/Job.tmpl b/iso/empanadas/empanadas/templates/kube/Job.tmpl index bfcc20a..1ddf1f2 100644 --- a/iso/empanadas/empanadas/templates/kube/Job.tmpl +++ b/iso/empanadas/empanadas/templates/kube/Job.tmpl @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: build-iso-{{ major }}-{{ architecture }} + name: {{ jobname }}-{{ major }}-{{ architecture }} namespace: {{ namespace }} spec: template: @@ -11,15 +11,18 @@ spec: peridot.rockylinux.org/workflow-tolerates-arch: {{ architecture }} spec: containers: - - name: buildiso-{{ major }}-{{ architecture }} + - name: {{ jobname }}-{{ major }}-{{ architecture }} image: {{ imageName }} command: ["/bin/bash", "-c"] args: - | - {{ command | join(' ') }} - aws s3 cp --recursive --exclude=* --include=lorax* \ - /var/lib/mock/rocky-{{ major }}-$(uname -m)/root/builddir/ \ - "s3://resf-empanadas/buildiso-{{ major }}-{{ architecture }}/{{ buildTime }}/" +{%- for c in command -%} +{%- if c is string %} + {{ c }} +{%- else %} + {{ ' '.join(c) }} +{%- endif %} +{%- endfor %} securityContext: runAsUser: 0 runAsGroup: 0 diff --git a/iso/empanadas/pyproject.toml b/iso/empanadas/pyproject.toml index a43a91d..35460be 100644 --- a/iso/empanadas/pyproject.toml +++ b/iso/empanadas/pyproject.toml @@ -19,6 +19,7 @@ kobo = "^0.24.1" [tool.poetry.dev-dependencies] pytest = "~5" +types-Jinja2 = "^2.11.9" # Remove when upgrading past Jinja 2.x as type annotations are in-tree [tool.poetry.scripts] sync_from_peridot = "empanadas.scripts.sync_from_peridot:run" @@ -28,6 +29,7 @@ build-iso = "empanadas.scripts.build_iso:run" build-iso-extra = "empanadas.scripts.build_iso_extra:run" pull-unpack-tree = "empanadas.scripts.pull_unpack_tree:run" launch-builds = "empanadas.scripts.launch_builds:run" +build-image = "empanadas.scripts.build_image:run" [build-system] requires = ["poetry-core>=1.0.0"] From e60f6524a22e03ee6799b2828d36656b1a7809be Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Mon, 20 Jun 2022 20:12:20 -0400 Subject: [PATCH 2/3] Implement a feature to assist in generating various images * use a flag to determine if we want an RC or not * Convert rldict and sigdict to an AttributeDict to allow access via __getattr__ * add fedora_release variable to configs for controlling icicle templates * build_image.py script to generate per-architecture XML files used by imagefactory * refactor time to call utcnow() once * add jinja types to development dependencies until we move past jinja 2.x * Generate TDL templates per architecture for each image variant on demand * Generate imagefactory and copy commands to execute image build * Refactor Kubernetes job template to be generic for all current jobs --- iso/empanadas/empanadas/scripts/build_image.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/iso/empanadas/empanadas/scripts/build_image.py b/iso/empanadas/empanadas/scripts/build_image.py index c9f7782..6222253 100644 --- a/iso/empanadas/empanadas/scripts/build_image.py +++ b/iso/empanadas/empanadas/scripts/build_image.py @@ -103,15 +103,14 @@ def generate_imagefactory_commands(tdl_template: Template, architecture: Archite out_type = get_image_format(results.type) package_command = ["imagefactory", "target_image", *args, template_path, - "--id", "$(awk '$1==\"UUID\":{print $NF}'"+f" /tmp/{outname}/base.meta)", + "--id", "$(awk '$1==\"UUID:\"{print $NF}'"+f" /tmp/{outname}/base.meta)", *package_args, "--parameter", "repository", outname, out_type, "|", "tee", "-a", f"{outdir}/base_image-{outname}.out", "|", "tail", "-n4", ">", f"{outdir}/target.meta", "||", "exit", "3" ] - copy_command = (f"aws s3 cp --recursive {outdir}/ s3://resf-empanadas/buildimage-{ outname }/{ BUILDTIME.strftime('%s') }/" - ) + copy_command = (f"aws s3 cp --recursive {outdir}/* s3://resf-empanadas/buildimage-{ outname }/{ BUILDTIME.strftime('%s') }/") commands = [build_command, package_command, copy_command] return commands From 79425e848e2232186ba4296cbea3031358c10096 Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Tue, 28 Jun 2022 09:29:49 -0400 Subject: [PATCH 3/3] Use devel branch for container, doc fixes --- iso/empanadas/Containerfile | 2 +- iso/empanadas/empanadas/scripts/launch_builds.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/iso/empanadas/Containerfile b/iso/empanadas/Containerfile index c3c4ed9..3eb45c2 100644 --- a/iso/empanadas/Containerfile +++ b/iso/empanadas/Containerfile @@ -56,7 +56,7 @@ RUN rm -rf /etc/yum.repos.d/*.repo RUN useradd -o -d /var/peridot -u 1002 peridotbuilder && usermod -a -G mock peridotbuilder RUN chown peridotbuilder:mock /etc/yum.conf && chown -R peridotbuilder:mock /etc/dnf && chown -R peridotbuilder:mock /etc/rpm && chown -R peridotbuilder:mock /etc/yum.repos.d -RUN pip install 'git+https://git.rockylinux.org/release-engineering/public/toolkit.git@feature/iso-kube#egg=empanadas&subdirectory=iso/empanadas' +RUN pip install 'git+https://git.rockylinux.org/release-engineering/public/toolkit.git@devel#egg=empanadas&subdirectory=iso/empanadas' RUN pip install awscli diff --git a/iso/empanadas/empanadas/scripts/launch_builds.py b/iso/empanadas/empanadas/scripts/launch_builds.py index abd01a4..86ae1e8 100755 --- a/iso/empanadas/empanadas/scripts/launch_builds.py +++ b/iso/empanadas/empanadas/scripts/launch_builds.py @@ -8,11 +8,11 @@ from empanadas.common import _rootdir from jinja2 import Environment, FileSystemLoader -parser = argparse.ArgumentParser(description="ISO Compose") +parser = argparse.ArgumentParser(description="Generate Kubernetes Jobs to run lorax in mock and upload the result. Pipe into kubectl for the appropriate cluster") -parser.add_argument('--release', type=str, help="Major Release Version", required=True) -parser.add_argument('--env', type=str, help="environment", required=True) -parser.add_argument('--rc', action='store_true', help="Release Candidate") +parser.add_argument('--release', type=str, help="Major Release Version: (8|9)", required=True) +parser.add_argument('--env', type=str, help="environment: one of (eks|ext|all). presently jobs are scheduled on different kubernetes clusters", required=True) +parser.add_argument('--rc', action='store_true', help="Release Candidate, Beta, RLN") results = parser.parse_args() rlvars = rldict[results.release] major = rlvars['major']