Implement a feature to assist in generating various images

* use a flag to determine if we want an RC or not
* Convert rldict and sigdict to an AttributeDict to allow access via
  __getattr__
* add fedora_release variable to configs for controlling icicle
  templates
* build_image.py script to generate per-architecture XML files used by
  imagefactory
* refactor time to call utcnow() once
* add jinja types to development dependencies until we move past jinja
  2.x
* Generate TDL templates per architecture for each image variant on
  demand
* Generate imagefactory and copy commands to execute image build
* Refactor Kubernetes job template to be generic for all current jobs
This commit is contained in:
Neil Hanlon 2022-06-20 20:12:20 -04:00
parent 190e1b4b22
commit 4bf6fb6618
Signed by untrusted user: neil
GPG Key ID: 705BC21EC3C70F34
10 changed files with 239 additions and 13 deletions

View File

@ -8,6 +8,24 @@ import yaml
import logging import logging
import hashlib import hashlib
from collections import defaultdict
from typing import Tuple
# An implementation from the Fabric python library
class AttributeDict(defaultdict):
def __init__(self):
super(AttributeDict, self).__init__(AttributeDict)
def __getattr__(self, key):
try:
return self[key]
except KeyError:
raise AttributeError(key)
def __setattr__(self, key, value):
self[key] = value
# These are a bunch of colors we may use in terminal output # These are a bunch of colors we may use in terminal output
class Color: class Color:
RED = '\033[91m' RED = '\033[91m'
@ -22,8 +40,8 @@ class Color:
END = '\033[0m' END = '\033[0m'
# vars and additional checks # vars and additional checks
rldict = {} rldict = AttributeDict()
sigdict = {} sigdict = AttributeDict()
config = { config = {
"rlmacro": rpm.expandMacro('%rhel'), "rlmacro": rpm.expandMacro('%rhel'),
"dist": 'el' + rpm.expandMacro('%rhel'), "dist": 'el' + rpm.expandMacro('%rhel'),
@ -77,3 +95,26 @@ for conf in glob.iglob(f"{_rootdir}/sig/*.yaml"):
#rlvars = rldict[rlver] #rlvars = rldict[rlver]
#rlvars = rldict[rlmacro] #rlvars = rldict[rlmacro]
#COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp #COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp
def valid_type_variant(_type: str, variant: str="") -> Tuple[bool, str]:
ALLOWED_TYPE_VARIANTS = {
"Container": ["Base", "Minimal"],
"GenericCloud": [],
}
if _type not in ALLOWED_TYPE_VARIANTS:
return False, f"Type is invalid: ({_type}, {variant})"
elif variant not in ALLOWED_TYPE_VARIANTS[_type]:
if variant.capitalize() in ALLOWED_TYPE_VARIANTS[_type]:
return False, f"Capitalization mismatch. Found: ({_type}, {variant}). Expected: ({_type}, {variant.capitalize()})"
return False, f"Type/Variant Combination is not allowed: ({_type}, {variant})"
return True, ""
class Architecture(str):
@staticmethod
def New(architecture: str, version: int):
if architecture not in rldict[version]["allowed_arches"]:
print("Invalid architecture/version combo, skipping")
exit()
return Architecture(architecture)

View File

@ -7,6 +7,7 @@
minor: '6' minor: '6'
profile: '8' profile: '8'
bugurl: 'https://bugs.rockylinux.org' bugurl: 'https://bugs.rockylinux.org'
fedora_release: 28
allowed_arches: allowed_arches:
- x86_64 - x86_64
- aarch64 - aarch64

View File

@ -8,6 +8,7 @@
profile: '9-beta' profile: '9-beta'
bugurl: 'https://bugs.rockylinux.org' bugurl: 'https://bugs.rockylinux.org'
checksum: 'sha256' checksum: 'sha256'
fedora_release: 34
allowed_arches: allowed_arches:
- x86_64 - x86_64
- aarch64 - aarch64

View File

@ -6,6 +6,7 @@
major: '9' major: '9'
minor: '0' minor: '0'
profile: '9' profile: '9'
fedora_release: 34
bugurl: 'https://bugs.rockylinux.org' bugurl: 'https://bugs.rockylinux.org'
checksum: 'sha256' checksum: 'sha256'
allowed_arches: allowed_arches:

View File

@ -8,6 +8,7 @@
profile: '9-lookahead' profile: '9-lookahead'
bugurl: 'https://bugs.rockylinux.org' bugurl: 'https://bugs.rockylinux.org'
checksum: 'sha256' checksum: 'sha256'
fedora_release: 34
allowed_arches: allowed_arches:
- x86_64 - x86_64
- aarch64 - aarch64

View File

@ -0,0 +1,145 @@
# Builds an image given a version, type, variant, and architecture
# Defaults to the running host's architecture
import argparse
import datetime
import os
import tempfile
import pathlib
from jinja2 import Environment, FileSystemLoader, Template
from typing import List, Tuple
from empanadas.common import Architecture, rldict, valid_type_variant
from empanadas.common import _rootdir
parser = argparse.ArgumentParser(description="ISO Compose")
parser.add_argument('--version', type=str, help="Release Version (8.6, 9.1)", required=True)
parser.add_argument('--rc', action='store_true', help="Release Candidate")
parser.add_argument('--kickstartdir', action='store_true', help="Use the kickstart dir instead of the os dir for repositories")
parser.add_argument('--debug', action='store_true', help="debug?")
parser.add_argument('--type', type=str, help="Image type (container, genclo, azure, aws, vagrant)", required=True)
parser.add_argument('--variant', type=str, help="", required=False)
parser.add_argument('--release', type=str, help="Image release for subsequent builds with the same date stamp (rarely needed)", required=False)
results = parser.parse_args()
rlvars = rldict[results.version]
major = rlvars["major"]
STORAGE_DIR = pathlib.Path("/var/lib/imagefactory/storage")
KICKSTART_PATH = pathlib.Path(os.environ.get("KICKSTART_PATH", "/kickstarts"))
BUILDTIME = datetime.datetime.utcnow()
def render_icicle_template(template: Template, architecture: Architecture) -> str:
handle, output = tempfile.mkstemp()
if not handle:
exit(3)
with os.fdopen(handle, "wb") as tmp:
_template = template.render(
architecture=architecture,
fedora_version=rlvars["fedora_release"],
iso8601date=BUILDTIME.strftime("%Y%m%d"),
installdir="kickstart" if results.kickstartdir else "os",
major=major,
release=results.release if results.release else 0,
size="10G",
type=results.type.capitalize(),
utcnow=BUILDTIME,
version_variant=rlvars["revision"] if not results.variant else f"{rlvars['revision']}-{results.variant.capitalize()}",
)
tmp.write(_template.encode())
return output
def generate_kickstart_imagefactory_args(debug: bool = False) -> str:
type_variant = results.type if not results.variant else f"{results.type}-{results.variant}" # todo -cleanup
kickstart_path = pathlib.Path(f"{KICKSTART_PATH}/Rocky-{major}-{type_variant}.ks")
if not kickstart_path.is_file():
print(f"Kickstart file is not available: {kickstart_path}")
if not debug:
exit(2)
return f"--file-parameter install_script {kickstart_path}"
def get_image_format(_type: str) -> str:
mapping = {
"Container": "docker"
}
return mapping[_type] if _type in mapping.keys() else ''
def generate_imagefactory_commands(tdl_template: Template, architecture: Architecture) -> List[List[str]]:
template_path = render_icicle_template(tdl_template, architecture)
if not template_path:
exit(2)
args_mapping = {
"debug": "--debug"
}
# only supports boolean flags right now?
args = [param for name, param in args_mapping.items() if getattr(results,name)]
package_args = []
kickstart_arg = generate_kickstart_imagefactory_args(True) # REMOVE DEBUG ARG
if results.type == "Container":
args += ["--parameter", "offline_icicle", "true"]
package_args += ["--parameter", "compress", "xz"]
tar_command = ["tar", "-Oxf", f"{STORAGE_DIR}/*.body" "./layer.tar"]
type_variant = results.type if not results.variant else f"{results.type}-{results.variant}" # todo -cleanup
outname = f"Rocky-{rlvars['major']}-{type_variant}.{BUILDTIME.strftime('%Y%m%d')}.{results.release if results.release else 0}.{architecture}"
outdir = pathlib.Path(f"/tmp/{outname}")
build_command = (f"imagefactory base_image {kickstart_arg} {' '.join(args)} {template_path}"
f" | tee -a {outdir}/logs/base_image-{outname}.out"
f" | tail -n4 > {outdir}/base.meta || exit 2"
)
out_type = get_image_format(results.type)
package_command = ["imagefactory", "target_image", *args, template_path,
"--id", "$(awk '$1==\"UUID\":{print $NF}'"+f" /tmp/{outname}/base.meta)",
*package_args,
"--parameter", "repository", outname, out_type,
"|", "tee", "-a", f"{outdir}/base_image-{outname}.out",
"|", "tail", "-n4", ">", f"{outdir}/target.meta", "||", "exit", "3"
]
copy_command = (f"aws s3 cp --recursive {outdir}/ s3://resf-empanadas/buildimage-{ outname }/{ BUILDTIME.strftime('%s') }/"
)
commands = [build_command, package_command, copy_command]
return commands
def run():
result, error = valid_type_variant(results.type, results.variant)
if not result:
print(error)
exit(2)
file_loader = FileSystemLoader(f"{_rootdir}/templates")
tmplenv = Environment(loader=file_loader)
tdl_template = tmplenv.get_template('icicle/tdl.xml.tmpl')
job_template = tmplenv.get_template('kube/Job.tmpl')
for architecture in rlvars["allowed_arches"]:
architecture = Architecture.New(architecture, major)
commands = generate_imagefactory_commands(tdl_template, architecture)
print(job_template.render(
architecture=architecture,
backoffLimit=4,
buildTime=datetime.datetime.utcnow().strftime("%s"),
command=commands,
imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest",
jobname="buildimage",
namespace="empanadas",
major=major,
restartPolicy="Never",
))

View File

@ -12,6 +12,7 @@ parser = argparse.ArgumentParser(description="ISO Compose")
parser.add_argument('--release', type=str, help="Major Release Version", required=True) parser.add_argument('--release', type=str, help="Major Release Version", required=True)
parser.add_argument('--env', type=str, help="environment", required=True) parser.add_argument('--env', type=str, help="environment", required=True)
parser.add_argument('--rc', action='store_true', help="Release Candidate")
results = parser.parse_args() results = parser.parse_args()
rlvars = rldict[results.release] rlvars = rldict[results.release]
major = rlvars['major'] major = rlvars['major']
@ -30,16 +31,25 @@ def run():
elif results.env == "all": elif results.env == "all":
arches = EKSARCH+EXTARCH arches = EKSARCH+EXTARCH
command = ["build-iso", "--release", f"{results.release}", "--rc", "--isolation", "simple"] command = ["build-iso", "--release", f"{results.release}", "--isolation", "simple"]
if results.rc:
command += ["--rc"]
buildstamp = datetime.datetime.utcnow()
out = "" out = ""
for arch in arches: for architecture in arches:
copy_command = (f"aws s3 cp --recursive --exclude=* --include=lorax* "
f"/var/lib/mock/rocky-{ major }-$(uname -m)/root/builddir/ "
f"s3://resf-empanadas/buildiso-{ major }-{ architecture }/{ buildstamp.strftime('%s') }/"
)
out += job_template.render( out += job_template.render(
architecture=arch, architecture=architecture,
backoffLimit=4, backoffLimit=4,
buildTime=datetime.datetime.utcnow().strftime("%s"), buildTime=buildstamp.strftime("%s"),
command=command, command=[command, copy_command],
imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest", imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest",
jobname="buildiso",
namespace="empanadas", namespace="empanadas",
major=major, major=major,
restartPolicy="Never", restartPolicy="Never",

View File

@ -0,0 +1,21 @@
<template>
<name>Rocky-{{major}}-{{type}}-{{version_variant}}.{{iso8601date}}.{{release}}.{{architecture}}</name>
<os>
<name>Fedora</name>
<version>{{fedora_version}}</version>
<arch>{{architecture}}</arch>
<install type='url'>
<url>https://dl.rockylinux.org/stg/rocky/{{major}}/BaseOS/{{architecture}}/{{installdir}}/</url>
</install>
<icicle>
<extra_command>rpm -qa --qf '%{NAME},%{VERSION},%{RELEASE},%{ARCH},%{EPOCH},%{SIZE},%{SIGMD5},%{BUILDTIME}
'</extra_command>
</icicle>
</os>
<description>Rocky-{{major}}-{{type}}-{{version_variant}}.{{iso8601date}}.{{release}}.{{architecture}} Generated on {{utcnow}}</description>
<disk>
<size>{{size}}</size>
</disk>
</template>

View File

@ -2,7 +2,7 @@
apiVersion: batch/v1 apiVersion: batch/v1
kind: Job kind: Job
metadata: metadata:
name: build-iso-{{ major }}-{{ architecture }} name: {{ jobname }}-{{ major }}-{{ architecture }}
namespace: {{ namespace }} namespace: {{ namespace }}
spec: spec:
template: template:
@ -11,15 +11,18 @@ spec:
peridot.rockylinux.org/workflow-tolerates-arch: {{ architecture }} peridot.rockylinux.org/workflow-tolerates-arch: {{ architecture }}
spec: spec:
containers: containers:
- name: buildiso-{{ major }}-{{ architecture }} - name: {{ jobname }}-{{ major }}-{{ architecture }}
image: {{ imageName }} image: {{ imageName }}
command: ["/bin/bash", "-c"] command: ["/bin/bash", "-c"]
args: args:
- | - |
{{ command | join(' ') }} {%- for c in command -%}
aws s3 cp --recursive --exclude=* --include=lorax* \ {%- if c is string %}
/var/lib/mock/rocky-{{ major }}-$(uname -m)/root/builddir/ \ {{ c }}
"s3://resf-empanadas/buildiso-{{ major }}-{{ architecture }}/{{ buildTime }}/" {%- else %}
{{ ' '.join(c) }}
{%- endif %}
{%- endfor %}
securityContext: securityContext:
runAsUser: 0 runAsUser: 0
runAsGroup: 0 runAsGroup: 0

View File

@ -19,6 +19,7 @@ kobo = "^0.24.1"
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]
pytest = "~5" pytest = "~5"
types-Jinja2 = "^2.11.9" # Remove when upgrading past Jinja 2.x as type annotations are in-tree
[tool.poetry.scripts] [tool.poetry.scripts]
sync_from_peridot = "empanadas.scripts.sync_from_peridot:run" sync_from_peridot = "empanadas.scripts.sync_from_peridot:run"
@ -28,6 +29,7 @@ build-iso = "empanadas.scripts.build_iso:run"
build-iso-extra = "empanadas.scripts.build_iso_extra:run" build-iso-extra = "empanadas.scripts.build_iso_extra:run"
pull-unpack-tree = "empanadas.scripts.pull_unpack_tree:run" pull-unpack-tree = "empanadas.scripts.pull_unpack_tree:run"
launch-builds = "empanadas.scripts.launch_builds:run" launch-builds = "empanadas.scripts.launch_builds:run"
build-image = "empanadas.scripts.build_image:run"
[build-system] [build-system]
requires = ["poetry-core>=1.0.0"] requires = ["poetry-core>=1.0.0"]