Merge branch 'feature/imagebuild' into 'devel'

Run builds in a container

See merge request release-engineering/public/toolkit!46
This commit is contained in:
Neil Hanlon 2022-06-28 13:34:36 +00:00
commit 56799df270
11 changed files with 242 additions and 17 deletions

View File

@ -56,7 +56,7 @@ RUN rm -rf /etc/yum.repos.d/*.repo
RUN useradd -o -d /var/peridot -u 1002 peridotbuilder && usermod -a -G mock peridotbuilder RUN useradd -o -d /var/peridot -u 1002 peridotbuilder && usermod -a -G mock peridotbuilder
RUN chown peridotbuilder:mock /etc/yum.conf && chown -R peridotbuilder:mock /etc/dnf && chown -R peridotbuilder:mock /etc/rpm && chown -R peridotbuilder:mock /etc/yum.repos.d RUN chown peridotbuilder:mock /etc/yum.conf && chown -R peridotbuilder:mock /etc/dnf && chown -R peridotbuilder:mock /etc/rpm && chown -R peridotbuilder:mock /etc/yum.repos.d
RUN pip install 'git+https://git.rockylinux.org/release-engineering/public/toolkit.git@feature/iso-kube#egg=empanadas&subdirectory=iso/empanadas' RUN pip install 'git+https://git.rockylinux.org/release-engineering/public/toolkit.git@devel#egg=empanadas&subdirectory=iso/empanadas'
RUN pip install awscli RUN pip install awscli

View File

@ -8,6 +8,24 @@ import yaml
import logging import logging
import hashlib import hashlib
from collections import defaultdict
from typing import Tuple
# An implementation from the Fabric python library
class AttributeDict(defaultdict):
def __init__(self):
super(AttributeDict, self).__init__(AttributeDict)
def __getattr__(self, key):
try:
return self[key]
except KeyError:
raise AttributeError(key)
def __setattr__(self, key, value):
self[key] = value
# These are a bunch of colors we may use in terminal output # These are a bunch of colors we may use in terminal output
class Color: class Color:
RED = '\033[91m' RED = '\033[91m'
@ -22,8 +40,8 @@ class Color:
END = '\033[0m' END = '\033[0m'
# vars and additional checks # vars and additional checks
rldict = {} rldict = AttributeDict()
sigdict = {} sigdict = AttributeDict()
config = { config = {
"rlmacro": rpm.expandMacro('%rhel'), "rlmacro": rpm.expandMacro('%rhel'),
"dist": 'el' + rpm.expandMacro('%rhel'), "dist": 'el' + rpm.expandMacro('%rhel'),
@ -77,3 +95,26 @@ for conf in glob.iglob(f"{_rootdir}/sig/*.yaml"):
#rlvars = rldict[rlver] #rlvars = rldict[rlver]
#rlvars = rldict[rlmacro] #rlvars = rldict[rlmacro]
#COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp #COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp
def valid_type_variant(_type: str, variant: str="") -> Tuple[bool, str]:
ALLOWED_TYPE_VARIANTS = {
"Container": ["Base", "Minimal"],
"GenericCloud": [],
}
if _type not in ALLOWED_TYPE_VARIANTS:
return False, f"Type is invalid: ({_type}, {variant})"
elif variant not in ALLOWED_TYPE_VARIANTS[_type]:
if variant.capitalize() in ALLOWED_TYPE_VARIANTS[_type]:
return False, f"Capitalization mismatch. Found: ({_type}, {variant}). Expected: ({_type}, {variant.capitalize()})"
return False, f"Type/Variant Combination is not allowed: ({_type}, {variant})"
return True, ""
class Architecture(str):
@staticmethod
def New(architecture: str, version: int):
if architecture not in rldict[version]["allowed_arches"]:
print("Invalid architecture/version combo, skipping")
exit()
return Architecture(architecture)

View File

@ -7,6 +7,7 @@
minor: '6' minor: '6'
profile: '8' profile: '8'
bugurl: 'https://bugs.rockylinux.org' bugurl: 'https://bugs.rockylinux.org'
fedora_release: 28
allowed_arches: allowed_arches:
- x86_64 - x86_64
- aarch64 - aarch64

View File

@ -8,6 +8,7 @@
profile: '9-beta' profile: '9-beta'
bugurl: 'https://bugs.rockylinux.org' bugurl: 'https://bugs.rockylinux.org'
checksum: 'sha256' checksum: 'sha256'
fedora_release: 34
allowed_arches: allowed_arches:
- x86_64 - x86_64
- aarch64 - aarch64

View File

@ -6,6 +6,7 @@
major: '9' major: '9'
minor: '0' minor: '0'
profile: '9' profile: '9'
fedora_release: 34
bugurl: 'https://bugs.rockylinux.org' bugurl: 'https://bugs.rockylinux.org'
checksum: 'sha256' checksum: 'sha256'
allowed_arches: allowed_arches:

View File

@ -8,6 +8,7 @@
profile: '9-lookahead' profile: '9-lookahead'
bugurl: 'https://bugs.rockylinux.org' bugurl: 'https://bugs.rockylinux.org'
checksum: 'sha256' checksum: 'sha256'
fedora_release: 34
allowed_arches: allowed_arches:
- x86_64 - x86_64
- aarch64 - aarch64

View File

@ -0,0 +1,144 @@
# Builds an image given a version, type, variant, and architecture
# Defaults to the running host's architecture
import argparse
import datetime
import os
import tempfile
import pathlib
from jinja2 import Environment, FileSystemLoader, Template
from typing import List, Tuple
from empanadas.common import Architecture, rldict, valid_type_variant
from empanadas.common import _rootdir
parser = argparse.ArgumentParser(description="ISO Compose")
parser.add_argument('--version', type=str, help="Release Version (8.6, 9.1)", required=True)
parser.add_argument('--rc', action='store_true', help="Release Candidate")
parser.add_argument('--kickstartdir', action='store_true', help="Use the kickstart dir instead of the os dir for repositories")
parser.add_argument('--debug', action='store_true', help="debug?")
parser.add_argument('--type', type=str, help="Image type (container, genclo, azure, aws, vagrant)", required=True)
parser.add_argument('--variant', type=str, help="", required=False)
parser.add_argument('--release', type=str, help="Image release for subsequent builds with the same date stamp (rarely needed)", required=False)
results = parser.parse_args()
rlvars = rldict[results.version]
major = rlvars["major"]
STORAGE_DIR = pathlib.Path("/var/lib/imagefactory/storage")
KICKSTART_PATH = pathlib.Path(os.environ.get("KICKSTART_PATH", "/kickstarts"))
BUILDTIME = datetime.datetime.utcnow()
def render_icicle_template(template: Template, architecture: Architecture) -> str:
handle, output = tempfile.mkstemp()
if not handle:
exit(3)
with os.fdopen(handle, "wb") as tmp:
_template = template.render(
architecture=architecture,
fedora_version=rlvars["fedora_release"],
iso8601date=BUILDTIME.strftime("%Y%m%d"),
installdir="kickstart" if results.kickstartdir else "os",
major=major,
release=results.release if results.release else 0,
size="10G",
type=results.type.capitalize(),
utcnow=BUILDTIME,
version_variant=rlvars["revision"] if not results.variant else f"{rlvars['revision']}-{results.variant.capitalize()}",
)
tmp.write(_template.encode())
return output
def generate_kickstart_imagefactory_args(debug: bool = False) -> str:
type_variant = results.type if not results.variant else f"{results.type}-{results.variant}" # todo -cleanup
kickstart_path = pathlib.Path(f"{KICKSTART_PATH}/Rocky-{major}-{type_variant}.ks")
if not kickstart_path.is_file():
print(f"Kickstart file is not available: {kickstart_path}")
if not debug:
exit(2)
return f"--file-parameter install_script {kickstart_path}"
def get_image_format(_type: str) -> str:
mapping = {
"Container": "docker"
}
return mapping[_type] if _type in mapping.keys() else ''
def generate_imagefactory_commands(tdl_template: Template, architecture: Architecture) -> List[List[str]]:
template_path = render_icicle_template(tdl_template, architecture)
if not template_path:
exit(2)
args_mapping = {
"debug": "--debug"
}
# only supports boolean flags right now?
args = [param for name, param in args_mapping.items() if getattr(results,name)]
package_args = []
kickstart_arg = generate_kickstart_imagefactory_args(True) # REMOVE DEBUG ARG
if results.type == "Container":
args += ["--parameter", "offline_icicle", "true"]
package_args += ["--parameter", "compress", "xz"]
tar_command = ["tar", "-Oxf", f"{STORAGE_DIR}/*.body" "./layer.tar"]
type_variant = results.type if not results.variant else f"{results.type}-{results.variant}" # todo -cleanup
outname = f"Rocky-{rlvars['major']}-{type_variant}.{BUILDTIME.strftime('%Y%m%d')}.{results.release if results.release else 0}.{architecture}"
outdir = pathlib.Path(f"/tmp/{outname}")
build_command = (f"imagefactory base_image {kickstart_arg} {' '.join(args)} {template_path}"
f" | tee -a {outdir}/logs/base_image-{outname}.out"
f" | tail -n4 > {outdir}/base.meta || exit 2"
)
out_type = get_image_format(results.type)
package_command = ["imagefactory", "target_image", *args, template_path,
"--id", "$(awk '$1==\"UUID:\"{print $NF}'"+f" /tmp/{outname}/base.meta)",
*package_args,
"--parameter", "repository", outname, out_type,
"|", "tee", "-a", f"{outdir}/base_image-{outname}.out",
"|", "tail", "-n4", ">", f"{outdir}/target.meta", "||", "exit", "3"
]
copy_command = (f"aws s3 cp --recursive {outdir}/* s3://resf-empanadas/buildimage-{ outname }/{ BUILDTIME.strftime('%s') }/")
commands = [build_command, package_command, copy_command]
return commands
def run():
result, error = valid_type_variant(results.type, results.variant)
if not result:
print(error)
exit(2)
file_loader = FileSystemLoader(f"{_rootdir}/templates")
tmplenv = Environment(loader=file_loader)
tdl_template = tmplenv.get_template('icicle/tdl.xml.tmpl')
job_template = tmplenv.get_template('kube/Job.tmpl')
for architecture in rlvars["allowed_arches"]:
architecture = Architecture.New(architecture, major)
commands = generate_imagefactory_commands(tdl_template, architecture)
print(job_template.render(
architecture=architecture,
backoffLimit=4,
buildTime=datetime.datetime.utcnow().strftime("%s"),
command=commands,
imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest",
jobname="buildimage",
namespace="empanadas",
major=major,
restartPolicy="Never",
))

View File

@ -8,10 +8,11 @@ from empanadas.common import _rootdir
from jinja2 import Environment, FileSystemLoader from jinja2 import Environment, FileSystemLoader
parser = argparse.ArgumentParser(description="ISO Compose") parser = argparse.ArgumentParser(description="Generate Kubernetes Jobs to run lorax in mock and upload the result. Pipe into kubectl for the appropriate cluster")
parser.add_argument('--release', type=str, help="Major Release Version", required=True) parser.add_argument('--release', type=str, help="Major Release Version: (8|9)", required=True)
parser.add_argument('--env', type=str, help="environment", required=True) parser.add_argument('--env', type=str, help="environment: one of (eks|ext|all). presently jobs are scheduled on different kubernetes clusters", required=True)
parser.add_argument('--rc', action='store_true', help="Release Candidate, Beta, RLN")
results = parser.parse_args() results = parser.parse_args()
rlvars = rldict[results.release] rlvars = rldict[results.release]
major = rlvars['major'] major = rlvars['major']
@ -30,16 +31,25 @@ def run():
elif results.env == "all": elif results.env == "all":
arches = EKSARCH+EXTARCH arches = EKSARCH+EXTARCH
command = ["build-iso", "--release", f"{results.release}", "--rc", "--isolation", "simple"] command = ["build-iso", "--release", f"{results.release}", "--isolation", "simple"]
if results.rc:
command += ["--rc"]
buildstamp = datetime.datetime.utcnow()
out = "" out = ""
for arch in arches: for architecture in arches:
copy_command = (f"aws s3 cp --recursive --exclude=* --include=lorax* "
f"/var/lib/mock/rocky-{ major }-$(uname -m)/root/builddir/ "
f"s3://resf-empanadas/buildiso-{ major }-{ architecture }/{ buildstamp.strftime('%s') }/"
)
out += job_template.render( out += job_template.render(
architecture=arch, architecture=architecture,
backoffLimit=4, backoffLimit=4,
buildTime=datetime.datetime.utcnow().strftime("%s"), buildTime=buildstamp.strftime("%s"),
command=command, command=[command, copy_command],
imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest", imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest",
jobname="buildiso",
namespace="empanadas", namespace="empanadas",
major=major, major=major,
restartPolicy="Never", restartPolicy="Never",

View File

@ -0,0 +1,21 @@
<template>
<name>Rocky-{{major}}-{{type}}-{{version_variant}}.{{iso8601date}}.{{release}}.{{architecture}}</name>
<os>
<name>Fedora</name>
<version>{{fedora_version}}</version>
<arch>{{architecture}}</arch>
<install type='url'>
<url>https://dl.rockylinux.org/stg/rocky/{{major}}/BaseOS/{{architecture}}/{{installdir}}/</url>
</install>
<icicle>
<extra_command>rpm -qa --qf '%{NAME},%{VERSION},%{RELEASE},%{ARCH},%{EPOCH},%{SIZE},%{SIGMD5},%{BUILDTIME}
'</extra_command>
</icicle>
</os>
<description>Rocky-{{major}}-{{type}}-{{version_variant}}.{{iso8601date}}.{{release}}.{{architecture}} Generated on {{utcnow}}</description>
<disk>
<size>{{size}}</size>
</disk>
</template>

View File

@ -2,7 +2,7 @@
apiVersion: batch/v1 apiVersion: batch/v1
kind: Job kind: Job
metadata: metadata:
name: build-iso-{{ major }}-{{ architecture }} name: {{ jobname }}-{{ major }}-{{ architecture }}
namespace: {{ namespace }} namespace: {{ namespace }}
spec: spec:
template: template:
@ -11,15 +11,18 @@ spec:
peridot.rockylinux.org/workflow-tolerates-arch: {{ architecture }} peridot.rockylinux.org/workflow-tolerates-arch: {{ architecture }}
spec: spec:
containers: containers:
- name: buildiso-{{ major }}-{{ architecture }} - name: {{ jobname }}-{{ major }}-{{ architecture }}
image: {{ imageName }} image: {{ imageName }}
command: ["/bin/bash", "-c"] command: ["/bin/bash", "-c"]
args: args:
- | - |
{{ command | join(' ') }} {%- for c in command -%}
aws s3 cp --recursive --exclude=* --include=lorax* \ {%- if c is string %}
/var/lib/mock/rocky-{{ major }}-$(uname -m)/root/builddir/ \ {{ c }}
"s3://resf-empanadas/buildiso-{{ major }}-{{ architecture }}/{{ buildTime }}/" {%- else %}
{{ ' '.join(c) }}
{%- endif %}
{%- endfor %}
securityContext: securityContext:
runAsUser: 0 runAsUser: 0
runAsGroup: 0 runAsGroup: 0

View File

@ -19,6 +19,7 @@ kobo = "^0.24.1"
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]
pytest = "~5" pytest = "~5"
types-Jinja2 = "^2.11.9" # Remove when upgrading past Jinja 2.x as type annotations are in-tree
[tool.poetry.scripts] [tool.poetry.scripts]
sync_from_peridot = "empanadas.scripts.sync_from_peridot:run" sync_from_peridot = "empanadas.scripts.sync_from_peridot:run"
@ -28,6 +29,7 @@ build-iso = "empanadas.scripts.build_iso:run"
build-iso-extra = "empanadas.scripts.build_iso_extra:run" build-iso-extra = "empanadas.scripts.build_iso_extra:run"
pull-unpack-tree = "empanadas.scripts.pull_unpack_tree:run" pull-unpack-tree = "empanadas.scripts.pull_unpack_tree:run"
launch-builds = "empanadas.scripts.launch_builds:run" launch-builds = "empanadas.scripts.launch_builds:run"
build-image = "empanadas.scripts.build_image:run"
[build-system] [build-system]
requires = ["poetry-core>=1.0.0"] requires = ["poetry-core>=1.0.0"]