From 43267a6ac66384b73e66ffd8fe3acfab53478b49 Mon Sep 17 00:00:00 2001 From: Neil Hanlon Date: Mon, 6 May 2024 15:43:03 -0400 Subject: [PATCH] WIP: refactor and support multiple image backends --- iso/empanadas/empanadas/backends/__init__.py | 5 + .../empanadas/backends/imagefactory.py | 313 ++++++++++++ iso/empanadas/empanadas/backends/interface.py | 37 ++ iso/empanadas/empanadas/backends/kiwi.py | 16 + iso/empanadas/empanadas/builders/__init__.py | 1 + .../empanadas/builders/imagebuild.py | 106 ++++ iso/empanadas/empanadas/builders/utils.py | 98 ++++ iso/empanadas/empanadas/common.py | 10 +- .../empanadas/scripts/build_image.py | 469 ++---------------- iso/empanadas/pyproject.toml | 11 + iso/empanadas/tox.ini | 2 + 11 files changed, 627 insertions(+), 441 deletions(-) create mode 100644 iso/empanadas/empanadas/backends/__init__.py create mode 100644 iso/empanadas/empanadas/backends/imagefactory.py create mode 100644 iso/empanadas/empanadas/backends/interface.py create mode 100644 iso/empanadas/empanadas/backends/kiwi.py create mode 100644 iso/empanadas/empanadas/builders/__init__.py create mode 100644 iso/empanadas/empanadas/builders/imagebuild.py create mode 100644 iso/empanadas/empanadas/builders/utils.py create mode 100644 iso/empanadas/tox.ini diff --git a/iso/empanadas/empanadas/backends/__init__.py b/iso/empanadas/empanadas/backends/__init__.py new file mode 100644 index 0000000..4c2ed94 --- /dev/null +++ b/iso/empanadas/empanadas/backends/__init__.py @@ -0,0 +1,5 @@ +"""Empanadas Backends (fillings)""" + +from .imagefactory import ImageFactoryBackend +from .kiwi import KiwiBackend +from .interface import BackendInterface diff --git a/iso/empanadas/empanadas/backends/imagefactory.py b/iso/empanadas/empanadas/backends/imagefactory.py new file mode 100644 index 0000000..41c88d0 --- /dev/null +++ b/iso/empanadas/empanadas/backends/imagefactory.py @@ -0,0 +1,313 @@ +"""Backend for ImageFactory""" + +import json +import os +import pathlib +import tempfile + +from .interface import BackendInterface +from empanadas.builders import utils + +from attrs import define, field + +from typing import List, Optional, Callable, Union + +KICKSTART_PATH = pathlib.Path(os.environ.get("KICKSTART_PATH", "/kickstarts")) +STORAGE_DIR = pathlib.Path("/var/lib/imagefactory/storage") + + +@define(kw_only=True) +class ImageFactoryBackend(BackendInterface): + """Build an image using ImageFactory""" + kickstart_arg: List[str] = field(factory=list) + kickstart_path: pathlib.Path = field(init=False) + base_uuid: Optional[str] = field(default="") + target_uuid: Optional[str] = field(default="") + tdl_path: pathlib.Path = field(init=False) + out_type: str = field(init=False) + command_args: List[str] = field(factory=list) + common_args: List[str] = field(factory=list) + package_args: List[str] = field(factory=list) + metadata: pathlib.Path = field(init=False) + ctx = field(init=False) + stage_commands: Optional[List[List[Union[str, Callable]]]] = field(init=False) + + # The url to use in the path when fetching artifacts for the build + kickstart_dir: str = field() # 'os' or 'kickstart' + + # The git repository to fetch kickstarts from + kickstart_repo: str = field() + + def prepare(self): + self.out_type = self.image_format() + + tdl_template = self.ctx.tmplenv.get_template('icicle/tdl.xml.tmpl') + + self.tdl_path = self.render_icicle_template(tdl_template) + if not self.tdl_path: + exit(2) + + self.metadata = pathlib.Path(self.ctx.outdir, ".imagefactory-metadata.json") + + self.kickstart_path = pathlib.Path(f"{KICKSTART_PATH}/Rocky-{self.ctx.architecture.major}-{self.ctx.type_variant}.ks") + + self.checkout_kickstarts() + self.kickstart_arg = self.kickstart_imagefactory_args() + + try: + os.mkdir(self.ctx.outdir) + except FileExistsError: + self.ctx.log.info("Directory already exists for this release. If possible, previously executed steps may be skipped") + except Exception as e: + self.ctx.log.exception("Some other exception occured while creating the output directory", e) + return 0 + + if os.path.exists(self.metadata): + self.ctx.log.info(f"Found metadata at {self.metadata}") + with open(self.metadata, "r") as f: + try: + o = json.load(f) + self.base_uuid = o['base_uuid'] + self.target_uuid = o['target_uuid'] + except json.decoder.JSONDecodeError as e: + self.ctx.log.exception("Couldn't decode metadata file", e) + finally: + f.flush() + + self.command_args = self._command_args() + self.package_args = self._package_args() + self.common_args = self._common_args() + + self.setup_staging() + + def build(self) -> int: + if self.base_uuid: + return 0 + + self.fix_ks() + + # TODO(neil): this should be a lambda which is called from the function + ret, out, err, uuid = self.ctx.prepare_and_run(self.build_command(), search=True) + if uuid: + self.base_uuid = uuid.rstrip() + self.save() + + if ret > 0: + return ret + + ret = self.package() + + if ret > 0: + return ret + + ret = self.copy() + return ret + + def clean(self): + pass + + def save(self): + with open(self.metadata, "w") as f: + try: + o = { + name: getattr(self, name) for name in [ + "base_uuid", "target_uuid" + ] + } + self.ctx.log.debug(o) + json.dump(o, f) + except AttributeError as e: + self.ctx.log.error("Couldn't find attribute in object. Something is probably wrong", e) + except Exception as e: + self.ctx.log.exception(e) + finally: + f.flush() + + def package(self) -> int: + # Some build types don't need to be packaged by imagefactory + # @TODO remove business logic if possible + if self.ctx.image_type in ["GenericCloud", "EC2", "Azure", "Vagrant", "OCP", "RPI", "GenericArm"]: + self.target_uuid = self.base_uuid if hasattr(self, 'base_uuid') else "" + + if self.target_uuid: + return 0 + + ret, out, err, uuid = self.ctx.prepare_and_run(self.package_command(), search=True) + if uuid: + self.target_uuid = uuid.rstrip() + self.save() + return ret + + def stage(self) -> int: + """ Stage the artifacst from wherever they are (unpacking and converting if needed)""" + if not hasattr(self, 'stage_commands'): + return 0 + + returns = [] + for command in self.stage_commands: # type: ignore + ret, out, err, _ = self.ctx.prepare_and_run(command, search=False) + returns.append(ret) + + return all(ret > 0 for ret in returns) + + def copy(self, skip=False) -> int: + # move or unpack if necessary + self.ctx.log.info("Executing staging commands") + if (stage := self.stage() > 0): + raise Exception(stage) + + if not skip: + self.ctx.log.info("Copying files to output directory") + ret, out, err, _ = self.ctx.prepare_and_run(self.copy_command(), search=False) + return ret + + self.ctx.log.info(f"Build complete! Output available in {self.ctx.outdir}/") + return 0 + + def checkout_kickstarts(self) -> int: + cmd = ["git", "clone", "--branch", f"r{self.ctx.architecture.major}", + self.kickstart_repo, f"{KICKSTART_PATH}"] + ret, out, err, _ = self.ctx.prepare_and_run(cmd, search=False) + self.ctx.log.debug(out) + self.ctx.log.debug(err) + if ret > 0: + ret = self.pull_kickstarts() + return ret + + def pull_kickstarts(self) -> int: + cmd: utils.CMD_PARAM_T = ["git", "-C", f"{KICKSTART_PATH}", "reset", "--hard", "HEAD"] + ret, out, err, _ = self.ctx.prepare_and_run(cmd, search=False) + self.ctx.log.debug(out) + self.ctx.log.debug(err) + if ret == 0: + cmd = ["git", "-C", f"{KICKSTART_PATH}", "pull"] + ret, out, err, _ = self.ctx.prepare_and_run(cmd, search=False) + self.ctx.log.debug(out) + self.ctx.log.debug(err) + return ret + + def _command_args(self): + args_mapping = { + "debug": "--debug", + } + # NOTE(neil): i'm intentionally leaving this as is; deprecated + return [param for name, param in args_mapping.items() if self.ctx.debug] + + def _package_args(self) -> List[str]: + if self.ctx.image_type in ["Container"]: + return ["--parameter", "compress", "xz"] + return [""] + + def _common_args(self) -> List[str]: + args = [] + if self.ctx.image_type in ["Container"]: + args = ["--parameter", "offline_icicle", "true"] + if self.ctx.image_type in ["GenericCloud", "EC2", "Vagrant", "Azure", "OCP", "RPI", "GenericArm"]: + args = ["--parameter", "generate_icicle", "false"] + return args + + def image_format(self) -> str: + mapping = { + "Container": "docker" + } + return mapping[self.ctx.image_type] if self.ctx.image_type in mapping.keys() else '' + + def kickstart_imagefactory_args(self) -> List[str]: + + if not self.kickstart_path.is_file(): + self.ctx.log.warning(f"Kickstart file is not available: {self.kickstart_path}") + if not self.ctx.debug: + self.ctx.log.warning("Exiting because debug mode is not enabled.") + exit(2) + + return ["--file-parameter", "install_script", str(self.kickstart_path)] + + def render_icicle_template(self, tdl_template) -> pathlib.Path: + output = tempfile.NamedTemporaryFile(delete=False).name + return utils.render_template(output, tdl_template, + architecture=self.ctx.architecture.name, + iso8601date=self.ctx.build_time.strftime("%Y%m%d"), + installdir=self.kickstart_dir, + major=self.ctx.architecture.major, + minor=self.ctx.architecture.minor, + release=self.ctx.release, + size="10G", + type=self.ctx.image_type, + utcnow=self.ctx.build_time, + version_variant=self.ctx.architecture.version if not self.ctx.variant else f"{self.ctx.architecture.version}-{self.ctx.variant}", + ) + + def build_command(self) -> List[str]: + build_command = ["imagefactory", "--timeout", self.ctx.timeout, + *self.command_args, "base_image", *self.common_args, + *self.kickstart_arg, self.tdl_path] + return build_command + + def package_command(self) -> List[str]: + package_command = ["imagefactory", *self.command_args, "target_image", + self.out_type, *self.common_args, + "--id", f"{self.base_uuid}", + *self.package_args, + "--parameter", "repository", self.ctx.outname] + return package_command + + def copy_command(self) -> List[str]: + + copy_command = ["aws", "s3", "cp", "--recursive", f"{self.ctx.outdir}/", + f"s3://resf-empanadas/buildimage-{self.ctx.architecture.version}-{self.ctx.architecture.name}/{self.ctx.outname}/{self.ctx.build_time.strftime('%s')}/" + ] + + return copy_command + + def fix_ks(self): + cmd: utils.CMD_PARAM_T = ["sed", "-i", f"s,$basearch,{self.ctx.architecture.name},", str(self.kickstart_path)] + self.ctx.prepare_and_run(cmd, search=False) + + def setup_staging(self): + # Yes, this is gross. I'll fix it later. + if self.ctx.image_type in ["Container"]: + self.stage_commands = [ + ["tar", "-C", f"{self.ctx.outdir}", "--strip-components=1", "-x", "-f", lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", "*/layer.tar"], + ["xz", f"{self.ctx.outdir}/layer.tar"] + ] + if self.ctx.image_type in ["RPI"]: + self.stage_commands = [ + ["cp", lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", f"{self.ctx.outdir}/{self.ctx.outname}.raw"], + ["xz", f"{self.ctx.outdir}/{self.ctx.outname}.raw"] + ] + if self.ctx.image_type in ["GenericCloud", "OCP", "GenericArm"]: + self.stage_commands = [ + ["qemu-img", "convert", "-c", "-f", "raw", "-O", "qcow2", + lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", f"{self.ctx.outdir}/{self.ctx.outname}.qcow2"] + ] + if self.ctx.image_type in ["EC2"]: + self.stage_commands = [ + ["qemu-img", "convert", "-f", "raw", "-O", "qcow2", lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", f"{self.ctx.outdir}/{self.ctx.outname}.qcow2"] + ] + if self.ctx.image_type in ["Azure"]: + self.stage_commands = [ + ["/prep-azure.sh", lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", f"{STORAGE_DIR}"], + ["cp", lambda: f"{STORAGE_DIR}/{self.target_uuid}.vhd", f"{self.ctx.outdir}/{self.ctx.outname}.vhd"] + ] + if self.ctx.image_type in ["Vagrant"]: + _map = { + "Vbox": {"format": "vmdk", "provider": "virtualbox"}, + "Libvirt": {"format": "qcow2", "provider": "libvirt", "virtual_size": 10}, + "VMware": {"format": "vmdk", "provider": "vmware_desktop"} + } + output = f"{_map[self.ctx.variant]['format']}" # type: ignore + provider = f"{_map[self.ctx.variant]['provider']}" # type: ignore + + # pop from the options map that will be passed to the vagrant metadata.json + convert_options = _map[self.ctx.variant].pop('convertOptions') if 'convertOptions' in _map[self.ctx.variant].keys() else '' # type: ignore + + self.stage_commands = [ + ["qemu-img", "convert", "-c", "-f", "raw", "-O", output, *convert_options, + lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", f"{self.ctx.outdir}/{self.ctx.outname}.{output}"], + ["tar", "-C", self.ctx.outdir, "-czf", f"/tmp/{self.ctx.outname}.box", '.'], + ["mv", f"/tmp/{self.ctx.outname}.box", self.ctx.outdir] + ] + self.prepare_vagrant(_map[self.ctx.variant]) + + if self.stage_commands: + self.stage_commands.append(["cp", "-v", lambda: f"{STORAGE_DIR}/{self.target_uuid}.meta", f"{self.ctx.outdir}/build.meta"]) diff --git a/iso/empanadas/empanadas/backends/interface.py b/iso/empanadas/empanadas/backends/interface.py new file mode 100644 index 0000000..a17fd5a --- /dev/null +++ b/iso/empanadas/empanadas/backends/interface.py @@ -0,0 +1,37 @@ +""" +empanadas backend interface +""" +from abc import ABC, abstractmethod + + +class BackendInterface(ABC): + """ + Interface to build images (or whatever) + """ + @abstractmethod + def prepare(self): + """ + Prepares the environment necessary for building the image. + This might include setting up directories, checking prerequisites, etc. + """ + + @abstractmethod + def build(self): + """ + Performs the image build operation. This is the core method + where the actual image building logic is implemented. + """ + + @abstractmethod + def stage(self): + """ + Transforms and copies artifacts from build directory to the + location expected by the builder (usually in /tmp/) + """ + + @abstractmethod + def clean(self): + """ + Cleans up any resources or temporary files created during + the image building process. + """ diff --git a/iso/empanadas/empanadas/backends/kiwi.py b/iso/empanadas/empanadas/backends/kiwi.py new file mode 100644 index 0000000..b1d5dd0 --- /dev/null +++ b/iso/empanadas/empanadas/backends/kiwi.py @@ -0,0 +1,16 @@ + +"""Backend for Kiwi""" +from .interface import BackendInterface + + +class KiwiBackend(BackendInterface): + """Build an image using Kiwi""" + + def prepare(self): + pass + + def build(self): + pass + + def clean(self): + pass diff --git a/iso/empanadas/empanadas/builders/__init__.py b/iso/empanadas/empanadas/builders/__init__.py new file mode 100644 index 0000000..07dc0ee --- /dev/null +++ b/iso/empanadas/empanadas/builders/__init__.py @@ -0,0 +1 @@ +from .imagebuild import ImageBuild diff --git a/iso/empanadas/empanadas/builders/imagebuild.py b/iso/empanadas/empanadas/builders/imagebuild.py new file mode 100644 index 0000000..0a3f81c --- /dev/null +++ b/iso/empanadas/empanadas/builders/imagebuild.py @@ -0,0 +1,106 @@ +"""Build an image with a given backend""" + +import datetime +import logging +import pathlib + +from attrs import define, field + +from empanadas.backends import BackendInterface, KiwiBackend +from empanadas.common import Architecture +from empanadas.common import _rootdir +from . import utils + +from jinja2 import Environment, FileSystemLoader, Template +from typing import List, Optional, Tuple, Callable + + +@define(kw_only=True) +class ImageBuild: # pylint: disable=too-few-public-methods + """Image builder using a given backend""" + tmplenv: Environment = field(init=False) + + # Only things we know we're keeping in this class here + architecture: Architecture = field() + backend: BackendInterface = field() + build_time: datetime.datetime = field() + debug: bool = field(default=False) + log: logging.Logger = field() + release: int = field(default=0) + timeout: str = field(default='3600') + + image_type: str = field() # the type of the image + type_variant: str = field(init=False) + variant: Optional[str] = field() + + # Kubernetes job template + job_template: Optional[Template] = field(init=False) # the kube Job tpl + + # Commands to stage artifacts + + # Where the artifacts should go to + outdir: pathlib.Path = field(init=False) + outname: str = field(init=False) + + def __attrs_post_init__(self): + self.backend.ctx = self + + file_loader = FileSystemLoader(f"{_rootdir}/templates") + self.tmplenv = Environment(loader=file_loader) + + self.job_template = self.tmplenv.get_template('kube/Job.tmpl') + + self.type_variant = self.type_variant_name() + self.outdir, self.outname = self.output_name() + + def output_name(self) -> Tuple[pathlib.Path, str]: + directory = f"Rocky-{self.architecture.major}-{self.type_variant}-{self.architecture.version}-{self.build_time.strftime('%Y%m%d')}.{self.release}" + name = f"{directory}.{self.architecture.name}" + outdir = pathlib.Path("/tmp/", directory) + return outdir, name + + def type_variant_name(self): + return self.image_type if not self.variant else f"{self.image_type}-{self.variant}" + + def prepare_and_run(self, command: utils.CMD_PARAM_T, search: Callable = None) -> utils.CMD_RESULT_T: + return utils.runCmd(self, self.prepare_command(command), search) + + def prepare_command(self, command_list: utils.CMD_PARAM_T) -> List[str]: + """ + Commands may be a callable, which should be a lambda to be evaluated at + preparation time with available locals. This can be used to, among + other things, perform lazy evaluations of f-strings which have values + not available at assignment time. e.g., filling in a second command + with a value extracted from the previous step or command. + + """ + + r = [] + for c in command_list: + if callable(c) and c.__name__ == '': + r.append(c()) + else: + r.append(str(c)) + return r + + def render_kubernetes_job(self): + # TODO(neil): should this be put in the builder class itself to return the right thing for us? + if self.backend == KiwiBackend: + self.log.error("Kube not implemented for Kiwi") + + commands = [self.backend.build_command(), self.backend.package_command(), self.backend.copy_command()] + if not self.job_template: + return None + template = self.job_template.render( + architecture=self.architecture.name, + backoffLimit=4, + buildTime=self.build_time.strftime("%s"), + command=commands, + imageName="ghcr.io/rockylinux/sig-core-toolkit:latest", + jobname="buildimage", + namespace="empanadas", + major=self.architecture.major, + minor=self.architecture.minor, + restartPolicy="Never", + ) + return template diff --git a/iso/empanadas/empanadas/builders/utils.py b/iso/empanadas/empanadas/builders/utils.py new file mode 100644 index 0000000..c287f7d --- /dev/null +++ b/iso/empanadas/empanadas/builders/utils.py @@ -0,0 +1,98 @@ +import pathlib +import subprocess + +from functools import partial +from typing import Callable, List, Tuple, Union + +CMD_PARAM_T = List[Union[str, Callable[..., str]]] + +STR_NONE_T = Union[bytes, None] +BYTES_NONE_T = Union[bytes, None] +# Tuple of int, stdout, stderr, uuid +CMD_RESULT_T = Tuple[int, BYTES_NONE_T, BYTES_NONE_T, STR_NONE_T] + + +# def prepare_vagrant(options): +# """Setup the output directory for the Vagrant type variant, dropping templates as required""" +# file_loader = FileSystemLoader(f"{_rootdir}/templates") +# tmplenv = Environment(loader=file_loader) +# +# templates = {} +# templates['Vagrantfile'] = tmplenv.get_template(f"vagrant/Vagrantfile.{self.variant}") +# templates['metadata.json'] = tmplenv.get_template('vagrant/metadata.tmpl.json') +# templates['info.json'] = tmplenv.get_template('vagrant/info.tmpl.json') +# +# if self.variant == "VMware": +# templates[f"{self.outname}.vmx"] = tmplenv.get_template('vagrant/vmx.tmpl') +# +# if self.variant == "Vbox": +# templates['box.ovf'] = tmplenv.get_template('vagrant/box.tmpl.ovf') +# +# if self.variant == "Libvirt": +# # Libvirt vagrant driver expects the qcow2 file to be called box.img. +# qemu_command_index = [i for i, d in enumerate(self.stage_commands) if d[0] == "qemu-img"][0] +# self.stage_commands.insert(qemu_command_index+1, ["mv", f"{self.outdir}/{self.outname}.qcow2", f"{self.outdir}/box.img"]) +# +# for name, template in templates.items(): +# self.render_template(f"{self.outdir}/{name}", template, +# name=self.outname, +# arch=self.architecture.name, +# options=options +# ) + + +def render_template(path, template, **kwargs) -> pathlib.Path: + with open(path, "wb") as f: + _template = template.render(**kwargs) + f.write(_template.encode()) + f.flush() + output = pathlib.Path(path) + if not output.exists(): + raise Exception("Failed to template") + return output + + +def runCmd(ctx, prepared_command: List[str], search: Callable = None) -> CMD_RESULT_T: + ctx.log.info(f"Running command: {' '.join(prepared_command)}") + + kwargs = { + "stderr": subprocess.PIPE, + "stdout": subprocess.PIPE + } + + if ctx.debug: + del kwargs["stderr"] + + with subprocess.Popen(prepared_command, **kwargs) as p: + uuid = None + # @TODO implement this as a callback? + if search: + for _, line in enumerate(p.stdout): # type: ignore + ln = line.decode() + if ln.startswith("UUID: "): + uuid = ln.split(" ")[-1] + ctx.log.debug(f"found uuid: {uuid}") + + out, err = p.communicate() + res = p.wait(), out, err, uuid + + if res[0] > 0: + ctx.log.error(f"Problem while executing command: '{prepared_command}'") + if search and not res[3]: + ctx.log.error("UUID not found in stdout. Dumping stdout and stderr") + log_subprocess(ctx, res) + + return res + + +def log_subprocess(ctx, result: CMD_RESULT_T): + def log_lines(title, lines): + ctx.log.info(f"====={title}=====") + ctx.log.info(lines.decode()) + ctx.log.info(f"Command return code: {result[0]}") + stdout = result[1] + stderr = result[2] + if stdout: + log_lines("Command STDOUT", stdout) + if stderr: + log_lines("Command STDERR", stderr) diff --git a/iso/empanadas/empanadas/common.py b/iso/empanadas/empanadas/common.py index bfb19b0..b22e556 100644 --- a/iso/empanadas/empanadas/common.py +++ b/iso/empanadas/empanadas/common.py @@ -1,12 +1,10 @@ # All imports are here import glob -import hashlib -import logging -import os import platform import time from collections import defaultdict -from typing import Tuple +from attrs import define, field + import rpm import yaml @@ -120,7 +118,7 @@ ALLOWED_TYPE_VARIANTS = { def valid_type_variant(_type: str, variant: str = "") -> bool: if _type not in ALLOWED_TYPE_VARIANTS: raise Exception(f"Type is invalid: ({_type}, {variant})") - if ALLOWED_TYPE_VARIANTS[_type] == None: + if ALLOWED_TYPE_VARIANTS[_type] is None: if variant is not None: raise Exception(f"{_type} Type expects no variant type.") return True @@ -135,8 +133,6 @@ def valid_type_variant(_type: str, variant: str = "") -> bool: return True -from attrs import define, field - @define(kw_only=True) class Architecture: diff --git a/iso/empanadas/empanadas/scripts/build_image.py b/iso/empanadas/empanadas/scripts/build_image.py index 79aca71..b3c317b 100644 --- a/iso/empanadas/empanadas/scripts/build_image.py +++ b/iso/empanadas/empanadas/scripts/build_image.py @@ -1,44 +1,43 @@ -# Builds an image given a version, type, variant, and architecture +# Builds an image given a version, type, variant, anctx.d architecture # Defaults to the running host's architecture import argparse import datetime -import json import logging -import os -import pathlib import platform -import subprocess import sys -import tempfile -import time - -from attrs import define, Factory, field, asdict -from botocore import args -from jinja2 import Environment, FileSystemLoader, Template -from typing import Callable, List, NoReturn, Optional, Tuple, IO, Union from empanadas.common import Architecture, rldict, valid_type_variant -from empanadas.common import _rootdir +from empanadas.builders import ImageBuild +from empanadas.backends import ImageFactoryBackend # , KiwiBackend parser = argparse.ArgumentParser(description="ISO Compose") -parser.add_argument('--version', type=str, help="Release Version (8.6, 9.1)", required=True) +parser.add_argument('--version', + type=str, help="Release Version (8.6, 9.1)", required=True) parser.add_argument('--rc', action='store_true', help="Release Candidate") -parser.add_argument('--kickstartdir', action='store_true', help="Use the kickstart dir instead of the os dir for repositories") +parser.add_argument('--kickstartdir', action='store_true', + help="Use the kickstart dir instead of the os dir") parser.add_argument('--debug', action='store_true', help="debug?") -parser.add_argument('--type', type=str, help="Image type (container, genclo, azure, aws, vagrant)", required=True) +parser.add_argument('--type', type=str, + help="Image type (container, genclo, azure, aws, vagrant)", + required=True) parser.add_argument('--variant', type=str, help="", required=False) -parser.add_argument('--release', type=str, help="Image release for subsequent builds with the same date stamp (rarely needed)", required=False) -parser.add_argument('--kube', action='store_true', help="output as a K8s job(s)", required=False) -parser.add_argument('--timeout', type=str, help="change timeout for imagefactory build process (default 3600)", required=False, default='3600') +parser.add_argument('--release', type=str, + help="Image release for builds with the same date stamp", + required=False) +parser.add_argument('--kube', action='store_true', + help="output as a K8s job(s)", + required=False) +parser.add_argument('--timeout', type=str, + help="change timeout for imagefactory build process", + required=False, default='3600') results = parser.parse_args() rlvars = rldict[results.version] major = rlvars["major"] - debug = results.debug log = logging.getLogger(__name__) @@ -52,405 +51,6 @@ formatter = logging.Formatter( handler.setFormatter(formatter) log.addHandler(handler) -STORAGE_DIR = pathlib.Path("/var/lib/imagefactory/storage") -KICKSTART_PATH = pathlib.Path(os.environ.get("KICKSTART_PATH", "/kickstarts")) -BUILDTIME = datetime.datetime.utcnow() - - -CMD_PARAM_T = List[Union[str, Callable[..., str]]] - -@define(kw_only=True) -class ImageBuild: - architecture: Architecture = field() - base_uuid: Optional[str] = field(default="") - cli_args: argparse.Namespace = field() - command_args: List[str] = field(factory=list) - common_args: List[str] = field(factory=list) - debug: bool = field(default=False) - image_type: str = field() - job_template: Optional[Template] = field(init=False) - kickstart_arg: List[str] = field(factory=list) - kickstart_path: pathlib.Path = field(init=False) - metadata: pathlib.Path = field(init=False) - out_type: str = field(init=False) - outdir: pathlib.Path = field(init=False) - outname: str = field(init=False) - package_args: List[str] = field(factory=list) - release: int = field(default=0) - stage_commands: Optional[List[List[Union[str,Callable]]]] = field(init=False) - target_uuid: Optional[str] = field(default="") - tdl_path: pathlib.Path = field(init=False) - template: Template = field() - timeout: str = field(default='3600') - type_variant: str = field(init=False) - variant: Optional[str] = field() - - def __attrs_post_init__(self): - self.tdl_path = self.render_icicle_template() - if not self.tdl_path: - exit(2) - self.type_variant = self.type_variant_name() - self.outdir, self.outname = self.output_name() - self.out_type = self.image_format() - self.command_args = self._command_args() - self.package_args = self._package_args() - self.common_args = self._common_args() - - self.metadata = pathlib.Path(self.outdir, ".imagefactory-metadata.json") - - self.kickstart_path = pathlib.Path(f"{KICKSTART_PATH}/Rocky-{self.architecture.major}-{self.type_variant}.ks") - - self.checkout_kickstarts() - self.kickstart_arg = self.kickstart_imagefactory_args() - - try: - os.mkdir(self.outdir) - except FileExistsError as e: - log.info("Directory already exists for this release. If possible, previously executed steps may be skipped") - except Exception as e: - log.exception("Some other exception occured while creating the output directory", e) - return 0 - - if os.path.exists(self.metadata): - with open(self.metadata, "r") as f: - try: - o = json.load(f) - self.base_uuid = o['base_uuid'] - self.target_uuid = o['target_uuid'] - except json.decoder.JSONDecodeError as e: - log.exception("Couldn't decode metadata file", e) - finally: - f.flush() - - # Yes, this is gross. I'll fix it later. - if self.image_type in ["Container"]: - self.stage_commands = [ - ["tar", "-C", f"{self.outdir}", "--strip-components=1", "-x", "-f", lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", "*/layer.tar"], - ["xz", f"{self.outdir}/layer.tar"] - ] - if self.image_type in ["RPI"]: - self.stage_commands = [ - ["cp", lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", f"{self.outdir}/{self.outname}.raw"], - ["xz", f"{self.outdir}/{self.outname}.raw"] - ] - if self.image_type in ["GenericCloud", "OCP", "GenericArm"]: - self.stage_commands = [ - ["qemu-img", "convert", "-c", "-f", "raw", "-O", "qcow2", lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", f"{self.outdir}/{self.outname}.qcow2"] - ] - if self.image_type in ["EC2"]: - self.stage_commands = [ - ["qemu-img", "convert", "-f", "raw", "-O", "qcow2", lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", f"{self.outdir}/{self.outname}.qcow2"] - ] - if self.image_type in ["Azure"]: - self.stage_commands = [ - ["/prep-azure.sh", lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", f"{STORAGE_DIR}"], - ["cp", lambda: f"{STORAGE_DIR}/{self.target_uuid}.vhd", f"{self.outdir}/{self.outname}.vhd"] - ] - if self.image_type in ["Vagrant"]: - _map = { - "Vbox": {"format": "vmdk", "provider": "virtualbox"}, - "Libvirt": {"format": "qcow2", "provider": "libvirt", "virtual_size": 10}, - "VMware": {"format": "vmdk", "provider": "vmware_desktop"} - } - output = f"{_map[self.variant]['format']}" #type: ignore - provider = f"{_map[self.variant]['provider']}" # type: ignore - - # pop from the options map that will be passed to the vagrant metadata.json - convert_options = _map[self.variant].pop('convertOptions') if 'convertOptions' in _map[self.variant].keys() else '' #type: ignore - - - self.stage_commands = [ - ["qemu-img", "convert", "-c", "-f", "raw", "-O", output, *convert_options, lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", f"{self.outdir}/{self.outname}.{output}"], - ["tar", "-C", self.outdir, "-czf", f"/tmp/{self.outname}.box", '.'], - ["mv", f"/tmp/{self.outname}.box", self.outdir] - ] - self.prepare_vagrant(_map[self.variant]) - - if self.stage_commands: - self.stage_commands.append(["cp", "-v", lambda: f"{STORAGE_DIR}/{self.target_uuid}.meta", f"{self.outdir}/build.meta"]) - - - def prepare_vagrant(self, options): - """Setup the output directory for the Vagrant type variant, dropping templates as required""" - file_loader = FileSystemLoader(f"{_rootdir}/templates") - tmplenv = Environment(loader=file_loader) - - templates = {} - templates['Vagrantfile'] = tmplenv.get_template(f"vagrant/Vagrantfile.{self.variant}") - templates['metadata.json'] = tmplenv.get_template('vagrant/metadata.tmpl.json') - templates['info.json'] = tmplenv.get_template('vagrant/info.tmpl.json') - - if self.variant == "VMware": - templates[f"{self.outname}.vmx"] = tmplenv.get_template('vagrant/vmx.tmpl') - - if self.variant == "Vbox": - templates['box.ovf'] = tmplenv.get_template('vagrant/box.tmpl.ovf') - - if self.variant == "Libvirt": - # Libvirt vagrant driver expects the qcow2 file to be called box.img. - qemu_command_index = [i for i, d in enumerate(self.stage_commands) if d[0] == "qemu-img"][0] - self.stage_commands.insert(qemu_command_index+1, ["mv", f"{self.outdir}/{self.outname}.qcow2", f"{self.outdir}/box.img"]) - - for name, template in templates.items(): - self.render_template(f"{self.outdir}/{name}", template, - name=self.outname, - arch=self.architecture.name, - options=options - ) - - def checkout_kickstarts(self) -> int: - cmd = ["git", "clone", "--branch", f"r{self.architecture.major}", rlvars['livemap']['git_repo'], f"{KICKSTART_PATH}"] - ret, out, err, _ = self.runCmd(cmd, search=False) - log.debug(out) - log.debug(err) - if ret > 0: - ret = self.pull_kickstarts() - return ret - - def pull_kickstarts(self) -> int: - cmd: CMD_PARAM_T = ["git", "-C", f"{KICKSTART_PATH}", "reset", "--hard", "HEAD"] - ret, out, err, _ = self.runCmd(cmd, search=False) - log.debug(out) - log.debug(err) - if ret == 0: - cmd = ["git", "-C", f"{KICKSTART_PATH}", "pull"] - ret, out, err, _ = self.runCmd(cmd, search=False) - log.debug(out) - log.debug(err) - return ret - - - def output_name(self) -> Tuple[pathlib.Path, str]: - directory = f"Rocky-{self.architecture.major}-{self.type_variant}-{self.architecture.version}-{BUILDTIME.strftime('%Y%m%d')}.{self.release}" - name = f"{directory}.{self.architecture.name}" - outdir = pathlib.Path(f"/tmp/", directory) - return outdir, name - - def type_variant_name(self): - return self.image_type if not self.variant else f"{self.image_type}-{self.variant}" - - def _command_args(self): - args_mapping = { - "debug": "--debug", - } - return [param for name, param in args_mapping.items() if getattr(self.cli_args, name)] - - def _package_args(self) -> List[str]: - if self.image_type in ["Container"]: - return ["--parameter", "compress", "xz"] - return [""] - - def _common_args(self) -> List[str]: - args = [] - if self.image_type in ["Container"]: - args = ["--parameter", "offline_icicle", "true"] - if self.image_type in ["GenericCloud", "EC2", "Vagrant", "Azure", "OCP", "RPI", "GenericArm"]: - args = ["--parameter", "generate_icicle", "false"] - return args - - def image_format(self) -> str: - mapping = { - "Container": "docker" - } - return mapping[self.image_type] if self.image_type in mapping.keys() else '' - - def kickstart_imagefactory_args(self) -> List[str]: - - if not self.kickstart_path.is_file(): - log.warning(f"Kickstart file is not available: {self.kickstart_path}") - if not debug: - log.warning("Exiting because debug mode is not enabled.") - exit(2) - - return ["--file-parameter", "install_script", str(self.kickstart_path)] - - def render_template(self, path, template, **kwargs) -> pathlib.Path: - with open(path, "wb") as f: - _template = template.render(**kwargs) - f.write(_template.encode()) - f.flush() - output = pathlib.Path(path) - if not output.exists(): - log.error("Failed to write template") - raise Exception("Failed to template") - return output - - def render_icicle_template(self) -> pathlib.Path: - output = tempfile.NamedTemporaryFile(delete=False).name - return self.render_template(output, self.template, - architecture=self.architecture.name, - iso8601date=BUILDTIME.strftime("%Y%m%d"), - installdir="kickstart" if self.cli_args.kickstartdir else "os", - major=self.architecture.major, - minor=self.architecture.minor, - release=self.release, - size="10G", - type=self.image_type, - utcnow=BUILDTIME, - version_variant=self.architecture.version if not self.variant else f"{self.architecture.version}-{self.variant}", - ) - - def build_command(self) -> List[str]: - build_command = ["imagefactory", "--timeout", self.timeout, *self.command_args, "base_image", *self.common_args, *self.kickstart_arg, self.tdl_path] - return build_command - def package_command(self) -> List[str]: - package_command = ["imagefactory", *self.command_args, "target_image", self.out_type, *self.common_args, - "--id", f"{self.base_uuid}", - *self.package_args, - "--parameter", "repository", self.outname, - ] - return package_command - - def copy_command(self) -> List[str]: - - copy_command = ["aws", "s3", "cp", "--recursive", f"{self.outdir}/", - f"s3://resf-empanadas/buildimage-{self.architecture.version}-{self.architecture.name}/{ self.outname }/{ BUILDTIME.strftime('%s') }/" - ] - - return copy_command - - def build(self) -> int: - if self.base_uuid: - return 0 - - self.fix_ks() - - ret, out, err, uuid = self.runCmd(self.build_command()) - if uuid: - self.base_uuid = uuid.rstrip() - self.save() - return ret - - def package(self) -> int: - # Some build types don't need to be packaged by imagefactory - # @TODO remove business logic if possible - if self.image_type in ["GenericCloud", "EC2", "Azure", "Vagrant", "OCP", "RPI", "GenericArm"]: - self.target_uuid = self.base_uuid if hasattr(self, 'base_uuid') else "" - - if self.target_uuid: - return 0 - - ret, out, err, uuid = self.runCmd(self.package_command()) - if uuid: - self.target_uuid = uuid.rstrip() - self.save() - return ret - - def stage(self) -> int: - """ Stage the artifacst from wherever they are (unpacking and converting if needed)""" - if not hasattr(self,'stage_commands'): - return 0 - - returns = [] - for command in self.stage_commands: #type: ignore - ret, out, err, _ = self.runCmd(command, search=False) - returns.append(ret) - - return all(ret > 0 for ret in returns) - - def copy(self, skip=False) -> int: - # move or unpack if necessary - log.info("Executing staging commands") - if (stage := self.stage() > 0): - raise Exception(stage) - - if not skip: - log.info("Copying files to output directory") - ret, out, err, _ = self.runCmd(self.copy_command(), search=False) - return ret - - log.info(f"Build complete! Output available in {self.outdir}/") - return 0 - - def runCmd(self, command: CMD_PARAM_T, search: bool = True) -> Tuple[int, Union[bytes,None], Union[bytes,None], Union[str,None]]: - prepared, _ = self.prepare_command(command) - log.info(f"Running command: {' '.join(prepared)}") - - kwargs = { - "stderr": subprocess.PIPE, - "stdout": subprocess.PIPE - } - if debug: del kwargs["stderr"] - - with subprocess.Popen(prepared, **kwargs) as p: - uuid = None - # @TODO implement this as a callback? - if search: - for _, line in enumerate(p.stdout): # type: ignore - ln = line.decode() - if ln.startswith("UUID: "): - uuid = ln.split(" ")[-1] - log.debug(f"found uuid: {uuid}") - - out, err = p.communicate() - res = p.wait(), out, err, uuid - - if res[0] > 0: - log.error(f"Problem while executing command: '{prepared}'") - if search and not res[3]: - log.error("UUID not found in stdout. Dumping stdout and stderr") - self.log_subprocess(res) - - return res - - def prepare_command(self, command_list: CMD_PARAM_T) -> Tuple[List[str],List[None]]: - """ - Commands may be a callable, which should be a lambda to be evaluated at - preparation time with available locals. This can be used to, among - other things, perform lazy evaluations of f-strings which have values - not available at assignment time. e.g., filling in a second command - with a value extracted from the previous step or command. - - """ - - r = [] - return r, [r.append(c()) if (callable(c) and c.__name__ == '') else r.append(str(c)) for c in command_list] - - def log_subprocess(self, result: Tuple[int, Union[bytes, None], Union[bytes, None], Union[str, None]]): - def log_lines(title, lines): - log.info(f"====={title}=====") - log.info(lines.decode()) - log.info(f"Command return code: {result[0]}") - stdout = result[1] - stderr = result[2] - if stdout: - log_lines("Command STDOUT", stdout) - if stderr: - log_lines("Command STDERR", stderr) - - def fix_ks(self): - cmd: CMD_PARAM_T = ["sed", "-i", f"s,$basearch,{self.architecture.name},", str(self.kickstart_path)] - self.runCmd(cmd, search=False) - - def render_kubernetes_job(self): - commands = [self.build_command(), self.package_command(), self.copy_command()] - if not self.job_template: - return None - template = self.job_template.render( - architecture=self.architecture.name, - backoffLimit=4, - buildTime=BUILDTIME.strftime("%s"), - command=commands, - imageName="ghcr.io/rockylinux/sig-core-toolkit:latest", - jobname="buildimage", - namespace="empanadas", - major=major, - restartPolicy="Never", - ) - return template - - def save(self): - with open(self.metadata, "w") as f: - try: - o = { name: getattr(self, name) for name in ["base_uuid", "target_uuid"] } - log.debug(o) - json.dump(o, f) - except AttributeError as e: - log.error("Couldn't find attribute in object. Something is probably wrong", e) - except Exception as e: - log.exception(e) - finally: - f.flush() def run(): try: @@ -459,28 +59,29 @@ def run(): log.exception(e) exit(2) - file_loader = FileSystemLoader(f"{_rootdir}/templates") - tmplenv = Environment(loader=file_loader) - tdl_template = tmplenv.get_template('icicle/tdl.xml.tmpl') - arches = rlvars['allowed_arches'] if results.kube else [platform.uname().machine] for architecture in arches: + backend = ImageFactoryBackend( + kickstart_dir="kickstart" if results.kickstartdir else "os", + kickstart_repo=rlvars['livemap']['git_repo'] + ) IB = ImageBuild( architecture=Architecture.from_version(architecture, rlvars['revision']), - cli_args=results, debug=results.debug, - image_type=results.type, + image_type=results.type, release=results.release if results.release else 0, - template=tdl_template, variant=results.variant, + build_time=datetime.datetime.utcnow(), + backend=backend, + log=log, ) - if results.kube: - IB.job_template = tmplenv.get_template('kube/Job.tmpl') - #commands = IB.kube_commands() - print(IB.render_kubernetes_job()) - else: - ret = IB.build() - ret = IB.package() - ret = IB.copy() + if results.kube: + # commands = IB.kube_commands() + print(IB.render_kubernetes_job()) + sys.exit(0) + + IB.backend.prepare() + IB.backend.build() + IB.backend.clean() diff --git a/iso/empanadas/pyproject.toml b/iso/empanadas/pyproject.toml index 184946e..3dedc50 100644 --- a/iso/empanadas/pyproject.toml +++ b/iso/empanadas/pyproject.toml @@ -20,6 +20,7 @@ attrs = "^23.1.0" [tool.poetry.dev-dependencies] pytest = "~5" +attrs = "^23.1.0" [tool.poetry.scripts] test_module = "empanadas.scripts.test_module:run" @@ -38,6 +39,16 @@ generate_compose = "empanadas.scripts.generate_compose:run" peridot_repoclosure = "empanadas.scripts.peridot_repoclosure:run" refresh_all_treeinfo = "empanadas.scripts.refresh_all_treeinfo:run" +[tool.pylint.main] +init-hook =""" +try: + import pylint_venv +except ImportError: + pass +else: + pylint_venv.inithook() +""" + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/iso/empanadas/tox.ini b/iso/empanadas/tox.ini new file mode 100644 index 0000000..e9587a9 --- /dev/null +++ b/iso/empanadas/tox.ini @@ -0,0 +1,2 @@ +[pycodestyle] +max-line-length = 160