#!/usr/bin/python3 import time import os import subprocess import boto3 import io import json import click import sys import requests from datetime import date import shutil import xml.etree.ElementTree as ET def fail(message): click.echo(message, err=True) sys.exit(1) tree = ET.parse("config.xml") for e in tree.getroot().iter("release-version"): RELEASE = e.text break if not RELEASE: fail("Could not parse release from config.xml") if os.path.exists("buildver"): with open("buildver", "r") as f: BUILDVER = f.read().strip() else: BUILDVER = date.today().strftime("%Y%m%d%H%m") with open("buildver", "w") as f: f.write(BUILDVER) # TODO: should be a class using abc TARGETS = { "kde": { "profile": "Workstation-KDE", "name": f"Fedora Linux {RELEASE.capitalize()} with KDE Plasma ({BUILDVER})", "os_name": "Fedora Linux with KDE Plasma", "id": "kde", }, "gnome": { "profile": "Workstation-GNOME", "name": f"Fedora Linux {RELEASE.capitalize()} with GNOME ({BUILDVER})", "os_name": "Fedora Linux with GNOME", "id": "gnome", }, "server": { "profile": "Server", "name": f"Fedora Linux {RELEASE.capitalize()} Server ({BUILDVER})", "os_name": "Fedora Linux Server", "id": "server", }, "minimal": { "profile": "Minimal", "name": f"Fedora Linux {RELEASE.capitalize()} Minimal ({BUILDVER})", "os_name": "Fedora Linux Minimal", "id": "minimal", }, } MANIFEST = os.getenv("MANIFEST", "installer_data.json") S3_BUCKET = os.getenv("S3_BUCKET") def requireCommands(commands): missing = [] for cmd in commands: if shutil.which(cmd) is None: missing.append(cmd) if len(missing) > 0: fail("Required commands not found: {}".format(" ".join(missing))) def runCommand(cmd, shell=False): p = subprocess.run( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, shell=shell, ) if p.returncode != 0: fail(p.stdout) return p def kiwiBuild(profile): requireCommands(["kiwi-ng"]) # TODO: check current status first # runCommand( # [ # "sudo", # "setenforce", # "0", # ] # ) # TODO: use the API instead of shelling out runCommand( [ "sudo", "kiwi-ng", "--debug", "--type=oem", f"--profile={profile}", "system", "build", "--description", "./", "--target-dir", "./outdir", ] ) # runCommand( # [ # "sudo", # "setenforce", # "1", # ] # ) def packageBuild(target): base = f"fedora-{RELEASE}-{target['id']}-{BUILDVER}" os.rename(f"fedora-{RELEASE}-{BUILDVER}.zip", f"{base}.zip") os.rename(f"fedora-{RELEASE}-{BUILDVER}.logs.zip", f"{base}.logs.zip") os.rename(f"fedora-{RELEASE}-{BUILDVER}.raw.zst", f"{base}.raw.zst") os.rename(f"fedora-{RELEASE}-{BUILDVER}.json", f"{base}.json") with open(f"{base}.json", "r") as f: data = json.load(f) data["name"] = target["name"] data["default_os_name"] = target["os_name"] data["package"] = f"{base}.zip" # mark rawhide builds as "expert", they are not supported if RELEASE == "rawhide": data["expert"] = True # Clobber extras as we don't want them on headless builds and # make-asahi-installer-package.sh doesn't have awareness of the target if target["id"] == "server" or target["id"] == "minimal": data["extras"] = {} with open(f"{base}.json", "w") as f: json.dump(data, f) if os.path.exists("installer_data.json"): with open("installer_data.json", "r") as f: data = json.load(f) data["os_list"][0]["name"] = target["name"] data["os_list"][0]["default_os_name"] = target["os_name"] data["os_list"][0]["package"] = f"{base}.zip" with open("installer_data.json", "w") as f: json.dump(data, f) def uploadToS3(source, destination): if S3_BUCKET is None: fail("S3_BUCKET is not set") s3 = boto3.client("s3") s3.upload_file(source, S3_BUCKET, destination) def packageUpload(target): base = f"fedora-{RELEASE}-{target['id']}-{BUILDVER}" package = f"{base}.zip" logs_package = f"{base}.logs.zip" image = f"{base}.raw.zst" manifest = f"{base}.json" uploadToS3(package, f"os/{package}") uploadToS3(logs_package, f"os/{logs_package}") uploadToS3(image, f"os/{image}") uploadToS3(manifest, f"os/{manifest}") def tmtCopy(target): base = f"fedora-{RELEASE}-{target['id']}-{BUILDVER}" package = f"{base}.zip" logs_package = f"{base}.logs.zip" image = f"{base}.raw.zst" manifest = f"{base}.json" artifacts = os.getenv("TMT_TEST_DATA") if not os.path.exists(artifacts): os.mkdir(artifacts) shutil.copyfile(package, f"{artifacts}/{package}") shutil.copyfile(logs_package, f"{artifacts}/{logs_package}") shutil.copyfile(image, f"{artifacts}/{image}") shutil.copyfile(manifest, f"{artifacts}/{manifest}") def getManifest(): if S3_BUCKET is None: fail("S3_BUCKET is not set") s3 = boto3.resource("s3") obj = s3.Object(S3_BUCKET, MANIFEST) data = io.BytesIO() obj.download_fileobj(data) return json.loads(data.getvalue().decode("utf-8")) def manifestUpdate(): old = getManifest() with open("installer_data.json", "r") as f: new = json.load(f) old["os_list"].insert(0, new["os_list"][0]) with open("merged_installer_data.json", "w") as f: json.dump(old, f) @click.group() def cli(): pass @cli.command() @click.argument("target") def build(target): if target not in TARGETS.keys(): fail(f"Unknown target: {target}") target = TARGETS[target] kiwiBuild(target["profile"]) @cli.command() @click.argument("target") def package(target): if target not in TARGETS.keys(): fail(f"Unknown target: {target}") target = TARGETS[target] packageBuild(target) @cli.command() @click.option("--manifest/--no-manifest", default=True) @click.argument("target") def upload(manifest, target): if target not in TARGETS.keys(): fail(f"Unknown target: {target}") target = TARGETS[target] packageUpload(target) if manifest: manifestUpdate() uploadToS3("merged_installer_data.json", MANIFEST) @cli.command() @click.argument("target") def tmt(target): if target not in TARGETS.keys(): fail(f"Unknown target: {target}") target = TARGETS[target] tmtCopy(target) @cli.command() @click.option("--update/--no-update", default=False) @click.option("--upload/--no-upload", default=False) def manifest(update, upload): if update: manifestUpdate() manifest = "merged_installer_data.json" if upload: uploadToS3(manifest, MANIFEST) else: manifest = getManifest() click.echo(json.dumps(manifest, indent=2)) @cli.command() def clean(): runCommand(["sudo", "rm", "-rf", "outdir"]) runCommand(["rm", "-f", "installer_data.json", "merged_installer_data.json"]) if __name__ == "__main__": cli()