asahi-kiwi-descriptions/builder.py

#!/usr/bin/python3

import time
import os
import subprocess
import boto3
import io
import json
import click
import sys
import requests
from datetime import date
import shutil
import xml.etree.ElementTree as ET


def fail(message):
    click.echo(message, err=True)
    sys.exit(1)


tree = ET.parse("config.xml")
for e in tree.getroot().iter("release-version"):
    RELEASE = e.text
    break


if not RELEASE:
    fail("Could not parse release from config.xml")

if os.path.exists("buildver"):
    with open("buildver", "r") as f:
        BUILDVER = f.read().strip()
else:
    BUILDVER = date.today().strftime("%Y%m%d%H%m")
    with open("buildver", "w") as f:
        f.write(BUILDVER)

# TODO: should be a class using abc
TARGETS = {
    "kde": {
        "profile": "Workstation-KDE",
        "name": f"Fedora Linux {RELEASE.capitalize()} with KDE Plasma ({BUILDVER})",
        "os_name": "Fedora Linux with KDE Plasma",
        "id": "kde",
    },
    "gnome": {
        "profile": "Workstation-GNOME",
        "name": f"Fedora Linux {RELEASE.capitalize()} with GNOME ({BUILDVER})",
        "os_name": "Fedora Linux with GNOME",
        "id": "gnome",
    },
    "server": {
        "profile": "Server",
        "name": f"Fedora Linux {RELEASE.capitalize()} Server ({BUILDVER})",
        "os_name": "Fedora Linux Server",
        "id": "server",
    },
    "minimal": {
        "profile": "Minimal",
        "name": f"Fedora Linux {RELEASE.capitalize()} Minimal ({BUILDVER})",
        "os_name": "Fedora Linux Minimal",
        "id": "minimal",
    },
}

MANIFEST = os.getenv("MANIFEST", "installer_data.json")
S3_BUCKET = os.getenv("S3_BUCKET")


def requireCommands(commands):
    missing = []
    for cmd in commands:
        if shutil.which(cmd) is None:
            missing.append(cmd)

    if len(missing) > 0:
        fail("Required commands not found: {}".format(" ".join(missing)))


def runCommand(cmd, shell=False):
    p = subprocess.run(
        cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        universal_newlines=True,
        shell=shell,
    )
    if p.returncode != 0:
        fail(p.stdout)

    return p


def kiwiBuild(profile):
    requireCommands(["kiwi-ng"])

    # TODO: check current status first
    # runCommand(
    #    [
    #        "sudo",
    #        "setenforce",
    #        "0",
    #    ]
    # )
    # TODO: use the API instead of shelling out
    runCommand(
        [
            "sudo",
            "kiwi-ng",
            "--debug",
            "--type=oem",
            f"--profile={profile}",
            "system",
            "build",
            "--description",
            "./",
            "--target-dir",
            "./outdir",
        ]
    )
    # runCommand(
    #    [
    #        "sudo",
    #        "setenforce",
    #        "1",
    #    ]
    # )


def packageBuild(target):
    # TODO: rewrite in python instead of shelling out
    runCommand(["./make-asahi-installer-package.sh"])

    base = f"fedora-{RELEASE}-{target['id']}-{BUILDVER}"
    os.rename(f"fedora-{RELEASE}-{BUILDVER}.zip", f"{base}.zip")
    os.rename(f"fedora-{RELEASE}-{BUILDVER}.logs.zip", f"{base}.logs.zip")
    os.rename(f"fedora-{RELEASE}-{BUILDVER}.raw.zst", f"{base}.raw.zst")
    os.rename(f"fedora-{RELEASE}-{BUILDVER}.json", f"{base}.json")
    with open(f"{base}.json", "r") as f:
        data = json.load(f)

    data["name"] = target["name"]
    data["default_os_name"] = target["os_name"]
    data["package"] = f"{base}.zip"
    # Clobber extras as we don't want them on headless builds and
    # make-asahi-installer-package.sh doesn't have awareness of the target
    if target["id"] == "server" or target["id"] == "minimal":
        data["extras"] = {}

    with open(f"{base}.json", "w") as f:
        json.dump(data, f)

    if os.path.exists("installer_data.json"):
        with open("installer_data.json", "r") as f:
            data = json.load(f)

        data["os_list"][0]["name"] = target["name"]
        data["os_list"][0]["default_os_name"] = target["os_name"]
        data["os_list"][0]["package"] = f"{base}.zip"

        with open("installer_data.json", "w") as f:
            json.dump(data, f)


def uploadToS3(source, destination):
    if S3_BUCKET is None:
        fail("S3_BUCKET is not set")

    s3 = boto3.client("s3")
    s3.upload_file(source, S3_BUCKET, destination)


def packageUpload(target):
    base = f"fedora-{RELEASE}-{target['id']}-{BUILDVER}"
    package = f"{base}.zip"
    logs_package = f"{base}.logs.zip"
    image = f"{base}.raw.zst"
    manifest = f"{base}.json"

    uploadToS3(package, f"os/{package}")
    uploadToS3(logs_package, f"os/{logs_package}")
    uploadToS3(image, f"os/{image}")
    uploadToS3(manifest, f"os/{manifest}")


def getManifest():
    if S3_BUCKET is None:
        fail("S3_BUCKET is not set")

    s3 = boto3.resource("s3")
    obj = s3.Object(S3_BUCKET, MANIFEST)
    data = io.BytesIO()
    obj.download_fileobj(data)

    return json.loads(data.getvalue().decode("utf-8"))


def manifestUpdate():
    old = getManifest()
    with open("installer_data.json", "r") as f:
        new = json.load(f)

    old["os_list"].insert(0, new["os_list"][0])
    with open("merged_installer_data.json", "w") as f:
        json.dump(old, f)


@click.group()
def cli():
    pass


@cli.command()
@click.argument("target")
def build(target):
    if target not in TARGETS.keys():
        fail(f"Unknown target: {target}")

    target = TARGETS[target]
    kiwiBuild(target["profile"])


@cli.command()
@click.argument("target")
def package(target):
    if target not in TARGETS.keys():
        fail(f"Unknown target: {target}")

    target = TARGETS[target]
    packageBuild(target)


@cli.command()
@click.option("--manifest/--no-manifest", default=True)
@click.argument("target")
def upload(manifest, target):
    if target not in TARGETS.keys():
        fail(f"Unknown target: {target}")

    target = TARGETS[target]

    packageUpload(target)
    if manifest:
        manifestUpdate()
        uploadToS3("merged_installer_data.json", MANIFEST)


@cli.command()
@click.option("--update/--no-update", default=False)
@click.option("--upload/--no-upload", default=False)
def manifest(update, upload):
    if update:
        manifestUpdate()
        manifest = "merged_installer_data.json"
        if upload:
            uploadToS3(manifest, MANIFEST)
    else:
        manifest = getManifest()

    click.echo(json.dumps(manifest, indent=2))


@cli.command()
def clean():
    runCommand(["sudo", "rm", "-rf", "outdir"])
    runCommand(["rm", "-f", "installer_data.json", "merged_installer_data.json"])


if __name__ == "__main__":
    cli()