0a91c4077b
The contents of TMT_PLAN_DATA does not get exposed in the CI artifacts output UI, whereas TMT_TEST_DATA does.
304 lines
7.4 KiB
Python
Executable File
304 lines
7.4 KiB
Python
Executable File
#!/usr/bin/python3
|
|
|
|
import time
|
|
import os
|
|
import subprocess
|
|
import boto3
|
|
import io
|
|
import json
|
|
import click
|
|
import sys
|
|
import requests
|
|
from datetime import date
|
|
import shutil
|
|
import xml.etree.ElementTree as ET
|
|
|
|
|
|
def fail(message):
|
|
click.echo(message, err=True)
|
|
sys.exit(1)
|
|
|
|
|
|
tree = ET.parse("config.xml")
|
|
for e in tree.getroot().iter("release-version"):
|
|
RELEASE = e.text
|
|
break
|
|
|
|
|
|
if not RELEASE:
|
|
fail("Could not parse release from config.xml")
|
|
|
|
if os.path.exists("buildver"):
|
|
with open("buildver", "r") as f:
|
|
BUILDVER = f.read().strip()
|
|
else:
|
|
BUILDVER = date.today().strftime("%Y%m%d%H%m")
|
|
with open("buildver", "w") as f:
|
|
f.write(BUILDVER)
|
|
|
|
# TODO: should be a class using abc
|
|
TARGETS = {
|
|
"kde": {
|
|
"profile": "Workstation-KDE",
|
|
"name": f"Fedora Linux {RELEASE.capitalize()} with KDE Plasma ({BUILDVER})",
|
|
"os_name": "Fedora Linux with KDE Plasma",
|
|
"id": "kde",
|
|
},
|
|
"gnome": {
|
|
"profile": "Workstation-GNOME",
|
|
"name": f"Fedora Linux {RELEASE.capitalize()} with GNOME ({BUILDVER})",
|
|
"os_name": "Fedora Linux with GNOME",
|
|
"id": "gnome",
|
|
},
|
|
"server": {
|
|
"profile": "Server",
|
|
"name": f"Fedora Linux {RELEASE.capitalize()} Server ({BUILDVER})",
|
|
"os_name": "Fedora Linux Server",
|
|
"id": "server",
|
|
},
|
|
"minimal": {
|
|
"profile": "Minimal",
|
|
"name": f"Fedora Linux {RELEASE.capitalize()} Minimal ({BUILDVER})",
|
|
"os_name": "Fedora Linux Minimal",
|
|
"id": "minimal",
|
|
},
|
|
}
|
|
|
|
MANIFEST = os.getenv("MANIFEST", "installer_data.json")
|
|
S3_BUCKET = os.getenv("S3_BUCKET")
|
|
|
|
|
|
def requireCommands(commands):
|
|
missing = []
|
|
for cmd in commands:
|
|
if shutil.which(cmd) is None:
|
|
missing.append(cmd)
|
|
|
|
if len(missing) > 0:
|
|
fail("Required commands not found: {}".format(" ".join(missing)))
|
|
|
|
|
|
def runCommand(cmd, shell=False):
|
|
p = subprocess.run(
|
|
cmd,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
universal_newlines=True,
|
|
shell=shell,
|
|
)
|
|
if p.returncode != 0:
|
|
fail(p.stdout)
|
|
|
|
return p
|
|
|
|
|
|
def kiwiBuild(profile):
|
|
requireCommands(["kiwi-ng"])
|
|
|
|
# TODO: check current status first
|
|
# runCommand(
|
|
# [
|
|
# "sudo",
|
|
# "setenforce",
|
|
# "0",
|
|
# ]
|
|
# )
|
|
# TODO: use the API instead of shelling out
|
|
runCommand(
|
|
[
|
|
"sudo",
|
|
"kiwi-ng",
|
|
"--debug",
|
|
"--type=oem",
|
|
f"--profile={profile}",
|
|
"system",
|
|
"build",
|
|
"--description",
|
|
"./",
|
|
"--target-dir",
|
|
"./outdir",
|
|
]
|
|
)
|
|
# runCommand(
|
|
# [
|
|
# "sudo",
|
|
# "setenforce",
|
|
# "1",
|
|
# ]
|
|
# )
|
|
|
|
|
|
def packageBuild(target):
|
|
# TODO: rewrite in python instead of shelling out
|
|
runCommand(["./make-asahi-installer-package.sh"])
|
|
|
|
base = f"fedora-{RELEASE}-{target['id']}-{BUILDVER}"
|
|
os.rename(f"fedora-{RELEASE}-{BUILDVER}.zip", f"{base}.zip")
|
|
os.rename(f"fedora-{RELEASE}-{BUILDVER}.logs.zip", f"{base}.logs.zip")
|
|
os.rename(f"fedora-{RELEASE}-{BUILDVER}.raw.zst", f"{base}.raw.zst")
|
|
os.rename(f"fedora-{RELEASE}-{BUILDVER}.json", f"{base}.json")
|
|
with open(f"{base}.json", "r") as f:
|
|
data = json.load(f)
|
|
|
|
data["name"] = target["name"]
|
|
data["default_os_name"] = target["os_name"]
|
|
data["package"] = f"{base}.zip"
|
|
# mark rawhide builds as "expert", they are not supported
|
|
if RELEASE == "rawhide":
|
|
data["expert"] = True
|
|
|
|
# Clobber extras as we don't want them on headless builds and
|
|
# make-asahi-installer-package.sh doesn't have awareness of the target
|
|
if target["id"] == "server" or target["id"] == "minimal":
|
|
data["extras"] = {}
|
|
|
|
with open(f"{base}.json", "w") as f:
|
|
json.dump(data, f)
|
|
|
|
if os.path.exists("installer_data.json"):
|
|
with open("installer_data.json", "r") as f:
|
|
data = json.load(f)
|
|
|
|
data["os_list"][0]["name"] = target["name"]
|
|
data["os_list"][0]["default_os_name"] = target["os_name"]
|
|
data["os_list"][0]["package"] = f"{base}.zip"
|
|
|
|
with open("installer_data.json", "w") as f:
|
|
json.dump(data, f)
|
|
|
|
|
|
def uploadToS3(source, destination):
|
|
if S3_BUCKET is None:
|
|
fail("S3_BUCKET is not set")
|
|
|
|
s3 = boto3.client("s3")
|
|
s3.upload_file(source, S3_BUCKET, destination)
|
|
|
|
|
|
def packageUpload(target):
|
|
base = f"fedora-{RELEASE}-{target['id']}-{BUILDVER}"
|
|
package = f"{base}.zip"
|
|
logs_package = f"{base}.logs.zip"
|
|
image = f"{base}.raw.zst"
|
|
manifest = f"{base}.json"
|
|
|
|
uploadToS3(package, f"os/{package}")
|
|
uploadToS3(logs_package, f"os/{logs_package}")
|
|
uploadToS3(image, f"os/{image}")
|
|
uploadToS3(manifest, f"os/{manifest}")
|
|
|
|
|
|
def tmtCopy(target):
|
|
base = f"fedora-{RELEASE}-{target['id']}-{BUILDVER}"
|
|
package = f"{base}.zip"
|
|
logs_package = f"{base}.logs.zip"
|
|
image = f"{base}.raw.zst"
|
|
manifest = f"{base}.json"
|
|
|
|
artifacts = os.getenv("TMT_TEST_DATA")
|
|
if not os.path.exists(artifacts):
|
|
os.mkdir(artifacts)
|
|
|
|
shutil.copyfile(package, f"{artifacts}/{package}")
|
|
shutil.copyfile(logs_package, f"{artifacts}/{logs_package}")
|
|
shutil.copyfile(image, f"{artifacts}/{image}")
|
|
shutil.copyfile(manifest, f"{artifacts}/{manifest}")
|
|
|
|
|
|
def getManifest():
|
|
if S3_BUCKET is None:
|
|
fail("S3_BUCKET is not set")
|
|
|
|
s3 = boto3.resource("s3")
|
|
obj = s3.Object(S3_BUCKET, MANIFEST)
|
|
data = io.BytesIO()
|
|
obj.download_fileobj(data)
|
|
|
|
return json.loads(data.getvalue().decode("utf-8"))
|
|
|
|
|
|
def manifestUpdate():
|
|
old = getManifest()
|
|
with open("installer_data.json", "r") as f:
|
|
new = json.load(f)
|
|
|
|
old["os_list"].insert(0, new["os_list"][0])
|
|
with open("merged_installer_data.json", "w") as f:
|
|
json.dump(old, f)
|
|
|
|
|
|
@click.group()
|
|
def cli():
|
|
pass
|
|
|
|
|
|
@cli.command()
|
|
@click.argument("target")
|
|
def build(target):
|
|
if target not in TARGETS.keys():
|
|
fail(f"Unknown target: {target}")
|
|
|
|
target = TARGETS[target]
|
|
kiwiBuild(target["profile"])
|
|
|
|
|
|
@cli.command()
|
|
@click.argument("target")
|
|
def package(target):
|
|
if target not in TARGETS.keys():
|
|
fail(f"Unknown target: {target}")
|
|
|
|
target = TARGETS[target]
|
|
packageBuild(target)
|
|
|
|
|
|
@cli.command()
|
|
@click.option("--manifest/--no-manifest", default=True)
|
|
@click.argument("target")
|
|
def upload(manifest, target):
|
|
if target not in TARGETS.keys():
|
|
fail(f"Unknown target: {target}")
|
|
|
|
target = TARGETS[target]
|
|
|
|
packageUpload(target)
|
|
if manifest:
|
|
manifestUpdate()
|
|
uploadToS3("merged_installer_data.json", MANIFEST)
|
|
|
|
|
|
@cli.command()
|
|
@click.argument("target")
|
|
def tmt(target):
|
|
if target not in TARGETS.keys():
|
|
fail(f"Unknown target: {target}")
|
|
|
|
target = TARGETS[target]
|
|
|
|
tmtCopy(target)
|
|
|
|
|
|
@cli.command()
|
|
@click.option("--update/--no-update", default=False)
|
|
@click.option("--upload/--no-upload", default=False)
|
|
def manifest(update, upload):
|
|
if update:
|
|
manifestUpdate()
|
|
manifest = "merged_installer_data.json"
|
|
if upload:
|
|
uploadToS3(manifest, MANIFEST)
|
|
else:
|
|
manifest = getManifest()
|
|
|
|
click.echo(json.dumps(manifest, indent=2))
|
|
|
|
|
|
@cli.command()
|
|
def clean():
|
|
runCommand(["sudo", "rm", "-rf", "outdir"])
|
|
runCommand(["rm", "-f", "installer_data.json", "merged_installer_data.json"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cli()
|