Merge branch 'feature/iso-kube' into 'devel'

* 76dba704 - Build a container to run the builds in * 4587287e - Github actions work for automatic building * 06036207 - Fixes for current RC images * 7a097fb3 - Ensure builds work properly - architectures must match See merge request release-engineering/public/toolkit!45
2022-06-20 22:04:28 +00:00 · 2022-06-20 22:04:28 +00:00 · 3251a7bc30
parent 22f920042e 7a097fb302
commit 3251a7bc30
11 changed files with 340 additions and 23 deletions
--- a/.github/workflows/mix-empanadas.yml
+++ b/.github/workflows/mix-empanadas.yml
@ -0,0 +1,47 @@
+---
+name: Build empanada container images
+
+on:
+  push:
+    branches: [ $default-branch ]
+  pull_request:
+    branches: [ $default-branch ]
+  workflow_dispatch:
+
+jobs:
+  buildx:
+    runs-on: 
+      - ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      # https://github.com/docker/setup-buildx-action
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          install: true
+
+      - name: Login to ghcr
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          builder: ${{ steps.buildx.outputs.name }}
+          platforms: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le
+          context: ./iso/empanadas
+          file: ./iso/empanadas/Containerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ghcr.io/neilhanlon/sig-core-toolkit:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
--- a/iso/empanadas/Containerfile
+++ b/iso/empanadas/Containerfile
@ -0,0 +1,66 @@
+FROM quay.io/centos/centos:stream9
+
+ADD images/get_arch /get_arch
+
+ENV TINI_VERSION v0.19.0
+RUN curl -o /tini -L "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-$(/get_arch)"
+RUN chmod +x /tini
+
+RUN rm -rf /etc/yum.repos.d/*.repo
+ADD images/epelkey.gpg /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-9
+ADD images/rhel.repo /etc/yum.repos.d/rhel.repo
+
+RUN dnf update -y && dnf install -y \
+    bash \
+    bzip2 \
+    cpio \
+    diffutils \
+    findutils \
+    gawk \
+    gcc \
+    gcc-c++ \
+    git \
+    grep \
+    gzip \
+    info \
+    make \
+    patch \
+    python3 \
+    redhat-rpm-config \
+    rpm-build \
+    scl-utils-build \
+    sed \
+    shadow-utils \
+    tar \
+    unzip \
+    util-linux \
+    which \
+    xz \
+    dnf-plugins-core \
+    createrepo_c \
+    rpm-sign \
+    sudo \
+    mock \
+    python-pip \
+    genisoimage \
+    isomd5sum \
+    lorax \
+    lorax-templates-rhel \
+    lorax-templates-generic
+
+RUN sed -i '/libreport-rhel-anaconda-bugzilla/ s/^/#/' /usr/share/lorax/templates.d/80-rhel/runtime-install.tmpl
+
+RUN ssh-keygen -t rsa -q -f "$HOME/.ssh/id_rsa" -N ""
+RUN dnf clean all
+RUN rm -rf /etc/yum.repos.d/*.repo
+RUN useradd -o -d /var/peridot -u 1002 peridotbuilder && usermod -a -G mock peridotbuilder
+RUN chown peridotbuilder:mock /etc/yum.conf && chown -R peridotbuilder:mock /etc/dnf && chown -R peridotbuilder:mock /etc/rpm && chown -R peridotbuilder:mock /etc/yum.repos.d
+
+RUN pip install 'git+https://git.rockylinux.org/release-engineering/public/toolkit.git@feature/iso-kube#egg=empanadas&subdirectory=iso/empanadas'
+
+RUN pip install awscli
+
+ENV USER=1002
+USER 1002
+
+ENTRYPOINT ["/tini", "--"]
--- a/iso/empanadas/build-container.sh
+++ b/iso/empanadas/build-container.sh
@ -0,0 +1,14 @@
+#!/bin/bash
+
+MANIFEST_NAME="peridotempanadas"
+BUILD_PATH="."
+REGISTRY="docker.io"
+USER="neilresf"
+IMAGE_TAG="v0.1.0"
+IMAGE_NAME="peridotempanadas"
+
+podman buildx build \
+  --platform linux/amd64,linux/arm64,linux/s390x,linux/ppc64le \
+  --tag "${REGISTRY}/${USER}/${IMAGE_NAME}:${IMAGE_TAG}" \
+  $PWD
+
--- a/iso/empanadas/empanadas/scripts/launch_builds.py
+++ b/iso/empanadas/empanadas/scripts/launch_builds.py
@ -0,0 +1,48 @@
+# Launches the builds of ISOs
+
+import argparse
+import datetime
+
+from empanadas.common import *
+from empanadas.common import _rootdir
+
+from jinja2 import Environment, FileSystemLoader
+
+parser = argparse.ArgumentParser(description="ISO Compose")
+
+parser.add_argument('--release', type=str, help="Major Release Version", required=True)
+parser.add_argument('--env', type=str, help="environment", required=True)
+results = parser.parse_args()
+rlvars = rldict[results.release]
+major = rlvars['major']
+
+EXTARCH=["s390x", "ppc64le"]
+EKSARCH=["amd64", "arm64"]
+
+def run():
+    file_loader = FileSystemLoader(f"{_rootdir}/templates")
+    tmplenv = Environment(loader=file_loader)
+    job_template = tmplenv.get_template('kube/Job.tmpl')
+
+    arches = EKSARCH
+    if results.env == "ext" and results.env != "all":
+        arches = EXTARCH
+    elif results.env == "all":
+        arches = EKSARCH+EXTARCH
+
+    command = ["build-iso", "--release", f"{results.release}", "--rc", "--isolation", "simple"]
+
+    out = ""
+    for arch in arches:
+        out += job_template.render(
+            architecture=arch,
+            backoffLimit=4,
+            buildTime=datetime.datetime.utcnow().strftime("%s"),
+            command=command,
+            imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest",
+            namespace="empanadas",
+            major=major,
+            restartPolicy="Never",
+        )
+
+    print(out)
--- a/iso/empanadas/empanadas/templates/kube/Job.tmpl
+++ b/iso/empanadas/empanadas/templates/kube/Job.tmpl
@ -0,0 +1,61 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: build-iso-{{ major }}-{{ architecture }}
+  namespace: {{ namespace }}
+spec:
+  template:
+    metadata:
+      labels:
+        peridot.rockylinux.org/workflow-tolerates-arch: {{ architecture }}
+    spec:
+      containers:
+      - name: buildiso-{{ major }}-{{ architecture }}
+        image: {{ imageName }}
+        command: ["/bin/bash", "-c"]
+        args:
+          - |
+            {{ command | join(' ') }}
+            aws s3 cp --recursive --exclude=* --include=lorax* \
+            /var/lib/mock/rocky-{{ major }}-$(uname -m)/root/builddir/ \
+            "s3://resf-empanadas/buildiso-{{ major }}-{{ architecture }}/{{ buildTime }}/"
+        securityContext:
+          runAsUser: 0
+          runAsGroup: 0
+          privileged: true
+          runAsNonRoot: false
+          allowPrivilegeEscalation: true
+        volumeMounts:
+          - mountPath: /etc/resolv.conf
+            name: resolv-conf
+          - mountPath: /var/lib/mock/
+            name: mock
+        env:
+        - name: AWS_REGION
+          value: us-east-2
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: empanadas-s3
+              key: ID
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: empanadas-s3
+              key: SECRET
+      tolerations:
+      - effect: NoSchedule
+        key: peridot.rockylinux.org/workflow-tolerates-arch
+        operator: Equal
+        value: {{ architecture }}
+      restartPolicy: {{ restartPolicy }}
+      volumes:
+      - name: resolv-conf
+        hostPath:
+          path: /etc/resolv.conf
+          type: File
+      - name: mock
+        emptyDir: {}
+  backoffLimit: {{ backoffLimit }}
+
--- a/iso/empanadas/images/epelkey.gpg
+++ b/iso/empanadas/images/epelkey.gpg
@ -0,0 +1,29 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBGE3mOsBEACsU+XwJWDJVkItBaugXhXIIkb9oe+7aadELuVo0kBmc3HXt/Yp
+CJW9hHEiGZ6z2jwgPqyJjZhCvcAWvgzKcvqE+9i0NItV1rzfxrBe2BtUtZmVcuE6
+2b+SPfxQ2Hr8llaawRjt8BCFX/ZzM4/1Qk+EzlfTcEcpkMf6wdO7kD6ulBk/tbsW
+DHX2lNcxszTf+XP9HXHWJlA2xBfP+Dk4gl4DnO2Y1xR0OSywE/QtvEbN5cY94ieu
+n7CBy29AleMhmbnx9pw3NyxcFIAsEZHJoU4ZW9ulAJ/ogttSyAWeacW7eJGW31/Z
+39cS+I4KXJgeGRI20RmpqfH0tuT+X5Da59YpjYxkbhSK3HYBVnNPhoJFUc2j5iKy
+XLgkapu1xRnEJhw05kr4LCbud0NTvfecqSqa+59kuVc+zWmfTnGTYc0PXZ6Oa3rK
+44UOmE6eAT5zd/ToleDO0VesN+EO7CXfRsm7HWGpABF5wNK3vIEF2uRr2VJMvgqS
+9eNwhJyOzoca4xFSwCkc6dACGGkV+CqhufdFBhmcAsUotSxe3zmrBjqA0B/nxIvH
+DVgOAMnVCe+Lmv8T0mFgqZSJdIUdKjnOLu/GRFhjDKIak4jeMBMTYpVnU+HhMHLq
+uDiZkNEvEEGhBQmZuI8J55F/a6UURnxUwT3piyi3Pmr2IFD7ahBxPzOBCQARAQAB
+tCdGZWRvcmEgKGVwZWw5KSA8ZXBlbEBmZWRvcmFwcm9qZWN0Lm9yZz6JAk4EEwEI
+ADgWIQT/itE0RZcQbs6BO5GKOHK/MihGfAUCYTeY6wIbDwULCQgHAgYVCgkICwIE
+FgIDAQIeAQIXgAAKCRCKOHK/MihGfFX/EACBPWv20+ttYu1A5WvtHJPzwbj0U4yF
+3zTQpBglQ2UfkRpYdipTlT3Ih6j5h2VmgRPtINCc/ZE28adrWpBoeFIS2YAKOCLC
+nZYtHl2nCoLq1U7FSttUGsZ/t8uGCBgnugTfnIYcmlP1jKKA6RJAclK89evDQX5n
+R9ZD+Cq3CBMlttvSTCht0qQVlwycedH8iWyYgP/mF0W35BIn7NuuZwWhgR00n/VG
+4nbKPOzTWbsP45awcmivdrS74P6mL84WfkghipdmcoyVb1B8ZP4Y/Ke0RXOnLhNe
+CfrXXvuW+Pvg2RTfwRDtehGQPAgXbmLmz2ZkV69RGIr54HJv84NDbqZovRTMr7gL
+9k3ciCzXCiYQgM8yAyGHV0KEhFSQ1HV7gMnt9UmxbxBE2pGU7vu3CwjYga5DpwU7
+w5wu1TmM5KgZtZvuWOTDnqDLf0cKoIbW8FeeCOn24elcj32bnQDuF9DPey1mqcvT
+/yEo/Ushyz6CVYxN8DGgcy2M9JOsnmjDx02h6qgWGWDuKgb9jZrvRedpAQCeemEd
+fhEs6ihqVxRFl16HxC4EVijybhAL76SsM2nbtIqW1apBQJQpXWtQwwdvgTVpdEtE
+r4ArVJYX5LrswnWEQMOelugUG6S3ZjMfcyOa/O0364iY73vyVgaYK+2XtT2usMux
+VL469Kj5m13T6w==
+=Mjs/
+-----END PGP PUBLIC KEY BLOCK-----
--- a/iso/empanadas/images/get_arch
+++ b/iso/empanadas/images/get_arch
@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+case "$(uname -m)" in
+    x86_64 | amd64)
+        echo -n "amd64"
+        ;;
+    arm64 | aarch64)
+        echo -n "arm64"
+        ;;
+    *)
+        echo -n "$(uname -m)"
+        ;;
+esac
--- a/iso/empanadas/images/rhel.repo
+++ b/iso/empanadas/images/rhel.repo
@ -0,0 +1,37 @@
+[baseos]
+name=CentOS Stream $releasever - BaseOS
+baseurl=https://ord.mirror.rackspace.com/centos-stream/9-stream/BaseOS/$arch/os
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
+gpgcheck=1
+repo_gpgcheck=0
+metadata_expire=6h
+countme=1
+enabled=1
+
+[appstream]
+name=CentOS Stream $releasever - AppStream
+baseurl=https://ord.mirror.rackspace.com/centos-stream/9-stream/AppStream/$arch/os
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
+gpgcheck=1
+repo_gpgcheck=0
+metadata_expire=6h
+countme=1
+enabled=1
+
+[extras-common]
+name=CentOS Stream $releasever - Extras packages
+baseurl=http://mirror.stream.centos.org/SIGs/9-stream/extras/$arch/extras-common
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Extras-SHA512
+gpgcheck=1
+repo_gpgcheck=0
+metadata_expire=6h
+countme=1
+enabled=1
+
+[epel]
+name=Extra Packages for Enterprise Linux $releasever - $basearch
+baseurl=https://download-ib01.fedoraproject.org/pub/epel/9/Everything/$arch
+enabled=1
+gpgcheck=1
+countme=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-9
--- a/iso/empanadas/images/yum-sudo
+++ b/iso/empanadas/images/yum-sudo
@ -0,0 +1,2 @@
+#!/bin/sh
+sudo yum $@
--- a/iso/empanadas/poetry.lock
+++ b/iso/empanadas/poetry.lock
@ -89,20 +89,20 @@ python-versions = ">=3.5"

 [[package]]
 name = "importlib-metadata"
-version = "4.8.3"
+version = "4.11.4"
 description = "Read metadata from Python packages"
 category = "dev"
 optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.7"

 [package.dependencies]
 typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""}
 zipp = ">=0.5"

 [package.extras]
-docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"]
+docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"]
 perf = ["ipython"]
-testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"]
+testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"]

 [[package]]
 name = "importlib-resources"
@ -203,14 +203,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"

 [[package]]
 name = "pyparsing"
-version = "3.0.7"
-description = "Python parsing module"
+version = "3.0.9"
+description = "pyparsing module - Classes and methods to define and execute parsing grammars"
 category = "dev"
 optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.6.8"

 [package.extras]
-diagrams = ["jinja2", "railroad-diagrams"]
+diagrams = ["railroad-diagrams", "jinja2"]

 [[package]]
 name = "pytest"
@ -304,11 +304,11 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"

 [[package]]
 name = "typing-extensions"
-version = "4.1.1"
-description = "Backported and Experimental Type Hints for Python 3.6+"
+version = "4.2.0"
+description = "Backported and Experimental Type Hints for Python 3.7+"
 category = "dev"
 optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.7"

 [[package]]
 name = "urllib3"
@ -341,15 +341,15 @@ python-versions = ">=3.4"

 [[package]]
 name = "zipp"
-version = "3.6.0"
+version = "3.8.0"
 description = "Backport of pathlib-compatible object wrapper for zip files"
 category = "main"
 optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.7"

 [package.extras]
-docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"]
-testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"]
+docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"]
+testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"]

 [metadata]
 lock-version = "1.1"
@ -390,8 +390,8 @@ idna = [
    {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"},
 ]
 importlib-metadata = [
-    {file = "importlib_metadata-4.8.3-py3-none-any.whl", hash = "sha256:65a9576a5b2d58ca44d133c42a241905cc45e34d2c06fd5ba2bafa221e5d7b5e"},
-    {file = "importlib_metadata-4.8.3.tar.gz", hash = "sha256:766abffff765960fcc18003801f7044eb6755ffae4521c8e8ce8e83b9c9b0668"},
+    {file = "importlib_metadata-4.11.4-py3-none-any.whl", hash = "sha256:c58c8eb8a762858f49e18436ff552e83914778e50e9d2f1660535ffb364552ec"},
+    {file = "importlib_metadata-4.11.4.tar.gz", hash = "sha256:5d26852efe48c0a32b0509ffbc583fda1a2266545a78d104a6f4aff3db17d700"},
 ]
 importlib-resources = [
    {file = "importlib_resources-5.8.0-py3-none-any.whl", hash = "sha256:7952325ffd516c05a8ad0858c74dff2c3343f136fe66a6002b2623dd1d43f223"},
@ -497,8 +497,8 @@ py = [
    {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"},
 ]
 pyparsing = [
-    {file = "pyparsing-3.0.7-py3-none-any.whl", hash = "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484"},
-    {file = "pyparsing-3.0.7.tar.gz", hash = "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea"},
+    {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"},
+    {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"},
 ]
 pytest = [
    {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"},
@ -559,8 +559,8 @@ six = [
    {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
 ]
 typing-extensions = [
-    {file = "typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"},
-    {file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"},
+    {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"},
+    {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"},
 ]
 urllib3 = [
    {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"},
@ -575,6 +575,6 @@ xmltodict = [
    {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"},
 ]
 zipp = [
-    {file = "zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"},
-    {file = "zipp-3.6.0.tar.gz", hash = "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832"},
+    {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"},
+    {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"},
 ]
--- a/iso/empanadas/pyproject.toml
+++ b/iso/empanadas/pyproject.toml
@ -25,6 +25,7 @@ sync_from_peridot_test = "empanadas.scripts.sync_from_peridot_test:run"
 sync_sig = "empanadas.scripts.sync_sig:run"
 build-iso = "empanadas.scripts.build_iso:run"
 pull-unpack-tree = "empanadas.scripts.pull_unpack_tree:run"
+launch-builds = "empanadas.scripts.launch_builds:run"

 [build-system]
 requires = ["poetry-core>=1.0.0"]