feat: build-push base-image-build

.devcontainer/compose.yaml

@@ -0,0 +1,18 @@
+services:
+  rocky:
+    image: emanuilov/gitlab-runner-image
+    cgroup: host
+    privileged: true
+    env_file: devcontainer.env
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup:rw
+      - ..:/base-image-build:rw
+      - ../secureFiles:/secureFiles:rw
+      - ../rockyDisks:/rockyDisks:rw
+  # nginx: # can be used for the useful-but-not-needed pxe steup
+  #   image: nginx
+  #   volumes:
+  #     - ./output:/usr/share/nginx/html
+  #   ports:
+  #     - 80:80
+  #   restart: unless-stopped

.devcontainer/devcontainer.env

@@ -0,0 +1,12 @@
+CI_COMMIT_SHA="1ecfd275763eff1d6b4844ea3168962458c9f27a"
+ROCKY_VERSION="9.4"
+VM_DISK_FILE_TO_DOWNLOAD="Rocky-9-GenericCloud-Base-9.4-20240609.0.x86_64.qcow2"
+VM_DISK_OUTPUT_DIR="/rockyDisks"
+BASE_DISK_FILEPATH="$VM_DISK_OUTPUT_DIR/$VM_DISK_FILE_TO_DOWNLOAD"
+OUTPUT_FILE_PATH="$VM_DISK_OUTPUT_DIR/$VM_DISK_FILE_TO_DOWNLOAD-$CI_COMMIT_SHA"
+SECURE_FILES_DOWNLOAD_PATH='/secureFiles'
+OUTPUT_BOOTC_IMAGE_NAME="rocky9-bootc:$CI_COMMIT_SHA"
+OUTPUT_BOOTC_IMAGE_NAME_LATEST="rocky9-bootc:latest"
+CONTAINER_REGISTRY_DOMAIN=docker.io
+CONTAINER_REGISTRY_ORG=emanuilov
+CONTAINER_REGISTRY_WITH_ORG=$CONTAINER_REGISTRY_DOMAIN/$CONTAINER_REGISTRY_ORG

.devcontainer/devcontainer.json

@@ -0,0 +1,31 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/debian
+{
+  "name": "Rocky9",
+  // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+  "dockerComposeFile": "./compose.yaml",
+  "service": "rocky",
+  "workspaceFolder": "/base-image-build",
+  "shutdownAction": "stopCompose",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-azuretools.vscode-docker",
+        "googlecloudtools.cloudcode"
+      ]
+    }
+  },
+  "postCreateCommand": "cd /secureFiles && ssh-keygen -t ed25519 -C 'your_email@example.com' -N '' -f id_ed25519 && chmod 600 id_ed25519"
+
+  // Features to add to the dev container. More info: https://containers.dev/features.
+  // "features": {},
+
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  // "forwardPorts": [],
+
+  // Configure tool-specific properties.
+  // "customizations": {},
+
+  // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+  // "remoteUser": "root"
+}

.devcontainer/useful-but-not-needed/Useful.md

@@ -0,0 +1,63 @@
+# Can be used within the VS Code dev container to debug
+
+### Create PXE network
+
+```
+virsh net-define ipxeboot.xml
+
+virsh net-start ipxeboot
+
+virsh net-autostart ipxeboot
+```
+
+### Start PXE machine
+
+```
+virt-install -n rocky --memory 4096 --vcpus 8 \
+    --pxe --disk size=25,bus=virtio --network network=ipxeboot,model=virtio \
+    --input tablet --os-variant rhel9.0 \
+    --noautoconsole
+```
+
+```
+virt-install -n rocky --memory 512 --vcpus 1 \
+ --pxe --disk size=5,bus=virtio --network network=alpineipxeboot,model=virtio \
+ --input tablet --os-variant id=http://alpinelinux.org/alpinelinux/3.19 \
+ --noautoconsole
+```
+
+### Inside a virt-install VM
+
+```
+[rocky@localhost ~]$ df -h
+Filesystem      Size  Used Avail Use% Mounted on
+devtmpfs        4.0M     0  4.0M   0% /dev
+tmpfs           2.0G     0  2.0G   0% /dev/shm
+tmpfs           783M  8.7M  775M   2% /run
+/dev/vda4       9.4G  1.4G  8.1G  15% /
+/dev/vda3       936M  220M  717M  24% /boot
+/dev/vda2       100M   11M   90M  11% /efi
+tmpfs           392M     0  392M   0% /run/user/1000
+```
+
+### Inside a virt-customize flow:
+
+```
+virt-df -h -a Rocky-9-GenericCloud-Base.latest.x86_64.qcow2
+Filesystem                                Size       Used  Available  Use%
+Rocky-9-GenericCloud-Base.latest.x86_64.qcow2:/dev/sda2
+                                          100M       7.0M        93M    8%
+Rocky-9-GenericCloud-Base.latest.x86_64.qcow2:/dev/sda3
+                                          936M       180M       756M   20%
+Rocky-9-GenericCloud-Base.latest.x86_64.qcow2:/dev/sda4
+                                          9.3G       1.4G       7.9G   16%
+```
+
+### Currently not needed, but if need in the future - systemctl/docker during build:
+
+```
+docker buildx create --use --name insecure-builder --buildkitd-flags '--allow-insecure-entitlement security.insecure'
+docker buildx use insecure-builder
+docker buildx build -t emanuilov/dev --load --allow security.insecure .
+docker buildx use default
+```

.devcontainer/useful-but-not-needed/ipxeboot.xml

@@ -0,0 +1,17 @@
+<network>
+  <name>ipxeboot</name>
+  <forward mode="nat">
+    <nat>
+      <port start="1024" end="65535"/>
+    </nat>
+  </forward>
+  <bridge name="virbr1" stp="on" delay="0"/>
+  <mac address="52:54:00:a4:10:b3"/>
+  <domain name="ipxeboot"/>
+  <ip address="192.168.129.1" netmask="255.255.255.0">
+    <dhcp>
+      <range start="192.168.129.128" end="192.168.129.254"/>
+      <bootp file="http://nginx/script.ipxe"/>
+    </dhcp>
+  </ip>
+</network>

.gitignore

@@ -0,0 +1,4 @@
+/rockyDisks
+/secureFiles
+local.qcow2
+debug.qcow2

.gitlab-ci.yml

@@ -0,0 +1,39 @@
+default:
+  image: $CONTAINER_REGISTRY_DOMAIN_WITH_ORG/gitlab-runner-image
+
+variables:
+    VM_DISK_OUTPUT_DIR: "/rockyDisks/"
+    SECURE_FILES_DOWNLOAD_PATH: "/secureFiles"
+
+build-builder-vm-disk:
+  stage: build
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        - scripts/utils/**/*
+        - scripts/setupBuilder/**/*
+  variables:
+    # Values should match the ones in .devcontainer/.env
+    ROCKY_VERSION: "9.4"
+    VM_DISK_FILE_TO_DOWNLOAD: "Rocky-9-GenericCloud-Base-9.4-20240609.0.x86_64.qcow2"
+    BASE_DISK_FILEPATH: "$VM_DISK_OUTPUT_DIR/$VM_DISK_FILE_TO_DOWNLOAD"
+    OUTPUT_FILE_PATH: "$VM_DISK_OUTPUT_DIR/$VM_DISK_FILE_TO_DOWNLOAD-$CI_COMMIT_SHA"
+  script:
+    - ./scripts/setupBuilder/main.sh
+
+# This job is NOT using the host's Docker, but the one within the VM that is generated from the job above
+# The VM is needed due to the bootc build process requiring a full Rocky host to be used for it
+build-push-base-image:
+  depends_on: build-builder-vm-disk
+  stage: build
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+        - scripts/utils/**/*
+        - scripts/setupBuilder/**/*
+        - scripts/buildPushBootcContainerImage/**/*
+  variables:
+    OUTPUT_BOOTC_IMAGE_NAME: "rocky9-bootc:$CI_COMMIT_SHA"
+    OUTPUT_BOOTC_IMAGE_NAME_LATEST: "rocky9-bootc:latest"
+  script:
+    - ./scripts/buildPushBootcContainerImage/main.sh

README.md

@@ -0,0 +1,35 @@
+## Overview of the pipeline
+1. A Cloud VM Rocky Linux disk is downloaded, then customized so it can run docker inside
+2. A Rocky Linux bootc repo is cloned onto that and the repo is used to build a Rocky Linux 9.4 bootc Docker image, which is then uploaded to a container registry, the pipeline for that is here due to `1.` and `2` sharing files and its easier to debug them if needed like this
+
+#### !!! IMPORTANT !!! The pipeline assumes that the Gitlab runner host has a shared a directory called `/rockyDisks` that can be used by all jobs and persists between runs/jobs, otherwise you'd need to make a change in some of the `build-builder-vm-disk` step's files so that the builder VM disk is generated/the step is triggered
+
+### To run a new build of the Cloud VM image change the 
+
+## To run this inside a VS Code dev container
+
+1. Make sure the env vars in `.devcontainer/devcontainer.env` match the ones in `.gitlab-ci.yml` for all jobs, and the CI_COMMIT_SHA var is present in the dev env file
+2. Setting up the builder VM disk:
+
+To download and setup a Rocky Linux VM Disk that is to be used for building a bootc container image run:
+
+```
+./scripts/setupBuilder/main.sh
+```
+
+3. To build a bootc container image and push it, fill out the CONTAINER_REGISTRY vars below, then run:
+```
+export CONTAINER_REGISTRY_RW_USER=someuser
+export CONTAINER_REGISTRY_RW_PASS=some_api_key
+./scripts/buildPushBootcContainerImage/main.sh
+```
+
+### If any debugging is needed you can connect to the latest builder disk run the following, note all changes will be saved:
+```
+./scripts/debug/main.sh
+```
+
+To connect to a specific builder disk run
+```
+./scripts/debug/main.sh /base-image-build/local.qcow2
+```

scripts/buildPushBootcContainerImage/buildPushBootcContainerImage.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -e
+
+source $utilsLocation/onStart.sh
+
+source $utilsLocation/getLatestRockyDiskPath.sh
+currentDir=$(pwd)
+tempDiskVM="$currentDir/local.qcow2"
+if [ ! -f $tempDiskVM ]; then
+  echo "Copying $latestRockyDiskPath into $tempDiskVM"
+  cp $latestRockyDiskPath $tempDiskVM
+else
+  echo "!!! ERROR $tempDiskVM - already exists !!!"
+  exit 1
+fi
+imagePath=$tempDiskVM
+
+echo "--- Using image: $imagePath ---"
+
+echo "--- Creating Rocky Cloud image KVM ---"
+virt-install --name=rocky --ram=4096 --vcpus=8 \
+    --disk $imagePath \
+    --boot hd --noautoconsole \
+    --os-variant rhel9.0
+echo "--- Finished creating Rocky Cloud image KVM ---"
+
+echo "--- Getting the Rocky Cloud image KVM's IP ---"
+source $utilsLocation/setVMIP.sh
+
+# Done here, and not through virt-customize due to it making the end image several hundred magabytes bigger
+outputImageNameWithOrg="$CONTAINER_REGISTRY_ORG/$OUTPUT_BOOTC_IMAGE_NAME"
+latestImageNameWithOrg="$CONTAINER_REGISTRY_ORG/$OUTPUT_BOOTC_IMAGE_NAME_LATEST"
+echo "--- Starting Build-Push for $outputImageNameWithOrg ---"
+gitDelimiter="&&"
+if [ "$CI" != "true" ]; then
+  gitDelimiter=";" # Soft fail git clone if running this locally
+fi
+
+resizeLog=$(ssh -o ConnectTimeout=300 -o StrictHostKeyChecking=no rocky@$vmIPaddress <<EOF
+git clone --recurse-submodules https://gitlab.com/marketso/base-images.git $gitDelimiter cd base-images && \
+echo "$CONTAINER_REGISTRY_RW_PASS" | podman login -u "$CONTAINER_REGISTRY_RW_USER" $CONTAINER_REGISTRY_DOMAIN --password-stdin && \
+podman build --security-opt=label=disable --cap-add=all \
+  --device /dev/fuse -t $outputImageNameWithOrg . && \
+podman tag $outputImageNameWithOrg $latestImageNameWithOrg && \
+podman push $outputImageNameWithOrg && \
+podman push $latestImageNameWithOrg
+EOF
+)
+
+echo "--- SUCCESS - container was pushed to $outputImageNameWithOrg ---"

scripts/buildPushBootcContainerImage/main.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+export scriptLocation="$(dirname "$0")/"
+export utilsLocation="$scriptLocation/../utils"
+
+forceExit() {
+  exitCode=$1
+
+  echo "--- ERROR - Script exited with an error or was interrupted ---"
+  $utilsLocation/cleanup.sh
+
+  exit $exitCode
+}
+
+trap "forceExit 0" SIGINT
+
+$scriptLocation/buildPushBootcContainerImage.sh
+exitCode=$?
+if [[ $exitCode -ne 0 ]];
+then
+  forceExit $exitCode
+else
+  $utilsLocation/cleanup.sh
+fi
+

scripts/debug/main.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -e
+
+export scriptLocation="$(dirname "$0")/"
+export utilsLocation="$scriptLocation/../utils"
+
+source $utilsLocation/onStart.sh
+
+source $utilsLocation/getLatestRockyDiskPath.sh
+currentDir=$(pwd)
+tempDiskVM="$currentDir/debug.qcow2"
+if [ -n "$1" ]; then
+    tempDiskVM="$1"
+else
+  if [ ! -f $tempDiskVM ]; then
+    echo "Copying $latestRockyDiskPath into $tempDiskVM"
+    cp $latestRockyDiskPath $tempDiskVM
+  else
+    echo "!!! Using cached $tempDiskVM - already exists !!!"
+    echo "!!! If you want to copy the latest file from /rockyDisks/ delete $tempDiskVM"
+  fi
+fi
+imagePath=$tempDiskVM
+
+echo "--- Current dir: $currentDir ---"
+echo "--- Using image: $imagePath ---"
+
+echo "--- Creating Rocky Cloud image KVM ---"
+virt-install --name=rocky --ram=4096 --vcpus=8 \
+    --disk $imagePath \
+    --boot hd --noautoconsole \
+    --os-variant rhel9.0
+echo "--- Finished creating Rocky Cloud image KVM ---"
+
+echo "+++ All changes will be saved +++"
+echo "--- Getting the Rocky Cloud image KVM's IP ---"
+source $utilsLocation/setVMIP.sh
+
+# Done here, and not through virt-customize due to it making the end image several hundred magabytes bigger
+echo "--- Resizing filesystem to match virtual disk image size ---"
+ssh -o ConnectTimeout=300 -o StrictHostKeyChecking=no rocky@$vmIPaddress
+
+$utilsLocation/cleanup.sh
+
+echo "+++ All changes were saved +++"
+
+echo "Finished successfully, output file is same as input: $imagePath"

scripts/setupBuilder/customizeBuilder.sh

@@ -0,0 +1,9 @@
+# Runs only on a new qcow2 VM image from Rocky (due to user creation)
+# Can be used only as virt-customize --commands-from-file input
+run-command useradd rocky
+run-command usermod -aG wheel rocky
+run-command dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+run-command dnf install -y git docker-ce docker-ce-cli containerd.io docker-compose-plugin buildah podman
+run-command dnf clean all
+run-command systemctl enable docker
+ssh-inject rocky:file:/secureFiles/id_ed25519.pub

scripts/setupBuilder/downloadGenericCloudDisk.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -e
+
+# Latest one can be found at https://dl.rockylinux.org/pub/rocky/9.4/images/x86_64/
+# It is marked as https://dl.rockylinux.org/pub/rocky/9.4/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2
+# The specific deployment it corresponds to can be found by searching for the file size (number on the right) in the page
+
+if [ ! -f $BASE_DISK_FILEPATH ]; then
+    vmDiskFileURL="https://dl.rockylinux.org/pub/rocky/$ROCKY_VERSION/images/x86_64/$VM_DISK_FILE_TO_DOWNLOAD"
+    checksumFileName="$VM_DISK_FILE_TO_DOWNLOAD.CHECKSUM"
+    vmDiskFileChecksumURL="https://dl.rockylinux.org/pub/rocky/$ROCKY_VERSION/images/x86_64/$checksumFileName"
+
+    echo "--- Downloading $vmDiskFileURL ---"
+    curl $vmDiskFileURL -o $BASE_DISK_FILEPATH
+    echo "--- Downloading $vmDiskFileChecksumURL ---"
+    curl $vmDiskFileChecksumURL -o $OUTPUT_DIR$checksumFileName
+
+    echo "--- Checking checksum ---"
+    expectedChecksum=$(cat $OUTPUT_DIR/$checksumFileName | awk 'NR==2 {print $4}')
+    if [ $(sha256sum $BASE_DISK_FILEPATH | awk '{print $1}') == "$expectedChecksum" ]; then
+        echo "--- CHECKSUM CONFIRMED ---"
+    else
+        echo "--- CHECKSUM DOES NOT MATCH! ---"
+        rm $BASE_DISK_FILEPATH
+        exit 1
+    fi
+    echo "--- Finished checking checksum ---"
+
+else
+    echo "--- Using cached base file $BASE_DISK_FILEPATH ---"
+fi

scripts/setupBuilder/main.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+export scriptLocation="$(dirname "$0")/"
+export utilsLocation="$scriptLocation/../utils"
+if [ ! -f $OUTPUT_FILE_PATH ]; then
+
+  forceExit() {
+    exitCode=$1
+
+    echo "--- ERROR - Script exited with an error or was interrupted ---"
+    $utilsLocation/cleanup.sh
+
+    echo "--- ERROR - Deleting generated files ---"
+    rm $OUTPUT_FILE_PATH
+
+    exit $exitCode
+  }
+
+  trap "forceExit 0" SIGINT
+
+  $scriptLocation/setupBuilder.sh
+  exitCode=$?
+  if [[ $exitCode -ne 0 ]];
+  then
+    forceExit $exitCode
+  else
+    echo "--- SUCCESS - Building the VM disk finished successfully: $OUTPUT_FILE_PATH ---"
+#    $utilsLocation/cleanup.sh
+  fi
+
+else
+
+  echo "SKIPPING - OUTPUT FILE ALREADY EXISTS - $OUTPUT_FILE_PATH"
+  echo 'If you need to regenerate it change the $OUTPUT_FILE_PATH make a new commit that changes some of the CI monitored files or delete the pre-generated file'
+  exit 0
+
+fi

scripts/setupBuilder/setupBuilder.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+
+source $utilsLocation/initSSHclient.sh
+
+customDiskPath=$1
+
+$scriptLocation/downloadGenericCloudDisk.sh
+
+imagePath=$BASE_DISK_FILEPATH
+
+echo "--- COPYING BASE IMAGE ---"
+cp $imagePath $OUTPUT_FILE_PATH
+imagePath=$OUTPUT_FILE_PATH
+
+echo "--- Using image: $imagePath ---"
+
+echo "--- Virtal disk resizing ---"
+tempDiskName="${imagePath}a"
+qemu-img create -f qcow2 $tempDiskName 50G
+virt-resize --expand /dev/vda4 $imagePath $tempDiskName
+rm $imagePath
+mv $tempDiskName $imagePath
+echo "--- Finished virtual disk resizing ---"
+
+echo "--- Setting up Rocky Cloud image for ssh access and docker/buildah/podman ---"
+echo "--- Creating rocky user; addng it to the wheel gorup; installing docker/buildah/podman; adding SSH access key to the rocky user ---"
+virt-customize -a $imagePath --commands-from-file $scriptLocation/customizeBuilder.sh
+echo "--- Finished setting up Rocky Rocky Cloud image for ssh access and docker/buildah/podman ---"

scripts/utils/cleanup.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+echo "--- RUNNING CLEANUP COMMAND ---"
+
+echo "--- Resetting existing KVMs ---"
+$utilsLocation/resetKVMs.sh
+echo "--- Finished resetting existing KVMs ---"

scripts/utils/getLatestRockyDiskPath.sh

@@ -0,0 +1,7 @@
+latestRockyDiskPath=$(find "$VM_DISK_OUTPUT_DIR" -type f -printf '%T@ %p\n' |
+  sort -rn |
+  head -1 |
+  sed 's/^[0-9.]\+ //' |
+  xargs readlink -f)
+
+export latestRockyDiskPath=$latestRockyDiskPath

scripts/utils/initSSHclient.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+if [ -n "$CI" ]; then
+    mkdir $SECURE_FILES_DOWNLOAD_PATH
+    echo $BUILDER_PUBLIC_KEY > $SECURE_FILES_DOWNLOAD_PATH/id_ed25519.pub
+    echo $BUILDER_PRIVATE_KEY > $SECURE_FILES_DOWNLOAD_PATH/id_ed25519
+fi
+
+eval "$(ssh-agent -s)"
+ssh-add $SECURE_FILES_DOWNLOAD_PATH/id_ed25519

scripts/utils/onStart.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+echo "--- Adding ssh key to client ---"
+source $utilsLocation/initSSHclient.sh
+echo "--- Finished adding ssh key to client ---"
+
+echo "--- Resetting existing KVMs ---"
+$utilsLocation/resetKVMs.sh
+echo "--- Finished resetting existing KVMs ---"

scripts/utils/resetKVMs.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+# Its fine if there are some errs here in most cases
+virsh reset rocky;virsh destroy rocky;virsh undefine rocky;virsh list;rm -f /var/lib/libvirt/images/*

scripts/utils/setVMIP.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+vmIPaddress=$(virsh domifaddr rocky | awk '/ipv4/ {print $4}' | awk -F'/' '{print $1}')
+
+while [ -z "$vmIPaddress" ]; do
+    sleep 3
+    vmIPaddress=$(virsh domifaddr rocky | awk '/ipv4/ {print $4}' | awk -F'/' '{print $1}')
+done
+
+export vmIPaddress="$vmIPaddress"
+
+echo "Builder VM IP (vmIPaddress) is: $vmIPaddress"