From 4ccdd47619a7fbd7cfd3ad8840b1ee9f71e66eeb Mon Sep 17 00:00:00 2001 From: Radostin Emanuilov Date: Sun, 11 Aug 2024 09:09:04 +0000 Subject: [PATCH] feat: build-push base-image-build --- .devcontainer/compose.yaml | 18 ++++++ .devcontainer/devcontainer.env | 12 ++++ .devcontainer/devcontainer.json | 31 +++++++++ .devcontainer/useful-but-not-needed/Useful.md | 63 +++++++++++++++++++ .../useful-but-not-needed/ipxeboot.xml | 17 +++++ .gitignore | 4 ++ .gitlab-ci.yml | 39 ++++++++++++ README.md | 35 +++++++++++ .../buildPushBootcContainerImage.sh | 50 +++++++++++++++ scripts/buildPushBootcContainerImage/main.sh | 24 +++++++ scripts/debug/main.sh | 47 ++++++++++++++ scripts/setupBuilder/customizeBuilder.sh | 9 +++ .../setupBuilder/downloadGenericCloudDisk.sh | 31 +++++++++ scripts/setupBuilder/main.sh | 36 +++++++++++ scripts/setupBuilder/setupBuilder.sh | 29 +++++++++ scripts/utils/cleanup.sh | 8 +++ scripts/utils/getLatestRockyDiskPath.sh | 7 +++ scripts/utils/initSSHclient.sh | 11 ++++ scripts/utils/onStart.sh | 10 +++ scripts/utils/resetKVMs.sh | 3 + scripts/utils/setVMIP.sh | 13 ++++ 21 files changed, 497 insertions(+) create mode 100644 .devcontainer/compose.yaml create mode 100644 .devcontainer/devcontainer.env create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/useful-but-not-needed/Useful.md create mode 100644 .devcontainer/useful-but-not-needed/ipxeboot.xml create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 README.md create mode 100644 scripts/buildPushBootcContainerImage/buildPushBootcContainerImage.sh create mode 100644 scripts/buildPushBootcContainerImage/main.sh create mode 100644 scripts/debug/main.sh create mode 100644 scripts/setupBuilder/customizeBuilder.sh create mode 100644 scripts/setupBuilder/downloadGenericCloudDisk.sh create mode 100644 scripts/setupBuilder/main.sh create mode 100644 scripts/setupBuilder/setupBuilder.sh create mode 100644 scripts/utils/cleanup.sh create mode 100644 scripts/utils/getLatestRockyDiskPath.sh create mode 100644 scripts/utils/initSSHclient.sh create mode 100644 scripts/utils/onStart.sh create mode 100644 scripts/utils/resetKVMs.sh create mode 100644 scripts/utils/setVMIP.sh diff --git a/.devcontainer/compose.yaml b/.devcontainer/compose.yaml new file mode 100644 index 0000000..7df4c24 --- /dev/null +++ b/.devcontainer/compose.yaml @@ -0,0 +1,18 @@ +services: + rocky: + image: emanuilov/gitlab-runner-image + cgroup: host + privileged: true + env_file: devcontainer.env + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - ..:/base-image-build:rw + - ../secureFiles:/secureFiles:rw + - ../rockyDisks:/rockyDisks:rw + # nginx: # can be used for the useful-but-not-needed pxe steup + # image: nginx + # volumes: + # - ./output:/usr/share/nginx/html + # ports: + # - 80:80 + # restart: unless-stopped diff --git a/.devcontainer/devcontainer.env b/.devcontainer/devcontainer.env new file mode 100644 index 0000000..3ce2237 --- /dev/null +++ b/.devcontainer/devcontainer.env @@ -0,0 +1,12 @@ +CI_COMMIT_SHA="1ecfd275763eff1d6b4844ea3168962458c9f27a" +ROCKY_VERSION="9.4" +VM_DISK_FILE_TO_DOWNLOAD="Rocky-9-GenericCloud-Base-9.4-20240609.0.x86_64.qcow2" +VM_DISK_OUTPUT_DIR="/rockyDisks" +BASE_DISK_FILEPATH="$VM_DISK_OUTPUT_DIR/$VM_DISK_FILE_TO_DOWNLOAD" +OUTPUT_FILE_PATH="$VM_DISK_OUTPUT_DIR/$VM_DISK_FILE_TO_DOWNLOAD-$CI_COMMIT_SHA" +SECURE_FILES_DOWNLOAD_PATH='/secureFiles' +OUTPUT_BOOTC_IMAGE_NAME="rocky9-bootc:$CI_COMMIT_SHA" +OUTPUT_BOOTC_IMAGE_NAME_LATEST="rocky9-bootc:latest" +CONTAINER_REGISTRY_DOMAIN=docker.io +CONTAINER_REGISTRY_ORG=emanuilov +CONTAINER_REGISTRY_WITH_ORG=$CONTAINER_REGISTRY_DOMAIN/$CONTAINER_REGISTRY_ORG \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..a8f39e3 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,31 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/debian +{ + "name": "Rocky9", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "dockerComposeFile": "./compose.yaml", + "service": "rocky", + "workspaceFolder": "/base-image-build", + "shutdownAction": "stopCompose", + "customizations": { + "vscode": { + "extensions": [ + "ms-azuretools.vscode-docker", + "googlecloudtools.cloudcode" + ] + } + }, + "postCreateCommand": "cd /secureFiles && ssh-keygen -t ed25519 -C 'your_email@example.com' -N '' -f id_ed25519 && chmod 600 id_ed25519" + + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Configure tool-specific properties. + // "customizations": {}, + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/.devcontainer/useful-but-not-needed/Useful.md b/.devcontainer/useful-but-not-needed/Useful.md new file mode 100644 index 0000000..9d58162 --- /dev/null +++ b/.devcontainer/useful-but-not-needed/Useful.md @@ -0,0 +1,63 @@ +# Can be used within the VS Code dev container to debug + +### Create PXE network + +``` +virsh net-define ipxeboot.xml + +virsh net-start ipxeboot + +virsh net-autostart ipxeboot +``` + +### Start PXE machine + +``` +virt-install -n rocky --memory 4096 --vcpus 8 \ + --pxe --disk size=25,bus=virtio --network network=ipxeboot,model=virtio \ + --input tablet --os-variant rhel9.0 \ + --noautoconsole +``` + +``` +virt-install -n rocky --memory 512 --vcpus 1 \ + --pxe --disk size=5,bus=virtio --network network=alpineipxeboot,model=virtio \ + --input tablet --os-variant id=http://alpinelinux.org/alpinelinux/3.19 \ + --noautoconsole +``` + +### Inside a virt-install VM + +``` +[rocky@localhost ~]$ df -h +Filesystem Size Used Avail Use% Mounted on +devtmpfs 4.0M 0 4.0M 0% /dev +tmpfs 2.0G 0 2.0G 0% /dev/shm +tmpfs 783M 8.7M 775M 2% /run +/dev/vda4 9.4G 1.4G 8.1G 15% / +/dev/vda3 936M 220M 717M 24% /boot +/dev/vda2 100M 11M 90M 11% /efi +tmpfs 392M 0 392M 0% /run/user/1000 +``` + +### Inside a virt-customize flow: + +``` +virt-df -h -a Rocky-9-GenericCloud-Base.latest.x86_64.qcow2 +Filesystem Size Used Available Use% +Rocky-9-GenericCloud-Base.latest.x86_64.qcow2:/dev/sda2 + 100M 7.0M 93M 8% +Rocky-9-GenericCloud-Base.latest.x86_64.qcow2:/dev/sda3 + 936M 180M 756M 20% +Rocky-9-GenericCloud-Base.latest.x86_64.qcow2:/dev/sda4 + 9.3G 1.4G 7.9G 16% +``` + +### Currently not needed, but if need in the future - systemctl/docker during build: + +``` +docker buildx create --use --name insecure-builder --buildkitd-flags '--allow-insecure-entitlement security.insecure' +docker buildx use insecure-builder +docker buildx build -t emanuilov/dev --load --allow security.insecure . +docker buildx use default +``` diff --git a/.devcontainer/useful-but-not-needed/ipxeboot.xml b/.devcontainer/useful-but-not-needed/ipxeboot.xml new file mode 100644 index 0000000..55e2572 --- /dev/null +++ b/.devcontainer/useful-but-not-needed/ipxeboot.xml @@ -0,0 +1,17 @@ + + ipxeboot + + + + + + + + + + + + + + + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7f6ffb1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/rockyDisks +/secureFiles +local.qcow2 +debug.qcow2 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..fae893b --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,39 @@ +default: + image: $CONTAINER_REGISTRY_DOMAIN_WITH_ORG/gitlab-runner-image + +variables: + VM_DISK_OUTPUT_DIR: "/rockyDisks/" + SECURE_FILES_DOWNLOAD_PATH: "/secureFiles" + +build-builder-vm-disk: + stage: build + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - scripts/utils/**/* + - scripts/setupBuilder/**/* + variables: + # Values should match the ones in .devcontainer/.env + ROCKY_VERSION: "9.4" + VM_DISK_FILE_TO_DOWNLOAD: "Rocky-9-GenericCloud-Base-9.4-20240609.0.x86_64.qcow2" + BASE_DISK_FILEPATH: "$VM_DISK_OUTPUT_DIR/$VM_DISK_FILE_TO_DOWNLOAD" + OUTPUT_FILE_PATH: "$VM_DISK_OUTPUT_DIR/$VM_DISK_FILE_TO_DOWNLOAD-$CI_COMMIT_SHA" + script: + - ./scripts/setupBuilder/main.sh + +# This job is NOT using the host's Docker, but the one within the VM that is generated from the job above +# The VM is needed due to the bootc build process requiring a full Rocky host to be used for it +build-push-base-image: + depends_on: build-builder-vm-disk + stage: build + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - scripts/utils/**/* + - scripts/setupBuilder/**/* + - scripts/buildPushBootcContainerImage/**/* + variables: + OUTPUT_BOOTC_IMAGE_NAME: "rocky9-bootc:$CI_COMMIT_SHA" + OUTPUT_BOOTC_IMAGE_NAME_LATEST: "rocky9-bootc:latest" + script: + - ./scripts/buildPushBootcContainerImage/main.sh \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..7b0a063 --- /dev/null +++ b/README.md @@ -0,0 +1,35 @@ +## Overview of the pipeline +1. A Cloud VM Rocky Linux disk is downloaded, then customized so it can run docker inside +2. A Rocky Linux bootc repo is cloned onto that and the repo is used to build a Rocky Linux 9.4 bootc Docker image, which is then uploaded to a container registry, the pipeline for that is here due to `1.` and `2` sharing files and its easier to debug them if needed like this + +#### !!! IMPORTANT !!! The pipeline assumes that the Gitlab runner host has a shared a directory called `/rockyDisks` that can be used by all jobs and persists between runs/jobs, otherwise you'd need to make a change in some of the `build-builder-vm-disk` step's files so that the builder VM disk is generated/the step is triggered + +### To run a new build of the Cloud VM image change the + +## To run this inside a VS Code dev container + +1. Make sure the env vars in `.devcontainer/devcontainer.env` match the ones in `.gitlab-ci.yml` for all jobs, and the CI_COMMIT_SHA var is present in the dev env file +2. Setting up the builder VM disk: + +To download and setup a Rocky Linux VM Disk that is to be used for building a bootc container image run: + +``` +./scripts/setupBuilder/main.sh +``` + +3. To build a bootc container image and push it, fill out the CONTAINER_REGISTRY vars below, then run: +``` +export CONTAINER_REGISTRY_RW_USER=someuser +export CONTAINER_REGISTRY_RW_PASS=some_api_key +./scripts/buildPushBootcContainerImage/main.sh +``` + +### If any debugging is needed you can connect to the latest builder disk run the following, note all changes will be saved: +``` +./scripts/debug/main.sh +``` + +To connect to a specific builder disk run +``` +./scripts/debug/main.sh /base-image-build/local.qcow2 +``` \ No newline at end of file diff --git a/scripts/buildPushBootcContainerImage/buildPushBootcContainerImage.sh b/scripts/buildPushBootcContainerImage/buildPushBootcContainerImage.sh new file mode 100644 index 0000000..34b92f7 --- /dev/null +++ b/scripts/buildPushBootcContainerImage/buildPushBootcContainerImage.sh @@ -0,0 +1,50 @@ +#!/bin/bash +set -e + +source $utilsLocation/onStart.sh + +source $utilsLocation/getLatestRockyDiskPath.sh +currentDir=$(pwd) +tempDiskVM="$currentDir/local.qcow2" +if [ ! -f $tempDiskVM ]; then + echo "Copying $latestRockyDiskPath into $tempDiskVM" + cp $latestRockyDiskPath $tempDiskVM +else + echo "!!! ERROR $tempDiskVM - already exists !!!" + exit 1 +fi +imagePath=$tempDiskVM + +echo "--- Using image: $imagePath ---" + +echo "--- Creating Rocky Cloud image KVM ---" +virt-install --name=rocky --ram=4096 --vcpus=8 \ + --disk $imagePath \ + --boot hd --noautoconsole \ + --os-variant rhel9.0 +echo "--- Finished creating Rocky Cloud image KVM ---" + +echo "--- Getting the Rocky Cloud image KVM's IP ---" +source $utilsLocation/setVMIP.sh + +# Done here, and not through virt-customize due to it making the end image several hundred magabytes bigger +outputImageNameWithOrg="$CONTAINER_REGISTRY_ORG/$OUTPUT_BOOTC_IMAGE_NAME" +latestImageNameWithOrg="$CONTAINER_REGISTRY_ORG/$OUTPUT_BOOTC_IMAGE_NAME_LATEST" +echo "--- Starting Build-Push for $outputImageNameWithOrg ---" +gitDelimiter="&&" +if [ "$CI" != "true" ]; then + gitDelimiter=";" # Soft fail git clone if running this locally +fi + +resizeLog=$(ssh -o ConnectTimeout=300 -o StrictHostKeyChecking=no rocky@$vmIPaddress < $SECURE_FILES_DOWNLOAD_PATH/id_ed25519.pub + echo $BUILDER_PRIVATE_KEY > $SECURE_FILES_DOWNLOAD_PATH/id_ed25519 +fi + +eval "$(ssh-agent -s)" +ssh-add $SECURE_FILES_DOWNLOAD_PATH/id_ed25519 \ No newline at end of file diff --git a/scripts/utils/onStart.sh b/scripts/utils/onStart.sh new file mode 100644 index 0000000..6469428 --- /dev/null +++ b/scripts/utils/onStart.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + +echo "--- Adding ssh key to client ---" +source $utilsLocation/initSSHclient.sh +echo "--- Finished adding ssh key to client ---" + +echo "--- Resetting existing KVMs ---" +$utilsLocation/resetKVMs.sh +echo "--- Finished resetting existing KVMs ---" \ No newline at end of file diff --git a/scripts/utils/resetKVMs.sh b/scripts/utils/resetKVMs.sh new file mode 100644 index 0000000..52c1063 --- /dev/null +++ b/scripts/utils/resetKVMs.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# Its fine if there are some errs here in most cases +virsh reset rocky;virsh destroy rocky;virsh undefine rocky;virsh list;rm -f /var/lib/libvirt/images/* \ No newline at end of file diff --git a/scripts/utils/setVMIP.sh b/scripts/utils/setVMIP.sh new file mode 100644 index 0000000..2961ef3 --- /dev/null +++ b/scripts/utils/setVMIP.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +vmIPaddress=$(virsh domifaddr rocky | awk '/ipv4/ {print $4}' | awk -F'/' '{print $1}') + +while [ -z "$vmIPaddress" ]; do + sleep 3 + vmIPaddress=$(virsh domifaddr rocky | awk '/ipv4/ {print $4}' | awk -F'/' '{print $1}') +done + +export vmIPaddress="$vmIPaddress" + +echo "Builder VM IP (vmIPaddress) is: $vmIPaddress" \ No newline at end of file