Merge imagefactory into main (#7)

* Build images using imagefactory, spawned by Makefile on Gitlab runners weekly on Sundays at 00:00 UTC
* Add details on creating builds to README
* add package diff script
* Don't include architecture in filename, so that it's easier to make images.. The compressed build artifacts are already named with the architecture
This commit is contained in:
Neil Hanlon 2022-03-14 17:22:26 -04:00 committed by GitHub
parent cb3ef4daad
commit 9009e76f10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 243 additions and 0 deletions

38
.github/workflows/build.yml vendored Normal file
View File

@ -0,0 +1,38 @@
---
name: Build container images
on:
push:
branches: [ $default-branch, imagefactory ]
pull_request:
branches: [ $default-branch ]
workflow_dispatch:
schedule:
- cron: "0 0 * * 0"
jobs:
Run-ImageFactory-Builds:
strategy:
matrix:
architecture:
- x64
- ARM64
runs-on:
- self-hosted
- ${{ matrix.architecture }}
steps:
- name: Checkout sig-cloud-instance-images
uses: actions/checkout@v2
- name: Build images
run: make
- name: Get image name
run: echo "ARTIFACT_NAME=$(make publish)" >> $GITHUB_ENV
- name: Upload Artifact
uses: actions/upload-artifact@v2
with:
name: ${{ env.ARTIFACT_NAME }}
path: out/
if-no-files-found: error

6
.gitignore vendored Normal file
View File

@ -0,0 +1,6 @@
*.swp
*.meta
*.xz
kickstarts/
logs/
output/

68
Makefile Normal file
View File

@ -0,0 +1,68 @@
ARCH = $(shell uname -m)
BUILDDATE = $(shell /bin/date +%Y%m%d_%H%M)
KICKSTART_DIR = kickstarts
KICKSTART_PATH = "${KICKSTART_DIR}/Rocky-8-Container.ks"
LOG_DIR = logs
OUT = out
RELEASE_VER = 8.5
MAJOR = $(shell v='$(RELEASE_VER)'; echo "$${v%.*}")
TEMPLATE_DIR = templates
TEMPLATE_PATH = "${TEMPLATE_DIR}/tdl-${ARCH}.xml"
OUTNAME := rocky-${RELEASE_VER}-docker
BASEIMAGE_META := base_image-$(OUTNAME).meta
TARGETIMAGE_META := target_image-$(OUTNAME).meta
STORAGEDIR := /var/lib/imagefactory/storage
.PHONY := all clean setup
.DEFAULT_GOAL := $(OUTNAME).tar.xz
BASEIMAGEUUID = $(shell awk '$$1=="UUID:"{print $$NF}' $(BASEIMAGE_META))
TARGETIMAGEUUID = $(shell awk '$$1=="UUID:"{print $$NF}' $(TARGETIMAGE_META))
ifneq ($(DEBUG),)
DEBUGPARAM := --debug
endif
# Basic type is just 'container'
TYPE=container
CONTAINER_NAME = rocky-$(MAJOR)-$(TYPE)-$(RELEASE_VER).$(BUILDDATE).$(ARCH)
clean:
-rm *.meta
publish:
@echo $(OUTNAME)-$(ARCH).tar.xz
$(KICKSTART_DIR):
git clone --branch r$(MAJOR) --single-branch https://git.rockylinux.org/rocky/kickstarts.git kickstarts
$(BASEIMAGE_META): $(KICKSTART_DIR)
sudo imagefactory $(DEBUGPARAM) base_image \
--parameter offline_icicle true \
--file-parameter install_script ${KICKSTART_PATH} \
${TEMPLATE_PATH} \
| tee -a logs/base_image-$(OUTNAME).out | tail -n4 > $(BASEIMAGE_META) || exit 2
$(TARGETIMAGE_META): $(BASEIMAGE_META)
sudo imagefactory $(DEBUGPARAM) target_image \
--id $(BASEIMAGEUUID) \
--parameter compress xz \
--parameter repository $(CONTAINER_NAME) \
docker | tee -a logs/target_image-$(OUTNAME).out | tail -n4 > $(TARGETIMAGE_META) || exit 3
$(OUT):
mkdir out
$(OUT)/packages.txt: $(OUT)
xmllint --xpath "//packages/*/@name" <(printf "$(jq '.icicle' < $(STORAGEDIR)/$(TARGETIMAGEUUID).meta)\n" | tr -d '\\' | tail -c +2 | head -c -2) | \
awk -F\= '{print substr($2,2,length($2)-2)}' | \
sort > $(OUT)/packages.txt
$(OUTNAME).tar.xz: $(OUT) $(TARGETIMAGE_META)
tar -Oxf $(STORAGEDIR)/$(TARGETIMAGEUUID).body */layer.tar | xz > $(OUT)/$(OUTNAME).tar.xz
tar -tf $(OUT)/$(OUTNAME).tar.xz > $(OUT)/filelist.txt
cp $(STORAGEDIR)/$(TARGETIMAGEUUID).meta $(OUT)/build.meta

View File

@ -8,3 +8,78 @@ Please see the following branches for the container filesystems and Dockerfiles.
* [Rocky-8.4-aarch64](https://github.com/rocky-linux/sig-cloud-instance-images/tree/Rocky-8.4-aarch64)
* [Rocky-8.4-x86_64](https://github.com/rocky-linux/sig-cloud-instance-images/tree/Rocky-8.4-x86_64)
* [Rocky Linux 8.4 RC1](https://github.com/rocky-linux/sig-cloud-instance-images/tree/Rocky-8.4-rc1-Container)
## Deployment
Rootfs tarballs are built weekly on Sunday at 00:00 UTC. There is not currently automation to auto-create releases, though that is under investigation (see #6).
There are two repositories on Dockerhub.com: the so-called "official" image, and the rockylinux organization. The former is facilitated by Docker, Inc., using information in the github.com/docker-library/official-images repository.
Builds are performed on Rocky Enterprise Software Foundation github runners that are spun up and down on demand for builds, as imagefactory requires bare metal hardware. Work is underway to not have to rely on bare metal installations.
### Updating the official Docker Hub image
Updating the official image is relatively simple, and could probably be automated, but at present it's not a high priority. To update the image, download the latest tested and verified build artifacts for every architecture. Anyone that is a maintainer of this repository (i.e., can merge changes into main) should be considered a maintainer of the official image and therefore allowed to request updates.
Make sure you have a fork of the rocky-linux/sig-cloud-instance-images repository that is checked out on your machine, and
#### Official Image Update Steps
1. Download build artifacts from the latest passing build for all available architectures. The next steps should be repeated for each architecture.
1. Change to the directory containing the clone of the sig-cloud-instance-images repository.
1. Create a new branch based off the library-template branch named using the template: "Rocky-$MAJOR.$MINOR.$ISO8601DATE-$ARCHITECTURE" e.g, `Rocky-8.5.20220314-x86_64` or `Rocky-8.5.20220314-aarch64`
```
git checkout -b Rocky-8.5.20220314-x86_64 library-template
```
1. Remove the history of the branch by dereferencing the current HEAD from the tree.
```
git update-ref -d HEAD
```
1. Unpack the build artifact zip file to the current directory and accept overwriting
```
unzip -d $PWD ~/Downloads/rocky-8.5-docker-x86_64.tar.xz.zip
```
1. Generate the packages.txt file using the instructions below. This step will parse the `build.meta` file included in the build artifacts, and write the list of packages out to `packages.txt`.
```shell
xmllint --xpath "//packages/*/@name" <(printf "$(jq '.icicle' < build.meta)\n" | tr -d '\\' | tail -c +2 | head -c -2) | \
awk -F\= '{print substr($2,2,length($2)-2)}' | \
sort >! packages.txt
```
1. Add the files to be tracked by git using `git add .`. Then, check the git repo using `git status`. It should look something like this, having the new build artifacts as ready to be committed.
```
On branch Rocky-8.5.20220314-x86_64
No commits yet
Changes to be committed:
(use "git rm --cached <file>..." to unstage)
new file: Dockerfile
new file: build.meta
new file: filelist.txt
new file: packages.txt
new file: rocky-8.5-docker.tar.xz
```
1. Create a commit with a message regarding the changes. Perhaps using tools/pkgdiff.sh to show a list of changed packages from the previous-latest.
1. Push the commit to your fork and open a pull request to merge it as a new branch upstream. If you have commit-level access, this can also be done directly without forking.
1. Note the commit hash (shasum), as it is needed later for requesting the update from Docker.
1. Repeat for every architecture being updated.
#### Open pull request to request update
> :warning: Consult **ALL** the documentation on the docker-hub/official-images README page about the format of the file the official-images repository uses to build and release images.
> * Branches that will be referenced by the official-images repository data **MUST** contain only a single commit. A discrete branch will be created for each distinct image-tag that is released.
Once the branches are prepared, a PR can be created against https://github.com/docker-library/official-images to push the new images out and tag them appropriately.
1. Fork and clone https://github.com/docker-hub/official-images to your machine. Cd into the directory containing the repository.
1. Create a new branch if preferred, or just commit against the latest master. Ensure your fork is up to date with upstream.
1. Edit the library/rockylinux file and rearrange any tags as needed. The `latest` and `MAJOR` tags (e.g., `8`) should always point to the most recent image, and the most recent image should also be tagged with a unique name containing an ISO8601 datestamp like 8.5.20220314. The MAJOR.MINOR tag **SHOULD NOT** change during a release cycle, and should instead always point to the initial container build post minor release.
1. Commit and create a pull request upstream requesting the change. If the change is a security one, ensure it is marked as such. Instructions for this are included in the README for the docker-hub/official-images repository.
#### Docker Hub Official Images Support
If support is required, or any questions about anything related to official images or our listing there, a great resource is the #docker-library channel on Libera.chat IRC. It's a relatively low traffic channel.
#### Official Image Readme
The readme for the official image is maintained in a separate repository - https://github.com/docker-library/docs. If any information on the README needs to be changed, submit a pull request on that repository.

20
templates/tdl-aarch64.xml Normal file
View File

@ -0,0 +1,20 @@
<template>
<name>Rocky-8-Container-8.5.20211114.1.aarch64</name>
<os>
<name>Fedora</name>
<version>20</version>
<arch>aarch64</arch>
<install type='url'>
<url>http://dl.rockylinux.org/stg/rocky/8/BaseOS/aarch64/kickstart/</url>
</install>
<icicle>
<extra_command>rpm -qa --qf '%{NAME},%{VERSION},%{RELEASE},%{ARCH},%{EPOCH},%{SIZE},%{SIGMD5},%{BUILDTIME}
'</extra_command>
</icicle>
</os>
<description>Rocky-8-Container-8.5.20211114.1.aarch64 OS</description>
<disk>
<size>10G</size>
</disk>
</template>

16
templates/tdl-x86_64.xml Normal file
View File

@ -0,0 +1,16 @@
<template>
<name>Rocky-8-Container-8.5.20211114.1.x86_64</name>
<os>
<name>Fedora</name>
<version>26</version>
<arch>x86_64</arch>
<install type='url'>
<url>http://dl.rockylinux.org/stg/rocky/8/BaseOS/x86_64/os/</url>
</install>
</os>
<description>Rocky-8-Container-8.5.20211114.1.x86_64 OS</description>
<disk>
<size>10G</size>
</disk>
</template>

20
tools/pkdiff.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/sh
base="$1"
compare="${2-$(git rev-parse --abbrev-ref HEAD)}"
file=${3:-packages.txt}
usage () {
echo "$0: <from> [to] (defaults to current HEAD)"
exit
}
if [[ -z $base || -z $compare ]]; then
usage
fi
git diff "${base}:${file}" "${compare}:${file}" \
| grep -E '^([+-]\w)' \
| awk '!(NR%2){print substr(p,2,length(p)),"=>",substr($0,2,length($0))}{p=$0}'\
| column -t