chore: merge devel -> main

* empanadas v0.3.0
This commit is contained in:
Neil Hanlon 2022-07-13 23:37:09 -04:00
commit 4a4c1c0aeb
Signed by: neil
GPG Key ID: 705BC21EC3C70F34
75 changed files with 4198 additions and 897 deletions

View File

@ -0,0 +1,47 @@
---
name: Build empanada images for imagefactory
on:
push:
branches: [ $default-branch, "devel" ]
pull_request:
branches: [ $default-branch ]
workflow_dispatch:
jobs:
buildx:
runs-on:
- ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
# https://github.com/docker/setup-buildx-action
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v1
with:
install: true
- name: Login to ghcr
if: github.event_name != 'pull_request'
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push
id: docker_build
uses: docker/build-push-action@v2
with:
builder: ${{ steps.buildx.outputs.name }}
platforms: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le
context: ./iso/empanadas
file: ./iso/empanadas/Containerfile.imagefactory
push: ${{ github.event_name != 'pull_request' }}
tags: ghcr.io/rocky-linux/empanadas-imagefactory:latest
cache-from: type=gha
cache-to: type=gha,mode=max

View File

@ -1,9 +1,9 @@
---
name: Build empanada container images
name: Build empanada container images for lorax
on:
push:
branches: [ $default-branch ]
branches: [ $default-branch, "devel" ]
pull_request:
branches: [ $default-branch ]
workflow_dispatch:
@ -42,6 +42,6 @@ jobs:
context: ./iso/empanadas
file: ./iso/empanadas/Containerfile
push: ${{ github.event_name != 'pull_request' }}
tags: ghcr.io/neilhanlon/sig-core-toolkit:latest
tags: ghcr.io/rocky-linux/sig-core-toolkit:latest
cache-from: type=gha
cache-to: type=gha,mode=max

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
.swp

View File

@ -3,10 +3,9 @@ sig-core-toolkit
Release Engineering toolkit for repeatable operations or functionality testing.
Currently mirrored at our [github](https://github.com/rocky-linux),
[Rocky Linux Git Service](https://git.rockylinux.org), and the
[RESF Git Service](https://git.resf.org). Changes either occur at the Rocky
Linux Git Service or RESF Git Service.
Currently mirrored at our [github](https://github.com/rocky-linux), and the
[RESF Git Service](https://git.resf.org). Changes will typically occur at the
RESF Git Service.
What does this have?
--------------------
@ -14,10 +13,10 @@ What does this have?
* analyze -> Analysis utilities (such as download stats)
* chat -> mattermost related utilities
* func -> (mostly defunct) testing scripts and tools to test base functionality
* iso -> ISO related utilities
* iso -> ISO, Compose, and Sync related utilities, primarily for Rocky Linux 9+
* live -> Live image related utilities
* mangle -> Manglers and other misc stuff
* sync -> Sync tools, primarily for Rocky Linux 8
* sync -> Sync tools, primarily for Rocky Linux 8 and will eventually be deprecated
How can I help?
---------------
@ -28,7 +27,7 @@ when you make changes:
* Have pre-commit installed
* Have shellcheck installed
* Shell Scripts: These must pass a shellcheck test!
* Python scripts: Try your best to follow PEP8 guidelines
* Python scripts: Try your best to follow PEP8 guidelines (even the best linters get things wrong)
Your PR should be against the devel branch at all times. PR's against the main
branch will be closed.

59
func/ipa.sh Normal file
View File

@ -0,0 +1,59 @@
#!/bin/bash
# Release Engineering Core Functionality Testing
# Louis Abel <label@rockylinux.org> @nazunalika
################################################################################
# Settings and variables
# Exits on any non-zero exit status - Disabled for now.
#set -e
# Undefined variables will cause an exit
set -u
COMMON_EXPORTS='./common/exports.sh'
COMMON_IMPORTS='./common/imports.sh'
SELINUX=$(getenforce)
# End
################################################################################
# shellcheck source=/dev/null disable=SC2015
[ -f $COMMON_EXPORTS ] && source $COMMON_EXPORTS || { echo -e "\n[-] $(date): Variables cannot be sourced."; exit 1; }
# shellcheck source=/dev/null disable=SC2015
[ -f $COMMON_IMPORTS ] && source $COMMON_IMPORTS || { echo -e "\n[-] $(date): Functions cannot be sourced."; exit 1; }
# Init log
# shellcheck disable=SC2015
[ -e "$LOGFILE" ] && m_recycleLog || touch "$LOGFILE"
# SELinux check
if [ "$SELINUX" != "Enforcing" ]; then
echo -e "\n[-] $(date): SELinux is not enforcing."
exit 1
fi
r_log "internal" "Starting Release Engineering Core Tests"
################################################################################
# Script Work
# Skip tests in a list - some tests are already -x, so it won't be an issue
if [ -e skip.list ]; then
r_log "internal" "Disabling tests"
# shellcheck disable=SC2162
grep -E "^${RL_VER}" skip.list | while read line; do
# shellcheck disable=SC2086
testFile="$(echo $line | cut -d '|' -f 2)"
r_log "internal" "SKIP ${testFile}"
chmod -x "${testFile}"
done
r_log "internal" "WARNING: Tests above were disabled."
fi
# TODO: should we let $1 judge what directory is ran?
# TODO: get some stacks and lib in there
#r_processor <(/usr/bin/find ./core -type f | sort -t'/')
#r_processor <(/usr/bin/find ./lib -type f | sort -t'/')
r_processor <(/usr/bin/find ./stacks/ipa -type f | sort -t'/')
r_log "internal" "Core Tests completed"
exit 0

0
func/stacks/ipa/00-ipa-pregame.sh Normal file → Executable file
View File

2
func/stacks/ipa/10-install-ipa.sh Normal file → Executable file
View File

@ -11,4 +11,4 @@ if [ "$RL_VER" -eq 8 ]; then
p_enableModule idm:DL1/{client,common,dns,server}
fi
p_installPackageNormal ipa-server ipa-server-dns
p_installPackageNormal ipa-server ipa-server-dns expect

0
func/stacks/ipa/11-configure-ipa.sh Normal file → Executable file
View File

0
func/stacks/ipa/12-verify-ipa.sh Normal file → Executable file
View File

35
func/stacks/ipa/20-ipa-user.sh Normal file → Executable file
View File

@ -13,42 +13,43 @@ kdestroy &> /dev/null
klist 2>&1 | grep -E "(No credentials|Credentials cache .* not found)" &> /dev/null
r_checkExitStatus $?
expect -f - <<EOF
set send_human {.1 .3 1 .05 2}
spawn kinit admin
sleep 1
expect "Password for admin@RLIPA.LOCAL:"
send -h "b1U3OnyX!\r"
sleep 5
close
EOF
echo "b1U3OnyX!" | kinit admin@RLIPA.LOCAL
klist | grep "admin@RLIPA.LOCAL" &> /dev/null
r_checkExitStatus $?
r_log "ipa" "Test adding a user"
userDetails="$(ipa user-add --first=test --last=user --random ipatestuser)"
echo "$userDetails" | grep -q 'Added user "ipatestuser"'
r_checkExitStatus $?
ipa user-add --first=test --last=user --random ipatestuser > /tmp/ipatestuser
grep -q 'Added user "ipatestuser"' /tmp/ipatestuser
echo "$userDetails" | grep -q 'First name: test'
ret_val=$?
if [ "$ret_val" -ne 0 ]; then
r_log "ipa" "User was not created, this is considered fatal"
r_checkExitStatus 1
exit 1
fi
sed -i 's|^ ||g' /tmp/ipatestuser
grep -q 'First name: test' /tmp/ipatestuser
r_checkExitStatus $?
echo "$userDetails" | grep -q 'Last name: user'
grep -q 'Last name: user' /tmp/ipatestuser
r_checkExitStatus $?
echo "$userDetails" | grep -q 'Full name: test user'
grep -q 'Full name: test user' /tmp/ipatestuser
r_checkExitStatus $?
echo "$userDetails" | grep -q 'Home directory: /home/ipatestuser'
grep -q 'Home directory: /home/ipatestuser' /tmp/ipatestuser
r_checkExitStatus $?
r_log "ipa" "Changing password of the user"
kdestroy &> /dev/null
userPassword="$(awk '/Random password/ { print $3 }' /tmp/ipatestuser)"
/bin/rm /tmp/ipatestuser
expect -f - <<EOF
set send_human {.1 .3 1 .05 2}
spawn kinit ipatestuser
sleep 1
expect "Password for ipatestuser@RLIPA.LOCAL: "
send -h -- "$(echo "$userDetails" | awk '$0 ~ /Random password/ {print $3}')\r"
send -h -- "$(echo "$userPassword")\r"
sleep 1
expect "Enter new password: "
send -h -- "gr@YAm3thy5st!\r"

18
func/stacks/ipa/21-ipa-service.sh Normal file → Executable file
View File

@ -13,29 +13,21 @@ kdestroy &> /dev/null
klist 2>&1 | grep -E "(No credentials|Credentials cache .* not found)" &> /dev/null
r_checkExitStatus $?
expect -f - <<EOF
set send_human {.1 .3 1 .05 2}
spawn kinit admin
sleep 1
expect "Password for admin@RLIPA.LOCAL:"
send -h "b1U3OnyX!\r"
sleep 5
close
EOF
echo "b1U3OnyX!" | kinit admin@RLIPA.LOCAL
klist | grep "admin@RLIPA.LOCAL" &> /dev/null
r_checkExitStatus $?
r_log "ipa" "Adding test service"
ipa service-add testservice/rltest.rlipa.local &> /dev/null
ipa service-add testservice/onyxtest.rlipa.local &> /dev/null
r_checkExitStatus $?
r_log "ipa" "Getting keytab for service"
ipa-getkeytab -s rltest.rlipa.local -p testservice/rltest.rlipa.local -k /tmp/testservice.keytab &> /dev/null
ipa-getkeytab -s onyxtest.rlipa.local -p testservice/onyxtest.rlipa.local -k /tmp/testservice.keytab &> /dev/null
r_checkExitStatus $?
r_log "ipa" "Getting a certificate for service"
ipa-getcert request -K testservice/rltest.rlipa.local -D rltest.rlipa.local -f /etc/pki/tls/certs/testservice.crt -k /etc/pki/tls/private/testservice.key &> /dev/null
ipa-getcert request -K testservice/onyxtest.rlipa.local -D onyxtest.rlipa.local -f /etc/pki/tls/certs/testservice.crt -k /etc/pki/tls/private/testservice.key &> /dev/null
r_checkExitStatus $?
while true; do
@ -57,7 +49,7 @@ while ! stat /etc/pki/tls/certs/testservice.crt &> /dev/null; do
done
r_log "ipa" "Verifying keytab"
klist -k /tmp/testservice.keytab | grep "testservice/rltest.rlipa.local" &> /dev/null
klist -k /tmp/testservice.keytab | grep "testservice/onyxtest.rlipa.local" &> /dev/null
r_checkExitStatus $?
r_log "ipa" "Verifying key matches the certificate"

16
func/stacks/ipa/22-ipa-dns.sh Normal file → Executable file
View File

@ -13,21 +13,13 @@ kdestroy &> /dev/null
klist 2>&1 | grep -qE "(No credentials|Credentials cache .* not found)" &> /dev/null
r_checkExitStatus $?
expect -f - <<EOF
set send_human {.1 .3 1 .05 2}
spawn kinit admin
sleep 1
expect "Password for admin@RLIPA.LOCAL:"
send -h "b1U3OnyX!\r"
sleep 5
close
EOF
echo "b1U3OnyX!" | kinit admin@RLIPA.LOCAL
klist | grep "admin@RLIPA.LOCAL" &> /dev/null
r_checkExitStatus $?
r_log "ipa" "Adding testzone subdomain"
ipa dnszone-add --name-server=rltest.rlipa.local. --admin-email=hostmaster.testzone.rlipa.local. testzone.rlipa.local &> /dev/null
ipa dnszone-add --name-server=onyxtest.rlipa.local. --admin-email=hostmaster.testzone.rlipa.local. testzone.rlipa.local &> /dev/null
r_checkExitStatus $?
sleep 5
@ -36,7 +28,7 @@ dig @localhost SOA testzone.rlipa.local | grep -q "status: NOERROR" &> /dev/null
r_checkExitStatus $?
r_log "ipa" "Adding a CNAME record to the primary domain"
ipa dnsrecord-add rlipa.local testrecord --cname-hostname=rltest &> /dev/null
ipa dnsrecord-add rlipa.local testrecord --cname-hostname=onyxtest &> /dev/null
r_checkExitStatus $?
sleep 5
@ -45,7 +37,7 @@ dig @localhost CNAME testrecord.rlipa.local | grep -q "status: NOERROR" &> /dev/
r_checkExitStatus $?
r_log "ipa" "Adding a CNAME to subdomain"
ipa dnsrecord-add testzone.rlipa.local testrecord --cname-hostname=rltest.rlipa.local. &> /dev/null
ipa dnsrecord-add testzone.rlipa.local testrecord --cname-hostname=onyxtest.rlipa.local. &> /dev/null
r_checkExitStatus $?
sleep 5

54
func/stacks/ipa/23-ipa-sudo.sh Normal file → Executable file
View File

@ -9,19 +9,51 @@ if [ "$IPAINSTALLED" -eq 1 ]; then
r_checkExitStatus 1
fi
kdestroy &> /dev/null
klist 2>&1 | grep -E "(No credentials|Credentials cache .* not found)" &> /dev/null
kdestroy -A
klist 2>&1 | grep -E "(No credentials|Credentials cache .* not found)"
r_checkExitStatus $?
expect -f - <<EOF
set send_human {.1 .3 1 .05 2}
spawn kinit admin
sleep 1
expect "Password for admin@RLIPA.LOCAL:"
send -h "b1U3OnyX!\r"
echo "b1U3OnyX!" | kinit admin@RLIPA.LOCAL
klist | grep -q "admin@RLIPA.LOCAL"
r_checkExitStatus $?
r_log "ipa" "Creating a test sudo rule"
ipa sudorule-add testrule --desc="Test rule in IPA" --hostcat=all --cmdcat=all --runasusercat=all --runasgroupcat=all &> /dev/null
r_checkExitStatus $?
r_log "ipa" "Adding user to test sudo rule"
ipa sudorule-add-user testrule --users="ipatestuser" &> /dev/null
r_checkExitStatus $?
r_log "ipa" "Verifying rule..."
ipa sudorule-show testrule > /tmp/testrule
grep -q 'Rule name: testrule' /tmp/testrule
r_checkExitStatus $?
grep -q 'Description: Test rule in IPA' /tmp/testrule
r_checkExitStatus $?
grep -q 'Enabled: TRUE' /tmp/testrule
r_checkExitStatus $?
grep -q 'Host category: all' /tmp/testrule
r_checkExitStatus $?
grep -q 'Command category: all' /tmp/testrule
r_checkExitStatus $?
grep -q 'RunAs User category: all' /tmp/testrule
r_checkExitStatus $?
grep -q 'RunAs Group category: all' /tmp/testrule
r_checkExitStatus $?
grep -q 'Users: ipatestuser' /tmp/testrule
r_checkExitStatus $?
m_serviceCycler sssd stop
rm -rf /var/lib/sss/db/*
m_serviceCycler sssd start
sleep 5
close
EOF
klist | grep "admin@RLIPA.LOCAL" &> /dev/null
r_log "ipa" "Verifying sudo abilities"
sudo -l -U ipatestuser > /tmp/sudooutput
grep -q 'ipatestuser may run the following commands' /tmp/sudooutput
r_checkExitStatus $?
grep -q 'ALL) ALL' /tmp/sudooutput
r_checkExitStatus $?

0
func/stacks/ipa/50-cleanup-ipa.sh Normal file → Executable file
View File

0
func/stacks/lamp/00-install-lamp.sh Executable file → Normal file
View File

0
func/stacks/lamp/01-verification.sh Executable file → Normal file
View File

0
func/stacks/lamp/10-test-lamp.sh Executable file → Normal file
View File

View File

@ -2,3 +2,4 @@ __pycache__/
*.py[cod]
*$py.class
*.so
Containerfile*.devel

View File

@ -56,7 +56,7 @@ RUN rm -rf /etc/yum.repos.d/*.repo
RUN useradd -o -d /var/peridot -u 1002 peridotbuilder && usermod -a -G mock peridotbuilder
RUN chown peridotbuilder:mock /etc/yum.conf && chown -R peridotbuilder:mock /etc/dnf && chown -R peridotbuilder:mock /etc/rpm && chown -R peridotbuilder:mock /etc/yum.repos.d
RUN pip install 'git+https://git.rockylinux.org/release-engineering/public/toolkit.git@feature/iso-kube#egg=empanadas&subdirectory=iso/empanadas'
RUN pip install 'git+https://git.resf.org/sig_core/toolkit.git@devel#egg=empanadas&subdirectory=iso/empanadas'
RUN pip install awscli

View File

@ -0,0 +1,70 @@
FROM docker.io/fedora:36
ADD images/get_arch /get_arch
ENV TINI_VERSION v0.19.0
RUN curl -o /tini -L "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-$(/get_arch)"
RUN chmod +x /tini
RUN dnf install -y \
bash \
bzip2 \
cpio \
diffutils \
findutils \
gawk \
gcc \
gcc-c++ \
git \
grep \
gzip \
info \
make \
patch \
python3 \
redhat-rpm-config \
rpm-build \
scl-utils-build \
sed \
shadow-utils \
tar \
unzip \
util-linux \
which \
xz \
dnf-plugins-core \
createrepo_c \
rpm-sign \
sudo \
mock \
python-pip \
imagefactory \
imagefactory-plugins*
RUN sed -i -e 's/# memory = 1024/memory = 2048/' /etc/oz/oz.cfg
COPY imagefactory.patch /
COPY oz.rpm /
RUN dnf -y install /oz.rpm
RUN (cd /usr/lib/python3.10/site-packages/; patch -p1 </imagefactory.patch)
RUN ssh-keygen -t rsa -q -f "$HOME/.ssh/id_rsa" -N ""
RUN dnf clean all
RUN rm -rf /etc/yum.repos.d/*.repo /get_arch
# RUN useradd -o -d /var/peridot -u 1002 peridotbuilder && usermod -a -G mock peridotbuilder
# RUN chown -R peridotbuilder:mock /etc/dnf && chown -R peridotbuilder:mock /etc/rpm && chown -R peridotbuilder:mock /etc/yum.repos.d && chown -R peridotbuilder:mock /var/lib/imagefactory/storage
RUN pip install awscli
ARG BRANCH r9
RUN git clone https://git.resf.org/sig_core/kickstarts.git --branch $BRANCH /kickstarts
RUN pip install 'git+https://git.resf.org/sig_core/toolkit.git@devel#egg=empanadas&subdirectory=iso/empanadas'
ENV LIBGUESTFS_BACKEND direct
COPY prep-azure.sh /prep-azure.sh
RUN chmod +x /prep-azure.sh
ENTRYPOINT ["/tini", "--"]

View File

@ -1,12 +1,21 @@
# iso
## Setup / Install
1. Install [Poetry](https://python-poetry.org/docs/)
2. Setup: `poetry install`
3. Install dependencies: `dnf install podman mock`
3. Have fun
## Reliance on podman and mock
### Why podman?
Podman is a requirement for performing reposyncs. This was done because it was found to be easier to spin up several podman containers than several mock chroots and it was faster than doing one at a time in a loop. Podman is also used to parallelize ISO builds.
### Why mock?
There are cases where running `mock` is the preferred go-to: For example, building lorax images. Since you cannot build a lorax image for an architecture your system does not support, trying to "parallelize" it was out of the question. Adding this support in was not only for local testing without podman, it was also done so it can be run in our peridot kube cluster for each architecture.
## Updating dependencies
@ -16,24 +25,35 @@ Changes to the poetry.lock should be commited if dependencies are added or updat
## TODO
Verbose mode should exist to output everything that's being called or ran.
There should be additional logging regardless, not just to stdout, but also to a file.
* Verbose mode should exist to output everything that's being called or ran.
* There should be additional logging regardless, not just to stdout, but also to a file.
## scripts
* sync-variant-pungi
* sync-variant-peridot
* sync-from-pungi
* sync-from-peridot
* sync-sig
* build-all-iso
* sign-repos-only
```
* sync_from_peridot -> Syncs repositories from Peridot
* sync_sig -> Syncs SIG repositories from Peridot
* build-iso -> Builds initial ISO's using Lorax
* build-iso-extra -> Builds DVD's and other images based on Lorax data
* build-iso-live -> Builds live images
* pull-unpack-tree -> Pulls the latest lorax data from an S3 bucket and configures treeinfo
* pull-cloud-image -> Pulls the latest cloud images from an S3 bucket
* finalize_compose -> Finalizes a compose with metadata and checksums, as well as copies images
* launch-builds -> Creates a kube config to run build-iso
* build-image -> Runs build-iso
* generate_compose -> Creates a compose directory right away and optionally links it as latest
(You should only use this if you are running into errors with images)
```
## wrappers
* lorax-generators
* sync-generators
```
* common -> The starting point
* iso_utils -> Does work for ISO building and generation
* dnf_utils -> Does work for repo building and generation
* check -> Checks if the architecture/release combination are valid
* shared -> Shared utilities between all wrappers
```
## rules
@ -43,8 +63,9 @@ When making a script, you *must* import common. This is insanely bad practice,
but we would prefer if we started out this way:
```
from common import *
import argparse
from empanadas.common import *
from empanadas.util import Checks
```
Whatever is imported in common will effectively be imported in your scripts as

View File

@ -1 +1 @@
__version__ = '0.1.0'
__version__ = '0.2.0'

View File

@ -8,6 +8,24 @@ import yaml
import logging
import hashlib
from collections import defaultdict
from typing import Tuple
# An implementation from the Fabric python library
class AttributeDict(defaultdict):
def __init__(self):
super(AttributeDict, self).__init__(AttributeDict)
def __getattr__(self, key):
try:
return self[key]
except KeyError:
raise AttributeError(key)
def __setattr__(self, key, value):
self[key] = value
# These are a bunch of colors we may use in terminal output
class Color:
RED = '\033[91m'
@ -20,10 +38,14 @@ class Color:
UNDERLINE = '\033[4m'
BOLD = '\033[1m'
END = '\033[0m'
INFO = '[' + BOLD + GREEN + 'INFO' + END + '] '
WARN = '[' + BOLD + YELLOW + 'WARN' + END + '] '
FAIL = '[' + BOLD + RED + 'FAIL' + END + '] '
STAT = '[' + BOLD + CYAN + 'STAT' + END + '] '
# vars and additional checks
rldict = {}
sigdict = {}
rldict = AttributeDict()
sigdict = AttributeDict()
config = {
"rlmacro": rpm.expandMacro('%rhel'),
"dist": 'el' + rpm.expandMacro('%rhel'),
@ -77,3 +99,40 @@ for conf in glob.iglob(f"{_rootdir}/sig/*.yaml"):
#rlvars = rldict[rlver]
#rlvars = rldict[rlmacro]
#COMPOSE_ISO_WORKDIR = COMPOSE_ROOT + "work/" + arch + "/" + date_stamp
ALLOWED_TYPE_VARIANTS = {
"Azure": None,
"Container": ["Base", "Minimal", "UBI"],
"EC2": None,
"GenericCloud": None,
"Vagrant": ["Libvirt", "Vbox"]
}
def valid_type_variant(_type: str, variant: str="") -> bool:
if _type not in ALLOWED_TYPE_VARIANTS:
raise Exception(f"Type is invalid: ({_type}, {variant})")
if ALLOWED_TYPE_VARIANTS[_type] == None:
if variant is not None:
raise Exception(f"{_type} Type expects no variant type.")
return True
if variant not in ALLOWED_TYPE_VARIANTS[_type]:
if variant.capitalize() in ALLOWED_TYPE_VARIANTS[_type]:
raise Exception(f"Capitalization mismatch. Found: ({_type}, {variant}). Expected: ({_type}, {variant.capitalize()})")
raise Exception(f"Type/Variant Combination is not allowed: ({_type}, {variant})")
return True
from attrs import define, field
@define(kw_only=True)
class Architecture:
name: str = field()
version: str = field()
major: int = field(converter=int)
minor: int = field(converter=int)
@classmethod
def from_version(cls, architecture: str, version: str):
major, minor = str.split(version, ".")
if architecture not in rldict[major]["allowed_arches"]:
print("Invalid architecture/version combo, skipping")
exit()
return cls(name=architecture, version=version, major=major, minor=minor)

View File

@ -44,64 +44,78 @@
has_modules:
- 'AppStream'
- 'PowerTools'
iso_map:
hosts:
x86_64: ''
aarch64: ''
ppc64le: ''
s390x: ''
images:
- dvd1
- minimal
- boot
repos:
- 'BaseOS'
- 'AppStream'
variant: 'BaseOS'
lorax_removes:
- 'libreport-rhel-anaconda-bugzilla'
required_packages:
- 'lorax'
- 'genisoimage'
- 'isomd5sum'
- 'lorax-templates-rhel'
- 'lorax-templates-generic'
structure:
packages: 'os/Packages'
repodata: 'os/repodata'
iso_map:
xorrisofs: False
iso_level: False
hosts:
x86_64: ''
aarch64: ''
images:
dvd:
disc: True
variant: 'AppStream'
repos:
- 'BaseOS'
- 'AppStream'
lorax_variants:
- dvd
- minimal
- BaseOS
repos:
- 'BaseOS'
- 'AppStream'
variant: 'BaseOS'
lorax_removes:
- 'libreport-rhel-anaconda-bugzilla'
minimal:
disc: True
isoskip: True
repos:
- 'minimal'
- 'BaseOS'
variant: 'minimal'
BaseOS:
disc: False
isoskip: True
variant: 'BaseOS'
repos:
- 'BaseOS'
- 'AppStream'
lorax:
repos:
- 'BaseOS'
- 'AppStream'
variant: 'BaseOS'
lorax_removes:
- 'libreport-rhel-anaconda-bugzilla'
required_pkgs:
- 'lorax'
- 'genisoimage'
- 'isomd5sum'
- 'lorax-templates-rhel'
- 'lorax-templates-generic'
- 'xorriso'
cloudimages:
images:
EC2:
format: raw
GenericCloud:
format: qcow2
livemap:
git_repo: 'https://git.resf.org/sig_core/kickstarts.git'
branch: 'r9'
ksentry:
Workstation: rocky-live-workstation.ks
Workstation-Lite: rocky-live-workstation-lite.ks
XFCE: rocky-live-xfce.ks
KDE: rocky-live-kde.ks
allowed_arches:
- x86_64
required_pkgs:
- 'lorax'
- 'genisoimage'
- 'isomd5sum'
- 'lorax-templates-rhel'
- 'lorax-templates-generic'
- 'lorax-lmc-novirt'
- 'vim-minimal'
- 'pykickstart'
- 'git'
variantmap:
git_repo: 'https://git.rockylinux.org/rocky/pungi-rocky.git'
branch: 'r8'
git_raw_path: 'https://git.rockylinux.org/rocky/pungi-rocky/-/raw/r8/'
repoclosure_map:
arches:
x86_64: '--arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'
aarch64: '--arch=aarch64 --arch=noarch'
ppc64le: '--arch=ppc64le --arch=noarch'
s390x: '--arch=s390x --arch=noarch'
x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'
aarch64: '--forcearch=aarch64 --arch=aarch64 --arch=noarch'
ppc64le: '--forcearch=ppc64le --arch=ppc64le --arch=noarch'
s390x: '--forcearch=s390x --arch=s390x --arch=noarch'
repos:
BaseOS: []
AppStream:

View File

@ -50,6 +50,7 @@
isoskip: True
repos:
- 'minimal'
- 'BaseOS'
variant: 'minimal'
BaseOS:
disc: False
@ -72,6 +73,31 @@
- 'lorax-templates-rhel'
- 'lorax-templates-generic'
- 'xorriso'
cloudimages:
images:
EC2:
format: raw
GenericCloud:
format: qcow2
livemap:
git_repo: 'https://git.resf.org/sig_core/kickstarts.git'
branch: 'r9-beta'
ksentry:
Workstation: rocky-live-workstation.ks
Workstation-Lite: rocky-live-workstation-lite.ks
XFCE: rocky-live-xfce.ks
KDE: rocky-live-kde.ks
allowed_arches:
- x86_64
required_pkgs:
- 'lorax-lmc-novirt'
- 'vim-minimal'
- 'pykickstart'
- 'git'
variantmap:
git_repo: 'https://git.rockylinux.org/rocky/pungi-rocky.git'
branch: 'r9-beta'
git_raw_path: 'https://git.rockylinux.org/rocky/pungi-rocky/-/raw/r9-beta/'
repoclosure_map:
arches:
x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'

View File

@ -2,7 +2,7 @@
'9':
fullname: 'Rocky Linux 9.0'
revision: '9.0'
rclvl: 'RC1'
rclvl: 'RC2'
major: '9'
minor: '0'
profile: '9'
@ -50,7 +50,9 @@
isoskip: True
repos:
- 'minimal'
- 'BaseOS'
variant: 'minimal'
volname: 'dvd'
BaseOS:
disc: False
isoskip: True
@ -72,6 +74,31 @@
- 'lorax-templates-rhel'
- 'lorax-templates-generic'
- 'xorriso'
cloudimages:
images:
EC2:
format: raw
GenericCloud:
format: qcow2
livemap:
git_repo: 'https://git.resf.org/sig_core/kickstarts.git'
branch: 'r9'
ksentry:
Workstation: rocky-live-workstation.ks
Workstation-Lite: rocky-live-workstation-lite.ks
XFCE: rocky-live-xfce.ks
KDE: rocky-live-kde.ks
allowed_arches:
- x86_64
required_pkgs:
- 'lorax-lmc-novirt'
- 'vim-minimal'
- 'pykickstart'
- 'git'
variantmap:
git_repo: 'https://git.rockylinux.org/rocky/pungi-rocky.git'
branch: 'r9'
git_raw_path: 'https://git.rockylinux.org/rocky/pungi-rocky/-/raw/r9/'
repoclosure_map:
arches:
x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'

View File

@ -50,6 +50,7 @@
isoskip: True
repos:
- 'minimal'
- 'BaseOS'
variant: 'minimal'
BaseOS:
disc: False
@ -72,6 +73,31 @@
- 'lorax-templates-rhel'
- 'lorax-templates-generic'
- 'xorriso'
cloudimages:
images:
EC2:
format: raw
GenericCloud:
format: qcow2
livemap:
git_repo: 'https://git.resf.org/sig_core/kickstarts.git'
branch: 'r9lh'
ksentry:
Workstation: rocky-live-workstation.ks
Workstation-Lite: rocky-live-workstation-lite.ks
XFCE: rocky-live-xfce.ks
KDE: rocky-live-kde.ks
allowed_arches:
- x86_64
required_pkgs:
- 'lorax-lmc-novirt'
- 'vim-minimal'
- 'pykickstart'
- 'git'
variantmap:
git_repo: 'https://git.rockylinux.org/rocky/pungi-rocky.git'
branch: 'r9lh'
git_raw_path: 'https://git.rockylinux.org/rocky/pungi-rocky/-/raw/r9lh/'
repoclosure_map:
arches:
x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'

View File

@ -40,18 +40,19 @@
iso_level: False
images:
dvd:
discnum: '1'
disc: True
variant: 'AppStream'
repos:
- 'BaseOS'
- 'AppStream'
minimal:
discnum: '1'
disc: True
isoskip: True
repos:
- 'minimal'
variant: 'minimal'
BaseOS:
disc: False
isoskip: True
variant: 'BaseOS'
repos:
@ -66,10 +67,32 @@
- 'libreport-rhel-anaconda-bugzilla'
required_pkgs:
- 'lorax'
- 'genisoimage'
- 'isomd5sum'
- 'lorax-templates-rhel'
- 'lorax-templates-generic'
- 'xorriso'
cloudimages:
images:
EC2:
format: raw
GenericCloud:
format: qcow2
livemap:
git_repo: 'https://git.resf.org/sig_core/kickstarts.git'
branch: 'rln'
ksentry:
Workstation: rocky-live-workstation.ks
Workstation-Lite: rocky-live-workstation-lite.ks
XFCE: rocky-live-xfce.ks
KDE: rocky-live-kde.ks
allowed_arches:
- x86_64
required_pkgs:
- 'lorax-lmc-novirt'
- 'vim-minimal'
- 'pykickstart'
- 'git'
repoclosure_map:
arches:
x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch'

View File

@ -0,0 +1,424 @@
# Builds an image given a version, type, variant, and architecture
# Defaults to the running host's architecture
import argparse
import datetime
import json
import logging
import os
import pathlib
import platform
import subprocess
import sys
import tempfile
import time
from attrs import define, Factory, field, asdict
from botocore import args
from jinja2 import Environment, FileSystemLoader, Template
from typing import Callable, List, NoReturn, Optional, Tuple, IO, Union
from empanadas.common import Architecture, rldict, valid_type_variant
from empanadas.common import _rootdir
parser = argparse.ArgumentParser(description="ISO Compose")
parser.add_argument('--version', type=str, help="Release Version (8.6, 9.1)", required=True)
parser.add_argument('--rc', action='store_true', help="Release Candidate")
parser.add_argument('--kickstartdir', action='store_true', help="Use the kickstart dir instead of the os dir for repositories")
parser.add_argument('--debug', action='store_true', help="debug?")
parser.add_argument('--type', type=str, help="Image type (container, genclo, azure, aws, vagrant)", required=True)
parser.add_argument('--variant', type=str, help="", required=False)
parser.add_argument('--release', type=str, help="Image release for subsequent builds with the same date stamp (rarely needed)", required=False)
parser.add_argument('--kube', action='store_true', help="output as a K8s job(s)", required=False)
results = parser.parse_args()
rlvars = rldict[results.version]
major = rlvars["major"]
debug = results.debug
log = logging.getLogger(__name__)
log.setLevel(logging.INFO if not debug else logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.INFO if not debug else logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s :: %(name)s :: %(message)s',
'%Y-%m-%d %H:%M:%S'
)
handler.setFormatter(formatter)
log.addHandler(handler)
STORAGE_DIR = pathlib.Path("/var/lib/imagefactory/storage")
KICKSTART_PATH = pathlib.Path(os.environ.get("KICKSTART_PATH", "/kickstarts"))
BUILDTIME = datetime.datetime.utcnow()
@define(kw_only=True)
class ImageBuild:
architecture: Architecture = field()
base_uuid: Optional[str] = field(default="")
cli_args: argparse.Namespace = field()
command_args: List[str] = field(factory=list)
common_args: List[str] = field(factory=list)
debug: bool = field(default=False)
image_type: str = field()
job_template: Optional[Template] = field(init=False)
kickstart_arg: List[str] = field(factory=list)
metadata: pathlib.Path = field(init=False)
out_type: str = field(init=False)
outdir: pathlib.Path = field(init=False)
outname: str = field(init=False)
package_args: List[str] = field(factory=list)
release: int = field(default=0)
stage_commands: Optional[List[List[Union[str,Callable]]]] = field(init=False)
target_uuid: Optional[str] = field(default="")
tdl_path: pathlib.Path = field(init=False)
template: Template = field()
type_variant: str = field(init=False)
variant: Optional[str] = field()
def __attrs_post_init__(self):
self.tdl_path = self.render_icicle_template()
if not self.tdl_path:
exit(2)
self.type_variant = self.type_variant_name()
self.outdir, self.outname = self.output_name()
self.out_type = self.image_format()
self.command_args = self._command_args()
self.package_args = self._package_args()
self.common_args = self._common_args()
self.kickstart_arg = self.kickstart_imagefactory_args()
self.metadata = pathlib.Path(self.outdir, "metadata.json")
# Yes, this is gross. I'll fix it later.
if self.image_type in ["Container"]:
self.stage_commands = [
["tar", "-C", f"{self.outdir}", "--strip-components=1", "-x", "-f", lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", "*/layer.tar"],
["xz", f"{self.outdir}/layer.tar"]
]
if self.image_type in ["GenericCloud"]:
self.stage_commands = [
["qemu-img", "convert", "-c", "-f", "raw", "-O", "qcow2", lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", f"{self.outdir}/{self.outname}.qcow2"]
]
if self.image_type in ["EC2"]:
self.stage_commands = [
["qemu-img", "convert", "-f", "raw", "-O", "qcow2", lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", f"{self.outdir}/{self.outname}.qcow2"]
]
if self.image_type in ["Azure"]:
self.stage_commands = [
["/prep-azure.sh", lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", f"{STORAGE_DIR}"],
["cp", lambda: f"{STORAGE_DIR}/{self.target_uuid}.vhd", f"{self.outdir}/{self.outname}.vhd"]
]
# ["qemu-img", "resize", "-f", "raw", lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", lambda: f"{self.rounded_size()}"],
# ["qemu-img", "convert", "-f", "raw", "-o", "subformat=fixed,force_size" ,"-O", "vpc", lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", f"{self.outdir}/{self.outname}.vhd"]
if self.image_type in ["Vagrant"]:
_map = {
"Vbox": "vmdk",
"Libvirt": "qcow2"
}
output = f"{_map[self.variant]}" #type: ignore
self.stage_commands = [
["qemu-img", "convert", "-c", "-f", "raw", "-O", output, lambda: f"{STORAGE_DIR}/{self.target_uuid}.body", f"{self.outdir}/{self.outname}.{output}"]
]
if self.stage_commands:
self.stage_commands.append(["cp", "-v", lambda: f"{STORAGE_DIR}/{self.target_uuid}.meta", f"{self.outdir}/build.meta"])
try:
os.mkdir(self.outdir)
except FileExistsError as e:
log.info("Directory already exists for this release. If possible, previously executed steps may be skipped")
except Exception as e:
log.exception("Some other exception occured while creating the output directory", e)
return 0
if os.path.exists(self.metadata):
with open(self.metadata, "r") as f:
try:
o = json.load(f)
self.base_uuid = o['base_uuid']
self.target_uuid = o['target_uuid']
except json.decoder.JSONDecodeError as e:
log.exception("Couldn't decode metadata file", e)
finally:
f.flush()
# def rounded_size(self) -> int:
# # Azure images need to be rounded to the nearest 1MB boundary.
# MB=1024*1024
#
# raw_size = pathlib.Path(STORAGE_DIR},f"{self.target_uuid}.body").stat().st_size
# rounded_size = raw
def output_name(self) -> Tuple[pathlib.Path, str]:
directory = f"Rocky-{self.architecture.major}-{self.type_variant}-{self.architecture.version}-{BUILDTIME.strftime('%Y%m%d')}.{self.release}"
name = f"{directory}.{self.architecture.name}"
outdir = pathlib.Path(f"/tmp/", directory)
return outdir, name
def type_variant_name(self):
return self.image_type if not self.variant else f"{self.image_type}-{self.variant}"
def _command_args(self):
args_mapping = {
"debug": "--debug"
}
return [param for name, param in args_mapping.items() if getattr(self.cli_args, name)]
def _package_args(self) -> List[str]:
if self.image_type == "Container":
return ["--parameter", "compress", "xz"]
return [""]
def _common_args(self) -> List[str]:
args = []
if self.image_type == "Container":
args = ["--parameter", "offline_icicle", "true"]
if self.image_type in ["GenericCloud", "EC2", "Vagrant", "Azure"]:
args = ["--parameter", "generate_icicle", "false"]
return args
def image_format(self) -> str:
mapping = {
"Container": "docker"
}
return mapping[self.image_type] if self.image_type in mapping.keys() else ''
def kickstart_imagefactory_args(self) -> List[str]:
kickstart_path = pathlib.Path(f"{KICKSTART_PATH}/Rocky-{self.architecture.major}-{self.type_variant}.ks")
if not kickstart_path.is_file():
log.warn(f"Kickstart file is not available: {kickstart_path}")
if not debug:
log.warn("Exiting because debug mode is not enabled.")
exit(2)
return ["--file-parameter", "install_script", str(kickstart_path)]
def render_icicle_template(self) -> pathlib.Path:
handle, output = tempfile.mkstemp()
if not handle:
exit(3)
with os.fdopen(handle, "wb") as tmp:
_template = self.template.render(
architecture=self.architecture.name,
iso8601date=BUILDTIME.strftime("%Y%m%d"),
installdir="kickstart" if self.cli_args.kickstartdir else "os",
major=self.architecture.major,
minor=self.architecture.minor,
release=self.release,
size="10G",
type=self.image_type,
utcnow=BUILDTIME,
version_variant=self.architecture.version if not self.variant else f"{self.architecture.version}-{self.variant}",
)
tmp.write(_template.encode())
tmp.flush()
output = pathlib.Path(output)
if not output.exists():
log.error("Failed to write TDL template")
raise Exception("Failed to write TDL template")
return output
def build_command(self) -> List[str]:
build_command = ["imagefactory", *self.command_args, "base_image", *self.common_args, *self.kickstart_arg, self.tdl_path
# "|", "tee", "-a", f"{outdir}/logs/base_image-{outname}.out",
# "|", "tail", "-n4", ">", f"{outdir}/base.meta", "||", "exit", "2"
]
return build_command
def package_command(self) -> List[str]:
package_command = ["imagefactory", *self.command_args, "target_image", self.out_type, *self.common_args,
"--id", f"{self.base_uuid}",
*self.package_args,
"--parameter", "repository", self.outname,
# "|", "tee", "-a", f"{outdir}/base_image-{outname}.out",
# "|", "tail", "-n4", ">", f"{outdir}/target.meta", "||", "exit", "3"
]
return package_command
def copy_command(self) -> List[str]:
copy_command = ["aws", "s3", "cp", "--recursive", f"{self.outdir}/",
f"s3://resf-empanadas/buildimage-{self.architecture.version}-{self.architecture.name}/{ self.outname }/{ BUILDTIME.strftime('%s') }/"
]
return copy_command
def build(self) -> int:
if self.base_uuid:
return 0
self.fix_ks()
ret, out, err, uuid = self.runCmd(self.build_command())
if uuid:
self.base_uuid = uuid.rstrip()
self.save()
return ret
def package(self) -> int:
# Some build types don't need to be packaged by imagefactory
# @TODO remove business logic if possible
if self.image_type in ["GenericCloud", "EC2", "Azure", "Vagrant"]:
self.target_uuid = self.base_uuid if hasattr(self, 'base_uuid') else ""
if self.target_uuid:
return 0
ret, out, err, uuid = self.runCmd(self.package_command())
if uuid:
self.target_uuid = uuid.rstrip()
self.save()
return ret
def stage(self) -> int:
""" Stage the artifacst from wherever they are (unpacking and converting if needed)"""
if not hasattr(self,'stage_commands'):
return 0
returns = []
for command in self.stage_commands: #type: ignore
ret, out, err, _ = self.runCmd(command, search=False)
returns.append(ret)
return all(ret > 0 for ret in returns)
def copy(self, skip=False) -> int:
# move or unpack if necessary
log.info("Executing staging commands")
if (stage := self.stage() > 0):
raise Exception(stage)
if not skip:
log.info("Copying files to output directory")
ret, out, err, _ = self.runCmd(self.copy_command(), search=False)
return ret
log.info(f"Build complete! Output available in {self.outdir}/")
return 0
def runCmd(self, command: List[Union[str, Callable]], search: bool = True) -> Tuple[int, Union[bytes,None], Union[bytes,None], Union[str,None]]:
prepared, _ = self.prepare_command(command)
log.info(f"Running command: {' '.join(prepared)}")
kwargs = {
"stderr": subprocess.PIPE,
"stdout": subprocess.PIPE
}
if debug: del kwargs["stderr"]
with subprocess.Popen(prepared, **kwargs) as p:
uuid = None
# @TODO implement this as a callback?
if search:
for _, line in enumerate(p.stdout): # type: ignore
ln = line.decode()
if ln.startswith("UUID: "):
uuid = ln.split(" ")[-1]
log.debug(f"found uuid: {uuid}")
out, err = p.communicate()
res = p.wait(), out, err, uuid
if res[0] > 0:
log.error(f"Problem while executing command: '{prepared}'")
if search and not res[3]:
log.error("UUID not found in stdout. Dumping stdout and stderr")
self.log_subprocess(res)
return res
def prepare_command(self, command_list: List[Union[str, Callable]]) -> Tuple[List[str],List[None]]:
"""
Commands may be a callable, which should be a lambda to be evaluated at
preparation time with available locals. This can be used to, among
other things, perform lazy evaluations of f-strings which have values
not available at assignment time. e.g., filling in a second command
with a value extracted from the previous step or command.
"""
r = []
return r, [r.append(c()) if (callable(c) and c.__name__ == '<lambda>') else r.append(str(c)) for c in command_list]
def log_subprocess(self, result: Tuple[int, Union[bytes, None], Union[bytes, None], Union[str, None]]):
def log_lines(title, lines):
log.info(f"====={title}=====")
log.info(lines.decode())
log.info(f"Command return code: {result[0]}")
stdout = result[1]
stderr = result[2]
if stdout:
log_lines("Command STDOUT", stdout)
if stderr:
log_lines("Command STDERR", stderr)
def fix_ks(self):
self.runCmd(["sed", "-i", f"s,$basearch,{self.architecture.name},", self.kickstart_arg[-1]], search=False)
def render_kubernetes_job(self):
commands = [self.build_command(), self.package_command(), self.copy_command()]
if not self.job_template:
return None
template = self.job_template.render(
architecture=self.architecture.name,
backoffLimit=4,
buildTime=BUILDTIME.strftime("%s"),
command=commands,
imageName="ghcr.io/rockylinux/sig-core-toolkit:latest",
jobname="buildimage",
namespace="empanadas",
major=major,
restartPolicy="Never",
)
return template
def save(self):
with open(pathlib.Path(self.outdir, "metadata.json"), "w") as f:
try:
o = { name: getattr(self, name) for name in ["base_uuid", "target_uuid"] }
log.debug(o)