diff --git a/.github/workflows/mix-empanadas.yml b/.github/workflows/mix-empanadas.yml new file mode 100644 index 0000000..2f408ca --- /dev/null +++ b/.github/workflows/mix-empanadas.yml @@ -0,0 +1,47 @@ +--- +name: Build empanada container images + +on: + push: + branches: [ $default-branch ] + pull_request: + branches: [ $default-branch ] + workflow_dispatch: + +jobs: + buildx: + runs-on: + - ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + with: + install: true + + - name: Login to ghcr + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + platforms: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le + context: ./iso/empanadas + file: ./iso/empanadas/Containerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ghcr.io/neilhanlon/sig-core-toolkit:latest + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/func/README.md b/func/README.md index 1d007a9..52b0062 100644 --- a/func/README.md +++ b/func/README.md @@ -28,9 +28,11 @@ How to Run There are two ways to run through the tests: * By running `/bin/bash runtests.sh` - * Runs all tests + * Runs all core tests +* By running `/bin/bash stacktests.sh` + * Runs all stack tests (eg, lamp, ipa) * By running `/bin/bash monotests.sh` - * Runs all tests one by one to help identify failures as they happen + * Supposed to runs all tests one by one to help identify failures as they happen (not functional) Adding Tests ------------ @@ -316,6 +318,9 @@ Current Tree │   │   ├── 00-install-lsof.sh │   │   ├── 10-test-lsof.sh │   │   └── README.md +│   ├── pkg_mdadm +│   │   ├── 00-install-mdadm.sh +│   │   └── 01-test-mdadm.sh │   ├── pkg_network │   │   ├── 00-install-packages.sh │   │   ├── 10-tracepath.sh @@ -348,6 +353,13 @@ Current Tree │   │   ├── 30-postfix-sasl.sh │   │   ├── 40-postfix-tls.sh │   │   └── README.md +│   ├── pkg_postgresql +│   │   ├── 00-install-postgresql.sh +│   │   ├── 01-configure-postgresql.sh +│   │   ├── 10-create-db.sh +│   │   ├── 11-create-user.sh +│   │   ├── 20-drop-db.sh +│   │   └── 21-drop-user.sh │   ├── pkg_python │   │   ├── 00-install-python.sh │   │   ├── 10-test-python3.sh @@ -423,6 +435,11 @@ Current Tree │   ├── pkg_telnet │   │   ├── 00-install-telnet.sh │   │   └── 10-test-telnet.sh +│   ├── pkg_tftp-server +│   │   ├── 00-install-tftp.sh +│   │   ├── 01-configure-tftp.sh +│   │   ├── 10-get-test.sh +│   │   └── 11-put-test.sh │   ├── pkg_vsftpd │   │   ├── 00-install-vsftpd.sh │   │   ├── 10-anonymous-vsftpd.sh @@ -447,20 +464,21 @@ Current Tree ├── README.md ├── runtests.sh ├── skip.list -└── stacks - ├── ipa - │   ├── 00-ipa-pregame.sh - │   ├── 10-install-ipa.sh - │   ├── 11-configure-ipa.sh - │   ├── 12-verify-ipa.sh - │   ├── 20-ipa-user.sh - │   ├── 21-ipa-service.sh - │   ├── 22-ipa-dns.sh - │   ├── 23-ipa-sudo.sh - │   ├── 50-cleanup-ipa.sh - │   └── README.md - └── lamp - ├── 00-install-lamp.sh - ├── 01-verification.sh - └── 10-test-lamp.sh +├── stacks +│   ├── ipa +│   │   ├── 00-ipa-pregame.sh +│   │   ├── 10-install-ipa.sh +│   │   ├── 11-configure-ipa.sh +│   │   ├── 12-verify-ipa.sh +│   │   ├── 20-ipa-user.sh +│   │   ├── 21-ipa-service.sh +│   │   ├── 22-ipa-dns.sh +│   │   ├── 23-ipa-sudo.sh +│   │   ├── 50-cleanup-ipa.sh +│   │   └── README.md +│   └── lamp +│   ├── 00-install-lamp.sh +│   ├── 01-verification.sh +│   └── 10-test-lamp.sh +└── stacks.sh ``` diff --git a/func/common/exports.sh b/func/common/exports.sh index 4ed75bc..2bb991f 100644 --- a/func/common/exports.sh +++ b/func/common/exports.sh @@ -11,5 +11,5 @@ export readonly RELEASE_NAME=rocky # A 0 means it was successful. It can be changed to 1 on failure. export IPAINSTALLED=0 -LOGFILE="./log/$(date +'%m-%d-%Y')-tests.log" +LOGFILE="$(pwd)/log/$(date +'%m-%d-%Y')-tests.log" export LOGFILE diff --git a/func/core/pkg_archive/00-install-formats.sh b/func/core/pkg_archive/00-install-formats.sh index 6525314..f7c8d77 100755 --- a/func/core/pkg_archive/00-install-formats.sh +++ b/func/core/pkg_archive/00-install-formats.sh @@ -2,4 +2,4 @@ r_log "archive" "Installing appropriate archive formats" # We might need expect for zmore - does anyone actually use zmore? -p_installPackageNormal bzip2 diffutils gzip less ncompress tar unzip util-linux-ng zip lzop +p_installPackageNormal bzip2 diffutils gzip less tar unzip util-linux-ng zip lzop diff --git a/func/core/pkg_archive/21-gzip-test.sh b/func/core/pkg_archive/21-gzip-test.sh index ce4a440..a19c96d 100755 --- a/func/core/pkg_archive/21-gzip-test.sh +++ b/func/core/pkg_archive/21-gzip-test.sh @@ -89,8 +89,12 @@ gzip $FILE $FILE.1 || r_checkExitStatus 1 r_log "archive" "Verify that .Z files can be handled" gunzip $FILE.gz ls -l /var/tmp >> $FILE -compress $FILE || r_checkExitStatus 1 -gunzip $FILE.Z || r_checkExitStatus 1 +if [ "$RL_VER" -eq 8 ]; then + compress $FILE || r_checkExitStatus 1 + gunzip $FILE.Z || r_checkExitStatus 1 +else + r_log "archive" "Skipping for 9" +fi # handle some zip files r_log "archive" "Verify that .zip files can be handled" diff --git a/func/core/pkg_firefox/10-check-firefox-start-page.sh b/func/core/pkg_firefox/10-check-firefox-start-page.sh index 33c59eb..3963fa1 100755 --- a/func/core/pkg_firefox/10-check-firefox-start-page.sh +++ b/func/core/pkg_firefox/10-check-firefox-start-page.sh @@ -9,7 +9,7 @@ fi COUNTS="$(grep -c rockylinux.org $FIREPATH)" -if [ "$COUNTS" -eq 2 ]; then +if [ "$COUNTS" -ge 2 ]; then r_checkExitStatus 0 else r_checkExitStatus 1 diff --git a/func/core/pkg_httpd/10-httpd-branding.sh b/func/core/pkg_httpd/10-httpd-branding.sh index 7508411..cbfe6ca 100755 --- a/func/core/pkg_httpd/10-httpd-branding.sh +++ b/func/core/pkg_httpd/10-httpd-branding.sh @@ -2,7 +2,7 @@ r_log "httpd" "Verify httpd branding" r_log "httpd" "Token" -curl -sI http://localhost/ | grep -i "Server:\ Apache.*\ (Rocky)" > /dev/null 2>&1 +curl -sI http://localhost/ | grep -i "Server:\ Apache.*\ (Rocky Linux)" > /dev/null 2>&1 r_checkExitStatus $? r_log "httpd" "index" diff --git a/func/core/pkg_lsb/00-install-lsb.sh b/func/core/pkg_lsb/00-install-lsb.sh index 452dd70..3733c79 100755 --- a/func/core/pkg_lsb/00-install-lsb.sh +++ b/func/core/pkg_lsb/00-install-lsb.sh @@ -1,3 +1,8 @@ #!/bin/bash r_log "lsb" "Install LSB package" +if [ "$RL_VER" -ge 8 ]; then + r_log "lsb" "redhat-lsb is not in EL9" + exit $PASS +fi + p_installPackageNormal redhat-lsb diff --git a/func/core/pkg_lsb/10-test-branding.sh b/func/core/pkg_lsb/10-test-branding.sh index 9be22cb..229fbcb 100755 --- a/func/core/pkg_lsb/10-test-branding.sh +++ b/func/core/pkg_lsb/10-test-branding.sh @@ -1,5 +1,10 @@ #!/bin/bash r_log "lsb" "Test LSB branding" +if [ "$RL_VER" -ge 8 ]; then + r_log "lsb" "redhat-lsb is not in EL9" + exit $PASS +fi + lsb_release -i | grep -q "Rocky" r_checkExitStatus $? lsb_release -d | grep -q "Rocky" diff --git a/func/core/pkg_mdadm/00-install-mdadm.sh b/func/core/pkg_mdadm/00-install-mdadm.sh new file mode 100755 index 0000000..eba00b5 --- /dev/null +++ b/func/core/pkg_mdadm/00-install-mdadm.sh @@ -0,0 +1,3 @@ +#!/bin/bash +r_log "mdadm" "Install mdadm" +p_installPackageNormal mdadm diff --git a/func/core/pkg_mdadm/01-test-mdadm.sh b/func/core/pkg_mdadm/01-test-mdadm.sh new file mode 100755 index 0000000..c8e0be3 --- /dev/null +++ b/func/core/pkg_mdadm/01-test-mdadm.sh @@ -0,0 +1,13 @@ +#!/bin/bash +r_log "mdadm" "Check that mdadm will operate and return the right exit codes" +[ ${EUID} -eq 0 ] || { r_log "mdadm" "Not running as root. Skipping." ; exit "$PASS"; } +MDADM=$(which mdadm) + +[ -z "${MDADM}" ] && { r_log "mdadm" "which reported the binary but it doesn't exist, why?"; exit "$FAIL"; } + +${MDADM} --detail --scan &> /dev/null +ret_val=$? + +[ "$ret_val" -eq 0 ] || { r_log "mdadm" "There was a non-zero exit. This is likely fatal."; exit "$FAIL"; } + +r_checkExitStatus $ret_val diff --git a/func/core/pkg_network/00-install-packages.sh b/func/core/pkg_network/00-install-packages.sh index 964c071..df109be 100755 --- a/func/core/pkg_network/00-install-packages.sh +++ b/func/core/pkg_network/00-install-packages.sh @@ -1,3 +1,7 @@ #!/bin/bash r_log "network" "Install necessary network packages and utilities" -p_installPackageNormal traceroute iputils iproute mtr arpwatch psmisc net-tools which iptraf +pkgs=(traceroute iputils iproute mtr psmisc net-tools which iptraf) +if [ "$RL_VER" -eq 8 ]; then + pkgs+=( arpwatch ) +fi +p_installPackageNormal "${pkgs[@]}" diff --git a/func/core/pkg_postgresql/00-install-postgresql.sh b/func/core/pkg_postgresql/00-install-postgresql.sh new file mode 100755 index 0000000..cc39dd3 --- /dev/null +++ b/func/core/pkg_postgresql/00-install-postgresql.sh @@ -0,0 +1,3 @@ +#!/bin/bash +r_log "postgresql" "Installing postgresql" +p_installPackageNormal postgresql-server postgresql diff --git a/func/core/pkg_postgresql/01-configure-postgresql.sh b/func/core/pkg_postgresql/01-configure-postgresql.sh new file mode 100755 index 0000000..0f1cb43 --- /dev/null +++ b/func/core/pkg_postgresql/01-configure-postgresql.sh @@ -0,0 +1,5 @@ +#!/bin/bash +r_log "postgresql" "Initialize postgresql" +postgresql-setup --initdb +m_serviceCycler postgresql cycle +sleep 15 diff --git a/func/core/pkg_postgresql/10-create-db.sh b/func/core/pkg_postgresql/10-create-db.sh new file mode 100755 index 0000000..c796d2c --- /dev/null +++ b/func/core/pkg_postgresql/10-create-db.sh @@ -0,0 +1,4 @@ +#!/bin/bash +r_log "postgresql" "Creating db" +su - postgres -c 'createdb pg_test' +r_checkExitStatus $? diff --git a/func/core/pkg_postgresql/11-create-user.sh b/func/core/pkg_postgresql/11-create-user.sh new file mode 100755 index 0000000..271eacf --- /dev/null +++ b/func/core/pkg_postgresql/11-create-user.sh @@ -0,0 +1,4 @@ +#!/bin/bash +r_log "postgresql" "Creating user" +su - postgres -c 'createuser -S -R -D testuser' > /dev/null 2>&1 +r_checkExitStatus $? diff --git a/func/core/pkg_postgresql/20-drop-db.sh b/func/core/pkg_postgresql/20-drop-db.sh new file mode 100755 index 0000000..a887545 --- /dev/null +++ b/func/core/pkg_postgresql/20-drop-db.sh @@ -0,0 +1,4 @@ +#!/bin/bash +r_log "postgresql" "Dropping database" +su - postgres -c 'dropdb pg_test' > /dev/null 2>&1 +r_checkExitStatus $? diff --git a/func/core/pkg_postgresql/21-drop-user.sh b/func/core/pkg_postgresql/21-drop-user.sh new file mode 100755 index 0000000..0679dc1 --- /dev/null +++ b/func/core/pkg_postgresql/21-drop-user.sh @@ -0,0 +1,4 @@ +#!/bin/bash +r_log "postgresql" "Dropping user" +su - postgres -c 'dropuser testuser' > /dev/null 2>&1 +r_checkExitStatus $? diff --git a/func/core/pkg_release/20-check-gpg-keys.sh b/func/core/pkg_release/20-check-gpg-keys.sh index d99b178..9a30354 100755 --- a/func/core/pkg_release/20-check-gpg-keys.sh +++ b/func/core/pkg_release/20-check-gpg-keys.sh @@ -4,8 +4,8 @@ if [ "$RL_VER" -eq 8 ]; then file /etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial > /dev/null 2>&1 && \ file /etc/pki/rpm-gpg/RPM-GPG-KEY-rockytesting > /dev/null 2>&1 else - file "/etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-${RL_VER}" > /ev/null 2>&1 && \ - file "/etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-${RL_VER}-Testing" > /ev/null 2>&1 + file "/etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-${RL_VER}" > /dev/null 2>&1 && \ + file "/etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-${RL_VER}-Testing" > /dev/null 2>&1 fi r_checkExitStatus $? diff --git a/func/core/pkg_release/30-os-release.sh b/func/core/pkg_release/30-os-release.sh index a35bfca..65b9ca7 100755 --- a/func/core/pkg_release/30-os-release.sh +++ b/func/core/pkg_release/30-os-release.sh @@ -4,10 +4,10 @@ r_log "rocky" "Check /etc/os-release stuff" r_log "rocky" "Verify support directives" for s in NAME=\"Rocky\ Linux\" \ ID=\"rocky\" \ - ROCKY_SUPPORT_PRODUCT=\"Rocky\ Linux\" \ - ROCKY_SUPPORT_PRODUCT_VERSION=\"$RL_VER\"; do + ROCKY_SUPPORT_PRODUCT=\"Rocky-Linux-$RL_VER\" \ + ROCKY_SUPPORT_PRODUCT_VERSION=\"$RL_VER\..*\"; do if ! grep -q "$s" /etc/os-release; then - r_log "rocky" "Missing string in /etc/os-release" + r_log "rocky" "Missing string ($s) in /etc/os-release" r_checkExitStatus 1 fi done diff --git a/func/core/pkg_shadow-utils/30-group-tests.sh b/func/core/pkg_shadow-utils/30-group-tests.sh index 4c30bba..b2e502d 100755 --- a/func/core/pkg_shadow-utils/30-group-tests.sh +++ b/func/core/pkg_shadow-utils/30-group-tests.sh @@ -110,6 +110,6 @@ r_log "shadow" "Test sg" sg onyxuser "touch /var/tmp/onyxsg" r_checkExitStatus $? r_log "shadow" "Verify sg worked" -stat --format="%U" /var/tmp/onyxsg | grep -q onyxuser +stat --format="%G" /var/tmp/onyxsg | grep -q onyxuser r_checkExitStatus $? rm /var/tmp/onyxsg diff --git a/func/core/pkg_tftp-server/00-install-tftp.sh b/func/core/pkg_tftp-server/00-install-tftp.sh new file mode 100755 index 0000000..34c47be --- /dev/null +++ b/func/core/pkg_tftp-server/00-install-tftp.sh @@ -0,0 +1,3 @@ +#!/bin/bash +r_log "tftp" "Installing packages" +p_installPackageNormal tftp-server tftp diff --git a/func/core/pkg_tftp-server/01-configure-tftp.sh b/func/core/pkg_tftp-server/01-configure-tftp.sh new file mode 100755 index 0000000..117d8ad --- /dev/null +++ b/func/core/pkg_tftp-server/01-configure-tftp.sh @@ -0,0 +1,23 @@ +#!/bin/bash +r_log "tftp" "Configure tftp" + +if [ "$RL_VER" -eq 8 ]; then + cat < /etc/xinetd.d/tftp +service tftp +{ + socket_type = dgram + protocol = udp + wait = yes + user = root + server = /usr/sbin/in.tftpd + server_args = -s /var/lib/tftpboot + disable = no + per_source = 11 + cps = 100 2 + flags = IPv4 +} +EOF + +fi + +m_serviceCycler tftp.socket start diff --git a/func/core/pkg_tftp-server/10-get-test.sh b/func/core/pkg_tftp-server/10-get-test.sh new file mode 100755 index 0000000..3113972 --- /dev/null +++ b/func/core/pkg_tftp-server/10-get-test.sh @@ -0,0 +1,10 @@ +#!/bin/bash +r_log "tftp" "Getting a file from tftp" + +chmod 777 /var/lib/tftpboot +echo "rocky func" > /var/lib/tftpboot/tftptest +tftp 127.0.0.1 -c get tftptest + +grep -q "rocky func" tftptest +r_checkExitStatus +/bin/rm tftptest diff --git a/func/core/pkg_tftp-server/11-put-test.sh b/func/core/pkg_tftp-server/11-put-test.sh new file mode 100755 index 0000000..afda5f1 --- /dev/null +++ b/func/core/pkg_tftp-server/11-put-test.sh @@ -0,0 +1,14 @@ +#!/bin/bash +r_log "tftp" "Testing anon write" +TFTPDIR=/var/lib/tftpboot +setsebool tftp_anon_write 1 +chmod 777 $TFTPDIR +echo "rocky func" > puttest +touch $TFTPDIR > $TFTPDIR/puttest +chmod 666 $TFTPDIR/puttest +tftp 127.0.0.1 -c put puttest +sleep 2 +grep -q 'rocky func' $TFTPDIR/puttest +r_checkExitStatus $? +/bin/rm puttest +/bin/rm $TFTPDIR/puttest diff --git a/func/skip.list b/func/skip.list index ccddc50..2ebe6ab 100644 --- a/func/skip.list +++ b/func/skip.list @@ -10,9 +10,12 @@ # -> Must be a URL to bugs.rl.o, a github issue number, or a code, such as: # * NEEDINFO # * NOTREADY +# * NOPKG 8|./core/pkg_archive/26-zmore.sh|nazunalika|NEEDINFO 8|./core/pkg_nfs/12-prepare-autofs.sh|nazunalika|NEEDINFO 8|./core/pkg_diffutils/00-install-diff.sh|nazunalika|NOTREADY 8|./core/pkg_snmp/12-test-snmp-3.sh|nazunalika|NOTWORKING 8|./core/pkg_samba/00-install-samba.sh|nazunalika|NOTWORKING 8|./core/pkg_samba/10-test-samba.sh|nazunalika|NOTWORKING +9|./core/pkg_archive/27-znew.sh|nazunalika|NOPKG +9|./core/pkg_network/30-test-arpwatch.sh|nazunalika|NOPKG diff --git a/func/stacks.sh b/func/stacks.sh new file mode 100644 index 0000000..cca6550 --- /dev/null +++ b/func/stacks.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Release Engineering Core Functionality Testing +# Louis Abel @nazunalika + +################################################################################ +# Settings and variables + +# Exits on any non-zero exit status - Disabled for now. +#set -e +# Undefined variables will cause an exit +set -u + +COMMON_EXPORTS='./common/exports.sh' +COMMON_IMPORTS='./common/imports.sh' +SELINUX=$(getenforce) + +# End +################################################################################ + +# shellcheck source=/dev/null disable=SC2015 +[ -f $COMMON_EXPORTS ] && source $COMMON_EXPORTS || { echo -e "\n[-] $(date): Variables cannot be sourced."; exit 1; } +# shellcheck source=/dev/null disable=SC2015 +[ -f $COMMON_IMPORTS ] && source $COMMON_IMPORTS || { echo -e "\n[-] $(date): Functions cannot be sourced."; exit 1; } +# Init log +# shellcheck disable=SC2015 +[ -e "$LOGFILE" ] && m_recycleLog || touch "$LOGFILE" +# SELinux check +if [ "$SELINUX" != "Enforcing" ]; then + echo -e "\n[-] $(date): SELinux is not enforcing." + exit 1 +fi + +r_log "internal" "Starting Release Engineering Core Tests" + +################################################################################ +# Script Work + +# Skip tests in a list - some tests are already -x, so it won't be an issue +if [ -e skip.list ]; then + r_log "internal" "Disabling tests" + # shellcheck disable=SC2162 + grep -E "^${RL_VER}" skip.list | while read line; do + # shellcheck disable=SC2086 + testFile="$(echo $line | cut -d '|' -f 2)" + r_log "internal" "SKIP ${testFile}" + chmod -x "${testFile}" + done + r_log "internal" "WARNING: Tests above were disabled." +fi + +# TODO: should we let $1 judge what directory is ran? +# TODO: get some stacks and lib in there + +#r_processor <(/usr/bin/find ./core -type f | sort -t'/') +#r_processor <(/usr/bin/find ./lib -type f | sort -t'/') +r_processor <(/usr/bin/find ./stacks -type f | sort -t'/') + +r_log "internal" "Core Tests completed" +exit 0 diff --git a/func/stacks/ipa/10-install-ipa.sh b/func/stacks/ipa/10-install-ipa.sh index 40aa006..0e5b029 100644 --- a/func/stacks/ipa/10-install-ipa.sh +++ b/func/stacks/ipa/10-install-ipa.sh @@ -9,5 +9,6 @@ fi # going to be the same thing or not so this check is there just in case. if [ "$RL_VER" -eq 8 ]; then p_enableModule idm:DL1/{client,common,dns,server} - p_installPackageNormal ipa-server ipa-server-dns fi + +p_installPackageNormal ipa-server ipa-server-dns diff --git a/func/stacks/ipa/20-ipa-user.sh b/func/stacks/ipa/20-ipa-user.sh index 884f0df..9965a56 100644 --- a/func/stacks/ipa/20-ipa-user.sh +++ b/func/stacks/ipa/20-ipa-user.sh @@ -4,8 +4,77 @@ if m_getArch aarch64 | grep -qE 'aarch64'; then exit 0 fi -if [ "$IPAINSTALLED" -eq 1 ]; then +if [ "$IPAINSTALLED" -eq 1 ]; then r_log "ipa" "IPA was not successfully installed. Aborting." r_checkExitStatus 1 fi +kdestroy &> /dev/null +klist 2>&1 | grep -E "(No credentials|Credentials cache .* not found)" &> /dev/null +r_checkExitStatus $? + +expect -f - < /dev/null +r_checkExitStatus $? + +r_log "ipa" "Test adding a user" +userDetails="$(ipa user-add --first=test --last=user --random ipatestuser)" +echo "$userDetails" | grep -q 'Added user "ipatestuser"' +r_checkExitStatus $? + +echo "$userDetails" | grep -q 'First name: test' +r_checkExitStatus $? +echo "$userDetails" | grep -q 'Last name: user' +r_checkExitStatus $? +echo "$userDetails" | grep -q 'Full name: test user' +r_checkExitStatus $? +echo "$userDetails" | grep -q 'Home directory: /home/ipatestuser' +r_checkExitStatus $? + +r_log "ipa" "Changing password of the user" +kdestroy &> /dev/null + +expect -f - < /dev/null +r_checkExitStatus $? + +kdestroy &> /dev/null + +r_log "ipa" "Testing for user in getent" +getent passwd ipatestuser &> /dev/null +r_checkExitStatus $? diff --git a/func/stacks/ipa/21-ipa-service.sh b/func/stacks/ipa/21-ipa-service.sh index 9f053a1..db50dd1 100644 --- a/func/stacks/ipa/21-ipa-service.sh +++ b/func/stacks/ipa/21-ipa-service.sh @@ -1,6 +1,6 @@ #!/bin/bash if m_getArch aarch64 | grep -qE 'aarch64'; then - r_log "ipa -bash" "Skipping for aarch64" + r_log "ipa" "Skipping for aarch64" exit 0 fi @@ -9,3 +9,61 @@ if [ "$IPAINSTALLED" -eq 1 ]; then r_checkExitStatus 1 fi +kdestroy &> /dev/null +klist 2>&1 | grep -E "(No credentials|Credentials cache .* not found)" &> /dev/null +r_checkExitStatus $? + +expect -f - < /dev/null +r_checkExitStatus $? + +r_log "ipa" "Adding test service" +ipa service-add testservice/rltest.rlipa.local &> /dev/null +r_checkExitStatus $? + +r_log "ipa" "Getting keytab for service" +ipa-getkeytab -s rltest.rlipa.local -p testservice/rltest.rlipa.local -k /tmp/testservice.keytab &> /dev/null +r_checkExitStatus $? + +r_log "ipa" "Getting a certificate for service" +ipa-getcert request -K testservice/rltest.rlipa.local -D rltest.rlipa.local -f /etc/pki/tls/certs/testservice.crt -k /etc/pki/tls/private/testservice.key &> /dev/null +r_checkExitStatus $? + +while true; do + entry="$(ipa-getcert list -r | sed -n '/Request ID/,/auto-renew: yes/p')" + if [[ $entry =~ "status:" ]] && [[ $entry =~ "CA_REJECTED" ]]; then + r_checkExitStatus 1 + break + fi + if [[ $entry =~ "" ]]; then + r_checkExitStatus 0 + break + fi + sleep 1 +done + +while ! stat /etc/pki/tls/certs/testservice.crt &> /dev/null; do + sync + sleep 1 +done + +r_log "ipa" "Verifying keytab" +klist -k /tmp/testservice.keytab | grep "testservice/rltest.rlipa.local" &> /dev/null +r_checkExitStatus $? + +r_log "ipa" "Verifying key matches the certificate" +diff <(openssl x509 -in /etc/pki/tls/certs/testservice.crt -noout -modulus 2>&1 ) <(openssl rsa -in /etc/pki/tls/private/testservice.key -noout -modulus 2>&1 ) +r_checkExitStatus $? + +r_log "ipa" "Verifying the certificate against our CA" +openssl verify -CAfile /etc/ipa/ca.crt /etc/pki/tls/certs/testservice.crt | grep "/etc/pki/tls/certs/testservice.crt: OK" &> /dev/null +r_checkExitStatus $? diff --git a/func/stacks/ipa/22-ipa-dns.sh b/func/stacks/ipa/22-ipa-dns.sh index 9f053a1..4d74174 100644 --- a/func/stacks/ipa/22-ipa-dns.sh +++ b/func/stacks/ipa/22-ipa-dns.sh @@ -9,3 +9,46 @@ if [ "$IPAINSTALLED" -eq 1 ]; then r_checkExitStatus 1 fi +kdestroy &> /dev/null +klist 2>&1 | grep -qE "(No credentials|Credentials cache .* not found)" &> /dev/null +r_checkExitStatus $? + +expect -f - < /dev/null +r_checkExitStatus $? + +r_log "ipa" "Adding testzone subdomain" +ipa dnszone-add --name-server=rltest.rlipa.local. --admin-email=hostmaster.testzone.rlipa.local. testzone.rlipa.local &> /dev/null +r_checkExitStatus $? +sleep 5 + +r_log "ipa" "Get SOA from testzone subdomain" +dig @localhost SOA testzone.rlipa.local | grep -q "status: NOERROR" &> /dev/null +r_checkExitStatus $? + +r_log "ipa" "Adding a CNAME record to the primary domain" +ipa dnsrecord-add rlipa.local testrecord --cname-hostname=rltest &> /dev/null +r_checkExitStatus $? +sleep 5 + +r_log "ipa" "Retrieving CNAME record" +dig @localhost CNAME testrecord.rlipa.local | grep -q "status: NOERROR" &> /dev/null +r_checkExitStatus $? + +r_log "ipa" "Adding a CNAME to subdomain" +ipa dnsrecord-add testzone.rlipa.local testrecord --cname-hostname=rltest.rlipa.local. &> /dev/null +r_checkExitStatus $? +sleep 5 + +r_log "ipa" "Testing can retrieve record from subdomain" +dig @localhost CNAME testrecord.testzone.rlipa.local | grep -q "status: NOERROR" &> /dev/null +r_checkExitStatus $? diff --git a/func/stacks/ipa/23-ipa-sudo.sh b/func/stacks/ipa/23-ipa-sudo.sh index 9f053a1..68e50ac 100644 --- a/func/stacks/ipa/23-ipa-sudo.sh +++ b/func/stacks/ipa/23-ipa-sudo.sh @@ -9,3 +9,19 @@ if [ "$IPAINSTALLED" -eq 1 ]; then r_checkExitStatus 1 fi +kdestroy &> /dev/null +klist 2>&1 | grep -E "(No credentials|Credentials cache .* not found)" &> /dev/null +r_checkExitStatus $? + +expect -f - < /dev/null +r_checkExitStatus $? diff --git a/func/stacks/ipa/50-cleanup-ipa.sh b/func/stacks/ipa/50-cleanup-ipa.sh index 9f053a1..9f17e3f 100644 --- a/func/stacks/ipa/50-cleanup-ipa.sh +++ b/func/stacks/ipa/50-cleanup-ipa.sh @@ -1,6 +1,6 @@ #!/bin/bash if m_getArch aarch64 | grep -qE 'aarch64'; then - r_log "ipa -bash" "Skipping for aarch64" + r_log "ipa" "Skipping for aarch64" exit 0 fi diff --git a/iso/py/.gitignore b/iso/empanadas/.gitignore similarity index 100% rename from iso/py/.gitignore rename to iso/empanadas/.gitignore diff --git a/iso/empanadas/Containerfile b/iso/empanadas/Containerfile new file mode 100644 index 0000000..c3c4ed9 --- /dev/null +++ b/iso/empanadas/Containerfile @@ -0,0 +1,66 @@ +FROM quay.io/centos/centos:stream9 + +ADD images/get_arch /get_arch + +ENV TINI_VERSION v0.19.0 +RUN curl -o /tini -L "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-$(/get_arch)" +RUN chmod +x /tini + +RUN rm -rf /etc/yum.repos.d/*.repo +ADD images/epelkey.gpg /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-9 +ADD images/rhel.repo /etc/yum.repos.d/rhel.repo + +RUN dnf update -y && dnf install -y \ + bash \ + bzip2 \ + cpio \ + diffutils \ + findutils \ + gawk \ + gcc \ + gcc-c++ \ + git \ + grep \ + gzip \ + info \ + make \ + patch \ + python3 \ + redhat-rpm-config \ + rpm-build \ + scl-utils-build \ + sed \ + shadow-utils \ + tar \ + unzip \ + util-linux \ + which \ + xz \ + dnf-plugins-core \ + createrepo_c \ + rpm-sign \ + sudo \ + mock \ + python-pip \ + genisoimage \ + isomd5sum \ + lorax \ + lorax-templates-rhel \ + lorax-templates-generic + +RUN sed -i '/libreport-rhel-anaconda-bugzilla/ s/^/#/' /usr/share/lorax/templates.d/80-rhel/runtime-install.tmpl + +RUN ssh-keygen -t rsa -q -f "$HOME/.ssh/id_rsa" -N "" +RUN dnf clean all +RUN rm -rf /etc/yum.repos.d/*.repo +RUN useradd -o -d /var/peridot -u 1002 peridotbuilder && usermod -a -G mock peridotbuilder +RUN chown peridotbuilder:mock /etc/yum.conf && chown -R peridotbuilder:mock /etc/dnf && chown -R peridotbuilder:mock /etc/rpm && chown -R peridotbuilder:mock /etc/yum.repos.d + +RUN pip install 'git+https://git.rockylinux.org/release-engineering/public/toolkit.git@feature/iso-kube#egg=empanadas&subdirectory=iso/empanadas' + +RUN pip install awscli + +ENV USER=1002 +USER 1002 + +ENTRYPOINT ["/tini", "--"] diff --git a/iso/py/README.md b/iso/empanadas/README.md similarity index 61% rename from iso/py/README.md rename to iso/empanadas/README.md index 41f3a41..e414f5e 100644 --- a/iso/py/README.md +++ b/iso/empanadas/README.md @@ -1,5 +1,19 @@ # iso + +## Setup / Install + +1. Install [Poetry](https://python-poetry.org/docs/) +2. Setup: `poetry install` +3. Have fun + + +## Updating dependencies + +Dependencies can be manipulated via the pyproject.toml file or with the poetry add/remove commands. + +Changes to the poetry.lock should be commited if dependencies are added or updated. + ## TODO Verbose mode should exist to output everything that's being called or ran. @@ -52,5 +66,6 @@ r.check_valid_arch() ### script names and permissions -* Callable scripts should *not* end in `.py` -* They should have at least `775` or `+x` permissions +* Callable scripts should always end in `.py` and live in the empanadas/scripts folder +* Poetry will handle the installation of these executables with setuptools for distribution, and they can be invoked by name using `poetry run script-name`, too. + * Configure the script and function to be executed in pyproject.toml (TODO: dynamically load scripts from this directory as well as standardize on the script input/outputs) diff --git a/iso/empanadas/build-container.sh b/iso/empanadas/build-container.sh new file mode 100644 index 0000000..5b0a53b --- /dev/null +++ b/iso/empanadas/build-container.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +MANIFEST_NAME="peridotempanadas" +BUILD_PATH="." +REGISTRY="docker.io" +USER="neilresf" +IMAGE_TAG="v0.1.0" +IMAGE_NAME="peridotempanadas" + +podman buildx build \ + --platform linux/amd64,linux/arm64,linux/s390x,linux/ppc64le \ + --tag "${REGISTRY}/${USER}/${IMAGE_NAME}:${IMAGE_TAG}" \ + $PWD + diff --git a/iso/empanadas/empanadas/__init__.py b/iso/empanadas/empanadas/__init__.py new file mode 100644 index 0000000..b794fd4 --- /dev/null +++ b/iso/empanadas/empanadas/__init__.py @@ -0,0 +1 @@ +__version__ = '0.1.0' diff --git a/iso/py/common.py b/iso/empanadas/empanadas/common.py similarity index 74% rename from iso/py/common.py rename to iso/empanadas/empanadas/common.py index 509e89e..c3619ce 100644 --- a/iso/py/common.py +++ b/iso/empanadas/empanadas/common.py @@ -6,6 +6,7 @@ import glob import rpm import yaml import logging +import hashlib # These are a bunch of colors we may use in terminal output class Color: @@ -25,6 +26,7 @@ rldict = {} sigdict = {} config = { "rlmacro": rpm.expandMacro('%rhel'), + "dist": 'el' + rpm.expandMacro('%rhel'), "arch": platform.machine(), "date_stamp": time.strftime("%Y%m%d.%H%M%S", time.localtime()), "compose_root": "/mnt/compose", @@ -33,16 +35,31 @@ config = { "category_stub": "mirror/pub/rocky", "sig_category_stub": "mirror/pub/sig", "repo_base_url": "https://yumrepofs.build.resf.org/v1/projects", - "container": "centos:stream9" + "mock_work_root": "/builddir", + "container": "centos:stream9", + "distname": "Rocky Linux", + "shortname": "Rocky", + "translators": { + "x86_64": "amd64", + "aarch64": "arm64", + "ppc64le": "ppc64le", + "s390x": "s390x" + }, + "aws_region": "us-east-2", + "bucket": "resf-empanadas", + "bucket_url": "https://resf-empanadas.s3.us-east-2.amazonaws.com" } # Importing the config from yaml -for conf in glob.iglob('configs/*.yaml'): +import importlib_resources +_rootdir = importlib_resources.files("empanadas") + +for conf in glob.iglob(f"{_rootdir}/configs/*.yaml"): with open(conf, 'r', encoding="utf-8") as file: rldict.update(yaml.safe_load(file)) # Import all SIG configs from yaml -for conf in glob.iglob('sig/*.yaml'): +for conf in glob.iglob(f"{_rootdir}/sig/*.yaml"): with open(conf, 'r', encoding="utf-8") as file: sigdict.update(yaml.safe_load(file)) diff --git a/iso/py/configs/el8.yaml b/iso/empanadas/empanadas/configs/el8.yaml similarity index 57% rename from iso/py/configs/el8.yaml rename to iso/empanadas/empanadas/configs/el8.yaml index c232bb0..eb80aff 100644 --- a/iso/py/configs/el8.yaml +++ b/iso/empanadas/empanadas/configs/el8.yaml @@ -1,16 +1,17 @@ --- '8': + fullname: 'Rocky Linux 8' revision: '8.6' rclvl: 'RC2' + major: '8' + minor: '6' + profile: '8' + bugurl: 'https://bugs.rockylinux.org' allowed_arches: - x86_64 - aarch64 provide_multilib: False project_id: '' - required_packages: - - 'lorax' - - 'genisoimage' - - 'isomd5sum' repo_symlinks: devel: 'Devel' NFV: 'nfv' @@ -53,6 +54,48 @@ - dvd1 - minimal - boot + repos: + - 'BaseOS' + - 'AppStream' + variant: 'BaseOS' + lorax_removes: + - 'libreport-rhel-anaconda-bugzilla' + required_packages: + - 'lorax' + - 'genisoimage' + - 'isomd5sum' + - 'lorax-templates-rhel' + - 'lorax-templates-generic' + structure: + packages: 'os/Packages' + repodata: 'os/repodata' + iso_map: + xorrisofs: False + iso_level: False + hosts: + x86_64: '' + aarch64: '' + images: + dvd: + repos: + - 'BaseOS' + - 'AppStream' + lorax_variants: + - dvd + - minimal + - BaseOS + repos: + - 'BaseOS' + - 'AppStream' + variant: 'BaseOS' + lorax_removes: + - 'libreport-rhel-anaconda-bugzilla' + required_pkgs: + - 'lorax' + - 'genisoimage' + - 'isomd5sum' + - 'lorax-templates-rhel' + - 'lorax-templates-generic' repoclosure_map: arches: x86_64: '--arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch' @@ -80,10 +123,15 @@ - AppStream extra_files: git_repo: 'https://git.rockylinux.org/staging/src/rocky-release.git' + git_raw_path: 'https://git.rockylinux.org/staging/src/rocky-release/-/raw/r8/' branch: 'r8' + gpg: + stable: 'SOURCES/RPM-GPG-KEY-rockyofficial' + testing: 'SOURCES/RPM-GPG-KEY-rockytesting' list: - 'SOURCES/COMMUNITY-CHARTER' - 'SOURCES/EULA' - 'SOURCES/LICENSE' - 'SOURCES/RPM-GPG-KEY-rockyofficial' + - 'SOURCES/RPM-GPG-KEY-rockytesting' ... diff --git a/iso/empanadas/empanadas/configs/el9-beta.yaml b/iso/empanadas/empanadas/configs/el9-beta.yaml new file mode 100644 index 0000000..19d6cd5 --- /dev/null +++ b/iso/empanadas/empanadas/configs/el9-beta.yaml @@ -0,0 +1,123 @@ +--- +'9-beta': + fullname: 'Rocky Linux 9.1' + revision: '9.1' + rclvl: 'BETA1' + major: '9' + minor: '1' + profile: '9-beta' + bugurl: 'https://bugs.rockylinux.org' + checksum: 'sha256' + allowed_arches: + - x86_64 + - aarch64 + - ppc64le + - s390x + provide_multilib: True + project_id: '' + repo_symlinks: + NFV: 'nfv' + renames: + all: 'devel' + all_repos: + - 'all' + - 'BaseOS' + - 'AppStream' + - 'CRB' + - 'HighAvailability' + - 'ResilientStorage' + - 'RT' + - 'NFV' + - 'SAP' + - 'SAPHANA' + - 'extras' + - 'plus' + structure: + packages: 'os/Packages' + repodata: 'os/repodata' + iso_map: + xorrisofs: True + iso_level: False + images: + dvd: + disc: True + variant: 'AppStream' + repos: + - 'BaseOS' + - 'AppStream' + minimal: + disc: True + isoskip: True + repos: + - 'minimal' + variant: 'minimal' + BaseOS: + disc: False + isoskip: True + variant: 'BaseOS' + repos: + - 'BaseOS' + - 'AppStream' + lorax: + repos: + - 'BaseOS' + - 'AppStream' + variant: 'BaseOS' + lorax_removes: + - 'libreport-rhel-anaconda-bugzilla' + required_pkgs: + - 'lorax' + - 'genisoimage' + - 'isomd5sum' + - 'lorax-templates-rhel' + - 'lorax-templates-generic' + - 'xorriso' + repoclosure_map: + arches: + x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch' + aarch64: '--forcearch=aarch64 --arch=aarch64 --arch=noarch' + ppc64le: '--forcearch=ppc64le --arch=ppc64le --arch=noarch' + s390x: '--forcearch=s390x --arch=s390x --arch=noarch' + repos: + devel: [] + BaseOS: [] + AppStream: + - BaseOS + CRB: + - BaseOS + - AppStream + HighAvailability: + - BaseOS + - AppStream + ResilientStorage: + - BaseOS + - AppStream + RT: + - BaseOS + - AppStream + NFV: + - BaseOS + - AppStream + SAP: + - BaseOS + - AppStream + - HighAvailability + SAPHANA: + - BaseOS + - AppStream + - HighAvailability + extra_files: + git_repo: 'https://git.rockylinux.org/staging/src/rocky-release.git' + git_raw_path: 'https://git.rockylinux.org/staging/src/rocky-release/-/raw/r9/' + branch: 'r9' + gpg: + stable: 'SOURCES/RPM-GPG-KEY-Rocky-9' + testing: 'SOURCES/RPM-GPG-KEY-Rocky-9-Testing' + list: + - 'SOURCES/Contributors' + - 'SOURCES/COMMUNITY-CHARTER' + - 'SOURCES/EULA' + - 'SOURCES/LICENSE' + - 'SOURCES/RPM-GPG-KEY-Rocky-9' + - 'SOURCES/RPM-GPG-KEY-Rocky-9-Testing' +... diff --git a/iso/py/configs/el9.yaml b/iso/empanadas/empanadas/configs/el9.yaml similarity index 56% rename from iso/py/configs/el9.yaml rename to iso/empanadas/empanadas/configs/el9.yaml index d6553a8..88a978b 100644 --- a/iso/py/configs/el9.yaml +++ b/iso/empanadas/empanadas/configs/el9.yaml @@ -1,7 +1,13 @@ --- '9': + fullname: 'Rocky Linux 9.0' revision: '9.0' rclvl: 'RC1' + major: '9' + minor: '0' + profile: '9' + bugurl: 'https://bugs.rockylinux.org' + checksum: 'sha256' allowed_arches: - x86_64 - aarch64 @@ -9,15 +15,10 @@ - s390x provide_multilib: True project_id: '55b17281-bc54-4929-8aca-a8a11d628738' - required_packages: - - 'lorax' - - 'genisoimage' - - 'isomd5sum' repo_symlinks: - devel: 'Devel' NFV: 'nfv' renames: - all: 'nplb' + all: 'devel' all_repos: - 'all' - 'BaseOS' @@ -30,39 +31,47 @@ - 'SAP' - 'SAPHANA' - 'extras' - - 'devel' - 'plus' - no_comps_or_groups: - - 'all' - - 'extras' - - 'devel' - - 'plus' - comps_or_groups: - - 'BaseOS' - - 'AppStream' - - 'CRB' - - 'HighAvailability' - - 'ResilientStorage' - - 'RT' - - 'NFV' - - 'SAP' - - 'SAPHANA' - has_modules: - - 'AppStream' - - 'CRB' + structure: + packages: 'os/Packages' + repodata: 'os/repodata' iso_map: - hosts: - x86_64: '' - aarch64: '' - ppc64le: '' - s390x: '' + xorrisofs: True + iso_level: False images: - - dvd1 - - minimal - - boot - repos: - - 'BaseOS' - - 'AppStream' + dvd: + disc: True + variant: 'AppStream' + repos: + - 'BaseOS' + - 'AppStream' + minimal: + disc: True + isoskip: True + repos: + - 'minimal' + variant: 'minimal' + BaseOS: + disc: False + isoskip: True + variant: 'BaseOS' + repos: + - 'BaseOS' + - 'AppStream' + lorax: + repos: + - 'BaseOS' + - 'AppStream' + variant: 'BaseOS' + lorax_removes: + - 'libreport-rhel-anaconda-bugzilla' + required_pkgs: + - 'lorax' + - 'genisoimage' + - 'isomd5sum' + - 'lorax-templates-rhel' + - 'lorax-templates-generic' + - 'xorriso' repoclosure_map: arches: x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch' @@ -70,7 +79,7 @@ ppc64le: '--forcearch=ppc64le --arch=ppc64le --arch=noarch' s390x: '--forcearch=s390x --arch=s390x --arch=noarch' repos: - nplb: [] + devel: [] BaseOS: [] AppStream: - BaseOS @@ -99,8 +108,13 @@ - HighAvailability extra_files: git_repo: 'https://git.rockylinux.org/staging/src/rocky-release.git' + git_raw_path: 'https://git.rockylinux.org/staging/src/rocky-release/-/raw/r9/' branch: 'r9' + gpg: + stable: 'SOURCES/RPM-GPG-KEY-Rocky-9' + testing: 'SOURCES/RPM-GPG-KEY-Rocky-9-Testing' list: + - 'SOURCES/Contributors' - 'SOURCES/COMMUNITY-CHARTER' - 'SOURCES/EULA' - 'SOURCES/LICENSE' diff --git a/iso/empanadas/empanadas/configs/el9lh.yaml b/iso/empanadas/empanadas/configs/el9lh.yaml new file mode 100644 index 0000000..4176f66 --- /dev/null +++ b/iso/empanadas/empanadas/configs/el9lh.yaml @@ -0,0 +1,123 @@ +--- +'9-lookahead': + fullname: 'Rocky Linux 9.1' + revision: '9.1' + rclvl: 'LH1' + major: '9' + minor: '1' + profile: '9-lookahead' + bugurl: 'https://bugs.rockylinux.org' + checksum: 'sha256' + allowed_arches: + - x86_64 + - aarch64 + - ppc64le + - s390x + provide_multilib: True + project_id: '' + repo_symlinks: + NFV: 'nfv' + renames: + all: 'devel' + all_repos: + - 'all' + - 'BaseOS' + - 'AppStream' + - 'CRB' + - 'HighAvailability' + - 'ResilientStorage' + - 'RT' + - 'NFV' + - 'SAP' + - 'SAPHANA' + - 'extras' + - 'plus' + structure: + packages: 'os/Packages' + repodata: 'os/repodata' + iso_map: + xorrisofs: True + iso_level: False + images: + dvd: + disc: True + variant: 'AppStream' + repos: + - 'BaseOS' + - 'AppStream' + minimal: + disc: True + isoskip: True + repos: + - 'minimal' + variant: 'minimal' + BaseOS: + disc: False + isoskip: True + variant: 'BaseOS' + repos: + - 'BaseOS' + - 'AppStream' + lorax: + repos: + - 'BaseOS' + - 'AppStream' + variant: 'BaseOS' + lorax_removes: + - 'libreport-rhel-anaconda-bugzilla' + required_pkgs: + - 'lorax' + - 'genisoimage' + - 'isomd5sum' + - 'lorax-templates-rhel' + - 'lorax-templates-generic' + - 'xorriso' + repoclosure_map: + arches: + x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch' + aarch64: '--forcearch=aarch64 --arch=aarch64 --arch=noarch' + ppc64le: '--forcearch=ppc64le --arch=ppc64le --arch=noarch' + s390x: '--forcearch=s390x --arch=s390x --arch=noarch' + repos: + devel: [] + BaseOS: [] + AppStream: + - BaseOS + CRB: + - BaseOS + - AppStream + HighAvailability: + - BaseOS + - AppStream + ResilientStorage: + - BaseOS + - AppStream + RT: + - BaseOS + - AppStream + NFV: + - BaseOS + - AppStream + SAP: + - BaseOS + - AppStream + - HighAvailability + SAPHANA: + - BaseOS + - AppStream + - HighAvailability + extra_files: + git_repo: 'https://git.rockylinux.org/staging/src/rocky-release.git' + git_raw_path: 'https://git.rockylinux.org/staging/src/rocky-release/-/raw/r9/' + branch: 'r9lh' + gpg: + stable: 'SOURCES/RPM-GPG-KEY-Rocky-9' + testing: 'SOURCES/RPM-GPG-KEY-Rocky-9-Testing' + list: + - 'SOURCES/Contributors' + - 'SOURCES/COMMUNITY-CHARTER' + - 'SOURCES/EULA' + - 'SOURCES/LICENSE' + - 'SOURCES/RPM-GPG-KEY-Rocky-9' + - 'SOURCES/RPM-GPG-KEY-Rocky-9-Testing' +... diff --git a/iso/empanadas/empanadas/configs/rln.yaml b/iso/empanadas/empanadas/configs/rln.yaml new file mode 100644 index 0000000..7544ce4 --- /dev/null +++ b/iso/empanadas/empanadas/configs/rln.yaml @@ -0,0 +1,121 @@ +--- +'rln': + fullname: 'Rocky Linux New' + revision: '10' + rclvl: 'RLN120' + major: '10' + minor: '0' + profile: 'rln' + bugurl: 'https://bugs.rockylinux.org' + checksum: 'sha256' + allowed_arches: + - x86_64 + - aarch64 + - ppc64le + - s390x + provide_multilib: True + project_id: '' + repo_symlinks: + NFV: 'nfv' + renames: + all: 'devel' + all_repos: + - 'all' + - 'BaseOS' + - 'AppStream' + - 'CRB' + - 'HighAvailability' + - 'ResilientStorage' + - 'RT' + - 'NFV' + - 'SAP' + - 'SAPHANA' + - 'extras' + - 'plus' + structure: + packages: 'os/Packages' + repodata: 'os/repodata' + iso_map: + xorrisofs: True + iso_level: False + images: + dvd: + discnum: '1' + variant: 'AppStream' + repos: + - 'BaseOS' + - 'AppStream' + minimal: + discnum: '1' + isoskip: True + repos: + - 'minimal' + variant: 'minimal' + BaseOS: + isoskip: True + variant: 'BaseOS' + repos: + - 'BaseOS' + - 'AppStream' + lorax: + repos: + - 'BaseOS' + - 'AppStream' + variant: 'BaseOS' + lorax_removes: + - 'libreport-rhel-anaconda-bugzilla' + required_pkgs: + - 'lorax' + - 'isomd5sum' + - 'lorax-templates-rhel' + - 'lorax-templates-generic' + - 'xorriso' + repoclosure_map: + arches: + x86_64: '--forcearch=x86_64 --arch=x86_64 --arch=athlon --arch=i686 --arch=i586 --arch=i486 --arch=i386 --arch=noarch' + aarch64: '--forcearch=aarch64 --arch=aarch64 --arch=noarch' + ppc64le: '--forcearch=ppc64le --arch=ppc64le --arch=noarch' + s390x: '--forcearch=s390x --arch=s390x --arch=noarch' + repos: + devel: [] + BaseOS: [] + AppStream: + - BaseOS + CRB: + - BaseOS + - AppStream + HighAvailability: + - BaseOS + - AppStream + ResilientStorage: + - BaseOS + - AppStream + RT: + - BaseOS + - AppStream + NFV: + - BaseOS + - AppStream + SAP: + - BaseOS + - AppStream + - HighAvailability + SAPHANA: + - BaseOS + - AppStream + - HighAvailability + extra_files: + git_repo: 'https://git.rockylinux.org/staging/src/rocky-release.git' + git_raw_path: 'https://git.rockylinux.org/staging/src/rocky-release/-/raw/rln/' + branch: 'rln' + gpg: + stable: 'SOURCES/RPM-GPG-KEY-Rocky-RLN' + testing: 'SOURCES/RPM-GPG-KEY-Rocky-RLN-Testing' + list: + - 'SOURCES/Contributors' + - 'SOURCES/COMMUNITY-CHARTER' + - 'SOURCES/EULA' + - 'SOURCES/LICENSE' + - 'SOURCES/RPM-GPG-KEY-Rocky-RLN' + - 'SOURCES/RPM-GPG-KEY-Rocky-RLN' +... diff --git a/iso/empanadas/empanadas/scripts/build_iso.py b/iso/empanadas/empanadas/scripts/build_iso.py new file mode 100755 index 0000000..fbf1d06 --- /dev/null +++ b/iso/empanadas/empanadas/scripts/build_iso.py @@ -0,0 +1,31 @@ +# builds ISO's + +import argparse + +from empanadas.common import * +from empanadas.util import Checks +from empanadas.util import IsoBuild + +parser = argparse.ArgumentParser(description="ISO Compose") + +parser.add_argument('--release', type=str, help="Major Release Version or major-type (eg 9-beta)", required=True) +parser.add_argument('--isolation', type=str, help="mock isolation mode") +parser.add_argument('--rc', action='store_true', help="Release Candidate, Beta, RLN") +parser.add_argument('--local-compose', action='store_true', help="Compose Directory is Here") +parser.add_argument('--logger', type=str) +results = parser.parse_args() +rlvars = rldict[results.release] +major = rlvars['major'] + +a = IsoBuild( + rlvars, + config, + major=major, + rc=results.rc, + isolation=results.isolation, + compose_dir_is_here=results.local_compose, + logger=results.logger, +) + +def run(): + a.run() diff --git a/iso/empanadas/empanadas/scripts/build_iso_extra.py b/iso/empanadas/empanadas/scripts/build_iso_extra.py new file mode 100755 index 0000000..9fe9c05 --- /dev/null +++ b/iso/empanadas/empanadas/scripts/build_iso_extra.py @@ -0,0 +1,37 @@ +# builds ISO's + +import argparse + +from empanadas.common import * +from empanadas.util import Checks +from empanadas.util import IsoBuild + +parser = argparse.ArgumentParser(description="ISO Compose") + +parser.add_argument('--release', type=str, help="Major Release Version or major-type (eg 9-beta)", required=True) +parser.add_argument('--rc', action='store_true', help="Release Candidate, Beta, RLN") +parser.add_argument('--arch', type=str, help="Architecture") +parser.add_argument('--isolation', type=str, help="Mock Isolation") +parser.add_argument('--local-compose', action='store_true', help="Compose Directory is Here") +parser.add_argument('--logger', type=str) +parser.add_argument('--extra-iso', type=str, help="Granular choice in which iso is built") +parser.add_argument('--extra-iso-mode', type=str, default='local') +results = parser.parse_args() +rlvars = rldict[results.release] +major = rlvars['major'] + +a = IsoBuild( + rlvars, + config, + major=major, + rc=results.rc, + arch=results.arch, + isolation=results.isolation, + extra_iso=results.extra_iso, + extra_iso_mode=results.extra_iso_mode, + compose_dir_is_here=results.local_compose, + logger=results.logger +) + +def run(): + a.run_build_extra_iso() diff --git a/iso/empanadas/empanadas/scripts/launch_builds.py b/iso/empanadas/empanadas/scripts/launch_builds.py new file mode 100755 index 0000000..f0f82f7 --- /dev/null +++ b/iso/empanadas/empanadas/scripts/launch_builds.py @@ -0,0 +1,48 @@ +# Launches the builds of ISOs + +import argparse +import datetime + +from empanadas.common import * +from empanadas.common import _rootdir + +from jinja2 import Environment, FileSystemLoader + +parser = argparse.ArgumentParser(description="ISO Compose") + +parser.add_argument('--release', type=str, help="Major Release Version", required=True) +parser.add_argument('--env', type=str, help="environment", required=True) +results = parser.parse_args() +rlvars = rldict[results.release] +major = rlvars['major'] + +EXTARCH=["s390x", "ppc64le"] +EKSARCH=["amd64", "arm64"] + +def run(): + file_loader = FileSystemLoader(f"{_rootdir}/templates") + tmplenv = Environment(loader=file_loader) + job_template = tmplenv.get_template('kube/Job.tmpl') + + arches = EKSARCH + if results.env == "ext" and results.env != "all": + arches = EXTARCH + elif results.env == "all": + arches = EKSARCH+EXTARCH + + command = ["build-iso", "--release", f"{results.release}", "--rc", "--isolation", "simple"] + + out = "" + for arch in arches: + out += job_template.render( + architecture=arch, + backoffLimit=4, + buildTime=datetime.datetime.utcnow().strftime("%s"), + command=command, + imageName="ghcr.io/neilhanlon/sig-core-toolkit:latest", + namespace="empanadas", + major=major, + restartPolicy="Never", + ) + + print(out) diff --git a/iso/empanadas/empanadas/scripts/pull_unpack_tree.py b/iso/empanadas/empanadas/scripts/pull_unpack_tree.py new file mode 100755 index 0000000..77f8100 --- /dev/null +++ b/iso/empanadas/empanadas/scripts/pull_unpack_tree.py @@ -0,0 +1,37 @@ +# builds ISO's + +import argparse + +from empanadas.common import * +from empanadas.util import Checks +from empanadas.util import IsoBuild + +parser = argparse.ArgumentParser(description="ISO Artifact Builder") + +parser.add_argument('--release', type=str, help="Major Release Version", required=True) +parser.add_argument('--s3', action='store_true', help="Release Candidate") +parser.add_argument('--rc', action='store_true', help="Release Candidate") +parser.add_argument('--arch', type=str, help="Architecture") +parser.add_argument('--local-compose', action='store_true', help="Compose Directory is Here") +parser.add_argument('--force-unpack', action='store_true', help="Force an unpack") +parser.add_argument('--force-download', action='store_true', help="Force a download") +parser.add_argument('--logger', type=str) +results = parser.parse_args() +rlvars = rldict[results.release] +major = rlvars['major'] + +a = IsoBuild( + rlvars, + config, + major=major, + rc=results.rc, + s3=results.s3, + arch=results.arch, + force_unpack=results.force_unpack, + force_download=results.force_download, + compose_dir_is_here=results.local_compose, + logger=results.logger, +) + +def run(): + a.run_pull_lorax_artifacts() diff --git a/iso/py/sync-from-peridot b/iso/empanadas/empanadas/scripts/sync_from_peridot.py similarity index 81% rename from iso/py/sync-from-peridot rename to iso/empanadas/empanadas/scripts/sync_from_peridot.py index d114a59..1e40ed8 100755 --- a/iso/py/sync-from-peridot +++ b/iso/empanadas/empanadas/scripts/sync_from_peridot.py @@ -1,21 +1,16 @@ -#!/usr/bin/env python3 - # This script can be called to do single syncs or full on syncs. import argparse -from common import * -from util import Checks -from util import RepoSync -#rlvars = rldict['9'] -#r = Checks(rlvars, config['arch']) -#r.check_valid_arch() +from empanadas.common import * +from empanadas.util import Checks +from empanadas.util import RepoSync # Start up the parser baby parser = argparse.ArgumentParser(description="Peridot Sync and Compose") # All of our options -parser.add_argument('--release', type=str, help="Major Release Version", required=True) +parser.add_argument('--release', type=str, help="Major Release Version or major-type (eg 9-beta)", required=True) parser.add_argument('--repo', type=str, help="Repository name") parser.add_argument('--arch', type=str, help="Architecture") parser.add_argument('--ignore-debug', action='store_true') @@ -26,14 +21,16 @@ parser.add_argument('--hashed', action='store_true') parser.add_argument('--dry-run', action='store_true') parser.add_argument('--full-run', action='store_true') parser.add_argument('--no-fail', action='store_true') +parser.add_argument('--refresh-extra-files', action='store_true') # I am aware this is confusing, I want podman to be the default option parser.add_argument('--simple', action='store_false') parser.add_argument('--logger', type=str) # Parse them results = parser.parse_args() - rlvars = rldict[results.release] +major = rlvars['major'] + r = Checks(rlvars, config['arch']) r.check_valid_arch() @@ -41,7 +38,7 @@ r.check_valid_arch() a = RepoSync( rlvars, config, - major=results.release, + major=major, repo=results.repo, arch=results.arch, ignore_debug=results.ignore_debug, @@ -53,7 +50,9 @@ a = RepoSync( dryrun=results.dry_run, fullrun=results.full_run, nofail=results.no_fail, - logger=results.logger + logger=results.logger, + refresh_extra_files=results.refresh_extra_files, ) -a.run() +def run(): + a.run() diff --git a/iso/empanadas/empanadas/scripts/sync_from_peridot_test.py b/iso/empanadas/empanadas/scripts/sync_from_peridot_test.py new file mode 100755 index 0000000..5057753 --- /dev/null +++ b/iso/empanadas/empanadas/scripts/sync_from_peridot_test.py @@ -0,0 +1,18 @@ +# This is a testing script to ensure the RepoSync class is working as intended. + +import argparse + +from empanadas.common import * +from empanadas.util import Checks +from empanadas.util import RepoSync + +rlvars = rldict['9-lookahead'] +r = Checks(rlvars, config['arch']) +r.check_valid_arch() + +#a = RepoSync(rlvars, config, major="9", repo="ResilientStorage", parallel=True, ignore_debug=False, ignore_source=False) +a = RepoSync(rlvars, config, major="9", repo="BaseOS", parallel=True, ignore_debug=False, ignore_source=False, hashed=True) + +def run(): + print(rlvars.keys()) + print(rlvars) diff --git a/iso/py/sync-sig b/iso/empanadas/empanadas/scripts/sync_sig.py similarity index 93% rename from iso/py/sync-sig rename to iso/empanadas/empanadas/scripts/sync_sig.py index ff9c8e1..3de8479 100755 --- a/iso/py/sync-sig +++ b/iso/empanadas/empanadas/scripts/sync_sig.py @@ -1,11 +1,9 @@ -#!/usr/bin/env python3 - # This script can be called to do single syncs or full on syncs. import argparse -from common import * -from util import Checks -from util import SigRepoSync +from empanadas.common import * +from empanadas.util import Checks +from empanadas.util import SigRepoSync #rlvars = rldict['9'] #r = Checks(rlvars, config['arch']) @@ -58,4 +56,6 @@ a = SigRepoSync( logger=results.logger ) -a.run() + +def run(): + a.run() diff --git a/iso/py/sig/altarch.yaml b/iso/empanadas/empanadas/sig/altarch.yaml similarity index 100% rename from iso/py/sig/altarch.yaml rename to iso/empanadas/empanadas/sig/altarch.yaml diff --git a/iso/py/sig/cloud.yaml b/iso/empanadas/empanadas/sig/cloud.yaml similarity index 100% rename from iso/py/sig/cloud.yaml rename to iso/empanadas/empanadas/sig/cloud.yaml diff --git a/iso/py/sig/core.yaml b/iso/empanadas/empanadas/sig/core.yaml similarity index 100% rename from iso/py/sig/core.yaml rename to iso/empanadas/empanadas/sig/core.yaml diff --git a/iso/empanadas/empanadas/templates/buildExtraImage.tmpl.sh b/iso/empanadas/empanadas/templates/buildExtraImage.tmpl.sh new file mode 100644 index 0000000..ae9b5cf --- /dev/null +++ b/iso/empanadas/empanadas/templates/buildExtraImage.tmpl.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -ex + +{% if extra_iso_mode == "podman" %} +{{ lorax_pkg_cmd }} +mkdir -p {{ compose_work_iso_dir }}/{{ arch }} +cd {{ compose_work_iso_dir }}/{{ arch }} +test -f {{ isoname }} || { echo "!! ISO ALREDY EXISTS !!"; exit 1; } +{% else %} +cd /builddir + +if ! TEMPLATE="$($(head -n1 $(which lorax) | cut -c3-) -c 'import pylorax; print(pylorax.find_templates())')"; then + TEMPLATE="/usr/share/lorax" +fi +{% endif %} + + +{{ make_image }} + +{{ isohybrid }} + +{{ implantmd5 }} + +{{ make_manifest }} + diff --git a/iso/empanadas/empanadas/templates/buildImage.tmpl.sh b/iso/empanadas/empanadas/templates/buildImage.tmpl.sh new file mode 100644 index 0000000..817d73b --- /dev/null +++ b/iso/empanadas/empanadas/templates/buildImage.tmpl.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +VOLID="{{ shortname }}-{{ major }}-{{ minor }}{{ rc }}-{{ arch }}-dvd" +VARIANT="{{ variant }}" +ARCH="{{ arch }}" +VERSION="{{ revision }}" +PRODUCT="{{ distname }}" +MOCKBLD="{{ builddir }}" +LORAXRES="{{ lorax_work_root }}" +LORAX_TAR="lorax-{{ revision }}-{{ arch }}.tar.gz" +LOGFILE="lorax-{{ arch }}.log" +BUGURL="{{ bugurl }}" + +{% for pkg in lorax %} +sed -i '/{{ pkg }}/ s/^/#/' /usr/share/lorax/templates.d/80-rhel/runtime-install.tmpl +{% endfor %} + +lorax --product="${PRODUCT}" \ + --version="${VERSION}" \ + --release="${VERSION}" \ +{%- if rc == '' %} + --isfinal \ +{%- endif %} +{%- for repo in repos %} + --source={{ repo.url }} \ +{%- endfor %} + --bugurl="${BUGURL}" \ + --variant="${VARIANT}" \ + --nomacboot \ + --buildarch="${ARCH}" \ + --volid="${VOLID}" \ + --logfile="${MOCKBLD}/${LOGFILE}" \ + --rootfs-size=3 \ + "${LORAXRES}" + +ret_val=$? +if [ $ret_val -ne 0 ]; then + echo "!! LORAX FAILED !!" + exit 1 +fi + +# If we didn't fail, let's pack up everything! +cd "${MOCKBLD}" + +# Get ISO manifest +if [ -f "/usr/bin/xorriso" ]; then + /usr/bin/xorriso -dev lorax/images/boot.iso --find | + tail -n+2 | + tr -d "'" | + cut -c2- | sort >> lorax/images/boot.iso.manifest +elif [ -f "/usr/bin/isoinfo" ]; then + /usr/bin/isoinfo -R -f -i lorax/images/boot.iso | + grep -v '/TRANS.TBL$' | sort >> lorax/images/boot.iso.manifest +fi + +tar czf "${LORAX_TAR}" lorax "${LOGFILE}" + +tar_ret_val=$? +if [ $ret_val -ne 0 ]; then + echo "!! PROBLEM CREATING ARCHIVE !!" + exit 1 +fi + +exit 0 diff --git a/iso/empanadas/empanadas/templates/extraisobuild.tmpl.sh b/iso/empanadas/empanadas/templates/extraisobuild.tmpl.sh new file mode 100644 index 0000000..4d42901 --- /dev/null +++ b/iso/empanadas/empanadas/templates/extraisobuild.tmpl.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# This is a template that is used to build extra ISO's for Rocky Linux. Only +# under extreme circumstances should you be filling this out and running +# manually. + +# Vars +MOCK_CFG="/var/tmp/lorax-{{ major }}.cfg" +MOCK_ROOT="/var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}" +MOCK_RESL="${MOCK_ROOT}/result" +MOCK_CHRO="${MOCK_ROOT}/root" +MOCK_LOG="${MOCK_RESL}/mock-output.log" +IMAGE_SCR="{{ entries_dir }}/buildExtraImage-{{ arch }}-{{ image }}.sh" +IMAGE_ISO="{{ isoname }}" +ISOLATION="{{ isolation }}" +BUILDDIR="{{ builddir }}" + +# Init the container +mock \ + -r "${MOCK_CFG}" \ + --isolation="${ISOLATION}" \ + --enable-network \ + --init + +init_ret_val=$? +if [ $init_ret_val -ne 0 ]; then + echo "!! MOCK INIT FAILED !!" + exit 1 +fi + +mkdir -p "${MOCK_RESL}" +cp "${IMAGE_SCR}" "${MOCK_CHRO}${IMAGE_SCR}" + +mock \ + -r "${MOCK_CFG}" \ + --shell \ + --isolation="${ISOLATION}" \ + --enable-network -- /bin/bash "${IMAGE_SCR}" | tee -a "${MOCK_LOG}" + +mock_ret_val=$? +if [ $mock_ret_val -eq 0 ]; then + # Copy resulting data to /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result + mkdir -p "${MOCK_RESL}" + cp "${MOCK_CHRO}${BUILDDIR}/${IMAGE_ISO}" "${MOCK_RESL}" + cp "${MOCK_CHRO}${BUILDDIR}/${IMAGE_ISO}.manifest" "${MOCK_RESL}" +else + echo "!! EXTRA ISO RUN FAILED !!" + exit 1 +fi + +# Clean up? diff --git a/iso/empanadas/empanadas/templates/isobuild.tmpl.sh b/iso/empanadas/empanadas/templates/isobuild.tmpl.sh new file mode 100644 index 0000000..95184b6 --- /dev/null +++ b/iso/empanadas/empanadas/templates/isobuild.tmpl.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# This is a template that is used to build ISO's for Rocky Linux. Only under +# extreme circumstances should you be filling this out and running manually. + +# Vars +MOCK_CFG="/var/tmp/lorax-{{ major }}.cfg" +MOCK_ROOT="/var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}" +MOCK_RESL="${MOCK_ROOT}/result" +MOCK_CHRO="${MOCK_ROOT}/root" +MOCK_LOG="${MOCK_RESL}/mock-output.log" +LORAX_SCR="/var/tmp/buildImage.sh" +LORAX_TAR="lorax-{{ revision }}-{{ arch }}.tar.gz" +ISOLATION="{{ isolation }}" +BUILDDIR="{{ builddir }}" + +# Init the container +mock \ + -r "${MOCK_CFG}" \ + --isolation="${ISOLATION}" \ + --enable-network \ + --init + +init_ret_val=$? +if [ $init_ret_val -ne 0 ]; then + echo "!! MOCK INIT FAILED !!" + exit 1 +fi + +mkdir -p "${MOCK_RESL}" +cp "${LORAX_SCR}" "${MOCK_CHRO}${LORAX_SCR}" + +mock \ + -r "${MOCK_CFG}" \ + --shell \ + --isolation="${ISOLATION}" \ + --enable-network -- /bin/bash "${LORAX_SCR}" | tee -a "${MOCK_LOG}" + +mock_ret_val=$? +if [ $mock_ret_val -eq 0 ]; then + # Copy resulting data to /var/lib/mock/{{ shortname|lower }}-{{ major }}-{{ arch }}/result + mkdir -p "${MOCK_RESL}" + cp "${MOCK_CHRO}${BUILDDIR}/${LORAX_TAR}" "${MOCK_RESL}" +else + echo "!! LORAX RUN FAILED !!" + exit 1 +fi + +# Clean up? diff --git a/iso/empanadas/empanadas/templates/isomock.tmpl.cfg b/iso/empanadas/empanadas/templates/isomock.tmpl.cfg new file mode 100644 index 0000000..53cd821 --- /dev/null +++ b/iso/empanadas/empanadas/templates/isomock.tmpl.cfg @@ -0,0 +1,49 @@ +config_opts['root'] = '{{ shortname|lower }}-{{ major }}-{{ arch }}' +config_opts['description'] = '{{ fullname }}' +config_opts['target_arch'] = '{{ arch }}' +config_opts['legal_host_arches'] = ('{{ arch }}',) + +config_opts['chroot_setup_cmd'] = 'install bash bzip2 coreutils cpio diffutils redhat-release findutils gawk glibc-minimal-langpack grep gzip info patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz {{ required_pkgs|join(' ') }}' +config_opts['dist'] = '{{ dist }}' # only useful for --resultdir variable subst +config_opts['releasever'] = '{{ major }}' +config_opts['package_manager'] = '{{ pkgmanager|default("dnf") }}' +config_opts['extra_chroot_dirs'] = [ '/run/lock', ] +# config_opts['bootstrap_image'] = 'quay.io/{{ shortname|lower }}/{{ shortname|lower }}:{{ major }}' + +# If compose is local, the bind mounts will be here +{% if compose_dir_is_here %} +config_opts['plugin_conf']['bind_mount_enable'] = True +config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('{{ compose_dir }}', '{{ compose_dir }}')) +{% endif %} + +config_opts['dnf.conf'] = """ +[main] +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= +metadata_expire=0 +mdpolicy=group:primary +best=1 +install_weak_deps=0 +protected_packages= +module_platform_id=platform:{{ dist }} +user_agent={{ user_agent }} + +{% for repo in repos %} +[{{ repo.name }}] +name={{ repo.name }} +baseurl={{ repo.url }} +enabled=1 +gpgcheck=0 + +{% endfor %} + +""" + diff --git a/iso/empanadas/empanadas/templates/kube/Job.tmpl b/iso/empanadas/empanadas/templates/kube/Job.tmpl new file mode 100644 index 0000000..bfcc20a --- /dev/null +++ b/iso/empanadas/empanadas/templates/kube/Job.tmpl @@ -0,0 +1,61 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: build-iso-{{ major }}-{{ architecture }} + namespace: {{ namespace }} +spec: + template: + metadata: + labels: + peridot.rockylinux.org/workflow-tolerates-arch: {{ architecture }} + spec: + containers: + - name: buildiso-{{ major }}-{{ architecture }} + image: {{ imageName }} + command: ["/bin/bash", "-c"] + args: + - | + {{ command | join(' ') }} + aws s3 cp --recursive --exclude=* --include=lorax* \ + /var/lib/mock/rocky-{{ major }}-$(uname -m)/root/builddir/ \ + "s3://resf-empanadas/buildiso-{{ major }}-{{ architecture }}/{{ buildTime }}/" + securityContext: + runAsUser: 0 + runAsGroup: 0 + privileged: true + runAsNonRoot: false + allowPrivilegeEscalation: true + volumeMounts: + - mountPath: /etc/resolv.conf + name: resolv-conf + - mountPath: /var/lib/mock/ + name: mock + env: + - name: AWS_REGION + value: us-east-2 + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: empanadas-s3 + key: ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: empanadas-s3 + key: SECRET + tolerations: + - effect: NoSchedule + key: peridot.rockylinux.org/workflow-tolerates-arch + operator: Equal + value: {{ architecture }} + restartPolicy: {{ restartPolicy }} + volumes: + - name: resolv-conf + hostPath: + path: /etc/resolv.conf + type: File + - name: mock + emptyDir: {} + backoffLimit: {{ backoffLimit }} + diff --git a/iso/empanadas/empanadas/templates/repoconfig.tmpl b/iso/empanadas/empanadas/templates/repoconfig.tmpl new file mode 100644 index 0000000..d322929 --- /dev/null +++ b/iso/empanadas/empanadas/templates/repoconfig.tmpl @@ -0,0 +1,26 @@ +{%- for repo in repos -%} +[{{ repo.name }}] +name={{repo.name}} +baseurl={{ repo.baseurl }} +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey={{ repo.gpgkey }} + +[{{ repo.name }}-debug] +name={{repo.name}} +baseurl={{ repo.baseurl }}-debug +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey={{ repo.gpgkey }} + +[{{ repo.name }}-source] +name={{repo.name}} +baseurl={{ repo.srcbaseurl }} +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey={{ repo.gpgkey }} + +{% endfor %} diff --git a/iso/empanadas/empanadas/templates/reposync-src.tmpl b/iso/empanadas/empanadas/templates/reposync-src.tmpl new file mode 100644 index 0000000..88836a3 --- /dev/null +++ b/iso/empanadas/empanadas/templates/reposync-src.tmpl @@ -0,0 +1,7 @@ +#!/bin/bash +set -o pipefail +{{ import_gpg_cmd }} | tee -a {{ sync_log }} +{{ dnf_plugin_cmd }} | tee -a {{ sync_log }} +{{ sync_cmd }} | tee -a {{ sync_log }} + +# {{ check_cmd }} | tee -a {{ sync_log }} diff --git a/iso/empanadas/empanadas/templates/reposync.tmpl b/iso/empanadas/empanadas/templates/reposync.tmpl new file mode 100644 index 0000000..22f71b4 --- /dev/null +++ b/iso/empanadas/empanadas/templates/reposync.tmpl @@ -0,0 +1,8 @@ +#!/bin/bash +set -o pipefail +{{ import_gpg_cmd }} | tee -a {{ sync_log }} +{{ arch_force_cp }} | tee -a {{ sync_log }} +{{ dnf_plugin_cmd }} | tee -a {{ sync_log }} +{{ sync_cmd }} | tee -a {{ sync_log }} + +# {{ check_cmd }} | tee -a {{ sync_log }} diff --git a/iso/empanadas/empanadas/templates/xorriso.tmpl.txt b/iso/empanadas/empanadas/templates/xorriso.tmpl.txt new file mode 100644 index 0000000..133aa11 --- /dev/null +++ b/iso/empanadas/empanadas/templates/xorriso.tmpl.txt @@ -0,0 +1,6 @@ +-indev {{ boot_iso }} +-outdev {{ isoname }} +-boot_image any replay +-volid {{ volid }} +{{ graft }} +-end diff --git a/iso/empanadas/empanadas/util/__init__.py b/iso/empanadas/empanadas/util/__init__.py new file mode 100644 index 0000000..f107a54 --- /dev/null +++ b/iso/empanadas/empanadas/util/__init__.py @@ -0,0 +1,27 @@ +""" +Imports all of our classes for this local module +""" + +from empanadas.util.check import ( + Checks, +) + +from empanadas.util.shared import ( + Shared, +) + +from empanadas.util.dnf_utils import ( + RepoSync, + SigRepoSync +) + +from empanadas.util.iso_utils import ( + IsoBuild, + LiveBuild +) + +__all__ = [ + 'Checks', + 'RepoSync', + 'Shared' +] diff --git a/iso/py/util/check.py b/iso/empanadas/empanadas/util/check.py similarity index 94% rename from iso/py/util/check.py rename to iso/empanadas/empanadas/util/check.py index a87d86f..b491a61 100644 --- a/iso/py/util/check.py +++ b/iso/empanadas/empanadas/util/check.py @@ -1,6 +1,7 @@ # Is our arch allowed for this particular release? Some previous releases do # not support ppc or s390x -from common import Color +from empanadas.common import Color + class Checks: """This class helps check some things""" def __init__(self, rlvars, arch): diff --git a/iso/py/util/dnf_utils.py b/iso/empanadas/empanadas/util/dnf_utils.py similarity index 73% rename from iso/py/util/dnf_utils.py rename to iso/empanadas/empanadas/util/dnf_utils.py index 4354e45..a54b4ee 100644 --- a/iso/py/util/dnf_utils.py +++ b/iso/empanadas/empanadas/util/dnf_utils.py @@ -10,10 +10,19 @@ import os import os.path import subprocess import shlex +import shutil import time import re +import json #import pipes -from common import Color + +from jinja2 import Environment, FileSystemLoader + +from empanadas.common import Color, _rootdir +from empanadas.util import Shared + +# initial treeinfo data is made here +import productmd.treeinfo #HAS_LIBREPO = True #try: @@ -37,12 +46,16 @@ class RepoSync: ignore_debug: bool = False, ignore_source: bool = False, repoclosure: bool = False, + refresh_extra_files: bool = False, + refresh_treeinfo: bool = False, skip_all: bool = False, hashed: bool = False, parallel: bool = False, dryrun: bool = False, fullrun: bool = False, nofail: bool = False, + gpgkey: str = 'stable', + rlmode: str = 'stable', logger=None ): self.nofail = nofail @@ -54,6 +67,8 @@ class RepoSync: self.skip_all = skip_all self.hashed = hashed self.repoclosure = repoclosure + self.refresh_extra_files = refresh_extra_files + self.refresh_treeinfo = refresh_treeinfo # Enables podman syncing, which should effectively speed up operations self.parallel = parallel # Relevant config items @@ -62,9 +77,12 @@ class RepoSync: self.repo_base_url = config['repo_base_url'] self.compose_root = config['compose_root'] self.compose_base = config['compose_root'] + "/" + major + self.profile = rlvars['profile'] # Relevant major version items + self.shortname = config['shortname'] self.revision = rlvars['revision'] + "-" + rlvars['rclvl'] + self.fullversion = rlvars['revision'] self.arches = rlvars['allowed_arches'] self.project_id = rlvars['project_id'] self.repo_renames = rlvars['renames'] @@ -72,6 +90,11 @@ class RepoSync: self.multilib = rlvars['provide_multilib'] self.repo = repo self.extra_files = rlvars['extra_files'] + self.gpgkey = gpgkey + + # Templates + file_loader = FileSystemLoader(f"{_rootdir}/templates") + self.tmplenv = Environment(loader=file_loader) # each el can have its own designated container to run stuff in, # otherwise we'll just default to the default config. @@ -91,7 +114,7 @@ class RepoSync: self.compose_latest_dir = os.path.join( config['compose_root'], major, - "latest-Rocky-{}".format(major) + "latest-Rocky-{}".format(self.profile) ) self.compose_latest_sync = os.path.join( @@ -104,6 +127,11 @@ class RepoSync: "work/logs" ) + self.compose_global_work_root = os.path.join( + self.compose_latest_dir, + "work/global" + ) + # This is temporary for now. if logger is None: self.log = logging.getLogger("reposync") @@ -169,27 +197,46 @@ class RepoSync: log_root = os.path.join( work_root, - "logs" + "logs", + self.date_stamp + ) + + global_work_root = os.path.join( + work_root, + "global", ) if self.dryrun: self.log.error('Dry Runs are not supported just yet. Sorry!') raise SystemExit() - self.sync(self.repo, sync_root, work_root, log_root, self.arch) + if self.fullrun and self.refresh_extra_files: + self.log.warn( + '[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' + + 'A full run implies extra files are also deployed.' + ) + + self.sync(self.repo, sync_root, work_root, log_root, global_work_root, self.arch) if self.fullrun: - self.deploy_extra_files() - self.symlink_to_latest() + self.deploy_extra_files(global_work_root) + self.deploy_treeinfo(self.repo, sync_root, self.arch) + self.symlink_to_latest(generated_dir) if self.repoclosure: self.repoclosure_work(sync_root, work_root, log_root) + if self.refresh_extra_files and not self.fullrun: + self.deploy_extra_files(global_work_root) + + if self.refresh_treeinfo and not self.fullrun: + self.deploy_treeinfo(self.repo, sync_root, self.arch) + self.log.info('Compose repo directory: %s' % sync_root) self.log.info('Compose logs: %s' % log_root) self.log.info('Compose completed.') - def sync(self, repo, sync_root, work_root, log_root, arch=None): + def sync(self, repo, sync_root, work_root, log_root, global_work_root, arch=None): """ Calls out syncing of the repos. We generally sync each component of a repo: @@ -200,7 +247,7 @@ class RepoSync: If parallel is true, we will run in podman. """ if self.parallel: - self.podman_sync(repo, sync_root, work_root, log_root, arch) + self.podman_sync(repo, sync_root, work_root, log_root, global_work_root, arch) else: self.dnf_sync(repo, sync_root, work_root, arch) @@ -212,7 +259,15 @@ class RepoSync: self.log.error('Please install podman and enable parallel') raise SystemExit() - def podman_sync(self, repo, sync_root, work_root, log_root, arch): + def podman_sync( + self, + repo, + sync_root, + work_root, + log_root, + global_work_root, + arch + ): """ This is for podman syncs @@ -230,6 +285,9 @@ class RepoSync: os.makedirs(entries_dir, exist_ok=True) # yeah, I know. + if not os.path.exists(global_work_root): + os.makedirs(global_work_root, exist_ok=True) + if not os.path.exists(log_root): os.makedirs(log_root, exist_ok=True) @@ -293,6 +351,11 @@ class RepoSync: 'debug/tree' ) + import_gpg_cmd = ("/usr/bin/rpm --import {}{}").format( + self.extra_files['git_raw_path'], + self.extra_files['gpg'][self.gpgkey] + ) + arch_force_cp = ("/usr/bin/sed 's|$basearch|{}|g' {} > {}.{}".format( a, self.dnf_config, @@ -300,48 +363,63 @@ class RepoSync: a )) + sync_log = ("{}/{}-{}.log").format( + log_root, + repo_name, + a + ) + + debug_sync_log = ("{}/{}-{}-debug.log").format( + log_root, + repo_name, + a + ) + sync_cmd = ("/usr/bin/dnf reposync -c {}.{} --download-metadata " - "--repoid={} -p {} --forcearch {} --norepopath 2>&1 " - "| tee -a {}/{}-{}-{}.log").format( + "--repoid={} -p {} --forcearch {} --norepopath " + "--gpgcheck --assumeyes 2>&1").format( self.dnf_config, a, r, os_sync_path, - a, - log_root, - repo_name, - a, - self.date_stamp + a ) debug_sync_cmd = ("/usr/bin/dnf reposync -c {}.{} " "--download-metadata --repoid={}-debug -p {} --forcearch {} " - "--norepopath 2>&1 | tee -a {}/{}-{}-debug-{}.log").format( + "--gpgcheck --norepopath --assumeyes 2>&1").format( self.dnf_config, a, r, debug_sync_path, - a, - log_root, - repo_name, - a, - self.date_stamp + a + ) + + dnf_plugin_cmd = "/usr/bin/dnf install dnf-plugins-core -y" + + sync_template = self.tmplenv.get_template('reposync.tmpl') + sync_output = sync_template.render( + import_gpg_cmd=import_gpg_cmd, + arch_force_cp=arch_force_cp, + dnf_plugin_cmd=dnf_plugin_cmd, + sync_cmd=sync_cmd, + sync_log=sync_log + ) + + debug_sync_template = self.tmplenv.get_template('reposync.tmpl') + debug_sync_output = debug_sync_template.render( + import_gpg_cmd=import_gpg_cmd, + arch_force_cp=arch_force_cp, + dnf_plugin_cmd=dnf_plugin_cmd, + sync_cmd=debug_sync_cmd, + sync_log=debug_sync_log ) entry_point_open = open(entry_point_sh, "w+") debug_entry_point_open = open(debug_entry_point_sh, "w+") - entry_point_open.write('#!/bin/bash\n') - entry_point_open.write('set -o pipefail\n') - entry_point_open.write(arch_force_cp + '\n') - entry_point_open.write('/usr/bin/dnf install dnf-plugins-core -y\n') - entry_point_open.write(sync_cmd + '\n') - - debug_entry_point_open.write('#!/bin/bash\n') - debug_entry_point_open.write('set -o pipefail\n') - debug_entry_point_open.write(arch_force_cp + '\n') - debug_entry_point_open.write('/usr/bin/dnf install dnf-plugins-core -y\n') - debug_entry_point_open.write(debug_sync_cmd + '\n') + entry_point_open.write(sync_output) + debug_entry_point_open.write(debug_sync_output) entry_point_open.close() debug_entry_point_open.close() @@ -365,21 +443,29 @@ class RepoSync: 'source/tree' ) + source_sync_log = ("{}/{}-source.log").format( + log_root, + repo_name + ) + source_sync_cmd = ("/usr/bin/dnf reposync -c {} " "--download-metadata --repoid={}-source -p {} " - "--norepopath | tee -a {}/{}-source-{}.log").format( + "--gpgcheck --norepopath --assumeyes 2>&1").format( self.dnf_config, r, - source_sync_path, - log_root, - repo_name, - self.date_stamp + source_sync_path ) + + source_sync_template = self.tmplenv.get_template('reposync-src.tmpl') + source_sync_output = source_sync_template.render( + import_gpg_cmd=import_gpg_cmd, + dnf_plugin_cmd=dnf_plugin_cmd, + sync_cmd=source_sync_cmd, + sync_log=source_sync_log + ) + source_entry_point_open = open(source_entry_point_sh, "w+") - source_entry_point_open.write('#!/bin/bash\n') - source_entry_point_open.write('set -o pipefail\n') - source_entry_point_open.write('/usr/bin/dnf install dnf-plugins-core -y\n') - source_entry_point_open.write(source_sync_cmd + '\n') + source_entry_point_open.write(source_sync_output) source_entry_point_open.close() os.chmod(source_entry_point_sh, 0o755) @@ -410,7 +496,10 @@ class RepoSync: join_all_pods = ' '.join(entry_name_list) time.sleep(3) - self.log.info('Syncing %s ...' % r) + self.log.info( + '[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' + + 'Syncing ' + r + ' ...' + ) pod_watcher = '{} wait {}'.format( cmd, join_all_pods @@ -439,10 +528,10 @@ class RepoSync: ) output, errors = podcheck.communicate() - if 'Exited (0)' in output.decode(): - self.log.info('%s seems ok' % pod) - else: - self.log.error('%s had issues syncing' % pod) + if 'Exited (0)' not in output.decode(): + self.log.error( + '[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + pod + ) bad_exit_list.append(pod) rmcmd = '{} rm {}'.format( @@ -458,7 +547,10 @@ class RepoSync: ) entry_name_list.clear() - self.log.info('Syncing %s completed' % r) + self.log.info( + '[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' + + 'Syncing ' + r + ' completed' + ) if len(bad_exit_list) > 0: self.log.error( @@ -467,6 +559,11 @@ class RepoSync: ) for issue in bad_exit_list: self.log.error(issue) + else: + self.log.info( + '[' + Color.BOLD + Color.GREEN + ' OK ' + Color.END + '] ' + 'No issues detected.' + ) def generate_compose_dirs(self) -> str: """ @@ -474,7 +571,7 @@ class RepoSync: """ compose_base_dir = os.path.join( self.compose_base, - "Rocky-{}-{}".format(self.major_version, self.date_stamp) + "Rocky-{}-{}".format(self.fullversion, self.date_stamp) ) self.log.info('Creating compose directory %s' % compose_base_dir) if not os.path.exists(compose_base_dir): @@ -482,7 +579,7 @@ class RepoSync: return compose_base_dir - def symlink_to_latest(self): + def symlink_to_latest(self, generated_dir): """ Emulates pungi and symlinks latest-Rocky-X @@ -490,7 +587,13 @@ class RepoSync: 'latest' directory is what is rsynced on to staging after completion. This link should not change often. """ - pass + try: + os.remove(self.compose_latest_dir) + except: + pass + + self.log.info('Symlinking to latest-{}-{}...'.format(self.shortname, self.major_version)) + os.symlink(generated_dir, self.compose_latest_dir) def generate_conf(self, dest_path='/var/tmp') -> str: """ @@ -520,15 +623,10 @@ class RepoSync: if not os.path.exists(dest_path): os.makedirs(dest_path, exist_ok=True) config_file = open(fname, "w+") + repolist = [] for repo in self.repos: - constructed_url = '{}/{}/repo/{}{}/$basearch'.format( - self.repo_base_url, - self.project_id, - prehashed, - repo, - ) - constructed_url_debug = '{}/{}/repo/{}{}/$basearch-debug'.format( + constructed_url = '{}/{}/repo/{}{}/$basearch'.format( self.repo_base_url, self.project_id, prehashed, @@ -542,27 +640,17 @@ class RepoSync: repo, ) - # normal - config_file.write('[%s]\n' % repo) - config_file.write('name=%s\n' % repo) - config_file.write('baseurl=%s\n' % constructed_url) - config_file.write("enabled=1\n") - config_file.write("gpgcheck=0\n\n") - - # debug - config_file.write('[%s-debug]\n' % repo) - config_file.write('name=%s debug\n' % repo) - config_file.write('baseurl=%s\n' % constructed_url_debug) - config_file.write("enabled=1\n") - config_file.write("gpgcheck=0\n\n") - - # src - config_file.write('[%s-source]\n' % repo) - config_file.write('name=%s source\n' % repo) - config_file.write('baseurl=%s\n' % constructed_url_src) - config_file.write("enabled=1\n") - config_file.write("gpgcheck=0\n\n") + repodata = { + 'name': repo, + 'baseurl': constructed_url, + 'srcbaseurl': constructed_url_src, + 'gpgkey': self.extra_files['git_raw_path'] + self.extra_files['gpg'][self.gpgkey] + } + repolist.append(repodata) + template = self.tmplenv.get_template('repoconfig.tmpl') + output = template.render(repos=repolist) + config_file.write(output) config_file.close() return fname @@ -605,6 +693,22 @@ class RepoSync: ) return cmd + def git_cmd(self) -> str: + """ + This generates the git command. This is when we need to pull down extra + files or do work from a git repository. + """ + cmd = None + if os.path.exists("/usr/bin/git"): + cmd = "/usr/bin/git" + else: + self.log.error('/usr/bin/git was not found. Good bye.') + raise SystemExit("\n\n/usr/bin/git was not found.\n\nPlease " + " ensure that you have installed the necessary packages on " + " this system. " + ) + return cmd + def repoclosure_work(self, sync_root, work_root, log_root): """ This is where we run repoclosures, based on the configuration of each @@ -662,7 +766,7 @@ class RepoSync: ) repoclosure_cmd = ('/usr/bin/dnf repoclosure {} ' '--repofrompath={},file://{}/{}/{}/os --repo={} --check={} {} ' - '| tee -a {}/{}-repoclosure-{}-{}.log').format( + '| tee -a {}/{}-repoclosure-{}.log').format( repoclosure_arch_list, repo, sync_root, @@ -673,8 +777,7 @@ class RepoSync: join_repo_comb, log_root, repo, - arch, - self.date_stamp + arch ) repoclosure_entry_point_open = open(repoclosure_entry_point_sh, "w+") repoclosure_entry_point_open.write('#!/bin/bash\n') @@ -735,10 +838,10 @@ class RepoSync: ) output, errors = podcheck.communicate() - if 'Exited (0)' in output.decode(): - self.log.info('%s seems ok' % pod) - else: - self.log.error('%s had issues closing' % pod) + if 'Exited (0)' not in output.decode(): + self.log.error( + '[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + pod + ) bad_exit_list.append(pod) rmcmd = '{} rm {}'.format( @@ -764,11 +867,84 @@ class RepoSync: for issue in bad_exit_list: self.log.error(issue) - def deploy_extra_files(self): + def deploy_extra_files(self, global_work_root): """ - deploys extra files based on info of rlvars + deploys extra files based on info of rlvars including a + extra_files.json + + might also deploy COMPOSE_ID and maybe in the future a metadata dir with + a bunch of compose-esque stuff. """ - pass + cmd = self.git_cmd() + tmpclone = '/tmp/clone' + extra_files_dir = os.path.join( + global_work_root, + 'extra-files' + ) + self.log.info( + '[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' + + 'Deploying extra files to work directory ...' + ) + + if not os.path.exists(extra_files_dir): + os.makedirs(extra_files_dir, exist_ok=True) + + clonecmd = '{} clone {} -b {} -q {}'.format( + cmd, + self.extra_files['git_repo'], + self.extra_files['branch'], + tmpclone + ) + + git_clone = subprocess.call( + shlex.split(clonecmd), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + # Copy files to work root + for extra in self.extra_files['list']: + src = '/tmp/clone/' + extra + # Copy extra files to root of compose here also - The extra files + # are meant to be picked up by our ISO creation process and also + # exist on our mirrors. + try: + shutil.copy2(src, extra_files_dir) + except: + self.log.warn( + '[' + Color.BOLD + Color.YELLOW + 'WARN' + Color.END + '] ' + + 'Extra file not copied: ' + src + ) + + try: + shutil.rmtree(tmpclone) + except OSError as e: + self.log.error( + '[' + Color.BOLD + Color.RED + 'FAIL' + Color.END + '] ' + + 'Directory ' + tmpclone + ' could not be removed: ' + + e.strerror + ) + + # Create metadata here? + + self.log.info( + '[' + Color.BOLD + Color.GREEN + 'INFO' + Color.END + '] ' + + 'Extra files phase completed.' + ) + + def deploy_treeinfo(self, repo, sync_root, arch): + """ + Deploys initial treeinfo files. These have the potential of being + overwritten by our ISO process, which is fine. + """ + arches_to_tree = self.arches + if arch: + arches_to_tree = [arch] + + repos_to_tree = self.repos + if repo and not self.fullrun: + repos_to_tree = [repo] + class SigRepoSync: """ @@ -785,6 +961,7 @@ class SigRepoSync: arch=None, ignore_source: bool = False, repoclosure: bool = False, + refresh_extra_files: bool = False, skip_all: bool = False, hashed: bool = False, parallel: bool = False, @@ -801,6 +978,7 @@ class SigRepoSync: self.skip_all = skip_all self.hashed = hashed self.repoclosure = repoclosure + self.refresh_extra_files = refresh_extra_files # Enables podman syncing, which should effectively speed up operations self.parallel = parallel # Relevant config items @@ -848,6 +1026,11 @@ class SigRepoSync: "work/logs" ) + self.compose_global_work_root = os.path.join( + self.compose_latest_dir, + "work/global" + ) + # This is temporary for now. if logger is None: self.log = logging.getLogger("sigreposync") diff --git a/iso/empanadas/empanadas/util/iso_utils.py b/iso/empanadas/empanadas/util/iso_utils.py new file mode 100644 index 0000000..5921df9 --- /dev/null +++ b/iso/empanadas/empanadas/util/iso_utils.py @@ -0,0 +1,1731 @@ +""" +Builds ISO's for Rocky Linux. + +Louis Abel