From ee46e2f9b76233b6eddf9e3a73e245e799bc4b99 Mon Sep 17 00:00:00 2001 From: Nir Magnezi Date: Sun, 17 Mar 2019 11:54:38 +0200 Subject: [PATCH] Add version-less RHEL element for RHEL7 and RHEL8 Make a version-less RHEL element to handle both '7' and '8' DIB_RELEASE. The element usage should align with other elements which operate in the same way such as the Fedora element. Additionally, this patch adds support for RHEL8 that operates with Python 3. As of now, users of diskimage-builder will still be able to use the 'rhel7' element, or migrate to 'rhel' and specify their respective DIB_RELEASE value. * mount the xfs file-system for extraction as read-only. vaguely based on explaination in [1] and the fact we only read the image data into a tar, so can ignore this. XFS (dm-1): Superblock has unknown read-only compatible features (0x4) enabled. * Use the redhat system python as the dib-python version. dib was ahead of it's time making an abstracted python interpreter for system work ;) the system python should work for running the various dib element scripts. [1] https://unix.stackexchange.com/questions/247550/unmountable-xfs-filesystem Redhat-Bug: https://bugzilla.redhat.com/show_bug.cgi?id=1700253 Co-Authored-By: Ian Wienand Change-Id: I90540675c70bb475d9db2ae24f81c648a31f3f95 --- .../environment.d/50-dib-python-version | 9 +++- .../dib-python/pre-install.d/01-dib-python | 10 ++++- .../04-install-pip | 22 +++++++++- .../elements/pip-and-virtualenv/pkg-map | 7 +++ .../elements/redhat-common/bin/extract-image | 7 ++- .../elements/redhat-common/pkg-map | 11 +++++ .../pre-install.d/00-rhel-registration | 11 ++++- diskimage_builder/elements/rhel/README.rst | 40 +++++++++++++++++ diskimage_builder/elements/rhel/element-deps | 6 +++ .../elements/rhel/element-provides | 1 + .../environment.d/10-rhel-distro-name.bash | 7 +++ .../rhel/environment.d/11-yum-dnf.bash | 7 +++ .../elements/rhel/root.d/10-rhel-cloud-image | 44 +++++++++++++++++++ .../elements/svc-map/bin/svc-map | 2 +- .../elements/svc-map/package-installs.yaml | 4 ++ diskimage_builder/elements/svc-map/pkg-map | 9 ++-- diskimage_builder/lib/img-defaults | 9 +++- ...on-less-rhel-element-82fac7f2609e16d3.yaml | 6 +++ 18 files changed, 201 insertions(+), 11 deletions(-) create mode 100644 diskimage_builder/elements/redhat-common/pkg-map create mode 100644 diskimage_builder/elements/rhel/README.rst create mode 100644 diskimage_builder/elements/rhel/element-deps create mode 100644 diskimage_builder/elements/rhel/element-provides create mode 100644 diskimage_builder/elements/rhel/environment.d/10-rhel-distro-name.bash create mode 100644 diskimage_builder/elements/rhel/environment.d/11-yum-dnf.bash create mode 100755 diskimage_builder/elements/rhel/root.d/10-rhel-cloud-image create mode 100644 releasenotes/notes/add-version-less-rhel-element-82fac7f2609e16d3.yaml diff --git a/diskimage_builder/elements/dib-python/environment.d/50-dib-python-version b/diskimage_builder/elements/dib-python/environment.d/50-dib-python-version index b75c9691..9e38f525 100644 --- a/diskimage_builder/elements/dib-python/environment.d/50-dib-python-version +++ b/diskimage_builder/elements/dib-python/environment.d/50-dib-python-version @@ -1,4 +1,4 @@ -# Pick which distros we need to force python2 +# Pick which distros we need to force python2 and which to python3 if [ -z "${DIB_PYTHON_VERSION:-}" ]; then if [ "$DISTRO_NAME" == "ubuntu" ]; then if [ "$DIB_RELEASE" == "trusty" ]; then @@ -15,7 +15,14 @@ if [ -z "${DIB_PYTHON_VERSION:-}" ]; then elif [ "$DISTRO_NAME" == "centos7" ]; then DIB_PYTHON_VERSION=2 elif [ "$DISTRO_NAME" == "rhel7" ]; then + # TODO(nmagnezi): Remove this when the 'rhel7' element gets replaced by 'rhel' DIB_PYTHON_VERSION=2 + elif [ "$DISTRO_NAME" == "rhel" ]; then + if [ "$DIB_RELEASE" -le 7 ]; then + DIB_PYTHON_VERSION=2 + elif [ "$DIB_RELEASE" -ge 8 ]; then + DIB_PYTHON_VERSION=3 + fi elif [ "$DISTRO_NAME" == "opensuse" ]; then DIB_PYTHON_VERSION=2 fi diff --git a/diskimage_builder/elements/dib-python/pre-install.d/01-dib-python b/diskimage_builder/elements/dib-python/pre-install.d/01-dib-python index bf4d1c81..3120fcd8 100755 --- a/diskimage_builder/elements/dib-python/pre-install.d/01-dib-python +++ b/diskimage_builder/elements/dib-python/pre-install.d/01-dib-python @@ -6,7 +6,15 @@ fi set -eu set -o pipefail -python_path=$(command -v python${DIB_PYTHON_VERSION}) +if [[ ${DISTRO_NAME} == "rhel" && ${DIB_RELEASE} == 8 ]]; then + # RHEL8 has a system python, separate from the user python. What + # a good idea, abstracting the python binary for system scripts! + # :) Use it for dib-python. + python_path=/usr/libexec/platform-python +else + python_path=$(command -v python${DIB_PYTHON_VERSION}) +fi + if [ -z "$python_path" ]; then echo "Could not find python${DIB_PYTHON_VERSION} executable." exit 1 diff --git a/diskimage_builder/elements/pip-and-virtualenv/install.d/pip-and-virtualenv-source-install/04-install-pip b/diskimage_builder/elements/pip-and-virtualenv/install.d/pip-and-virtualenv-source-install/04-install-pip index 19c7ea41..fa38f8f3 100755 --- a/diskimage_builder/elements/pip-and-virtualenv/install.d/pip-and-virtualenv-source-install/04-install-pip +++ b/diskimage_builder/elements/pip-and-virtualenv/install.d/pip-and-virtualenv-source-install/04-install-pip @@ -6,7 +6,7 @@ fi set -eu set -o pipefail -if [[ $DISTRO_NAME =~ (opensuse|fedora|centos|centos7|rhel7) ]]; then +if [[ $DISTRO_NAME =~ (opensuse|fedora|centos|centos7|rhel|rhel7) ]]; then # Default packages _do_py3=0 @@ -33,6 +33,25 @@ if [[ $DISTRO_NAME =~ (opensuse|fedora|centos|centos7|rhel7) ]]; then packages="python2-virtualenv python2-pip python2-setuptools" packages+=" python3-virtualenv python3-pip python3-setuptools" ;; + rhel) + _clear_old_files=1 + case "$DIB_RELEASE" in + 8) + _do_py3=1 + packages=" python3-virtualenv python3-pip python3-setuptools" + ;; + 7) + # note python2-pip in epel + _extra_repo="--enablerepo=epel" + packages="python-virtualenv python2-pip" + if [[ "$(rpm -q --qf '[%{obsoletes}\n]' python2-setuptools)" == "python-setuptools" ]]; then + packages+=" python2-setuptools" + else + packages+=" python-setuptools" + fi + ;; + esac + ;; opensuse) case "$DIB_RELEASE" in 42*) @@ -44,6 +63,7 @@ if [[ $DISTRO_NAME =~ (opensuse|fedora|centos|centos7|rhel7) ]]; then packages="python2-virtualenv python2-pip python2-setuptools" ;; esac + ;; esac # force things to happen so our assumptions hold diff --git a/diskimage_builder/elements/pip-and-virtualenv/pkg-map b/diskimage_builder/elements/pip-and-virtualenv/pkg-map index 0cfe5462..c0f4b4cf 100644 --- a/diskimage_builder/elements/pip-and-virtualenv/pkg-map +++ b/diskimage_builder/elements/pip-and-virtualenv/pkg-map @@ -1,4 +1,11 @@ { + "release": { + "rhel": { + "8": { + "python3-dev": "platform-python-devel" + } + } + }, "family": { "gentoo": { "python-pip": "dev-python/pip", diff --git a/diskimage_builder/elements/redhat-common/bin/extract-image b/diskimage_builder/elements/redhat-common/bin/extract-image index 9360fa1e..ef26f992 100755 --- a/diskimage_builder/elements/redhat-common/bin/extract-image +++ b/diskimage_builder/elements/redhat-common/bin/extract-image @@ -83,7 +83,12 @@ function extract_image() { mkdir $WORKING/mnt if [ "xfs" = "$(sudo blkid -o value -s TYPE /dev/mapper/$ROOT_LOOPDEV)" ]; then # mount xfs with nouuid, just in case that uuid is already mounted - MOUNTOPTS="-o nouuid" + # use ro to avoid/workaround xfs uuid issues on older + # kernels with newer rhel images which seem to set + # flags to generate unique uuid's: + # xfs superblock has incompatible features (0x4) + # we don't need to worry about this, we just want the data + MOUNTOPTS="-o nouuid,ro" else MOUNTOPTS="" fi diff --git a/diskimage_builder/elements/redhat-common/pkg-map b/diskimage_builder/elements/redhat-common/pkg-map new file mode 100644 index 00000000..84cb5786 --- /dev/null +++ b/diskimage_builder/elements/redhat-common/pkg-map @@ -0,0 +1,11 @@ +{ + "release": { + "rhel": { + "8": { + "libselinux-python": "python3-libselinux", + "policycoreutils": "python3-policycoreutils", + "policycoreutils-python": "policycoreutils-python-utils" + } + } + } +} diff --git a/diskimage_builder/elements/rhel-common/pre-install.d/00-rhel-registration b/diskimage_builder/elements/rhel-common/pre-install.d/00-rhel-registration index 5f402a38..ff98c610 100755 --- a/diskimage_builder/elements/rhel-common/pre-install.d/00-rhel-registration +++ b/diskimage_builder/elements/rhel-common/pre-install.d/00-rhel-registration @@ -9,8 +9,15 @@ set -o pipefail opts= attach_opts= -repos="repos --enable rhel-7-server-rpms" -satellite_repo="rhel-7-server-rh-common-rpms" + +if [ "${DIB_RELEASE:-7}" == "7" ]; then + repos="repos --enable rhel-7-server-rpms" + satellite_repo="rhel-7-server-rh-common-rpms" +elif [ "${DIB_RELEASE}" == "8" ]; then + repos="repos --enable rhel-8-for-x86_64-appstream-rpms --enable rhel-8-for-x86_64-baseos-rpms" + satellite_repo="satellite-tools-6.5-for-rhel-8-x86_64-rpms" +fi + REG_SAT_CERT=${REG_SAT_CERT:-"katello-ca-consumer-latest.noarch.rpm"} if [ -n "${REG_AUTO_ATTACH:-}" ]; then diff --git a/diskimage_builder/elements/rhel/README.rst b/diskimage_builder/elements/rhel/README.rst new file mode 100644 index 00000000..fb550e41 --- /dev/null +++ b/diskimage_builder/elements/rhel/README.rst @@ -0,0 +1,40 @@ +==== +rhel +==== + +Use RHEL cloud images as the baseline for built disk images. + +Because RHEL base images are not publicly available, it is necessary to first +download the RHEL cloud image from the Red Hat Customer Portal and pass the +path to the resulting file to disk-image-create as the ``DIB_LOCAL_IMAGE`` +environment variable. + +The cloud image can be found at (login required): +RHEL8: https://access.redhat.com/downloads/content/479/ver=/rhel---8/8.0/x86_64/product-software +RHEL7: https://access.redhat.com/downloads/content/69/ver=/rhel---7/7.1/x86_64/product-downloads + + +Then before running the image build, define DIB_LOCAL_IMAGE (replace the file +name with the one downloaded, if it differs from the example): + +.. code-block:: bash + + export DIB_LOCAL_IMAGE=rhel-8.0-x86_64-kvm.qcow2 + +The downloaded file will then be used as the basis for any subsequent image +builds. + +For further details about building RHEL images, see the rhel-common and +redhat-common element README files. + +Environment Variables +--------------------- + +DIB_LOCAL_IMAGE + :Required: Yes + :Default: None + :Description: The RHEL 8 base image you have downloaded. See the element + description above for more details. + :Example: ``DIB_LOCAL_IMAGE=/tmp/rhel8-cloud.qcow2`` + + diff --git a/diskimage_builder/elements/rhel/element-deps b/diskimage_builder/elements/rhel/element-deps new file mode 100644 index 00000000..81585c43 --- /dev/null +++ b/diskimage_builder/elements/rhel/element-deps @@ -0,0 +1,6 @@ +cache-url +redhat-common +rhel-common +rpm-distro +source-repositories +yum diff --git a/diskimage_builder/elements/rhel/element-provides b/diskimage_builder/elements/rhel/element-provides new file mode 100644 index 00000000..a72e0496 --- /dev/null +++ b/diskimage_builder/elements/rhel/element-provides @@ -0,0 +1 @@ +operating-system diff --git a/diskimage_builder/elements/rhel/environment.d/10-rhel-distro-name.bash b/diskimage_builder/elements/rhel/environment.d/10-rhel-distro-name.bash new file mode 100644 index 00000000..54c2a3de --- /dev/null +++ b/diskimage_builder/elements/rhel/environment.d/10-rhel-distro-name.bash @@ -0,0 +1,7 @@ +export DISTRO_NAME=rhel +export DIB_RELEASE=${DIB_RELEASE:-8} + +if [ "${DISTRO_NAME}" = "rhel" ] && [ "${DIB_RELEASE}" = "8" ] && [ "${FS_TYPE}" != "xfs" ]; then + echo "ERROR: RHEL8 images file-system type must be set to xfs, FS_TYPE is currently set to" $FS_TYPE + exit 1 +fi diff --git a/diskimage_builder/elements/rhel/environment.d/11-yum-dnf.bash b/diskimage_builder/elements/rhel/environment.d/11-yum-dnf.bash new file mode 100644 index 00000000..e3e648dc --- /dev/null +++ b/diskimage_builder/elements/rhel/environment.d/11-yum-dnf.bash @@ -0,0 +1,7 @@ +# since RHEL8, dnf is the yum replacement. + +if [[ ${DIB_RELEASE} == '8' ]]; then + export YUM=dnf +elif [[ ${DIB_RELEASE} == '7' ]]; then + export YUM=yum +fi diff --git a/diskimage_builder/elements/rhel/root.d/10-rhel-cloud-image b/diskimage_builder/elements/rhel/root.d/10-rhel-cloud-image new file mode 100755 index 00000000..b1043d89 --- /dev/null +++ b/diskimage_builder/elements/rhel/root.d/10-rhel-cloud-image @@ -0,0 +1,44 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +[ -n "$ARCH" ] +[ -n "$TARGET_ROOT" ] + +if [[ "amd64 x86_64" =~ "$ARCH" ]]; then + ARCH="x86_64" +elif [[ "ppc64le" =~ "$ARCH" ]]; then + # We don't need to do anything here other than avoid the else clause + : +else + echo 'rhel root element only supports x86_64 and ppc64le values for $ARCH' + exit 1 +fi + +DIB_LOCAL_IMAGE=${DIB_LOCAL_IMAGE:-""} + +if [ -n "$DIB_LOCAL_IMAGE" ]; then + IMAGE_LOCATION=$DIB_LOCAL_IMAGE + # No need to copy a local image into the cache directory, so just specify + # the cached path as the original path. + CACHED_IMAGE=$IMAGE_LOCATION + BASE_IMAGE_FILE=`basename $DIB_LOCAL_IMAGE` + BASE_IMAGE_TAR=$BASE_IMAGE_FILE.tgz +else + if [ -z "${BASE_IMAGE_FILE:-}" -o -z "${DIB_CLOUD_IMAGES:-}" ]; then + echo "No source for a base image file configured." + echo "See rhel element readme for details on how to obtain and use a base image." + exit 1 + fi + DIB_RELEASE=${DIB_RELEASE:-latest} + BASE_IMAGE_TAR=$DIB_RELEASE-rhel-server-$ARCH-latest.tgz + IMAGE_LOCATION=$DIB_CLOUD_IMAGES/$BASE_IMAGE_FILE + CACHED_IMAGE=$DIB_IMAGE_CACHE/$BASE_IMAGE_FILE + +fi + +$TMP_HOOKS_PATH/bin/extract-image $BASE_IMAGE_FILE $BASE_IMAGE_TAR $IMAGE_LOCATION $CACHED_IMAGE diff --git a/diskimage_builder/elements/svc-map/bin/svc-map b/diskimage_builder/elements/svc-map/bin/svc-map index 2df6ba91..ce04ee05 100755 --- a/diskimage_builder/elements/svc-map/bin/svc-map +++ b/diskimage_builder/elements/svc-map/bin/svc-map @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/local/bin/dib-python # Copyright 2012 Hewlett-Packard Development Company, L.P. # Copyright 2014 Red Hat, Inc. diff --git a/diskimage_builder/elements/svc-map/package-installs.yaml b/diskimage_builder/elements/svc-map/package-installs.yaml index b7303da6..6f4d9ec5 100644 --- a/diskimage_builder/elements/svc-map/package-installs.yaml +++ b/diskimage_builder/elements/svc-map/package-installs.yaml @@ -1,2 +1,6 @@ PyYAML: phase: pre-install.d + dib_python_version: 2 +python3-PyYAML: + phase: pre-install.d + dib_python_version: 3 diff --git a/diskimage_builder/elements/svc-map/pkg-map b/diskimage_builder/elements/svc-map/pkg-map index fdda5ae5..e5fb5bd1 100644 --- a/diskimage_builder/elements/svc-map/pkg-map +++ b/diskimage_builder/elements/svc-map/pkg-map @@ -1,16 +1,19 @@ { "family": { "redhat": { - "PyYAML": "PyYAML" + "PyYAML": "PyYAML", + "python3-PyYAML": "python3-pyyaml" }, "debian": { - "PyYAML": "python-yaml" + "PyYAML": "python-yaml", + "python3-PyYAML": "python3-yaml" }, "suse": { "PyYAML": "python-PyYAML" } }, "default": { - "PyYAML": "PyYAML" + "PyYAML": "PyYAML", + "python3-PyYAML": "python3-PyYAML" } } diff --git a/diskimage_builder/lib/img-defaults b/diskimage_builder/lib/img-defaults index 1bdf6431..a36698e6 100644 --- a/diskimage_builder/lib/img-defaults +++ b/diskimage_builder/lib/img-defaults @@ -17,7 +17,14 @@ source $_LIB/common-defaults # options for create-baremetal-image.sh -export FS_TYPE=${FS_TYPE:-ext4} +if [ "${DISTRO_NAME}" = "rhel" ] && [ "${DIB_RELEASE}" = "8" ]; then + # xfs is the default file-system for RHEL8 + export DEFAULT_FS_TYPE=xfs +else + export DEFAULT_FS_TYPE=ext4 +fi + +export FS_TYPE=${FS_TYPE:-$DEFAULT_FS_TYPE} # Used to set the file extension only at this stage. export IMAGE_TYPE=${IMAGE_TYPE:-qcow2} export IMAGE_NAME=${IMAGE_NAME:-image} diff --git a/releasenotes/notes/add-version-less-rhel-element-82fac7f2609e16d3.yaml b/releasenotes/notes/add-version-less-rhel-element-82fac7f2609e16d3.yaml new file mode 100644 index 00000000..0b1329a1 --- /dev/null +++ b/releasenotes/notes/add-version-less-rhel-element-82fac7f2609e16d3.yaml @@ -0,0 +1,6 @@ +--- +features: + - Adds a new element ``rhel`` to handle all supported RHEL releases, which + are currently '7' and '8'. As of now, users of diskimage-builder will + still be able to use the 'rhel7' element, or migrate to 'rhel' and specify + their respective DIB_RELEASE value.