Merge "Add version-less RHEL element for RHEL7 and RHEL8"

diskimage_builder/elements/dib-python/environment.d/50-dib-python-version

@@ -1,4 +1,4 @@
-# Pick which distros we need to force python2
+# Pick which distros we need to force python2 and which to python3
 if [ -z "${DIB_PYTHON_VERSION:-}" ]; then
     if [ "$DISTRO_NAME" == "ubuntu" ]; then
         if [ "$DIB_RELEASE" == "trusty" ]; then
@@ -15,7 +15,14 @@ if [ -z "${DIB_PYTHON_VERSION:-}" ]; then
     elif [ "$DISTRO_NAME" == "centos7" ]; then
         DIB_PYTHON_VERSION=2
     elif [ "$DISTRO_NAME" == "rhel7" ]; then
+        # TODO(nmagnezi): Remove this when the 'rhel7' element gets replaced by 'rhel'
         DIB_PYTHON_VERSION=2
+    elif [ "$DISTRO_NAME" == "rhel" ]; then
+        if [ "$DIB_RELEASE" -le 7 ]; then
+            DIB_PYTHON_VERSION=2
+        elif [ "$DIB_RELEASE" -ge 8 ]; then
+            DIB_PYTHON_VERSION=3
+        fi
     elif [ "$DISTRO_NAME" == "opensuse" ]; then
         DIB_PYTHON_VERSION=2
     fi

diskimage_builder/elements/dib-python/pre-install.d/01-dib-python

@@ -6,7 +6,15 @@ fi
 set -eu
 set -o pipefail
 
-python_path=$(command -v python${DIB_PYTHON_VERSION})
+if [[ ${DISTRO_NAME} == "rhel" && ${DIB_RELEASE} == 8 ]]; then
+    # RHEL8 has a system python, separate from the user python.  What
+    # a good idea, abstracting the python binary for system scripts!
+    # :)  Use it for dib-python.
+    python_path=/usr/libexec/platform-python
+else
+    python_path=$(command -v python${DIB_PYTHON_VERSION})
+fi
+
 if [ -z "$python_path" ]; then
     echo "Could not find python${DIB_PYTHON_VERSION} executable."
     exit 1

diskimage_builder/elements/pip-and-virtualenv/install.d/pip-and-virtualenv-source-install/04-install-pip

@@ -6,7 +6,7 @@ fi
 set -eu
 set -o pipefail
 
-if [[ $DISTRO_NAME =~ (opensuse|fedora|centos|centos7|rhel7) ]]; then
+if [[ $DISTRO_NAME =~ (opensuse|fedora|centos|centos7|rhel|rhel7) ]]; then
 
     # Default packages
     _do_py3=0
@@ -33,6 +33,25 @@ if [[ $DISTRO_NAME =~ (opensuse|fedora|centos|centos7|rhel7) ]]; then
             packages="python2-virtualenv python2-pip python2-setuptools"
             packages+=" python3-virtualenv python3-pip python3-setuptools"
             ;;
+        rhel)
+            _clear_old_files=1
+            case "$DIB_RELEASE" in
+                8)
+                    _do_py3=1
+                    packages=" python3-virtualenv python3-pip python3-setuptools"
+                    ;;
+                7)
+                    # note python2-pip in epel
+                    _extra_repo="--enablerepo=epel"
+                    packages="python-virtualenv python2-pip"
+                    if [[ "$(rpm -q --qf '[%{obsoletes}\n]' python2-setuptools)" == "python-setuptools" ]]; then
+                        packages+=" python2-setuptools"
+                    else
+                        packages+=" python-setuptools"
+                    fi
+                    ;;
+            esac
+            ;;
         opensuse)
             case "$DIB_RELEASE" in
                 42*)
@@ -44,6 +63,7 @@ if [[ $DISTRO_NAME =~ (opensuse|fedora|centos|centos7|rhel7) ]]; then
                     packages="python2-virtualenv python2-pip python2-setuptools"
                     ;;
             esac
+            ;;
     esac
 
     # force things to happen so our assumptions hold

diskimage_builder/elements/pip-and-virtualenv/pkg-map

@@ -1,4 +1,11 @@
 {
+  "release": {
+    "rhel": {
+      "8": {
+        "python3-dev": "platform-python-devel"
+      }
+    }
+  },
   "family": {
     "gentoo": {
       "python-pip": "dev-python/pip",

diskimage_builder/elements/redhat-common/bin/extract-image

@@ -83,7 +83,12 @@ function extract_image() {
             mkdir $WORKING/mnt
             if [ "xfs" = "$(sudo blkid -o value -s TYPE /dev/mapper/$ROOT_LOOPDEV)" ]; then
                 # mount xfs with nouuid, just in case that uuid is already mounted
-                MOUNTOPTS="-o nouuid"
+                # use ro to avoid/workaround xfs uuid issues on older
+                # kernels with newer rhel images which seem to set
+                # flags to generate unique uuid's:
+                #  xfs superblock has incompatible features (0x4)
+                # we don't need to worry about this, we just want the data
+                MOUNTOPTS="-o nouuid,ro"
             else
                 MOUNTOPTS=""
             fi

diskimage_builder/elements/redhat-common/pkg-map

@@ -0,0 +1,11 @@
+{
+  "release": {
+    "rhel": {
+      "8": {
+        "libselinux-python": "python3-libselinux",
+        "policycoreutils": "python3-policycoreutils",
+        "policycoreutils-python": "policycoreutils-python-utils"
+      }
+    }
+  }
+}

diskimage_builder/elements/rhel-common/pre-install.d/00-rhel-registration

@@ -9,8 +9,15 @@ set -o pipefail
 
 opts=
 attach_opts=
-repos="repos --enable rhel-7-server-rpms"
+
-satellite_repo="rhel-7-server-rh-common-rpms"
+if [ "${DIB_RELEASE:-7}" == "7" ]; then
+    repos="repos --enable rhel-7-server-rpms"
+    satellite_repo="rhel-7-server-rh-common-rpms"
+elif [ "${DIB_RELEASE}" == "8" ]; then
+    repos="repos --enable rhel-8-for-x86_64-appstream-rpms --enable rhel-8-for-x86_64-baseos-rpms"
+    satellite_repo="satellite-tools-6.5-for-rhel-8-x86_64-rpms"
+fi
+
 REG_SAT_CERT=${REG_SAT_CERT:-"katello-ca-consumer-latest.noarch.rpm"}
 
 if [ -n "${REG_AUTO_ATTACH:-}" ]; then

diskimage_builder/elements/rhel/README.rst

@@ -0,0 +1,40 @@
+====
+rhel
+====
+
+Use RHEL cloud images as the baseline for built disk images.
+
+Because RHEL base images are not publicly available, it is necessary to first
+download the RHEL cloud image from the Red Hat Customer Portal and pass the
+path to the resulting file to disk-image-create as the ``DIB_LOCAL_IMAGE``
+environment variable.
+
+The cloud image can be found at (login required):
+RHEL8: https://access.redhat.com/downloads/content/479/ver=/rhel---8/8.0/x86_64/product-software
+RHEL7: https://access.redhat.com/downloads/content/69/ver=/rhel---7/7.1/x86_64/product-downloads
+
+
+Then before running the image build, define DIB_LOCAL_IMAGE (replace the file
+name with the one downloaded, if it differs from the example):
+
+.. code-block:: bash
+
+   export DIB_LOCAL_IMAGE=rhel-8.0-x86_64-kvm.qcow2
+
+The downloaded file will then be used as the basis for any subsequent image
+builds.
+
+For further details about building RHEL images, see the rhel-common and
+redhat-common element README files.
+
+Environment Variables
+---------------------
+
+DIB_LOCAL_IMAGE
+  :Required: Yes
+  :Default: None
+  :Description: The RHEL 8 base image you have downloaded. See the element
+                description above for more details.
+  :Example: ``DIB_LOCAL_IMAGE=/tmp/rhel8-cloud.qcow2``
+
+

diskimage_builder/elements/rhel/element-deps

@@ -0,0 +1,6 @@
+cache-url
+redhat-common
+rhel-common
+rpm-distro
+source-repositories
+yum

diskimage_builder/elements/rhel/element-provides

@@ -0,0 +1 @@
+operating-system

diskimage_builder/elements/rhel/environment.d/10-rhel-distro-name.bash

@@ -0,0 +1,7 @@
+export DISTRO_NAME=rhel
+export DIB_RELEASE=${DIB_RELEASE:-8}
+
+if [ "${DISTRO_NAME}" = "rhel" ] && [ "${DIB_RELEASE}" = "8" ] && [ "${FS_TYPE}" != "xfs" ]; then
+    echo "ERROR: RHEL8 images file-system type must be set to xfs, FS_TYPE is currently set to" $FS_TYPE
+    exit 1
+fi

diskimage_builder/elements/rhel/environment.d/11-yum-dnf.bash

@@ -0,0 +1,7 @@
+# since RHEL8, dnf is the yum replacement.
+
+if [[ ${DIB_RELEASE} == '8' ]]; then
+    export YUM=dnf
+elif [[ ${DIB_RELEASE} == '7' ]]; then
+    export YUM=yum
+fi

diskimage_builder/elements/rhel/root.d/10-rhel-cloud-image

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+[ -n "$ARCH" ]
+[ -n "$TARGET_ROOT" ]
+
+if [[ "amd64 x86_64" =~ "$ARCH" ]]; then
+    ARCH="x86_64"
+elif [[ "ppc64le" =~ "$ARCH" ]]; then
+    # We don't need to do anything here other than avoid the else clause
+    :
+else
+    echo 'rhel root element only supports x86_64 and ppc64le values for $ARCH'
+    exit 1
+fi
+
+DIB_LOCAL_IMAGE=${DIB_LOCAL_IMAGE:-""}
+
+if [ -n "$DIB_LOCAL_IMAGE" ]; then
+    IMAGE_LOCATION=$DIB_LOCAL_IMAGE
+    # No need to copy a local image into the cache directory, so just specify
+    # the cached path as the original path.
+    CACHED_IMAGE=$IMAGE_LOCATION
+    BASE_IMAGE_FILE=`basename $DIB_LOCAL_IMAGE`
+    BASE_IMAGE_TAR=$BASE_IMAGE_FILE.tgz
+else
+    if [ -z "${BASE_IMAGE_FILE:-}" -o -z "${DIB_CLOUD_IMAGES:-}" ]; then
+        echo "No source for a base image file configured."
+        echo "See rhel element readme for details on how to obtain and use a base image."
+        exit 1
+    fi
+    DIB_RELEASE=${DIB_RELEASE:-latest}
+    BASE_IMAGE_TAR=$DIB_RELEASE-rhel-server-$ARCH-latest.tgz
+    IMAGE_LOCATION=$DIB_CLOUD_IMAGES/$BASE_IMAGE_FILE
+    CACHED_IMAGE=$DIB_IMAGE_CACHE/$BASE_IMAGE_FILE
+
+fi
+
+$TMP_HOOKS_PATH/bin/extract-image $BASE_IMAGE_FILE $BASE_IMAGE_TAR $IMAGE_LOCATION $CACHED_IMAGE

diskimage_builder/elements/svc-map/bin/svc-map

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/local/bin/dib-python
 
 # Copyright 2012 Hewlett-Packard Development Company, L.P.
 # Copyright 2014 Red Hat, Inc.

diskimage_builder/elements/svc-map/package-installs.yaml

@@ -1,2 +1,6 @@
 PyYAML:
     phase: pre-install.d
+    dib_python_version: 2
+python3-PyYAML:
+    phase: pre-install.d
+    dib_python_version: 3

diskimage_builder/elements/svc-map/pkg-map

@@ -1,16 +1,19 @@
 {
   "family": {
     "redhat": {
-      "PyYAML": "PyYAML"
+      "PyYAML": "PyYAML",
+      "python3-PyYAML": "python3-pyyaml"
     },
     "debian": {
-      "PyYAML": "python-yaml"
+      "PyYAML": "python-yaml",
+      "python3-PyYAML": "python3-yaml"
     },
     "suse": {
       "PyYAML": "python-PyYAML"
     }
   },
   "default": {
-    "PyYAML": "PyYAML"
+    "PyYAML": "PyYAML",
+    "python3-PyYAML": "python3-PyYAML"
   }
 }

diskimage_builder/lib/img-defaults

@@ -17,7 +17,14 @@
 source $_LIB/common-defaults
 # options for create-baremetal-image.sh
 
-export FS_TYPE=${FS_TYPE:-ext4}
+if [ "${DISTRO_NAME}" = "rhel" ] && [ "${DIB_RELEASE}" = "8" ]; then
+    # xfs is the default file-system for RHEL8
+    export DEFAULT_FS_TYPE=xfs
+else
+    export DEFAULT_FS_TYPE=ext4
+fi
+
+export FS_TYPE=${FS_TYPE:-$DEFAULT_FS_TYPE}
 # Used to set the file extension only at this stage.
 export IMAGE_TYPE=${IMAGE_TYPE:-qcow2}
 export IMAGE_NAME=${IMAGE_NAME:-image}

releasenotes/notes/add-version-less-rhel-element-82fac7f2609e16d3.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - Adds a new element ``rhel`` to handle all supported RHEL releases, which
+    are currently '7' and '8'. As of now, users of diskimage-builder will
+    still be able to use the 'rhel7' element, or migrate to 'rhel' and specify
+    their respective DIB_RELEASE value.