Merge remote-tracking branch 'origin/master' into merge-branch

Change-Id: Ibab1bb95521292ae818bd91f7073c3749a2cc0cb

MANIFEST.in

@@ -1 +0,0 @@
-include README.rst

bin/dib-lint

@@ -153,6 +153,12 @@ for i in $(find $ELEMENTS_DIR -type f   \
         fi
     fi
 
+    # check that environment files don't "set -x"
+    if [[ "$i" =~ (environment.d) ]]; then
+        if grep -q "set -x" $i; then
+            error "Environment file $i should not set tracing"
+        fi
+    fi
 
     # check that sudo calls in phases run outside the chroot look
     # "safe"; meaning that they seem to operate within the chroot

diskimage_builder/elements/centos/environment.d/00-bootloader.bash

@@ -1,9 +1 @@
-#!/bin/bash
-
-if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
-    set -x
-fi
-set -eu
-set -o pipefail
-
 export DIB_EXTLINUX=1

diskimage_builder/elements/centos7/root.d/10-centos7-cloud-image

@@ -9,10 +9,10 @@ set -o pipefail
 [ -n "$ARCH" ]
 [ -n "$TARGET_ROOT" ]
 
-if [ 'amd64' = "$ARCH" ] ; then
+if [[ "amd64 x86_64" =~ "$ARCH" ]]; then
     ARCH="x86_64"
 else
-    echo 'centos7 root element only support the amd64 $ARCH value.'
+    echo 'centos7 root element only support the x86_64 $ARCH value.'
     exit 1
 fi
 

diskimage_builder/elements/debian/package-installs.yaml

@@ -10,3 +10,4 @@ net-tools:
 cloud-init:
 cloud-utils:
 cloud-initramfs-growroot:
+dialog:

diskimage_builder/elements/dhcp-all-interfaces/install.d/50-dhcp-all-interfaces

@@ -15,7 +15,13 @@ fi
 
 DIB_INIT_SYSTEM=$(dib-init-system)
 if [ "$DIB_INIT_SYSTEM" == "upstart" ]; then
+    if [ -e "/etc/redhat-release" ] ; then
+        # the init system is upstart but networking is using sysv compatabiliy (i.e. Centos/RHEL 6)
+        install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.init /etc/init.d/dhcp-all-interfaces
+        chkconfig dhcp-all-interfaces on
+    else
         install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.conf /etc/init/dhcp-all-interfaces.conf
+    fi
 elif [ "$DIB_INIT_SYSTEM" == "systemd" ]; then
     install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-interface@.service /usr/lib/systemd/system/dhcp-interface@.service
     install -D -g root -o root -m 0644 ${SCRIPTDIR}/dhcp-all-interfaces-udev.rules /etc/udev/rules.d/99-dhcp-all-interfaces.rules

diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.init

@@ -6,6 +6,7 @@
 # Default-Start:     S
 # Default-Stop:      0 6
 # X-Start-Before:    networking
+# chkconfig:         3 9 50
 # Short-Description: Autodetect network interfaces
 # Description:       Autodetect network interfaces during boot and configure them for DHCP
 ### END INIT INFO

diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.sh

@@ -14,7 +14,12 @@ PATH=/sbin:$PATH
 if [ -d "/etc/network" ]; then
     CONF_TYPE="eni"
 elif [ -d "/etc/sysconfig/network-scripts/" ]; then
-    CONF_TYPE="netscripts"
+    CONF_TYPE="rhel-netscripts"
+    SCRIPTS_PATH="/etc/sysconfig/network-scripts/"
+elif [ -d "/etc/sysconfig/network/" ]; then
+    # SUSE network scripts location
+    CONF_TYPE="suse-netscripts"
+    SCRIPTS_PATH="/etc/sysconfig/network/"
 else
     echo "Unsupported network configuration type!"
     exit 1
@@ -42,8 +47,10 @@ function enable_interface() {
     serialize_me
     if [ "$CONF_TYPE" == "eni" ]; then
         printf "auto $interface\niface $interface inet dhcp\n\n" >>$ENI_FILE
-    elif [ "$CONF_TYPE" == "netscripts" ]; then
+    elif [ "$CONF_TYPE" == "rhel-netscripts" ]; then
-        printf "DEVICE=\"$interface\"\nBOOTPROTO=\"dhcp\"\nONBOOT=\"yes\"\nTYPE=\"Ethernet\"" >"/etc/sysconfig/network-scripts/ifcfg-$interface"
+        printf "DEVICE=\"$interface\"\nBOOTPROTO=\"dhcp\"\nONBOOT=\"yes\"\nTYPE=\"Ethernet\"" >"${SCRIPTS_PATH}ifcfg-$interface"
+    elif [ "$CONF_TYPE" == "suse-netscripts" ]; then
+        printf "BOOTPROTO=\"dhcp\"\nSTARTMODE=\"auto\"" >"${SCRIPTS_PATH}ifcfg-$interface"
     fi
     echo "Configured $1"
 
@@ -51,8 +58,8 @@ function enable_interface() {
 
 function config_exists() {
     local interface=$1
-    if [ "$CONF_TYPE" == "netscripts" ]; then
+    if [[ "$CONF_TYPE" =~ "netscripts" ]]; then
-        if [ -f "/etc/sysconfig/network-scripts/ifcfg-$interface" ]; then
+        if [ -f "${SCRIPTS_PATH}ifcfg-$interface" ]; then
             return 0
         fi
     else

diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.service

@@ -3,6 +3,7 @@ Description=Ironic Python Agent
 After=network-online.target
 
 [Service]
+ExecStartPre=/usr/sbin/modprobe vfat
 ExecStart=/usr/local/bin/ironic-python-agent
 Restart=always
 RestartSec=30s

diskimage_builder/elements/manifests/environment.d/14-manifests

@@ -1,5 +1,3 @@
-#!/bin/bash
-#
 # Copyright 2014 Hewlett-Packard Development Company, L.P.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -15,11 +13,5 @@
 # under the License.
 #
 
-if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
-    set -x
-fi
-set -eu
-set -o pipefail
-
 export DIB_MANIFEST_IMAGE_DIR=${DIB_MANIFEST_IMAGE_DIR:-/etc/dib-manifests}
 export DIB_MANIFEST_SAVE_DIR=${DIB_MANIFEST_SAVE_DIR:-${IMAGE_NAME}.d/}

diskimage_builder/elements/redhat-common/finalise.d/99-cleanup-tmp-grub

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then
-    set -x
-fi
-set -eu
-set -o pipefail
-
-rm -rf /tmp/grub

diskimage_builder/elements/redhat-common/package-installs.yaml

@@ -6,14 +6,3 @@ traceroute:
 which:
 gettext:
     phase: pre-install.d
-
-# these are being installed to satisfy the dependencies of grub2.  See
-# 15-remove-grub for more details
-grub2-tools:
-    phase: pre-install.d
-os-prober:
-    phase: pre-install.d
-redhat-lsb-core:
-    phase: pre-install.d
-system-logos:
-    phase: pre-install.d

diskimage_builder/elements/redhat-common/pre-install.d/15-remove-grub

@@ -1,47 +0,0 @@
-#!/bin/bash
-if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
-    set -x
-fi
-set -eu
-set -o pipefail
-
-# grub2 isn't available on rhel6/centos6; they are setup to use
-# extlinux.  skip this
-# you would think we could match on $DISTRO or something else; but
-# we can't because the rhel/centos elements are a bit mixed up;
-# centos-minimal for example sets distro to "centos".  so the best
-# check is just for the original "grub-install" script
-if [ -f /sbin/grub-install ]; then
-    exit 0
-fi
-
-# remove grub2 package.  As described in
-# elements/ubuntu/pre-install.d/00-remove-grub; the grub post-kernel
-# install hook will barf if the block device can't be found (as
-# happens in a chroot).
-#
-# XXX : it is not clear this is necessary for fedora/centos7 and it's
-# install hooks.  Investigation is required.
-if rpm -q grub2; then
-    install-packages -e grub-pc
-fi
-
-# now configure things to re-install grub at the end.  We don't want
-# to rely on vm/finalise.d/51-bootloader to simply reinstall the
-# package via the package-manager, because at that point (during
-# finalise) the build-time yum-cache has been unmounted (hence the
-# local-cache looks empty) and yum may try to repopulate the
-# local-cache with all the grub2 dependencies.  This is slow, and
-# potentially fills up the disk.
-#
-# XXX : At this point, keepcache=0 *should* probably be set for
-# yum/dnf.  We have not standarised/documented that this will be done,
-# however.  This would *probably* stop dependencies being populated
-# into the cache.  We could investigate this, and possibly remove this
-# all together if we standardise some of these behaviours.
-
-# So we download the latest grub2 package and setup the install script
-# to just install the single-package, which will be called later by
-# vm/finalise.d/51-bootloader
-install-packages -d /tmp/grub grub-pc
-echo "rpm -i /tmp/grub/*.rpm" > /tmp/grub/install

diskimage_builder/elements/rhel-common/os-refresh-config/pre-configure.d/06-rhel-registration

@@ -131,6 +131,8 @@ case "${REG_METHOD:-}" in
         rpm -Uvh "$REG_SAT_URL/pub/katello-ca-consumer-latest.noarch.rpm" || true
         echo "Registering with options: $sanitized_opts"
         subscription-manager register $opts
+        echo "Disabling all previous repos"
+        subscription-manager repos --disable=\*
         echo "Enabling repos: $user_repos"
         subscription-manager $repos
         echo "Installing katello-agent"

diskimage_builder/elements/rhel-common/pre-install.d/00-rhel-registration

@@ -114,8 +114,6 @@ case "${REG_METHOD:-}" in
         subscription-manager repos --disable=\*
         echo "Enabling repos: $user_repos"
         subscription-manager $repos
-        echo "Disabling satellite repo because it is no longer needed"
-        subscription-manager repos --disable ${satellite_repo}
         ;;
     disable)
         echo "Disabling RHEL registration"

diskimage_builder/elements/rhel7/root.d/10-rhel7-cloud-image

@@ -9,10 +9,10 @@ set -o pipefail
 [ -n "$ARCH" ]
 [ -n "$TARGET_ROOT" ]
 
-if [ 'amd64' = "$ARCH" ] ; then
+if [[ "amd64 x86_64" =~ "$ARCH" ]]; then
     ARCH="x86_64"
 else
-    echo 'rhel7 root element only support the amd64 $ARCH value.'
+    echo 'rhel7 root element only support the x86_64 $ARCH value.'
     exit 1
 fi
 

diskimage_builder/elements/rpm-distro/pre-install.d/01-override-yum-arch

@@ -9,7 +9,7 @@ set -o pipefail
 if [ "i386" = "$ARCH" ]; then
     basearch=i386
     arch=i686
-elif [ "amd64" = "$ARCH" ]; then
+elif [[ "amd64 x86_64" =~ "$ARCH" ]]; then
     basearch=x86_64
     arch=x86_64
 elif [[ "$ARCH" = "ppc64" ]]; then
@@ -28,7 +28,7 @@ fi
 if [[ $DISTRO_NAME == "fedora" && $DIB_RELEASE -ge 22 ]]; then
     mkdir -p /etc/dnf/vars
     echo $basearch > /etc/dnf/vars/basearch
-    echo $arch > /etc/dnf/vars/basearch
+    echo $arch > /etc/dnf/vars/arch
 else
     echo $basearch > /etc/yum/vars/basearch
     echo $arch > /etc/yum/vars/arch

diskimage_builder/elements/runtime-ssh-host-keys/element-deps

@@ -1 +1,2 @@
 dib-init-system
+install-static

diskimage_builder/elements/runtime-ssh-host-keys/init-scripts/systemd/ssh-keygen.service

@@ -2,19 +2,8 @@
 Description=OpenSSH Server Key Generation
 Before=ssh.service
 
-ConditionPathExists=|!/etc/ssh/ssh_host_key
-ConditionPathExists=|!/etc/ssh/ssh_host_key.pub
-ConditionPathExists=|!/etc/ssh/ssh_host_rsa_key
-ConditionPathExists=|!/etc/ssh/ssh_host_rsa_key.pub
-ConditionPathExists=|!/etc/ssh/ssh_host_dsa_key
-ConditionPathExists=|!/etc/ssh/ssh_host_dsa_key.pub
-ConditionPathExists=|!/etc/ssh/ssh_host_ecdsa_key
-ConditionPathExists=|!/etc/ssh/ssh_host_ecdsa_key.pub
-ConditionPathExists=|!/etc/ssh/ssh_host_ed25519_key
-ConditionPathExists=|!/etc/ssh/ssh_host_ed25519_key.pub
-
 [Service]
-ExecStart=/usr/bin/ssh-keygen -A
+ExecStart=/usr/local/sbin/runtime-ssh-host-keys.sh
 Type=oneshot
 RemainAfterExit=yes
 

diskimage_builder/elements/runtime-ssh-host-keys/init-scripts/upstart/ssh-keygen.conf

@@ -5,4 +5,4 @@ console output
 
 task
 
-exec /usr/bin/ssh-keygen -A
+exec /usr/local/sbin/runtime-ssh-host-keys.sh

diskimage_builder/elements/runtime-ssh-host-keys/pkg-map

@@ -5,6 +5,9 @@
     },
     "gentoo": {
       "openssh-client": ""
+    },
+    "suse": {
+      "openssh-client": "openssh"
     }
   }
 }

diskimage_builder/elements/simple-init/install.d/60-simple-init-remove-interfaces

@@ -12,6 +12,9 @@ set -o pipefail
 # Fedora
 rm -f /etc/sysconfig/network-scripts/ifcfg-eth*
 
+# SUSE
+rm -f /etc/sysconfig/network/ifcfg-eth*
+
 # Ubuntu
 rm -f /etc/network/interfaces.d/eth*
 

diskimage_builder/elements/ubuntu-minimal/environment.d/10-ubuntu-distro-name.bash

@@ -1,4 +1,4 @@
 export DISTRO_NAME=ubuntu
-export DIB_RELEASE=${DIB_RELEASE:-trusty}
+export DIB_RELEASE=${DIB_RELEASE:-xenial}
 export DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main,restricted,universe}
 export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://archive.ubuntu.com/ubuntu}

diskimage_builder/elements/ubuntu/environment.d/10-ubuntu-distro-name.bash

@@ -1,2 +1,2 @@
 export DISTRO_NAME=ubuntu
-export DIB_RELEASE=${DIB_RELEASE:-trusty}
+export DIB_RELEASE=${DIB_RELEASE:-xenial}

diskimage_builder/elements/yum-minimal/package-installs.yaml

@@ -4,7 +4,7 @@ grubby:
 kernel:
 initscripts:
 man-pages:
-redhat-lsb-core:
+lsb_release:
 selinux-policy:
 selinux-policy-targeted:
 libselinux-python:

diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot

@@ -56,6 +56,7 @@ _RPM="rpm --dbpath=/var/lib/rpm"
 # has yum/yumdownloader
 function _install_repos {
     local packages
+    local rc
 
     # pre-install the base system packages via rpm.  We previously
     # just left it up to yum to drag these in when we "yum install
@@ -79,11 +80,21 @@ function _install_repos {
         packages+="${DISTRO_NAME}-repos "
     fi
 
-    yumdownloader \
+    # yumdownloader puts repo xml files and such into a directory
+    # ${TMPDIR}/yum-$USER-random.  Since we don't need this once the
+    # initial download happens, redirect TMPDIR for this call so we
+    # can clean it up nicely
+    local temp_tmp
+    temp_tmp=$(mktemp -d)
+    TMPDIR=${temp_tmp} yumdownloader \
         --releasever=$DIB_RELEASE \
         --setopt=reposdir=$TMP_HOOKS_PATH/yum.repos.d \
         --destdir=$WORKING \
-        ${packages}
+        ${packages} && rc=$? || rc=$?
+    rm -rf ${temp_tmp}
+    if [[ ${rc} != 0 ]]; then
+        die "Failed to download initial packages: ${packages}"
+    fi
 
     # --nodeps works around these wanting /bin/sh in some fedora
     # releases, see rhbz#1265873

diskimage_builder/lib/common-functions

@@ -142,21 +142,48 @@ function eval_run_d () {
     trap - ERR
 }
 
+function kill_chroot_processes () {
+    if [ -z "${1}" ]; then
+        echo "ERROR: no chroot directory specified"
+        exit 1
+    fi
+    for piddir in /proc/[0-9]*; do
+        pid=${piddir##/proc/}
+        pidname=$(cat $piddir/comm 2>/dev/null || echo "unknown")
+        # If there are open files from the chroot, just kill the process using
+        # these files.
+        if sudo readlink -f $piddir/root | grep -q $TMP_BUILD_DIR; then
+            echo "Killing chroot process: '${pidname}($pid)'"
+            sudo kill $pid
+        fi
+    done
+}
+
 function cleanup_build_dir () {
     if ! timeout 5  sh -c " while ! sudo rm -rf $TMP_BUILD_DIR/built; do sleep 1; done"; then
         echo "ERROR: unable to cleanly remove $TMP_BUILD_DIR/built"
         exit 1
     fi
     sudo rm -rf $TMP_BUILD_DIR/mnt
+    kill_chroot_processes $TMP_BUILD_DIR
     if tmpfs_check 0; then
-        sudo umount -f $TMP_BUILD_DIR || true
+        # If kill_chroot_processes did not succeed then we have to wait for
+        # init to reap the orphaned chroot processes
+        if ! timeout 120 sh -c "while ! sudo umount -f $TMP_BUILD_DIR; do sleep 1; done"; then
+            echo "ERROR: failed to umount the $TMP_BUILD_DIR tmpfs mount point"
+            exit 1
+        fi
     fi
     rm -rf --one-file-system $TMP_BUILD_DIR
 }
 
 function cleanup_image_dir () {
+    kill_chroot_processes $TMP_IMAGE_DIR
     if tmpfs_check 0; then
-        sudo umount -f $TMP_IMAGE_DIR || true
+        if ! timeout 120 sh -c "while ! sudo umount -f $TMP_IMAGE_DIR; do sleep 1; done"; then
+            echo "ERROR: failed to umount the $TMP_IMAGE_DIR tmpfs mount point"
+            exit 1
+        fi
     fi
     rm -rf --one-file-system $TMP_IMAGE_DIR
 }

diskimage_builder/lib/disk-image-create

@@ -79,7 +79,7 @@ function show_options () {
     echo "    --docker-target -- specify the repo and tag to use if the output type is docker. Defaults to the value of output imagename"
     if [ "$IS_RAMDISK" == "0" ]; then
         echo "    -n skip the default inclusion of the 'base' element"
-        echo "    -p package[,package,package] -- list of packages to install in the image"
+        echo "    -p package[,package,package] -- list of packages to install in the image. If specified multiple times the packages are appended to the list."
     fi
     echo "    -h|--help -- display this help and exit"
     echo "    --version -- display version and exit"
@@ -136,7 +136,7 @@ while true ; do
         -u) shift; export COMPRESS_IMAGE="";;
         -c) shift ; export CLEAR_ENV=1;;
         -n) shift; export SKIP_BASE="1";;
-        -p) IFS="," read -a INSTALL_PACKAGES <<< "$2"; export INSTALL_PACKAGES ; shift 2 ;;
+        -p) IFS="," read -a _INSTALL_PACKAGES <<< "$2"; export INSTALL_PACKAGES=( ${INSTALL_PACKAGES[@]} ${_INSTALL_PACKAGES[@]} ) ; shift 2 ;;
         --checksum) shift; export DIB_CHECKSUM=1;;
         --image-size) export DIB_IMAGE_SIZE=$2; shift 2;;
         --image-cache) export DIB_IMAGE_CACHE=$2; shift 2;;

doc/source/developer/developing_elements.rst

@@ -172,11 +172,12 @@ the image as executable files.
 Environment Variables
 ^^^^^^^^^^^^^^^^^^^^^
 
-To set environment variables for other hooks, add a file to your element
+To set environment variables for other hooks, add a file to your
-``environment.d``.
+element ``environment.d``.  This directory contains bash script
-
+snippets that are sourced before running scripts in each phase.  Note
-This directory contains bash script snippets that are sourced before running
+that because environment includes are sourced together, they should
-scripts in each phase.
+not set global flags like ``set -x`` because they will affect all
+preceeding imports.
 
 
 Dependencies

elements/opensuse-minimal/environment.d/10-opensuse-distro-name.bash

@@ -1,5 +1,6 @@
 export DISTRO_NAME=opensuse
-export DIB_RELEASE=${DIB_RELEASE:-42.1}
+DIB_RELEASE=${DIB_RELEASE:-42.1}
+export DIB_RELEASE=${DIB_RELEASE,,}
 export DIB_OPENSUSE_MIRROR=${DIB_OPENSUSE_MIRROR:-http://download.opensuse.org}
 case ${DIB_RELEASE} in
     # We are using "=>" as the assignment symbol since "@" "=" etc could be used in the URI itself.
@@ -15,6 +16,11 @@ case ${DIB_RELEASE} in
         ZYPPER_REPOS="update=>${DIB_OPENSUSE_MIRROR}/update/leap/${DIB_RELEASE}/oss/ "
         ZYPPER_REPOS+="oss=>${DIB_OPENSUSE_MIRROR}/distribution/leap/${DIB_RELEASE}/repo/oss/"
         ;;
+    # Tumbleweed
+    tumbleweed)
+        ZYPPER_REPOS="update=>${DIB_OPENSUSE_MIRROR}/update/${DIB_RELEASE}/ "
+        ZYPPER_REPOS+="oss=>${DIB_OPENSUSE_MIRROR}/${DIB_RELEASE}/repo/oss/"
+        ;;
     *) echo "Unsupported openSUSE release: ${DIB_RELEASE}"; exit 1 ;;
 esac
 export ZYPPER_REPOS

elements/runtime-ssh-host-keys/static/usr/local/sbin/runtime-ssh-host-keys.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+# dib-lint: disable=dibdebugtrace
+
+set -exu
+set -o pipefail
+
+# We are running into race conditions with glean, which ssh-keygen -A is
+# not handling properly.  So, create a new script to first check if the
+# file exists, then use 'yes' to disable overwriting of existing files.
+
+for key in dsa ecdsa ed25519 rsa; do
+    FILE=/etc/ssh/ssh_host_${key}_key
+    if ! [ -e $FILE ]; then
+        /usr/bin/yes n | /usr/bin/ssh-keygen -f $FILE -N '' -t $key
+    fi
+done

elements/sysctl/README.rst

@@ -0,0 +1,12 @@
+======
+sysctl
+======
+
+Add a sysctl-set-value command which can be run from within an element.
+Running this command will cause the sysctl value to be set on boot (by
+writing the value to /etc/sysctl.d).
+
+Example usage
+
+::
+    sysctl-set-value net.ipv4.ip_forward 1

elements/sysctl/bin/sysctl-set-value

@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Copied from tripleo-image-element's sysctl element
+#
+# Validate and manage setting sysctl settings.
+#
+# The script is called with name/value pairs which are stored
+# in the system default sysctl.d directory. Before adding new
+# settings a validation is done to ensure that conflicting
+# sysctl settings have not been requested. Once finished sysctl
+# is used to activate the changes.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+NAME=${1:-}
+VALUE=${2:-}
+# Optional comment used to describe the setting
+COMMENT=${3:-"This file was created by diskimage-builder."}
+
+if [ -z "$NAME" -o -z "$VALUE" ]; then
+    echo "NAME and VALUE are required."
+    exit 1
+fi
+
+FILENAME="/etc/sysctl.d/${NAME}.conf"
+
+if [ -f $FILENAME ]; then
+    # check to make sure the settings match... otherwise fail
+    if ! grep -q "^$NAME = $VALUE" $FILENAME; then
+        echo "Conflicting sysctl.conf setting for $NAME == $VALUE. Found:"
+        grep "^$NAME" $FILENAME
+        exit 1
+    fi
+else
+
+    if ! sysctl -a | grep -q "^$NAME"; then
+        echo "Invalid sysctl key: $NAME"
+        exit 1
+    fi
+
+    sysctl-write-value $NAME "$VALUE" "$COMMENT"
+
+    sysctl -p $FILENAME
+
+fi

elements/sysctl/bin/sysctl-write-value

@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Copied from tripleo-image-element's sysctl element
+#
+# Validate and manage setting sysctl settings.
+#
+# The script is called with name/value pairs which are stored
+# in the system default sysctl.d directory. This script performs
+# no checking, just writing out the file.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+NAME=${1:-}
+VALUE=${2:-}
+# Optional comment used to describe the setting
+COMMENT=${3:-"This file was created by diskimage-builder."}
+
+if [ -z "$NAME" -o -z "$VALUE" ]; then
+    echo "Usage: sysctl-write-value <name> <value> [comment]"
+    exit 1
+fi
+
+FILENAME="/etc/sysctl.d/${NAME}.conf"
+
+cat > $FILENAME <<EOF_CAT
+# $COMMENT
+$NAME = $VALUE
+EOF_CAT

elements/zypper-minimal/root.d/08-zypper-chroot

@@ -27,6 +27,10 @@ set -o pipefail
 [ -n "${ZYPPER_REPOS}" ]
 
 function cleanup() {
+    sudo umount $TARGET_ROOT/proc
+    sudo umount $TARGET_ROOT/dev/pts
+    sudo umount $TARGET_ROOT/dev
+    sudo umount $TARGET_ROOT/sys
     sudo umount $TMP_MOUNT_PATH/var/cache/zypp
 }
 
@@ -41,9 +45,6 @@ for repo in ${ZYPPER_REPOS}; do
     sudo zypper ${ZYPPER_TARGET_OPTS} addrepo --name ${reponame} --keep-packages ${repouri} ${reponame}
 done
 
-# Refresh it
-sudo zypper ${ZYPPER_TARGET_OPTS} refresh
-
 # It appears that zypper will clean up the repo's cache when it (re-)adds the
 # repo so we need to add the cache now, once the repos are added. This is
 # similar to what the zypper/50-zypper-cache script does
@@ -53,6 +54,20 @@ mkdir -p $ZYPPER_CACHE_DIR
 sudo mkdir -p $TMP_MOUNT_PATH/var/cache/zypp
 sudo mount --bind $ZYPPER_CACHE_DIR $TMP_MOUNT_PATH/var/cache/zypp
 
+# Refresh it so we get updated data in cased we switched DIB_RELEASE
+# since last run.
+sudo zypper ${ZYPPER_TARGET_OPTS} refresh
+
+# Note this is not usually done for root.d elements (see
+# lib/common-functions:mount_proc_dev_sys) but it's important that
+# we have things like /dev/urandom around inside the chroot for
+# the rpm [pre|post]inst scripts within the packages.
+sudo mkdir -p $TARGET_ROOT/proc $TARGET_ROOT/dev $TARGET_ROOT/sys
+sudo mount -t proc none $TARGET_ROOT/proc
+sudo mount --bind /dev $TARGET_ROOT/dev
+sudo mount --bind /dev/pts $TARGET_ROOT/dev/pts
+sudo mount -t sysfs none $TARGET_ROOT/sys
+
 # Install filesystem, base and useful tools
 sudo zypper ${ZYPPER_TARGET_OPTS} install ${ZYPPER_INSTALL_OPTS} filesystem
 # Install basic components in order
@@ -83,5 +98,5 @@ for newfile in $(sudo find $TARGET_ROOT -type f -name '*rpmnew') ; do
     sudo mv $newfile $(echo $newfile | sed 's/.rpmnew$//')
 done
 
-# Unmounting of the /var/cache/zypp is handled by the cleanup EXIT
+# Unmounting of all the mount points is handled by the cleanup EXIT
 # handler so there is nothing else to do here

requirements.txt

@@ -2,7 +2,7 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 Babel>=2.3.4 # BSD
-pbr>=1.6 # Apache-2.0
+pbr>=1.8 # Apache-2.0
 PyYAML>=3.10.0 # MIT
 flake8<2.6.0,>=2.5.4 # MIT
 six>=1.9.0 # MIT

setup.cfg

@@ -6,7 +6,7 @@ description-file =
 author = HP Cloud Services
 author_email = openstack-dev@lists.openstack.org
 license: Apache License (2.0)
-home-page = https://git.openstack.org/cgit/openstack/diskimage-builder
+home-page = http://docs.openstack.org/developer/diskimage-builder/
 classifier =
     Development Status :: 3 - Alpha
     License :: OSI Approved :: Apache Software License

test-requirements.txt

@@ -11,4 +11,4 @@ sphinx!=1.3b1,<1.4,>=1.2.1 # BSD
 oslosphinx>=4.7.0 # Apache-2.0
 
 # releasenotes
-reno>=1.8.0 # Apache2
+reno>=1.8.0 # Apache-2.0

tests/run_functests.sh

@@ -29,16 +29,26 @@ DEFAULT_SKIP_TESTS=(
 
 function log_with_prefix {
     local pr=$1
+    local log
 
     while read a; do
-        echo $(date +"%Y%m%d-%H%M%S.%N") "[$pr] $a"
+        log="[$pr] $a"
+        if [[ ${LOG_DATESTAMP} -ne 0 ]]; then
+            log="$(date +"%Y%m%d-%H%M%S.%N") ${log}"
+        fi
+        echo "${log}"
     done
 }
 
 # Log job control messages
 function log_jc {
     local msg="$1"
-    printf "[JOB-CONTROL] %s %s\n" "$(date)" "${msg}"
+    local log="[JOB-CONTROL] ${msg}"
+
+    if [[ ${LOG_DATESTAMP} -ne 0 ]]; then
+        log="$(date +"%Y%m%d-%H%M%S.%N") ${log}"
+    fi
+    echo "${log}"
 }
 
 function job_cnt {
@@ -158,15 +168,23 @@ for e in $DIB_ELEMENTS/*/test-elements/*; do
     TESTS+=("$element/$test_element")
 done
 
+#
+# Default values
+#
 JOB_MAX_CNT=1
+LOG_DATESTAMP=0
 
-while getopts ":hlpj:" opt; do
+#
+# Parse args
+#
+while getopts ":hlj:t" opt; do
     case $opt in
         h)
             echo "run_functests.sh [-h] [-l] <test> <test> ..."
             echo "  -h : show this help"
             echo "  -l : list available tests"
-            echo "  -p : run all tests in parallel"
+            echo "  -j : parallel job count (default to 1)"
+            echo "  -t : prefix log messages with timestamp"
             echo "  <test> : functional test to run"
             echo "           Special test 'all' will run all tests"
             exit 0
@@ -184,6 +202,9 @@ while getopts ":hlpj:" opt; do
             JOB_MAX_CNT=${OPTARG}
             echo "Running parallel - using [${JOB_MAX_CNT}] jobs"
             ;;
+        t)
+            LOG_DATESTAMP=1
+            ;;
         \?)
             echo "Invalid option: -$OPTARG"
             exit 1

tox.ini

@@ -37,4 +37,4 @@ commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasen
 
 [flake8]
 ignore = E125,H202,H302,H803
-exclude =  .venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,conf.py
+exclude =  .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,conf.py