Merge branch 'master' into feature/v2

Change-Id: I0747a27560161ef869868967ab140b5d0be9d4da

bin/disk-image-create

@@ -109,6 +109,7 @@ function show_options () {
     echo "    -x -- turn on tracing (use -x -x for very detailed tracing)"
     echo "    -u -- uncompressed; do not compress the image - larger but faster"
     echo "    -c -- clear environment before starting work"
+    echo "    --checksum -- generate MD5 and SHA256 checksum files for the created image"
     echo "    --image-size size -- image size in GB for the created image"
     echo "    --image-cache directory -- location for cached images(default ~/.cache/image-create)"
     echo "    --max-online-resize size -- max number of filesystem blocks to support when resizing."
@@ -172,7 +173,7 @@ DIB_DEFAULT_INSTALLTYPE=${DIB_DEFAULT_INSTALLTYPE:-"source"}
 MKFS_OPTS=""
 ACI_MANIFEST=${ACI_MANIFEST:-}
 DOCKER_TARGET=""
-TEMP=`getopt -o a:ho:t:xucnp: -l no-tmpfs,offline,help,version,min-tmpfs:,image-size:,image-cache:,max-online-resize:,mkfs-options:,qemu-img-options:,ramdisk-element:,root-label:,install-type:,docker-target: -n $SCRIPTNAME -- "$@"`
+TEMP=`getopt -o a:ho:t:xucnp: -l checksum,no-tmpfs,offline,help,version,min-tmpfs:,image-size:,image-cache:,max-online-resize:,mkfs-options:,qemu-img-options:,ramdisk-element:,root-label:,install-type:,docker-target: -n $SCRIPTNAME -- "$@"`
 if [ $? -ne 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
 
 # Note the quotes around `$TEMP': they are essential!
@@ -190,7 +191,8 @@ while true ; do
         -c) shift ; export CLEAR_ENV=1;;
         -n) shift; export SKIP_BASE="1";;
         -p) IFS="," read -a INSTALL_PACKAGES <<< "$2"; export INSTALL_PACKAGES ; shift 2 ;;
-        --image-size) DIB_IMAGE_SIZE=$2; shift 2;;
+        --checksum) shift; export DIB_CHECKSUM=1;;
+        --image-size) export DIB_IMAGE_SIZE=$2; shift 2;;
         --image-cache) export DIB_IMAGE_CACHE=$2; shift 2;;
         --max-online-resize) export MAX_ONLINE_RESIZE=$2; shift 2;;
         --mkfs-options) MKFS_OPTS=$2; shift 2;;
@@ -338,26 +340,28 @@ mv $TMP_BUILD_DIR/mnt $TMP_BUILD_DIR/built
 # logs with du output below.
 xtrace=$(set +o | grep xtrace)
 
+# temp file for holding du output
+du_output=${TMP_BUILD_DIR}/du_output.tmp
+
 if [ -n "$DIB_IMAGE_SIZE" ]; then
     du_size=$(echo "$DIB_IMAGE_SIZE" | awk '{printf("%d\n",$1 * 1024 *1024)}')
 else
     set +o xtrace
     echo "Calculating image size (this may take a minute)..."
-    du_output=$(sudo du -a -c -x ${TMP_BUILD_DIR}/built)
+    sudo du -a -c -x ${TMP_BUILD_DIR}/built > ${du_output}
     # the last line is the total size from "-c".
     # scale this by 0.6 to create a slightly bigger image
-    du_size=$(echo "$du_output" | tail -n1 | cut -f1 | \
+    du_size=$(tail -n1 ${du_output} | cut -f1 | awk '{print int($1 / 0.6)}')
-                     awk '{print int($1 / 0.6)}')
     $xtrace
 fi
 
 if [[ "${DIB_SHOW_IMAGE_USAGE:-0}" != 0 ]]; then
     set +o xtrace
-    if [ -z "$du_output" ]; then
+    if [ ! -f "$du_output" ]; then
-        du_output=$(sudo du -a -c -x ${TMP_BUILD_DIR}/built)
+        sudo du -a -c -x ${TMP_BUILD_DIR}/built > ${du_output}
     fi
 
-    du_output_show="sort -nr |
+    du_output_show="sort -nr ${du_output} |
                      numfmt --to=iec-i --padding=7
                        --suffix=B --field=1 --from-unit=1024"
 
@@ -376,7 +380,7 @@ if [[ "${DIB_SHOW_IMAGE_USAGE:-0}" != 0 ]]; then
         echo "================="
     fi
 
-    eval ${du_output_show} <<< "$du_output"
+    eval ${du_output_show}
 
     echo
     echo "===== end image size report ====="
@@ -385,6 +389,8 @@ if [[ "${DIB_SHOW_IMAGE_USAGE:-0}" != 0 ]]; then
     $xtrace
 fi
 
+rm -f ${du_output}
+
 if [ "$FS_TYPE" = "ext4" ] ; then
   # Very conservative to handle images being resized a lot
   # We set journal size to 64M so our journal is large enough when we

bindep.txt

@@ -0,0 +1,2 @@
+# This is a cross-platform list tracking distribution packages needed by tests;
+# see http://docs.openstack.org/infra/bindep/ for additional information.

doc/source/conf.py

@@ -124,7 +124,7 @@ html_theme = 'default'
 #html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+html_sidebars = { '**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html'], }
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.

doc/source/copyright.rst

@@ -1,20 +0,0 @@
-Copyright
-=========
-
-Copyright 2012 Hewlett-Packard Development Company, L.P.
-
-Copyright (c) 2012 NTT DOCOMO, INC.
-
-All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may
-not use this file except in compliance with the License. You may obtain
-a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-License for the specific language governing permissions and limitations
-under the License.

doc/source/developer/developing_elements.rst

@@ -412,6 +412,11 @@ line to run it.  If it should not be run as part of the default CI
 run, you can submit a change with it added to ``DEFAULT_SKIP_TESTS``
 in that file.
 
+Running the functional tests is time consuming.  Multiple parallel
+jobs can be started by specifying ``-j <job count>``.  Each of the
+jobs uses a lot resources (CPU, disk space, RAM) - therefore the job
+count must carefully be chosen.
+
 python
 """"""
 

doc/source/developer/index.rst

@@ -1,5 +1,16 @@
-Developer Documentation
+Developer Guide
-=======================
+===============
+
+.. toctree::
+   :maxdepth: 1
+
+   design
+   components
+   invocation
+   caches
+   developing_elements
+   dib_lint
+   stable_interfaces
 
 This documentation explains how to get started with creating your own
 disk-image-builder elements as well as some high level concepts for element
@@ -13,7 +24,7 @@ To get started developing with ``diskimage-builder``, install to a
 
  $ mkdir dib
  $ cd dib
- $ virtualenv create env
+ $ virtualenv env
  $ source env/bin/activate
  $ git clone https://git.openstack.org/openstack/diskimage-builder
  $ cd diskimage-builder
@@ -24,13 +35,10 @@ and testing your changes.  When you are done editing, use ``git
 review`` to submit changes to the upstream gerrit.
 
 
-.. toctree::
+Finding Work
-   :maxdepth: 2
+------------
 
-   design
+We maintain a list of low-hanging-fruit tags on launchpad:
-   components
+
-   invocation
+ * `https://bugs.launchpad.net/diskimage-builder/+bugs?field.tag=low-hanging-fruit`
-   caches
+   <https://bugs.launchpad.net/diskimage-builder/+bugs?field.tag=low-hanging-fruit>
-   developing_elements
-   dib_lint
-   stable_interfaces

doc/source/index.rst

@@ -30,6 +30,15 @@ The code is available at:
    <https://git.openstack.org/cgit/openstack/diskimage-builder/>`__
 
 
+Issues
+------
+
+Issues are tracked on launchpad at:
+
+ * `https://bugs.launchpad.net/diskimage-builder/+bugs`
+   <https://bugs.launchpad.net/diskimage-builder/+bugs>
+
+
 Communication
 -------------
 
@@ -46,4 +55,3 @@ Table of Contents
    user_guide/index
    developer/index
    elements
-   copyright

doc/source/user_guide/installation.rst

@@ -55,3 +55,13 @@ Installing via pip is as simple as:
 
     pip install diskimage-builder
 
+
+Package Installation
+--------------------
+
+On Gentoo you can emerge diskimage-builder directly.
+
+::
+
+    emerge app-emulation/diskimage-builder
+

elements/cloud-init/post-install.d/20-enable-cloud-init

@@ -10,6 +10,6 @@ set -o pipefail
 if [[ "${DISTRO_NAME}" == "gentoo" ]]; then
     rc-update add cloud-config default
     rc-update add cloud-final default
-    rc-update add cloud-init-local default
+    rc-update add cloud-init-local boot
     rc-update add cloud-init default
 fi

elements/debian-minimal/README.rst

@@ -11,16 +11,16 @@ There are two ways to configure apt-sources:
    and security repositories is the default. In this case you can
    overwrite the two environment variables to adapt the behavior:
    `DIB_DISTRIBUTION_MIRROR`: the mirror to use
-      default: http://httpredir.debian.org/debian
+      default: http://ftp.us.debian.org/debian
    `DIB_DEBIAN_COMPONENTS`: (default) `main`
       a comma separated list of components. For Debian this can be
       e.g. `main,contrib,non-free`.
 
-   Note that the default Debian series is `unstable`, and the default
+   Note it is not recommended to use http://httpredir.debian.org/ for
-   mirrors for Debian can be problematic for `unstable`. Because apt
+   `DIB_DISTRIBUTION_MIRROR` due to how unreliable it is.  Be sure to
-   does not handle changing Packages files well across multiple out of
+   select a mirror from the official mirror list:
-   sync mirrors, it is recommended that you choose a single mirror of
+
-   Debian, and pass it in via `DIB_DISTRIBUTION_MIRROR`.
+       https://www.debian.org/mirror/list
 
    By default only `main` component is used. If
    `DIB_DEBIAN_COMPONENTS` (comma separated) from the `debootstrap`

elements/debian-minimal/environment.d/10-debian-minimal.bash

@@ -1,6 +1,6 @@
 export DISTRO_NAME=debian
 export DIB_RELEASE=${DIB_RELEASE:-stable}
-export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://httpredir.debian.org/debian}
+export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://ftp.us.debian.org/debian}
 export DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main}
 export DIB_DEBIAN_COMPONENTS_WS=${DIB_DEBIAN_COMPONENTS//,/ }
 

elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces-udev.rules

@@ -1 +1 @@
-SUBSYSTEM=="net", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}+="dhcp-interface@$name.service"
+SUBSYSTEM=="net", KERNEL!="lo", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}+="dhcp-interface@$name.service"

elements/dhcp-all-interfaces/install.d/dhcp-interface@.service

@@ -11,6 +11,7 @@ User=root
 ExecStartPre=/usr/local/sbin/dhcp-all-interfaces.sh %I
 ExecStart=/sbin/ifup %I
 RemainAfterExit=true
+TimeoutStartSec=30s
 
 [Install]
 WantedBy=multi-user.target

elements/growroot/README.rst

@@ -9,3 +9,4 @@ This is only supported on:
 * ubuntu trusty or later
 * gentoo
 * fedora & centos
+* suse & opensuse

elements/growroot/pkg-map

@@ -8,6 +8,10 @@
             "growpart": "cloud-utils",
             "e2fsprogs": "e2fsprogs"
         },
+        "suse": {
+            "growpart": "growpart",
+            "e2fsprogs": "e2fsprogs"
+        },
         "gentoo": {
             "growpart": "sys-fs/growpart",
             "e2fsprogs": "sys-fs/e2fsprogs"

elements/ironic-agent/cleanup.d/99-ramdisk-create

@@ -11,6 +11,8 @@ set -o pipefail
 
 [ -n "$TARGET_ROOT" ]
 
+USER=${USER:-$(whoami)}
+
 source $_LIB/img-functions
 
 IMAGE_PATH=$(readlink -f $IMAGE_NAME)

elements/opensuse-minimal/README.rst

@@ -0,0 +1,28 @@
+================
+opensuse-minimal
+================
+
+This element will build a minimal openSUSE image. It requires 'zypper' to be
+installed on the host.
+
+These images should be considered experimental. There are curently only x86_64
+images.
+
+Environment Variables
+---------------------
+
+DIB_RELEASE
+  :Required: No
+  :Default: 42.1
+  :Description: Set the desired openSUSE release.
+
+DIB_OPENSUSE_MIRROR:
+   :Required: No
+   :Default: http://download.opensuse.org
+   :Description: To use a specific openSUSE mirror, set this variable to the
+                 mirror URL before running bin/disk-image-create. This URL
+                 should point to the root directory as indicated in the
+                 http://mirrors.opensuse.org/ webpage. You normally
+                 don't want to change that since the default setting will
+                 pick the mirror closest to you.
+   :Example: ``DIB_OPENSUSE_MIRROR=http://ftp.cc.uoc.gr/mirrors/linux/opensuse/opensuse/``

elements/opensuse-minimal/element-deps

@@ -0,0 +1 @@
+zypper-minimal

elements/opensuse-minimal/element-provides

@@ -0,0 +1 @@
+operating-system

elements/opensuse-minimal/environment.d/10-opensuse-distro-name.bash

@@ -0,0 +1,20 @@
+export DISTRO_NAME=opensuse
+export DIB_RELEASE=${DIB_RELEASE:-42.1}
+export DIB_OPENSUSE_MIRROR=${DIB_OPENSUSE_MIRROR:-http://download.opensuse.org}
+case ${DIB_RELEASE} in
+    # We are using "=>" as the assignment symbol since "@" "=" etc could be used in the URI itself.
+    # Remember, we can't export an array in bash so we use a string instead.
+    # Repo format: {name}=>{uri}
+    # Old openSUSE releases
+    13*)
+        ZYPPER_REPOS="update=>${DIB_OPENSUSE_MIRROR}/update/${DIB_RELEASE}/ "
+        ZYPPER_REPOS+="oss=>${DIB_OPENSUSE_MIRROR}/distribution/${DIB_RELEASE}/repo/oss/"
+        ;;
+    # New Leap releases
+    42*)
+        ZYPPER_REPOS="update=>${DIB_OPENSUSE_MIRROR}/update/leap/${DIB_RELEASE}/oss/ "
+        ZYPPER_REPOS+="oss=>${DIB_OPENSUSE_MIRROR}/distribution/leap/${DIB_RELEASE}/repo/oss/"
+        ;;
+    *) echo "Unsupported openSUSE release: ${DIB_RELEASE}"; exit 1 ;;
+esac
+export ZYPPER_REPOS

elements/opensuse/README.rst

@@ -13,6 +13,19 @@ For example, the images of openSUSE 13.2 can be found here:
 These images should be considered experimental. There are curently only x86_64
 images.
 
+Environment Variables
+---------------------
+
+DIB_RELEASE
+  :Required: No
+  :Default: 13.1
+  :Description: Set the desired openSUSE release.
+
+DIB_CLOUD_IMAGES
+  :Required: No
+  :Default: http://download.opensuse.org/repositories/Cloud:/Images:/(openSUSE|Leap)_${DIB_RELEASE}
+  :Description: Set the desired URL to fetch the images from.
+
 Notes:
 
 * There are very frequently new automated builds that include changes that
@@ -21,8 +34,3 @@ Notes:
   point to the latest image, but will frequently change its content. The versioned
   one will never change content, but will frequently be deleted and replaced
   by a newer build with a higher version-release number.
-
-* Building with DIB\_EXTLINUX=1 doesn't work.  It fails with:
-  /tmp/in\_target.d/finalise.d/51-bootloader: line 14: 16286 Segmentation fault
-  extlinux --install /boot/syslinux
-  (https://bugzilla.novell.com/show_bug.cgi?id=852856)

elements/opensuse/element-deps

@@ -1,5 +1,4 @@
 cache-url
 dib-run-parts
-install-bin
 package-installs
 zypper

elements/opensuse/environment.d/10-opensuse-distro-name.bash

@@ -1 +1,9 @@
 export DISTRO_NAME=opensuse
+export DIB_RELEASE=${DIB_RELEASE:-13.1}
+case ${DIB_RELEASE} in
+    # Old openSUSE releases
+    13*) export OPENSUSE_REPO_DIR=openSUSE_${DIB_RELEASE} ;;
+    # New Leap releases
+    42*) export OPENSUSE_REPO_DIR=openSUSE_Leap_${DIB_RELEASE} ;;
+    *) echo "Unsupported openSUSE release: ${DIB_RELEASE}"; exit 1 ;;
+esac

elements/opensuse/pre-install.d/00-opensuse-setup

@@ -7,4 +7,4 @@ fi
 set -eu
 set -o pipefail
 
-zypper ar -f http://download.opensuse.org/repositories/X11:/Bumblebee/openSUSE_13.1/X11:Bumblebee.repo
+zypper ar -f http://download.opensuse.org/repositories/X11:/Bumblebee/${OPENSUSE_REPO_DIR}/X11:Bumblebee.repo

elements/opensuse/root.d/10-opensuse-cloud-image

@@ -18,11 +18,24 @@ if ! [ 'x86_64' = "$ARCH" ] ; then
     exit 1
 fi
 
-DIB_RELEASE=${DIB_RELEASE:-13.1}
+# Set some image defaults
+case ${DIB_RELEASE} in
+    # Old openSUSE releases
+    13*)
+        OPENSUSE_IMAGE_BASEDIR=openSUSE
+        OPENSUSE_IMAGE_FILE=openSUSE-${DIB_RELEASE}-OS
+        ;;
+    # New Leap releases
+    42*)
+        OPENSUSE_IMAGE_BASEDIR=Leap
+        OPENSUSE_IMAGE_FILE=openSUSE-Leap-${DIB_RELEASE}-OpenStack
+        ;;
+    # We handle unknown cases in environment.d/10-opensuse-distro-name.bash
+esac
 # NOTE(toabctl): if something changes here on the buildservice side, please
 # first ask in #opensuse-cloud on freenode before you change the format here!
-DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-http://download.opensuse.org/repositories/Cloud:/Images:/openSUSE_${DIB_RELEASE}/images/}
+DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-http://download.opensuse.org/repositories/Cloud:/Images:/${OPENSUSE_IMAGE_BASEDIR}_${DIB_RELEASE}/images/}
-BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-openSUSE-${DIB_RELEASE}-OS-rootfs.${ARCH}.tbz}
+BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-${OPENSUSE_IMAGE_FILE}-rootfs.${ARCH}.tbz}
 SHA256SUMS_FILE=${SHA256SUMS_FILE:-${BASE_IMAGE_FILE}.sha256}
 
 CACHED_FILE=$DIB_IMAGE_CACHE/$BASE_IMAGE_FILE

elements/redhat-common/bin/extract-image

@@ -59,7 +59,7 @@ function extract_image() {
 
             qemu-img convert -f qcow2 -O raw $CACHED_IMAGE $RAW_FILE
 
-            ROOT_PARTITON=p$(sudo kpartx -l $RAW_FILE | awk "/loop[0-9]+p/"|wc -l)
+            ROOT_PARTITION=p$(sudo kpartx -l $RAW_FILE | awk "/loop[0-9]+p/"|wc -l)
             sudo udevadm settle
 
             # kpartx fails if no /dev/loop* exists, "losetup -f" prints first unused
@@ -69,7 +69,7 @@ function extract_image() {
             # XXX: Parsing stdout is dangerous, would like a better way to discover
             #      the device used for the image.
             ROOT_LOOPDEV=$(sudo kpartx -av $RAW_FILE | \
-                awk "/loop[0-9]+$ROOT_PARTITON/ {print \$3}")
+                awk "/loop[0-9]+$ROOT_PARTITION/ {print \$3}")
             # If running inside Docker, make our nodes manually, because udev will not be working.
             if [ -f /.dockerenv ]; then
                 sudo dmsetup --noudevsync mknodes

elements/redhat-common/pre-install.d/15-remove-grub

@@ -23,7 +23,7 @@ fi
 # XXX : it is not clear this is necessary for fedora/centos7 and it's
 # install hooks.  Investigation is required.
 if rpm -q grub2; then
-    install-packages -e grub2
+    install-packages -e grub-pc
 fi
 
 # now configure things to re-install grub at the end.  We don't want
@@ -43,5 +43,5 @@ fi
 # So we download the latest grub2 package and setup the install script
 # to just install the single-package, which will be called later by
 # vm/finalise.d/51-bootloader
-install-packages -d /tmp/grub grub2
+install-packages -d /tmp/grub grub-pc
 echo "rpm -i /tmp/grub/*.rpm" > /tmp/grub/install

elements/rhel-common/pre-install.d/00-rhel-registration

@@ -97,6 +97,8 @@ case "${REG_METHOD:-}" in
             echo "Attaching with options: $attach_opts"
             subscription-manager attach $attach_opts
         fi
+        echo "Disabling all previous repos"
+        subscription-manager repos --disable=\*
         echo "Enabling repos: $repos"
         subscription-manager $repos
         ;;
@@ -108,6 +110,8 @@ case "${REG_METHOD:-}" in
         rpm -Uvh "$REG_SAT_URL/pub/katello-ca-consumer-latest.noarch.rpm" || true
         echo "Registering with options: $sanitized_opts"
         subscription-manager register $opts
+        echo "Disabling all previous repos"
+        subscription-manager repos --disable=\*
         echo "Enabling repos: $user_repos"
         subscription-manager $repos
         echo "Disabling satellite repo because it is no longer needed"

elements/runtime-ssh-host-keys/README.rst

@@ -0,0 +1,10 @@
+=====================
+runtime-ssh-host-keys
+=====================
+An element to generate SSH host keys on first boot.
+
+Since ssh key generation is not yet common to all operating systems, we need to
+create a DIB element to manage this. We force the removal of the SSH host keys,
+then add init scripts to generate them on first boot.
+
+This element currently supports Debian and Ubuntu (both systemd and upstart).

elements/runtime-ssh-host-keys/cleanup.d/90-remove-ssh-host-keys

@@ -10,9 +10,6 @@ set -o pipefail
 # in so that they are regenerated on first boot and
 # are unique.
 
-# TODO(greghaynes) This should be a thing we do for all images, not just
-# simple-init.
-
 if [ -d $TARGET_ROOT/etc/ssh ] ; then
     sudo find $TARGET_ROOT/etc/ssh -name 'ssh_host*' -type f -delete
 fi

elements/runtime-ssh-host-keys/element-deps

@@ -0,0 +1 @@
+dib-init-system

elements/runtime-ssh-host-keys/init-scripts/systemd/ssh-keygen.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=OpenSSH Server Key Generation
+Before=ssh.service
+
+ConditionPathExists=|!/etc/ssh/ssh_host_key
+ConditionPathExists=|!/etc/ssh/ssh_host_key.pub
+ConditionPathExists=|!/etc/ssh/ssh_host_rsa_key
+ConditionPathExists=|!/etc/ssh/ssh_host_rsa_key.pub
+ConditionPathExists=|!/etc/ssh/ssh_host_dsa_key
+ConditionPathExists=|!/etc/ssh/ssh_host_dsa_key.pub
+ConditionPathExists=|!/etc/ssh/ssh_host_ecdsa_key
+ConditionPathExists=|!/etc/ssh/ssh_host_ecdsa_key.pub
+ConditionPathExists=|!/etc/ssh/ssh_host_ed25519_key
+ConditionPathExists=|!/etc/ssh/ssh_host_ed25519_key.pub
+
+[Service]
+ExecStart=/usr/bin/ssh-keygen -A
+Type=oneshot
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target

elements/runtime-ssh-host-keys/init-scripts/upstart/ssh-keygen.conf

@@ -0,0 +1,8 @@
+description "OpenSSH Server Key Generation"
+
+start on starting ssh
+console output
+
+task
+
+exec /usr/bin/ssh-keygen -A

elements/runtime-ssh-host-keys/package-installs.yaml

@@ -0,0 +1 @@
+openssh-client:

elements/runtime-ssh-host-keys/pkg-map

@@ -0,0 +1,10 @@
+{
+  "family": {
+    "redhat": {
+      "openssh-client": "openssh"
+    },
+    "gentoo": {
+      "openssh-client": ""
+    }
+  }
+}

elements/runtime-ssh-host-keys/post-install.d/80-ssh-keygen

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+case "$DIB_INIT_SYSTEM" in
+    upstart)
+        # nothing to do
+        exit 0
+        ;;
+    systemd)
+        if [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then
+            # NOTE(pabelanger): Only support ubuntu / debian today.
+            systemctl enable ssh-keygen.service
+        else
+            # Since we are not enabling it, delete it.
+            rm /usr/lib/systemd/system/ssh-keygen.service
+        fi
+        ;;
+    openrc)
+        # let dib-init-system's postinstall handle enabling init scripts
+        exit 0
+        ;;
+    *)
+        echo "Unsupported init system"
+        exit 1
+        ;;
+esac

elements/simple-init/README.rst

@@ -33,3 +33,28 @@ not there.
 
 Finally, glean will handle ssh-keypair-injection from config
 drive if cloud-init is not installed.
+
+Chosing glean installation source
+---------------------------------
+
+By default glean is installed using pip using the latest release on pypi.
+It is also possible to install glean from a specified git repository
+location. This is useful for debugging and testing new glean changes
+for example. To do this you need to set these variables::
+
+  DIB_INSTALLTYPE_simple_init=repo
+  DIB_REPOLOCATION_glean=/path/to/glean/repo
+  DIB_REPOREF_glean=name_of_git_ref
+
+For example to test glean change 364516 do::
+
+  git clone https://git.openstack.org/openstack-infra/glean /tmp/glean
+  cd /tmp/glean
+  git review -d 364516
+  git checkout -b my-test-ref
+
+Then set your DIB env vars like this before running DIB::
+
+  DIB_INSTALLTYPE_simple_init=repo
+  DIB_REPOLOCATION_glean=/tmp/glean
+  DIB_REPOREF_glean=my-test-ref

elements/simple-init/element-deps

@@ -1,5 +1,5 @@
 cloud-init-datasources
-dib-init-system
 install-types
 pip-and-virtualenv
+runtime-ssh-host-keys
 source-repositories

elements/source-repositories/pkg-map

@@ -2,6 +2,9 @@
   "family": {
     "gentoo": {
       "git": "dev-vcs/git"
+    },
+    "suse": {
+      "git": "git-core"
     }
   },
   "default": {

elements/yum/bin/install-packages

@@ -14,7 +14,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+if [ ${DIB_DEBUG_TRACE:-0} -gt 1 ]; then
     set -x
 fi
 set -eu

elements/zypper-minimal/README.rst

@@ -0,0 +1,19 @@
+==============
+zypper-minimal
+==============
+Base element for creating minimal SUSE-based images
+
+This element is incomplete by itself so you probaby want to use it along
+with the opensuse-minimal one. It requires 'zypper' to be installed on the
+host.
+
+Repositories
+------------
+
+This element expects the `ZYPPER_REPOS` variable to be exported by the
+operating system element. This variable contains repository mappings in 
+the following format: `${repo_name}==>${repo_url}`. For example::
+
+ ZYPPER_REPOS="update=>http://download.opensuse.org/update/leap/42.1/oss/ "
+ ZYPPER_REPOS+="oss=>http://download.opensuse.org/distribution/leap/42.1/repo/oss/"
+ export ZYPPER_REPOS

elements/zypper-minimal/element-deps

@@ -0,0 +1,3 @@
+dib-run-parts
+package-installs
+zypper

elements/zypper-minimal/install.d/15-zypper-fstab

@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Copyright 2015 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+cat << EOF > /etc/fstab
+proc /proc proc nodev,noexec,nosuid 0 0
+LABEL=${DIB_ROOT_LABEL} / ${FS_TYPE} errors=remount-ro 0 1
+EOF

elements/zypper-minimal/package-installs.yaml

@@ -0,0 +1,10 @@
+# kernel
+linux-image-generic:
+# And a few useful tools. Some are pulled
+# as dependencies but that may change so lets
+# be explicit.
+bash:
+lsb-release:
+openssl:
+sed:
+sudo:

elements/zypper-minimal/root.d/08-zypper-chroot

@@ -0,0 +1,87 @@
+#!/bin/bash
+#
+# Copyright 2016 SUSE Linux Products Gmb
+# Copyright 2015 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+# dib-lint: disable=safe_sudo
+
+if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+[ -n "${ZYPPER_REPOS}" ]
+
+function cleanup() {
+    sudo umount $TMP_MOUNT_PATH/var/cache/zypp
+}
+
+trap cleanup EXIT
+
+ZYPPER_TARGET_OPTS="--non-interactive --gpg-auto-import-keys --root $TARGET_ROOT"
+ZYPPER_INSTALL_OPTS="--no-confirm --no-recommends"
+
+for repo in ${ZYPPER_REPOS}; do
+    reponame=repo-${repo%%=>*}
+    repouri=${repo##*=>}
+    sudo zypper ${ZYPPER_TARGET_OPTS} addrepo --name ${reponame} --keep-packages ${repouri} ${reponame}
+done
+
+# Refresh it
+sudo zypper ${ZYPPER_TARGET_OPTS} refresh
+
+# It appears that zypper will clean up the repo's cache when it (re-)adds the
+# repo so we need to add the cache now, once the repos are added. This is
+# similar to what the zypper/50-zypper-cache script does
+ZYPPER_CACHE_DIR=$DIB_IMAGE_CACHE/zypper
+mkdir -p $ZYPPER_CACHE_DIR
+
+sudo mkdir -p $TMP_MOUNT_PATH/var/cache/zypp
+sudo mount --bind $ZYPPER_CACHE_DIR $TMP_MOUNT_PATH/var/cache/zypp
+
+# Install filesystem, base and useful tools
+sudo zypper ${ZYPPER_TARGET_OPTS} install ${ZYPPER_INSTALL_OPTS} filesystem
+# Install basic components in order
+sudo zypper ${ZYPPER_TARGET_OPTS} install ${ZYPPER_INSTALL_OPTS} -t pattern base
+# Install a few useful tools
+sudo zypper ${ZYPPER_TARGET_OPTS} install ${ZYPPER_INSTALL_OPTS} python zypper
+
+# Put in a dummy /etc/resolv.conf over the temporary one we used
+# to bootstrap.  systemd has a bug/feature [1] that it will assume
+# you want systemd-networkd as the network manager and create a
+# broken symlink to /run/... if the base image doesn't have one.
+# This broken link confuses things like dhclient.
+# [1] https://bugzilla.redhat.com/show_bug.cgi?id=1197204
+echo -e "# This file intentionally left blank\n" | \
+    sudo tee $TARGET_ROOT/etc/resolv.conf
+
+# set the most reliable UTF-8 locale
+echo -e 'LANG="en_US.UTF-8"' | \
+sudo tee $TARGET_ROOT/etc/locale.conf
+# default to UTC
+sudo -E chroot $TARGET_ROOT ln -sf /usr/share/zoneinfo/UTC \
+    /etc/localtime
+
+# RPM doesn't know whether files have been changed since install
+# At this point though, we know for certain that we have changed no
+# config files, so anything marked .rpmnew is just a bug.
+for newfile in $(sudo find $TARGET_ROOT -type f -name '*rpmnew') ; do
+    sudo mv $newfile $(echo $newfile | sed 's/.rpmnew$//')
+done
+
+# Unmounting of the /var/cache/zypp is handled by the cleanup EXIT
+# handler so there is nothing else to do here

elements/zypper/bin/install-packages

@@ -20,7 +20,7 @@ fi
 set -eu
 set -o pipefail
 
-EXTRA_ARGS=""
+EXTRA_ARGS="--no-recommends"
 MAP_ELEMENT=""
 ACTION=install
 
@@ -47,7 +47,7 @@ eval set -- "$TEMP"
 
 while true ; do
     case "$1" in
-        -u) run_zypper dist-upgrade; exit 0;;
+        -u) run_zypper dist-upgrade --no-recommends; exit 0;;
         -e) ACTION="remove"; shift;;
         -d) EXTRA_ARGS="--download-only"; shift;;
         -m) MAP_ELEMENT=$2; shift 2;;

elements/zypper/bin/map-services

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# dib-lint: disable=indent
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+# Copyright 2014 SUSE, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import print_function
+import os
+import sys
+
+# Manually maintained for brevity; consider making this compiled from
+# distromatch or other rich data sources.
+# Debian name on the left, openSUSE on the right.
+service_map = {
+    # openstack mapping
+    'cinder-api': 'openstack-cinder-api',
+    'cinder-backup': 'openstack-cinder-backup',
+    'cinder-scheduler': 'openstack-cinder-scheduler',
+    'cinder-volume': 'openstack-cinder-volume',
+    'glance-api': 'openstack-glance-api',
+    'glance-reg': 'openstack-glance-registry',
+    'heat-api-cfn': 'openstack-heat-api-cfn',
+    'heat-api-cloudwatch': 'openstack-heat-api-cloudwatch',
+    'heat-api': 'openstack-heat-api',
+    'heat-engine': 'openstack-heat-engine',
+    'keystone': 'openstack-keystone',
+    'libvirt-bin': 'libvirtd',
+    'neutron-dhcp-agent': 'openstack-neutron-dhcp-agent',
+    'neutron-openvswitch-agent': 'openstack-neutron-openvswitch-agent',
+    'neutron-l3-agent': 'openstack-neutron-l3-agent',
+    'neutron-metadata-agent': 'openstack-neutron-metadata-agent',
+    'neutron-ovs-cleanup': 'openstack-neutron-ovs-cleanup',
+    'neutron-server': 'openstack-neutron',
+    'nova-api': 'openstack-nova-api',
+    'nova-cert': 'openstack-nova-cert',
+    'nova-compute': 'openstack-nova-compute',
+    'nova-conductor': 'openstack-nova-conductor',
+    'nova-consoleauth': 'openstack-nova-console',
+    'nova-baremetal-deploy-helper': 'openstack-nova-baremetal-deploy-helper',
+    'nova-novncproxy': 'openstack-nova-novncproxy',
+    'nova-scheduler': 'openstack-nova-scheduler',
+}
+
+print("WARNING: map-services has been deprecated.  "
+      "Please use the svc-map element.", file=sys.stderr)
+
+for arg in sys.argv[1:]:
+    # We need to support the service name being different when installing from
+    # source vs. packages. So, if the requested service file already exists,
+    # just use that.
+    if os.path.exists('/usr/lib/systemd/system/%s.service' % arg):
+        print(arg)
+    else:
+        print(service_map.get(arg, arg))
+sys.exit(0)

elements/zypper/element-deps

@@ -0,0 +1 @@
+install-bin

lib/common-defaults

@@ -34,6 +34,7 @@ fi
 ARCH=${ARCH:-$_ARCH}
 export ARCH
 
+export DIB_CHECKSUM=${DIB_CHECKSUM:-0}
 export DIB_NO_TMPFS=${DIB_NO_TMPFS:-0}
 export DIB_MIN_TMPFS=${DIB_MIN_TMPFS:-2}
 # Set via the CLI normally.

lib/common-functions

@@ -51,9 +51,19 @@ function finish_image () {
       old_image="${1%.*}"-$(date +%Y.%m.%d-%H.%M.%S).${1##*.}
       echo "Old image found. Renaming it to $old_image"
       mv "$1" "$old_image"
+      if [ -f "$1.md5" ]; then
+        mv "$1.md5" "$old_image.md5"
+      fi
+      if [ -f "$1.sha256" ]; then
+        mv "$1.sha256" "$old_image.sha256"
+      fi
     fi
 
     mv $OUT_IMAGE_PATH $1
+    if [ "$DIB_CHECKSUM" == "1" ]; then
+      md5sum $1 > $1.md5
+      sha256sum $1 > $1.sha256
+    fi
     echo "Image file $1 created..."
 }
 

releasenotes/notes/opensuse-minimal-45267f5be1112c22.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - New zypper-minimal and opensuse-minimal elements to create basic
+    openSUSE images. These two new elements are also making use of the
+    existing zypper element which has been extended to include the
+    functionality previously present in the opensuse element.

releasenotes/notes/runtime-ssh-host-keys-7a2fc873cc90d33e.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - New element (runtime-ssh-host-keys) to manage SSH host keys at boot. Since
+    SSH host key generation is not standard across operating systems, add
+    support for both Debian and Ubuntu to handle it. While this is a new
+    element, simple-init has been updated to depend on it.

releasenotes/source/conf.py

@@ -235,3 +235,6 @@ texinfo_documents = [
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
 #texinfo_no_detailmenu = False
+
+# -- Options for Internationalization output ------------------------------
+locale_dirs = ['locale/']

requirements.txt

@@ -4,7 +4,7 @@
 Babel>=2.3.4 # BSD
 dib-utils # Apache-2.0
 pbr>=1.6 # Apache-2.0
-PyYAML>=3.1.0 # MIT
+PyYAML>=3.10.0 # MIT
 flake8<2.6.0,>=2.5.4 # MIT
 six>=1.9.0 # MIT
-oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
+oslosphinx>=4.7.0 # Apache-2.0

test-requirements.txt

@@ -7,8 +7,8 @@ oslotest>=1.10.0 # Apache-2.0
 testrepository>=0.0.18 # Apache-2.0/BSD
 
 # Doc requirements
-sphinx!=1.3b1,<1.3,>=1.2.1 # BSD
+sphinx!=1.3b1,<1.4,>=1.2.1 # BSD
-oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
+oslosphinx>=4.7.0 # Apache-2.0
 
 # releasenotes
 reno>=1.8.0 # Apache2

tests/run_functests.sh

@@ -25,21 +25,65 @@ DEFAULT_SKIP_TESTS=(
     centos-minimal/build-succeeds
 )
 
+function log_with_prefix {
+    local pr=$1
+
+    while read a; do
+        echo $(date +"%Y%m%d-%H%M%S.%N") "[$pr] $a"
+    done
+}
+
+# Log job control messages
+function log_jc {
+    local msg="$1"
+    printf "[JOB-CONTROL] %s %s\n" "$(date)" "${msg}"
+}
+
+function job_cnt {
+    running_jobs=$(jobs -p)
+    echo ${running_jobs} | wc -w
+}
+
+# This is needed, because the better 'wait -n' is
+# available since bash 4.3 only.
+function wait_minus_n {
+    if [ "${BASH_VERSINFO[0]}" -gt 4 \
+                               -o "${BASH_VERSINFO[0]}" = 4 \
+                               -a "${BASH_VERSINFO[1]}" -ge 3 ]; then
+        # Good way: wait on any job
+        wait -n
+        return $?
+    else
+        # Not that good way: wait on one specific job
+        # (others may be finished in the mean time)
+        local wait_for_pid=$(jobs -p | head -1)
+        wait ${wait_for_pid}
+        return $?
+    fi
+}
+
 # run_disk_element_test <test_element> <element>
 #  Run a disk-image-build .tar build of ELEMENT including any elements
 #  specified by TEST_ELEMENT
 function run_disk_element_test() {
     local test_element=$1
     local element=$2
+    local dont_use_tmp=$3
+    local use_tmp_flag=""
     local dest_dir=$(mktemp -d)
 
-    trap "rm -rf $dest_dir /tmp/dib-test-should-fail" EXIT
+    trap "rm -rf $dest_dir" EXIT
+
+    if [ "${dont_use_tmp}" = "yes" ]; then
+        use_tmp_flag="--no-tmpfs"
+    fi
 
     if break="after-error" break_outside_target=1 \
-        break_cmd="cp \$TMP_MOUNT_PATH/tmp/dib-test-should-fail /tmp/ 2>&1 > /dev/null || true" \
+        break_cmd="cp -v \$TMP_MOUNT_PATH/tmp/dib-test-should-fail ${dest_dir} || true" \
         DIB_SHOW_IMAGE_USAGE=1 \
         ELEMENTS_PATH=$DIB_ELEMENTS:$DIB_ELEMENTS/$element/test-elements \
-        $DIB_CMD -x -t tar,qcow2 -o $dest_dir/image -n $element $test_element; then
+        $DIB_CMD -x -t tar,qcow2 ${use_tmp_flag} -o $dest_dir/image -n $element $test_element 2>&1 \
+           | log_with_prefix "${element}/${test_element}"; then
 
         if ! [ -f "$dest_dir/image.qcow2" ]; then
             echo "Error: qcow2 build failed for element: $element, test-element: $test_element."
@@ -60,7 +104,7 @@ function run_disk_element_test() {
             fi
         fi
     else
-        if [ -f "/tmp/dib-test-should-fail" ]; then
+        if [ -f "${dest_dir}/dib-test-should-fail" ]; then
             echo "PASS: Element $element, test-element: $test_element"
         else
             echo "Error: Build failed for element: $element, test-element: $test_element."
@@ -81,7 +125,8 @@ function run_ramdisk_element_test() {
     local dest_dir=$(mktemp -d)
 
     if ELEMENTS_PATH=$DIB_ELEMENTS/$element/test-elements \
-        $DIB_CMD -x -o $dest_dir/image $element $test_element; then
+        $DIB_CMD -x -o $dest_dir/image $element $test_element \
+            | log_with_prefix "${element}/${test_element}"; then
         # TODO(dtantsur): test also kernel presence once we sort out its naming
         # problem (vmlinuz vs kernel)
         if ! [ -f "$dest_dir/image.initramfs" ]; then
@@ -111,12 +156,15 @@ for e in $DIB_ELEMENTS/*/test-elements/*; do
     TESTS+=("$element/$test_element")
 done
 
-while getopts ":hl" opt; do
+JOB_MAX_CNT=1
+
+while getopts ":hlpj:" opt; do
     case $opt in
         h)
             echo "run_functests.sh [-h] [-l] <test> <test> ..."
             echo "  -h : show this help"
             echo "  -l : list available tests"
+            echo "  -p : run all tests in parallel"
             echo "  <test> : functional test to run"
             echo "           Special test 'all' will run all tests"
             exit 0
@@ -130,6 +178,10 @@ while getopts ":hl" opt; do
             echo
             exit 0
             ;;
+        j)
+            JOB_MAX_CNT=${OPTARG}
+            echo "Running parallel - using [${JOB_MAX_CNT}] jobs"
+            ;;
         \?)
             echo "Invalid option: -$OPTARG"
             exit 1
@@ -138,6 +190,15 @@ while getopts ":hl" opt; do
 done
 shift $((OPTIND-1))
 
+DONT_USE_TMP="no"
+if [ "${JOB_MAX_CNT}" -gt 1 ]; then
+    # switch off using tmp dir for image building
+    # (The mem check using the tmp dir is currently done
+    #  based on the available memory - and not on the free.
+    #  See #1618124 for more details)
+    DONT_USE_TMP="yes"
+fi
+
 # cull the list of tests to run into TESTS_TO_RUN
 TESTS_TO_RUN=()
 title=""
@@ -173,7 +234,36 @@ for test in "${TESTS_TO_RUN[@]}"; do
 done
 echo "------"
 
+function wait_and_exit_on_failure {
+    local pid=$1
+
+    wait ${pid}
+    result=$?
+
+    if [ "${result}" -ne 0 ]; then
+        exit ${result}
+    fi
+    return 0
+}
+
+EXIT_CODE=0
 for test in "${TESTS_TO_RUN[@]}"; do
+    running_jobs_cnt=$(job_cnt)
+    log_jc "Number of running jobs [${running_jobs_cnt}] max jobs [${JOB_MAX_CNT}]"
+    if [ "${running_jobs_cnt}" -ge "${JOB_MAX_CNT}" ]; then
+        log_jc "Waiting for job to finish"
+        wait_minus_n
+        result=$?
+
+        if [ "${result}" -ne 0 ]; then
+            EXIT_CODE=1
+            # If a job fails, do not start any new ones.
+            break
+        fi
+    fi
+
+    log_jc "Starting new job"
+
     # from above; each array value is element/test_element.  split it
     # back up
     element=${test%/*}
@@ -188,7 +278,30 @@ for test in "${TESTS_TO_RUN[@]}"; do
     fi
 
     echo "Running $test ($element_type)"
-    run_${element_type}_element_test $test_element $element
+    run_${element_type}_element_test $test_element $element ${DONT_USE_TMP} &
 done
 
+# Wait for the rest of the jobs
+while true; do
+    running_jobs_cnt=$(job_cnt)
+    log_jc "Number of running jobs left [${running_jobs_cnt}]"
+
+    if [ "${running_jobs_cnt}" -eq 0 ]; then
+        break;
+    fi
+
+    wait_minus_n
+    result=$?
+
+    if [ "${result}" -ne 0 ]; then
+        EXIT_CODE=1
+    fi
+done
+
+if [ "${EXIT_CODE}" -eq 0 ]; then
     echo "Tests passed!"
+    exit 0
+else
+    echo "At least one test failed"
+    exit 1
+fi