Release 1.25.2

-----BEGIN PGP SIGNATURE-----
 
 iQEcBAABAgAGBQJYV1yqAAoJEBty/58O8cX8hLwIAKP66w6MdPN8PDgUOteui/Sx
 N0UFKJ9yR4GQOAP0NffPLjch5/g0iJLs3eFKOhtGC1LjbDjpVgjX8vW18ib8wBZK
 GemOZPF3uxg8FROrZF1vpoDy/cHgL1YV10hCnwdjN/r9rb8zOuSabqjW+Dennj2n
 fZ0SJfa8Owfudn3YxGuOymVb/wMtEloDmVGBEI1Y+h7osELCCDi3OXmwsA8qMsdl
 cTwbeugBs4PlOVbZUK/JKGuwIHKgPnDYzYu5KpXw77/MdjGT0fo5Tlq5AOBDI2sC
 9JOFEBDli4Ro05VwvI58ADMpvvOax+9EvOhLbB1dRPdZl21Iyb6gOdy2PUbFO0c=
 =aKxq
 -----END PGP SIGNATURE-----

Merge tag '1.25.2' into merge-branch

Release 1.25.2

Change-Id: I698bcf2e82117bd81649cd065a7af5cac85990c7

bin/dib-lint

@@ -158,11 +158,24 @@ for i in $(find $ELEMENTS_DIR -type f   \
         fi
     fi
 
-    # check that environment files don't "set -x"
+    # check that environment files don't "set -x" and they have no executable
+    # bits set
     if [[ "$i" =~ (environment.d) ]]; then
         if grep -q "set -x" $i; then
             error "Environment file $i should not set tracing"
         fi
+        if [[ -x $i ]]; then
+            error "Environment file $i should not be marked as executable"
+        fi
+    fi
+
+    # check for
+    #  export FOO=$(bar)
+    # calls.  These are dangerous, because the export hides the return
+    # code of the $(bar) call.  Split this into 2 lines and -e will
+    # fail on the assignment
+    if grep -q 'export .*\$(' $i; then
+        error "Split export and assignments in $i"
     fi
 
     # check that sudo calls in phases run outside the chroot look

diskimage_builder/elements/apt-sources/test-elements/test-sources/environment.d/00-set-apt-sources

@@ -1 +1,2 @@
-export DIB_APT_SOURCES=$(mktemp)
+DIB_APT_SOURCES=$(mktemp)
+export DIB_APT_SOURCES

diskimage_builder/elements/cloud-init/README.rst

@@ -6,3 +6,12 @@ Install's and enables cloud-init for systems that don't come with it
 pre-installed
 
 Currently only supports Gentoo.
+
+Environment Variables
+---------------------
+
+DIB_CLOUD_INIT_ALLOW_PASS_AUTH
+  :Required: No
+  :Default: password authentication disabled when cloud-init installed
+  :Description: customize cloud-init to allow ssh password
+    authentication.

diskimage_builder/elements/cloud-init/post-install.d/21-cloud-init-allow-password-auth

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+# Set cloud-init to allow password authentication
+if [ -n "${DIB_CLOUD_INIT_ALLOW_SSH_PWAUTH:-}" ]; then
+    if [ -f "/etc/cloud/cloud.cfg" ]; then
+        if [ -z "$(cat /etc/cloud/cloud.cfg | grep ssh_pwauth)" ]; then
+            echo "ssh_pwauth not exist. append to EOF"
+            sudo sh -c 'echo "ssh_pwauth: 1" >> /etc/cloud/cloud.cfg'
+        else
+            echo "ssh_pwauth exist. make sure ssh_pwauth enabled"
+            sed -i -e 's/ssh_pwauth: *0/ssh_pwauth: 1/g' /etc/cloud/cloud.cfg
+            sed -i -e 's/ssh_pwauth: *False/ssh_pwauth: True/g' /etc/cloud/cloud.cfg
+            sed -i -e 's/ssh_pwauth: *false/ssh_pwauth: true/g' /etc/cloud/cloud.cfg
+        fi
+    fi
+fi

diskimage_builder/elements/debian-minimal/element-deps

@@ -1,2 +1,3 @@
 debootstrap
+dib-python
 pkg-map

diskimage_builder/elements/debian-minimal/root.d/75-debian-minimal-baseinstall

@@ -54,4 +54,13 @@ $apt_get update
 $apt_get clean
 $apt_get dist-upgrade -y
 
-$apt_get install -y busybox python sudo
+$apt_get install -y busybox sudo
+
+if [ "$DIB_PYTHON_VERSION" == "2" ]; then
+    $apt_get install -y python
+elif [ "$DIB_PYTHON_VERSION" == "3" ]; then
+    $apt_get install -y python3
+else
+    echo "ERROR: DIB_PYTHON_VERSION is '$DIB_PYTHON_VERSION' but needs to be 2 or 3"
+    exit 1
+fi

diskimage_builder/elements/dhcp-all-interfaces/install.d/50-dhcp-all-interfaces

@@ -13,7 +13,6 @@ if [ "$DISTRO_NAME" != "gentoo" ]; then
     install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.sh /usr/local/sbin/dhcp-all-interfaces.sh
 fi
 
-DIB_INIT_SYSTEM=$(dib-init-system)
 if [ "$DIB_INIT_SYSTEM" == "upstart" ]; then
     if [ -e "/etc/redhat-release" ] ; then
         # the init system is upstart but networking is using sysv compatabiliy (i.e. Centos/RHEL 6)
@@ -23,7 +22,7 @@ if [ "$DIB_INIT_SYSTEM" == "upstart" ]; then
         install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.conf /etc/init/dhcp-all-interfaces.conf
     fi
 elif [ "$DIB_INIT_SYSTEM" == "systemd" ]; then
-    install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-interface@.service /usr/lib/systemd/system/dhcp-interface@.service
+    install -D -g root -o root -m 0644 ${SCRIPTDIR}/dhcp-interface@.service /usr/lib/systemd/system/dhcp-interface@.service
     install -D -g root -o root -m 0644 ${SCRIPTDIR}/dhcp-all-interfaces-udev.rules /etc/udev/rules.d/99-dhcp-all-interfaces.rules
 elif [ "$DIB_INIT_SYSTEM" == "sysv" ]; then
     install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.init /etc/init.d/dhcp-all-interfaces

diskimage_builder/elements/dib-init-system/environment.d/10-dib-init-system.bash

@@ -1 +1,2 @@
-export DIB_INIT_SYSTEM=$(PATH="$PATH:$(dirname $BASH_SOURCE)/.." dib-init-system)
+DIB_INIT_SYSTEM=$(PATH="$PATH:$(dirname $BASH_SOURCE)/.." dib-init-system)
+export DIB_INIT_SYSTEM

diskimage_builder/elements/dib-python/README.rst

@@ -7,3 +7,6 @@ or python3 executable. This is useful for creating #! lines for scripts that
 are compatible with both python2 and python3.
 
 This does not install a python if one does not exist, and instead fails.
+
+This also exports a variable DIB_PYTHON_VERSION which will either be '2' or
+'3' depending on the python version which dib-python points to.

diskimage_builder/elements/dib-python/environment.d/50-dib-python-version

@@ -0,0 +1,28 @@
+# Pick which distros we need to force python2
+if [ -z "${DIB_PYTHON_VERSION:-}" ]; then
+    if [ "$DISTRO_NAME" == "ubuntu" ]; then
+        if [ "$DIB_RELEASE" == "precise" ]; then
+            DIB_PYTHON_VERSION=2
+        elif [ "$DIB_RELEASE" == "trusty" ]; then
+            DIB_PYTHON_VERSION=2
+        fi
+    elif [ "$DISTRO_NAME" == "debian" ]; then
+        DIB_PYTHON_VERSION=2
+    elif [ "$DISTRO_NAME" == "fedora" ]; then
+        if [ "$DIB_RELEASE" -le 22 ]; then
+            DIB_PYTHON_VERSION=2
+        fi
+    elif [ "$DISTRO_NAME" == "centos" ]; then
+        DIB_PYTHON_VERSION=2
+    elif [ "$DISTRO_NAME" == "centos7" ]; then
+        DIB_PYTHON_VERSION=2
+    elif [ "$DISTRO_NAME" == "opensuse" ]; then
+        DIB_PYTHON_VERSION=2
+    fi
+fi
+
+if [ -z "${DIB_PYTHON_VERSION:-}" ]; then
+    DIB_PYTHON_VERSION=3
+fi
+
+export DIB_PYTHON_VERSION

diskimage_builder/elements/dib-python/pre-install.d/01-dib-python

@@ -6,9 +6,9 @@ fi
 set -eu
 set -o pipefail
 
-python_path=$(command -v python2 || command -v python3)
+python_path=$(command -v python${DIB_PYTHON_VERSION})
 if [ -z "$python_path" ]; then
-    echo "Could not find python2 or python3 executable."
+    echo "Could not find python${DIB_PYTHON_VERSION} executable."
     exit 1
 fi
 

diskimage_builder/elements/enable-serial-console/install.d/20-stty

@@ -8,7 +8,6 @@ set -o pipefail
 
 SCRIPTDIR=$(dirname $0)
 
-DIB_INIT_SYSTEM=$(dib-init-system)
 if [ "$DIB_INIT_SYSTEM" == "upstart" ]; then
     install -m 0644 -o root -g root ${SCRIPTDIR}/ttySx.conf /etc/init/ttySx.conf
 elif [ "$DIB_INIT_SYSTEM" == "systemd" ]; then

diskimage_builder/elements/package-installs/README.rst

@@ -26,6 +26,10 @@ example ``package-installs.yaml``
     not-arch: ppc64, ppc64le
   lshw:
     arch: ppc64, ppc64le
+  python-dev:
+    dib_python_version: 2
+  python3-dev:
+    dib_python_version: 3
 
 example package-installs.json
 
@@ -46,6 +50,7 @@ the following default values::
     uninstall: False
     installtype: * (Install package for all installtypes)
     arch: * (Install package for all architectures)
+    dib_python_version: (2 or 3 depending on DIB_PYTHON_VERSION, see dib-python)
 
 Setting the installtype property causes the package only to be installed if
 the specified installtype would be used for the element. See the

diskimage_builder/elements/package-installs/bin/package-installs-squash

@@ -75,8 +75,12 @@ def collect_data(data, filename, element_name):
                              installtype == elem_installtype)
         valid_arch = _valid_for_arch(pkg_name, params.get('arch', None),
                                      params.get('not-arch', None))
+        dib_py_version = str(params.get('dib_python_version', ''))
+        dib_py_version_env = os.environ.get('DIB_PYTHON_VERSION', '')
+        valid_dib_python_version = (dib_py_version == '' or
+                                    dib_py_version == dib_py_version_env)
 
-        if valid_installtype and valid_arch:
+        if valid_installtype and valid_arch and valid_dib_python_version:
             data[phase][install].append((pkg_name, element_name))
 
     return data

diskimage_builder/elements/pip-and-virtualenv/README.rst

@@ -10,3 +10,6 @@ from get-pip.py and pip (respectively).
 To install pip and virtualenv from package:
 
   export DIB_INSTALLTYPE_pip_and_virtualenv=package
+
+Any element that uses these commands must be designated as
+05-* or higher to ensure that they are first installed.

diskimage_builder/elements/pip-and-virtualenv/element-deps

@@ -1,2 +1,4 @@
+dib-python
+install-types
 package-installs
 source-repositories

diskimage_builder/elements/pip-and-virtualenv/package-installs.yaml

@@ -2,7 +2,22 @@
 # early enough because pip depends on it.
 python-xml:
   phase: pre-install.d
+  dib_python_version: 2
 python-pip:
   installtype: package
+  dib_python_version: 2
+python3-pip:
+  installtype: package
+  dib_python_version: 3
 python-virtualenv:
   installtype: package
+  dib_python_version: 2
+python3-virtualenv:
+  installtype: package
+  dib_python_version: 3
+python-dev:
+  installtype: source
+  dib_python_version: 2
+python3-dev:
+  installtype: source
+  dib_python_version: 3

diskimage_builder/elements/pip-and-virtualenv/pkg-map

@@ -2,10 +2,17 @@
   "family": {
     "gentoo": {
       "python-pip": "dev-python/pip",
-      "python-virtualenv": "dev-python/virtualenv"
+      "python-virtualenv": "dev-python/virtualenv",
+      "python3-dev": "dev-lang/python"
     },
     "suse": {
-      "python-xml": "python-xml"
+      "python-xml": "python-xml",
+      "python2-dev": "python2-devel",
+      "python3-dev": "python3-devel"
+    },
+    "redhat": {
+      "python2-dev": "python2-devel",
+      "python3-dev": "python3-devel"
     }
   },
   "default": {

diskimage_builder/elements/python-brickclient/README.rst

@@ -0,0 +1,34 @@
+python-brickclient
+==================
+
+* This element is aimed for providing cinder local attach/detach functionality.
+* Currently the feature has a dependency on a known bug
+  "https://launchpad.net/bugs/1623549", which has been resolved and will be part
+  of the upstream with the next release of python-brick-cinderclient-ext.
+  Note: Current version of python-brick-cinderclient-ext i.e. 0.2.0 requires and update
+  to be made in Line32 fo below script.
+  /usr/share/python-brickclient/venv/lib/python2.7/site-packages/brick_cinderclient_ext/__init__.py
+  update "brick-python-cinderclient-ext" to "python-brick-cinderclient-ext".
+
+* Usage:
+  Pass the below shell script to parameter 'user-data' and set 'config-drive=true'
+  at the time of provisioning the node via nova-boot to make cinder local
+  attach/detach commands talk to your cloud controller.
+  [Example of Config Drive Script]
+    #!/bin/bash
+    FILE="/etc/bash.bashrc"
+    [ ! -f "$FILE" ] && touch "$FILE"
+    echo 'export OS_AUTH_URL="http://<controller_ip>:5000/v2.0"' >> "$FILE"
+    echo 'export OS_PASSWORD="password"'  >> "$FILE"
+    echo 'export OS_USERNAME="demo"' >> "$FILE"
+    echo 'export OS_TENANT_NAME="demo"'  >> "$FILE"
+    echo 'export OS_PROJECT_NAME="demo"' >> "$FILE"
+    exec bash
+  To attach: /usr/share/python-brickclient/venv/bin/cinder local-attach <volume_id>
+  To detach: /usr/share/python-brickclient/venv/bin/cinder local-detach <volume_id>
+
+* Alternatively, the same action can be completed manually at the node which does
+  not require setting up of config drive such as:
+  /usr/share/python-brickclient/venv/bin/cinder --os-username demo --os-password \
+  password  --os-tenant-name demo --os-project-name demo \
+  --os-auth-url=http://<controller_ip>:5000/v2.0 local-attach <volume_id>

diskimage_builder/elements/python-brickclient/element-deps

@@ -0,0 +1,2 @@
+package-installs
+pip-and-virtualenv

diskimage_builder/elements/python-brickclient/package-installs.yaml

@@ -0,0 +1,3 @@
+libssl-dev:
+libffi-dev:
+python-dev:

diskimage_builder/elements/python-brickclient/pkg-map

@@ -0,0 +1,14 @@
+{
+  "family": {
+    "redhat": {
+      "python-dev": "python2-devel",
+      "libssl-dev": "openssl-devel",
+      "libffi-dev": "libffi-devel"
+    }
+  },
+  "default": {
+    "python-dev": "python-dev",
+    "libssl-dev": "libssl-dev",
+    "libffi-dev": "libffi-dev"
+  }
+}

diskimage_builder/elements/python-brickclient/post-install.d/55-brick-client-install

@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# Copyright 2016 Hewlett Packard Enterprise Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then
+    set -x
+fi
+
+set -eu
+set -o pipefail
+
+PBCDIR=/usr/share/python-brickclient
+mkdir $PBCDIR
+
+# create the virtual environment
+virtualenv $PBCDIR/venv
+
+# Install the required packages inside virtual env
+$PBCDIR/venv/bin/pip install python-brick-cinderclient-ext
+
+ln -s $PBCDIR/venv/bin/python-brickclient /usr/local/bin/python-brickclient

diskimage_builder/elements/svc-map/extra-data.d/10-merge-svc-map-files

@@ -15,7 +15,6 @@
 import os
 import sys
 
-import six
 import yaml
 
 
@@ -39,7 +38,7 @@ def merge_data(source, destination, distro):
     result = dict()
     result.update(destination)
     family = os_family(distro)
-    for servicename, mapping in six.iteritems(source):
+    for servicename, mapping in source.items():
         if servicename in result:
             raise Exception("%s already found in services list" % servicename)
         if distro in mapping:

diskimage_builder/elements/sysctl/bin/sysctl-write-value

@@ -26,7 +26,18 @@ fi
 
 FILENAME="/etc/sysctl.d/${NAME}.conf"
 
+if [ -f $FILENAME ]; then
+    # check to make sure the settings match... otherwise fail
+    if ! grep -q "^$NAME = $VALUE" $FILENAME; then
+        echo "Conflicting sysctl.conf setting for $NAME == $VALUE. Found:"
+        grep "^$NAME" $FILENAME
+        exit 1
+    fi
+else
+
 cat > $FILENAME <<EOF_CAT
 # $COMMENT
 $NAME = $VALUE
 EOF_CAT
+
+fi

diskimage_builder/elements/ubuntu-minimal/root.d/75-ubuntu-minimal-baseinstall

@@ -43,4 +43,13 @@ $apt_get update
 $apt_get clean
 $apt_get dist-upgrade -y
 
-$apt_get install -y busybox python sudo
+$apt_get install -y busybox sudo
+
+if [ "$DIB_PYTHON_VERSION" == "2" ]; then
+    $apt_get install -y python
+elif [ "$DIB_PYTHON_VERSION" == "3" ]; then
+    $apt_get install -y python3
+else
+    echo "ERROR: DIB_PYTHON_VERSION is '$DIB_PYTHON_VERSION' but needs to be 2 or 3"
+    exit 1
+fi

diskimage_builder/elements/yum-minimal/package-installs.yaml

@@ -1,5 +1,6 @@
 audit:
 dracut:
+dracut-config-generic:
 grubby:
 kernel:
 initscripts:

diskimage_builder/lib/disk-image-create

@@ -423,7 +423,7 @@ fi
 
 # End: Creation of the partitions
 
-sudo mkfs $MKFS_OPTS -t $FS_TYPE -L ${DIB_ROOT_LABEL} ${IMAGE_BLOCK_DEVICE}
+sudo mkfs -t $FS_TYPE $MKFS_OPTS -L ${DIB_ROOT_LABEL} ${IMAGE_BLOCK_DEVICE}
 # Tuning the rootfs uuid works only for ext filesystems.
 if echo "$FS_TYPE" | grep -q "^ext"; then
     sudo tune2fs -U ${DIB_IMAGE_ROOT_FS_UUID} ${IMAGE_BLOCK_DEVICE}

doc/source/user_guide/building_an_image.rst

@@ -265,18 +265,25 @@ three partitions are created: all are about 1/3 of the disk size.
 Filesystem Caveat
 -----------------
 
-By default, disk-image-create uses a 4k byte-to-inode ratio when creating the
-filesystem in the image. This allows large 'whole-system' images to utilize
-several TB disks without exhausting inodes. In contrast, when creating images
-intended for tenant instances, this ratio consumes more disk space than an
-end-user would expect (e.g. a 50GB root disk has 47GB avail.). If the image is
-intended to run within a tens to hundrededs of gigabyte disk, setting the
-byte-to-inode ratio to the ext4 default of 16k will allow for more usable space
-on the instance. The default can be overridden by passing --mkfs-options like
-this::
+By default, disk-image-create uses a 4k byte-to-inode ratio when
+creating the filesystem in the image. This allows large 'whole-system'
+images to utilize several TB disks without exhausting inodes. In
+contrast, when creating images intended for tenant instances, this
+ratio consumes more disk space than an end-user would expect (e.g. a
+50GB root disk has 47GB avail.). If the image is intended to run
+within a tens to hundrededs of gigabyte disk, setting the
+byte-to-inode ratio to the ext4 default of 16k will allow for more
+usable space on the instance. The default can be overridden by passing
+``--mkfs-options`` like this::
 
     disk-image-create --mkfs-options '-i 16384' <distro> vm
 
+You can also select a different filesystem by setting the ``FS_TYPE``
+environment variable.
+
+Note ``--mkfs-options`` are options passed to the mfks *driver*,
+rather than ``mkfs`` itself (i.e. after the initial `-t` argument).
+
 Speedups
 --------
 If you have 4GB of available physical RAM (as reported by /proc/meminfo

setup.cfg

@@ -3,8 +3,8 @@ name = diskimage-builder
 summary = Golden Disk Image builder.
 description-file =
     README.rst
-author = HP Cloud Services
-author_email = openstack-dev@lists.openstack.org
+author = OpenStack
+author-email = openstack-dev@lists.openstack.org
 license: Apache License (2.0)
 home-page = http://docs.openstack.org/developer/diskimage-builder/
 classifier =

tests/run_functests.sh

@@ -32,6 +32,8 @@ DEFAULT_SKIP_TESTS=(
     debian-minimal/testing-build-succeeds
     # Currently failing due to bug in locale generation
     centos-minimal/build-succeeds
+    # No longer reasonable to test upstream (lacks a mirror in infra)
+    centos/build-succeeds
 )
 
 function log_with_prefix {

tox.ini

@@ -39,5 +39,5 @@ commands = python setup.py build_sphinx
 commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
 
 [flake8]
-ignore = E126,E127,E125,H202,H302,H803
+ignore = E125,E126,E127,H202,H803
 exclude =  .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,conf.py