diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 9561fb10..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include README.rst diff --git a/bin/dib-lint b/bin/dib-lint index b54f5921..964c982f 100755 --- a/bin/dib-lint +++ b/bin/dib-lint @@ -153,6 +153,12 @@ for i in $(find $ELEMENTS_DIR -type f \ fi fi + # check that environment files don't "set -x" + if [[ "$i" =~ (environment.d) ]]; then + if grep -q "set -x" $i; then + error "Environment file $i should not set tracing" + fi + fi # check that sudo calls in phases run outside the chroot look # "safe"; meaning that they seem to operate within the chroot diff --git a/diskimage_builder/elements/centos/environment.d/00-bootloader.bash b/diskimage_builder/elements/centos/environment.d/00-bootloader.bash index 6478bc96..d406155d 100755 --- a/diskimage_builder/elements/centos/environment.d/00-bootloader.bash +++ b/diskimage_builder/elements/centos/environment.d/00-bootloader.bash @@ -1,9 +1 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - export DIB_EXTLINUX=1 diff --git a/diskimage_builder/elements/centos7/root.d/10-centos7-cloud-image b/diskimage_builder/elements/centos7/root.d/10-centos7-cloud-image index cd79a418..7db1557d 100755 --- a/diskimage_builder/elements/centos7/root.d/10-centos7-cloud-image +++ b/diskimage_builder/elements/centos7/root.d/10-centos7-cloud-image @@ -9,10 +9,10 @@ set -o pipefail [ -n "$ARCH" ] [ -n "$TARGET_ROOT" ] -if [ 'amd64' = "$ARCH" ] ; then +if [[ "amd64 x86_64" =~ "$ARCH" ]]; then ARCH="x86_64" else - echo 'centos7 root element only support the amd64 $ARCH value.' + echo 'centos7 root element only support the x86_64 $ARCH value.' exit 1 fi diff --git a/diskimage_builder/elements/debian/package-installs.yaml b/diskimage_builder/elements/debian/package-installs.yaml index 862c274f..92d0131d 100644 --- a/diskimage_builder/elements/debian/package-installs.yaml +++ b/diskimage_builder/elements/debian/package-installs.yaml @@ -10,3 +10,4 @@ net-tools: cloud-init: cloud-utils: cloud-initramfs-growroot: +dialog: diff --git a/diskimage_builder/elements/dhcp-all-interfaces/install.d/50-dhcp-all-interfaces b/diskimage_builder/elements/dhcp-all-interfaces/install.d/50-dhcp-all-interfaces index f17b23a7..24c109eb 100755 --- a/diskimage_builder/elements/dhcp-all-interfaces/install.d/50-dhcp-all-interfaces +++ b/diskimage_builder/elements/dhcp-all-interfaces/install.d/50-dhcp-all-interfaces @@ -15,7 +15,13 @@ fi DIB_INIT_SYSTEM=$(dib-init-system) if [ "$DIB_INIT_SYSTEM" == "upstart" ]; then - install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.conf /etc/init/dhcp-all-interfaces.conf + if [ -e "/etc/redhat-release" ] ; then + # the init system is upstart but networking is using sysv compatabiliy (i.e. Centos/RHEL 6) + install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.init /etc/init.d/dhcp-all-interfaces + chkconfig dhcp-all-interfaces on + else + install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-all-interfaces.conf /etc/init/dhcp-all-interfaces.conf + fi elif [ "$DIB_INIT_SYSTEM" == "systemd" ]; then install -D -g root -o root -m 0755 ${SCRIPTDIR}/dhcp-interface@.service /usr/lib/systemd/system/dhcp-interface@.service install -D -g root -o root -m 0644 ${SCRIPTDIR}/dhcp-all-interfaces-udev.rules /etc/udev/rules.d/99-dhcp-all-interfaces.rules diff --git a/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.init b/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.init index 8023e154..89fb7919 100755 --- a/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.init +++ b/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.init @@ -6,6 +6,7 @@ # Default-Start: S # Default-Stop: 0 6 # X-Start-Before: networking +# chkconfig: 3 9 50 # Short-Description: Autodetect network interfaces # Description: Autodetect network interfaces during boot and configure them for DHCP ### END INIT INFO diff --git a/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.sh b/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.sh index cb30ceed..4884c435 100755 --- a/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.sh +++ b/diskimage_builder/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces.sh @@ -14,7 +14,12 @@ PATH=/sbin:$PATH if [ -d "/etc/network" ]; then CONF_TYPE="eni" elif [ -d "/etc/sysconfig/network-scripts/" ]; then - CONF_TYPE="netscripts" + CONF_TYPE="rhel-netscripts" + SCRIPTS_PATH="/etc/sysconfig/network-scripts/" +elif [ -d "/etc/sysconfig/network/" ]; then + # SUSE network scripts location + CONF_TYPE="suse-netscripts" + SCRIPTS_PATH="/etc/sysconfig/network/" else echo "Unsupported network configuration type!" exit 1 @@ -42,8 +47,10 @@ function enable_interface() { serialize_me if [ "$CONF_TYPE" == "eni" ]; then printf "auto $interface\niface $interface inet dhcp\n\n" >>$ENI_FILE - elif [ "$CONF_TYPE" == "netscripts" ]; then - printf "DEVICE=\"$interface\"\nBOOTPROTO=\"dhcp\"\nONBOOT=\"yes\"\nTYPE=\"Ethernet\"" >"/etc/sysconfig/network-scripts/ifcfg-$interface" + elif [ "$CONF_TYPE" == "rhel-netscripts" ]; then + printf "DEVICE=\"$interface\"\nBOOTPROTO=\"dhcp\"\nONBOOT=\"yes\"\nTYPE=\"Ethernet\"" >"${SCRIPTS_PATH}ifcfg-$interface" + elif [ "$CONF_TYPE" == "suse-netscripts" ]; then + printf "BOOTPROTO=\"dhcp\"\nSTARTMODE=\"auto\"" >"${SCRIPTS_PATH}ifcfg-$interface" fi echo "Configured $1" @@ -51,8 +58,8 @@ function enable_interface() { function config_exists() { local interface=$1 - if [ "$CONF_TYPE" == "netscripts" ]; then - if [ -f "/etc/sysconfig/network-scripts/ifcfg-$interface" ]; then + if [[ "$CONF_TYPE" =~ "netscripts" ]]; then + if [ -f "${SCRIPTS_PATH}ifcfg-$interface" ]; then return 0 fi else diff --git a/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.service b/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.service index 50e9d4aa..da970966 100644 --- a/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.service +++ b/diskimage_builder/elements/ironic-agent/install.d/ironic-agent-source-install/ironic-python-agent.service @@ -3,6 +3,7 @@ Description=Ironic Python Agent After=network-online.target [Service] +ExecStartPre=/usr/sbin/modprobe vfat ExecStart=/usr/local/bin/ironic-python-agent Restart=always RestartSec=30s diff --git a/diskimage_builder/elements/manifests/environment.d/14-manifests b/diskimage_builder/elements/manifests/environment.d/14-manifests index c2ae96dc..a125a6f2 100755 --- a/diskimage_builder/elements/manifests/environment.d/14-manifests +++ b/diskimage_builder/elements/manifests/environment.d/14-manifests @@ -1,5 +1,3 @@ -#!/bin/bash -# # Copyright 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -15,11 +13,5 @@ # under the License. # -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - export DIB_MANIFEST_IMAGE_DIR=${DIB_MANIFEST_IMAGE_DIR:-/etc/dib-manifests} export DIB_MANIFEST_SAVE_DIR=${DIB_MANIFEST_SAVE_DIR:-${IMAGE_NAME}.d/} diff --git a/diskimage_builder/elements/redhat-common/finalise.d/99-cleanup-tmp-grub b/diskimage_builder/elements/redhat-common/finalise.d/99-cleanup-tmp-grub deleted file mode 100755 index 6a790911..00000000 --- a/diskimage_builder/elements/redhat-common/finalise.d/99-cleanup-tmp-grub +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -rm -rf /tmp/grub diff --git a/diskimage_builder/elements/redhat-common/package-installs.yaml b/diskimage_builder/elements/redhat-common/package-installs.yaml index 025158f1..81e19aa9 100644 --- a/diskimage_builder/elements/redhat-common/package-installs.yaml +++ b/diskimage_builder/elements/redhat-common/package-installs.yaml @@ -6,14 +6,3 @@ traceroute: which: gettext: phase: pre-install.d - -# these are being installed to satisfy the dependencies of grub2. See -# 15-remove-grub for more details -grub2-tools: - phase: pre-install.d -os-prober: - phase: pre-install.d -redhat-lsb-core: - phase: pre-install.d -system-logos: - phase: pre-install.d diff --git a/diskimage_builder/elements/redhat-common/pre-install.d/15-remove-grub b/diskimage_builder/elements/redhat-common/pre-install.d/15-remove-grub deleted file mode 100755 index ba1d922a..00000000 --- a/diskimage_builder/elements/redhat-common/pre-install.d/15-remove-grub +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -# grub2 isn't available on rhel6/centos6; they are setup to use -# extlinux. skip this -# you would think we could match on $DISTRO or something else; but -# we can't because the rhel/centos elements are a bit mixed up; -# centos-minimal for example sets distro to "centos". so the best -# check is just for the original "grub-install" script -if [ -f /sbin/grub-install ]; then - exit 0 -fi - -# remove grub2 package. As described in -# elements/ubuntu/pre-install.d/00-remove-grub; the grub post-kernel -# install hook will barf if the block device can't be found (as -# happens in a chroot). -# -# XXX : it is not clear this is necessary for fedora/centos7 and it's -# install hooks. Investigation is required. -if rpm -q grub2; then - install-packages -e grub-pc -fi - -# now configure things to re-install grub at the end. We don't want -# to rely on vm/finalise.d/51-bootloader to simply reinstall the -# package via the package-manager, because at that point (during -# finalise) the build-time yum-cache has been unmounted (hence the -# local-cache looks empty) and yum may try to repopulate the -# local-cache with all the grub2 dependencies. This is slow, and -# potentially fills up the disk. -# -# XXX : At this point, keepcache=0 *should* probably be set for -# yum/dnf. We have not standarised/documented that this will be done, -# however. This would *probably* stop dependencies being populated -# into the cache. We could investigate this, and possibly remove this -# all together if we standardise some of these behaviours. - -# So we download the latest grub2 package and setup the install script -# to just install the single-package, which will be called later by -# vm/finalise.d/51-bootloader -install-packages -d /tmp/grub grub-pc -echo "rpm -i /tmp/grub/*.rpm" > /tmp/grub/install diff --git a/diskimage_builder/elements/rhel-common/os-refresh-config/pre-configure.d/06-rhel-registration b/diskimage_builder/elements/rhel-common/os-refresh-config/pre-configure.d/06-rhel-registration index 21f386f2..851aa08e 100755 --- a/diskimage_builder/elements/rhel-common/os-refresh-config/pre-configure.d/06-rhel-registration +++ b/diskimage_builder/elements/rhel-common/os-refresh-config/pre-configure.d/06-rhel-registration @@ -131,6 +131,8 @@ case "${REG_METHOD:-}" in rpm -Uvh "$REG_SAT_URL/pub/katello-ca-consumer-latest.noarch.rpm" || true echo "Registering with options: $sanitized_opts" subscription-manager register $opts + echo "Disabling all previous repos" + subscription-manager repos --disable=\* echo "Enabling repos: $user_repos" subscription-manager $repos echo "Installing katello-agent" diff --git a/diskimage_builder/elements/rhel-common/pre-install.d/00-rhel-registration b/diskimage_builder/elements/rhel-common/pre-install.d/00-rhel-registration index 60c449d8..a21ef041 100755 --- a/diskimage_builder/elements/rhel-common/pre-install.d/00-rhel-registration +++ b/diskimage_builder/elements/rhel-common/pre-install.d/00-rhel-registration @@ -114,8 +114,6 @@ case "${REG_METHOD:-}" in subscription-manager repos --disable=\* echo "Enabling repos: $user_repos" subscription-manager $repos - echo "Disabling satellite repo because it is no longer needed" - subscription-manager repos --disable ${satellite_repo} ;; disable) echo "Disabling RHEL registration" diff --git a/diskimage_builder/elements/rhel7/root.d/10-rhel7-cloud-image b/diskimage_builder/elements/rhel7/root.d/10-rhel7-cloud-image index be38caa4..bc54b101 100755 --- a/diskimage_builder/elements/rhel7/root.d/10-rhel7-cloud-image +++ b/diskimage_builder/elements/rhel7/root.d/10-rhel7-cloud-image @@ -9,10 +9,10 @@ set -o pipefail [ -n "$ARCH" ] [ -n "$TARGET_ROOT" ] -if [ 'amd64' = "$ARCH" ] ; then +if [[ "amd64 x86_64" =~ "$ARCH" ]]; then ARCH="x86_64" else - echo 'rhel7 root element only support the amd64 $ARCH value.' + echo 'rhel7 root element only support the x86_64 $ARCH value.' exit 1 fi diff --git a/diskimage_builder/elements/rpm-distro/pre-install.d/01-override-yum-arch b/diskimage_builder/elements/rpm-distro/pre-install.d/01-override-yum-arch index ff84375c..c1347ec9 100755 --- a/diskimage_builder/elements/rpm-distro/pre-install.d/01-override-yum-arch +++ b/diskimage_builder/elements/rpm-distro/pre-install.d/01-override-yum-arch @@ -9,7 +9,7 @@ set -o pipefail if [ "i386" = "$ARCH" ]; then basearch=i386 arch=i686 -elif [ "amd64" = "$ARCH" ]; then +elif [[ "amd64 x86_64" =~ "$ARCH" ]]; then basearch=x86_64 arch=x86_64 elif [[ "$ARCH" = "ppc64" ]]; then @@ -28,7 +28,7 @@ fi if [[ $DISTRO_NAME == "fedora" && $DIB_RELEASE -ge 22 ]]; then mkdir -p /etc/dnf/vars echo $basearch > /etc/dnf/vars/basearch - echo $arch > /etc/dnf/vars/basearch + echo $arch > /etc/dnf/vars/arch else echo $basearch > /etc/yum/vars/basearch echo $arch > /etc/yum/vars/arch diff --git a/diskimage_builder/elements/runtime-ssh-host-keys/element-deps b/diskimage_builder/elements/runtime-ssh-host-keys/element-deps index 3a027762..69a71fde 100644 --- a/diskimage_builder/elements/runtime-ssh-host-keys/element-deps +++ b/diskimage_builder/elements/runtime-ssh-host-keys/element-deps @@ -1 +1,2 @@ dib-init-system +install-static diff --git a/diskimage_builder/elements/runtime-ssh-host-keys/init-scripts/systemd/ssh-keygen.service b/diskimage_builder/elements/runtime-ssh-host-keys/init-scripts/systemd/ssh-keygen.service index 90a83136..ef2201bc 100644 --- a/diskimage_builder/elements/runtime-ssh-host-keys/init-scripts/systemd/ssh-keygen.service +++ b/diskimage_builder/elements/runtime-ssh-host-keys/init-scripts/systemd/ssh-keygen.service @@ -2,19 +2,8 @@ Description=OpenSSH Server Key Generation Before=ssh.service -ConditionPathExists=|!/etc/ssh/ssh_host_key -ConditionPathExists=|!/etc/ssh/ssh_host_key.pub -ConditionPathExists=|!/etc/ssh/ssh_host_rsa_key -ConditionPathExists=|!/etc/ssh/ssh_host_rsa_key.pub -ConditionPathExists=|!/etc/ssh/ssh_host_dsa_key -ConditionPathExists=|!/etc/ssh/ssh_host_dsa_key.pub -ConditionPathExists=|!/etc/ssh/ssh_host_ecdsa_key -ConditionPathExists=|!/etc/ssh/ssh_host_ecdsa_key.pub -ConditionPathExists=|!/etc/ssh/ssh_host_ed25519_key -ConditionPathExists=|!/etc/ssh/ssh_host_ed25519_key.pub - [Service] -ExecStart=/usr/bin/ssh-keygen -A +ExecStart=/usr/local/sbin/runtime-ssh-host-keys.sh Type=oneshot RemainAfterExit=yes diff --git a/diskimage_builder/elements/runtime-ssh-host-keys/init-scripts/upstart/ssh-keygen.conf b/diskimage_builder/elements/runtime-ssh-host-keys/init-scripts/upstart/ssh-keygen.conf index 3fa2c012..0f85e22a 100644 --- a/diskimage_builder/elements/runtime-ssh-host-keys/init-scripts/upstart/ssh-keygen.conf +++ b/diskimage_builder/elements/runtime-ssh-host-keys/init-scripts/upstart/ssh-keygen.conf @@ -5,4 +5,4 @@ console output task -exec /usr/bin/ssh-keygen -A +exec /usr/local/sbin/runtime-ssh-host-keys.sh diff --git a/diskimage_builder/elements/runtime-ssh-host-keys/pkg-map b/diskimage_builder/elements/runtime-ssh-host-keys/pkg-map index ce9fd939..79312403 100644 --- a/diskimage_builder/elements/runtime-ssh-host-keys/pkg-map +++ b/diskimage_builder/elements/runtime-ssh-host-keys/pkg-map @@ -5,6 +5,9 @@ }, "gentoo": { "openssh-client": "" + }, + "suse": { + "openssh-client": "openssh" } } } diff --git a/diskimage_builder/elements/simple-init/install.d/60-simple-init-remove-interfaces b/diskimage_builder/elements/simple-init/install.d/60-simple-init-remove-interfaces index 21bbb88a..5e47a854 100755 --- a/diskimage_builder/elements/simple-init/install.d/60-simple-init-remove-interfaces +++ b/diskimage_builder/elements/simple-init/install.d/60-simple-init-remove-interfaces @@ -12,6 +12,9 @@ set -o pipefail # Fedora rm -f /etc/sysconfig/network-scripts/ifcfg-eth* +# SUSE +rm -f /etc/sysconfig/network/ifcfg-eth* + # Ubuntu rm -f /etc/network/interfaces.d/eth* diff --git a/diskimage_builder/elements/ubuntu-minimal/environment.d/10-ubuntu-distro-name.bash b/diskimage_builder/elements/ubuntu-minimal/environment.d/10-ubuntu-distro-name.bash index ba6783f7..e9ae083e 100644 --- a/diskimage_builder/elements/ubuntu-minimal/environment.d/10-ubuntu-distro-name.bash +++ b/diskimage_builder/elements/ubuntu-minimal/environment.d/10-ubuntu-distro-name.bash @@ -1,4 +1,4 @@ export DISTRO_NAME=ubuntu -export DIB_RELEASE=${DIB_RELEASE:-trusty} +export DIB_RELEASE=${DIB_RELEASE:-xenial} export DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main,restricted,universe} export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://archive.ubuntu.com/ubuntu} diff --git a/diskimage_builder/elements/ubuntu/environment.d/10-ubuntu-distro-name.bash b/diskimage_builder/elements/ubuntu/environment.d/10-ubuntu-distro-name.bash index 2860427c..fa19f207 100644 --- a/diskimage_builder/elements/ubuntu/environment.d/10-ubuntu-distro-name.bash +++ b/diskimage_builder/elements/ubuntu/environment.d/10-ubuntu-distro-name.bash @@ -1,2 +1,2 @@ export DISTRO_NAME=ubuntu -export DIB_RELEASE=${DIB_RELEASE:-trusty} +export DIB_RELEASE=${DIB_RELEASE:-xenial} diff --git a/diskimage_builder/elements/yum-minimal/package-installs.yaml b/diskimage_builder/elements/yum-minimal/package-installs.yaml index e81126df..b7cf1e36 100644 --- a/diskimage_builder/elements/yum-minimal/package-installs.yaml +++ b/diskimage_builder/elements/yum-minimal/package-installs.yaml @@ -4,7 +4,7 @@ grubby: kernel: initscripts: man-pages: -redhat-lsb-core: +lsb_release: selinux-policy: selinux-policy-targeted: libselinux-python: diff --git a/diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot b/diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot index 50c0a6b2..10fe1d25 100755 --- a/diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot +++ b/diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot @@ -56,6 +56,7 @@ _RPM="rpm --dbpath=/var/lib/rpm" # has yum/yumdownloader function _install_repos { local packages + local rc # pre-install the base system packages via rpm. We previously # just left it up to yum to drag these in when we "yum install @@ -79,11 +80,21 @@ function _install_repos { packages+="${DISTRO_NAME}-repos " fi - yumdownloader \ + # yumdownloader puts repo xml files and such into a directory + # ${TMPDIR}/yum-$USER-random. Since we don't need this once the + # initial download happens, redirect TMPDIR for this call so we + # can clean it up nicely + local temp_tmp + temp_tmp=$(mktemp -d) + TMPDIR=${temp_tmp} yumdownloader \ --releasever=$DIB_RELEASE \ --setopt=reposdir=$TMP_HOOKS_PATH/yum.repos.d \ --destdir=$WORKING \ - ${packages} + ${packages} && rc=$? || rc=$? + rm -rf ${temp_tmp} + if [[ ${rc} != 0 ]]; then + die "Failed to download initial packages: ${packages}" + fi # --nodeps works around these wanting /bin/sh in some fedora # releases, see rhbz#1265873 diff --git a/diskimage_builder/lib/common-functions b/diskimage_builder/lib/common-functions index cb3a1a9e..d4f65f5e 100644 --- a/diskimage_builder/lib/common-functions +++ b/diskimage_builder/lib/common-functions @@ -142,21 +142,48 @@ function eval_run_d () { trap - ERR } +function kill_chroot_processes () { + if [ -z "${1}" ]; then + echo "ERROR: no chroot directory specified" + exit 1 + fi + for piddir in /proc/[0-9]*; do + pid=${piddir##/proc/} + pidname=$(cat $piddir/comm 2>/dev/null || echo "unknown") + # If there are open files from the chroot, just kill the process using + # these files. + if sudo readlink -f $piddir/root | grep -q $TMP_BUILD_DIR; then + echo "Killing chroot process: '${pidname}($pid)'" + sudo kill $pid + fi + done +} + function cleanup_build_dir () { if ! timeout 5 sh -c " while ! sudo rm -rf $TMP_BUILD_DIR/built; do sleep 1; done"; then echo "ERROR: unable to cleanly remove $TMP_BUILD_DIR/built" exit 1 fi sudo rm -rf $TMP_BUILD_DIR/mnt + kill_chroot_processes $TMP_BUILD_DIR if tmpfs_check 0; then - sudo umount -f $TMP_BUILD_DIR || true + # If kill_chroot_processes did not succeed then we have to wait for + # init to reap the orphaned chroot processes + if ! timeout 120 sh -c "while ! sudo umount -f $TMP_BUILD_DIR; do sleep 1; done"; then + echo "ERROR: failed to umount the $TMP_BUILD_DIR tmpfs mount point" + exit 1 + fi fi rm -rf --one-file-system $TMP_BUILD_DIR } function cleanup_image_dir () { + kill_chroot_processes $TMP_IMAGE_DIR if tmpfs_check 0; then - sudo umount -f $TMP_IMAGE_DIR || true + if ! timeout 120 sh -c "while ! sudo umount -f $TMP_IMAGE_DIR; do sleep 1; done"; then + echo "ERROR: failed to umount the $TMP_IMAGE_DIR tmpfs mount point" + exit 1 + fi fi rm -rf --one-file-system $TMP_IMAGE_DIR } diff --git a/diskimage_builder/lib/disk-image-create b/diskimage_builder/lib/disk-image-create index 816858b3..1288582d 100644 --- a/diskimage_builder/lib/disk-image-create +++ b/diskimage_builder/lib/disk-image-create @@ -79,7 +79,7 @@ function show_options () { echo " --docker-target -- specify the repo and tag to use if the output type is docker. Defaults to the value of output imagename" if [ "$IS_RAMDISK" == "0" ]; then echo " -n skip the default inclusion of the 'base' element" - echo " -p package[,package,package] -- list of packages to install in the image" + echo " -p package[,package,package] -- list of packages to install in the image. If specified multiple times the packages are appended to the list." fi echo " -h|--help -- display this help and exit" echo " --version -- display version and exit" @@ -136,7 +136,7 @@ while true ; do -u) shift; export COMPRESS_IMAGE="";; -c) shift ; export CLEAR_ENV=1;; -n) shift; export SKIP_BASE="1";; - -p) IFS="," read -a INSTALL_PACKAGES <<< "$2"; export INSTALL_PACKAGES ; shift 2 ;; + -p) IFS="," read -a _INSTALL_PACKAGES <<< "$2"; export INSTALL_PACKAGES=( ${INSTALL_PACKAGES[@]} ${_INSTALL_PACKAGES[@]} ) ; shift 2 ;; --checksum) shift; export DIB_CHECKSUM=1;; --image-size) export DIB_IMAGE_SIZE=$2; shift 2;; --image-cache) export DIB_IMAGE_CACHE=$2; shift 2;; diff --git a/doc/source/developer/developing_elements.rst b/doc/source/developer/developing_elements.rst index ae58c51f..8c4b89fb 100644 --- a/doc/source/developer/developing_elements.rst +++ b/doc/source/developer/developing_elements.rst @@ -172,11 +172,12 @@ the image as executable files. Environment Variables ^^^^^^^^^^^^^^^^^^^^^ -To set environment variables for other hooks, add a file to your element -``environment.d``. - -This directory contains bash script snippets that are sourced before running -scripts in each phase. +To set environment variables for other hooks, add a file to your +element ``environment.d``. This directory contains bash script +snippets that are sourced before running scripts in each phase. Note +that because environment includes are sourced together, they should +not set global flags like ``set -x`` because they will affect all +preceeding imports. Dependencies diff --git a/elements/opensuse-minimal/environment.d/10-opensuse-distro-name.bash b/elements/opensuse-minimal/environment.d/10-opensuse-distro-name.bash index 58fff6d2..c6aafbbc 100644 --- a/elements/opensuse-minimal/environment.d/10-opensuse-distro-name.bash +++ b/elements/opensuse-minimal/environment.d/10-opensuse-distro-name.bash @@ -1,5 +1,6 @@ export DISTRO_NAME=opensuse -export DIB_RELEASE=${DIB_RELEASE:-42.1} +DIB_RELEASE=${DIB_RELEASE:-42.1} +export DIB_RELEASE=${DIB_RELEASE,,} export DIB_OPENSUSE_MIRROR=${DIB_OPENSUSE_MIRROR:-http://download.opensuse.org} case ${DIB_RELEASE} in # We are using "=>" as the assignment symbol since "@" "=" etc could be used in the URI itself. @@ -15,6 +16,11 @@ case ${DIB_RELEASE} in ZYPPER_REPOS="update=>${DIB_OPENSUSE_MIRROR}/update/leap/${DIB_RELEASE}/oss/ " ZYPPER_REPOS+="oss=>${DIB_OPENSUSE_MIRROR}/distribution/leap/${DIB_RELEASE}/repo/oss/" ;; + # Tumbleweed + tumbleweed) + ZYPPER_REPOS="update=>${DIB_OPENSUSE_MIRROR}/update/${DIB_RELEASE}/ " + ZYPPER_REPOS+="oss=>${DIB_OPENSUSE_MIRROR}/${DIB_RELEASE}/repo/oss/" + ;; *) echo "Unsupported openSUSE release: ${DIB_RELEASE}"; exit 1 ;; esac export ZYPPER_REPOS diff --git a/elements/runtime-ssh-host-keys/static/usr/local/sbin/runtime-ssh-host-keys.sh b/elements/runtime-ssh-host-keys/static/usr/local/sbin/runtime-ssh-host-keys.sh new file mode 100755 index 00000000..4fa2374d --- /dev/null +++ b/elements/runtime-ssh-host-keys/static/usr/local/sbin/runtime-ssh-host-keys.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright 2016 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# dib-lint: disable=dibdebugtrace + +set -exu +set -o pipefail + +# We are running into race conditions with glean, which ssh-keygen -A is +# not handling properly. So, create a new script to first check if the +# file exists, then use 'yes' to disable overwriting of existing files. + +for key in dsa ecdsa ed25519 rsa; do + FILE=/etc/ssh/ssh_host_${key}_key + if ! [ -e $FILE ]; then + /usr/bin/yes n | /usr/bin/ssh-keygen -f $FILE -N '' -t $key + fi +done diff --git a/elements/sysctl/README.rst b/elements/sysctl/README.rst new file mode 100644 index 00000000..d6e37d1b --- /dev/null +++ b/elements/sysctl/README.rst @@ -0,0 +1,12 @@ +====== +sysctl +====== + +Add a sysctl-set-value command which can be run from within an element. +Running this command will cause the sysctl value to be set on boot (by +writing the value to /etc/sysctl.d). + +Example usage + +:: + sysctl-set-value net.ipv4.ip_forward 1 diff --git a/elements/sysctl/bin/sysctl-set-value b/elements/sysctl/bin/sysctl-set-value new file mode 100755 index 00000000..cbeb6eff --- /dev/null +++ b/elements/sysctl/bin/sysctl-set-value @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Copied from tripleo-image-element's sysctl element +# +# Validate and manage setting sysctl settings. +# +# The script is called with name/value pairs which are stored +# in the system default sysctl.d directory. Before adding new +# settings a validation is done to ensure that conflicting +# sysctl settings have not been requested. Once finished sysctl +# is used to activate the changes. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +NAME=${1:-} +VALUE=${2:-} +# Optional comment used to describe the setting +COMMENT=${3:-"This file was created by diskimage-builder."} + +if [ -z "$NAME" -o -z "$VALUE" ]; then + echo "NAME and VALUE are required." + exit 1 +fi + +FILENAME="/etc/sysctl.d/${NAME}.conf" + +if [ -f $FILENAME ]; then + # check to make sure the settings match... otherwise fail + if ! grep -q "^$NAME = $VALUE" $FILENAME; then + echo "Conflicting sysctl.conf setting for $NAME == $VALUE. Found:" + grep "^$NAME" $FILENAME + exit 1 + fi +else + + if ! sysctl -a | grep -q "^$NAME"; then + echo "Invalid sysctl key: $NAME" + exit 1 + fi + + sysctl-write-value $NAME "$VALUE" "$COMMENT" + + sysctl -p $FILENAME + +fi diff --git a/elements/sysctl/bin/sysctl-write-value b/elements/sysctl/bin/sysctl-write-value new file mode 100755 index 00000000..e60f5535 --- /dev/null +++ b/elements/sysctl/bin/sysctl-write-value @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Copied from tripleo-image-element's sysctl element +# +# Validate and manage setting sysctl settings. +# +# The script is called with name/value pairs which are stored +# in the system default sysctl.d directory. This script performs +# no checking, just writing out the file. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +NAME=${1:-} +VALUE=${2:-} +# Optional comment used to describe the setting +COMMENT=${3:-"This file was created by diskimage-builder."} + +if [ -z "$NAME" -o -z "$VALUE" ]; then + echo "Usage: sysctl-write-value [comment]" + exit 1 +fi + +FILENAME="/etc/sysctl.d/${NAME}.conf" + +cat > $FILENAME <=2.3.4 # BSD -pbr>=1.6 # Apache-2.0 +pbr>=1.8 # Apache-2.0 PyYAML>=3.10.0 # MIT flake8<2.6.0,>=2.5.4 # MIT six>=1.9.0 # MIT diff --git a/setup.cfg b/setup.cfg index 4adae9d6..80ac6fc2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,7 +6,7 @@ description-file = author = HP Cloud Services author_email = openstack-dev@lists.openstack.org license: Apache License (2.0) -home-page = https://git.openstack.org/cgit/openstack/diskimage-builder +home-page = http://docs.openstack.org/developer/diskimage-builder/ classifier = Development Status :: 3 - Alpha License :: OSI Approved :: Apache Software License diff --git a/test-requirements.txt b/test-requirements.txt index 8671a449..f9f2a6b8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,4 +11,4 @@ sphinx!=1.3b1,<1.4,>=1.2.1 # BSD oslosphinx>=4.7.0 # Apache-2.0 # releasenotes -reno>=1.8.0 # Apache2 +reno>=1.8.0 # Apache-2.0 diff --git a/tests/run_functests.sh b/tests/run_functests.sh index c04d551e..accc6247 100755 --- a/tests/run_functests.sh +++ b/tests/run_functests.sh @@ -29,16 +29,26 @@ DEFAULT_SKIP_TESTS=( function log_with_prefix { local pr=$1 + local log while read a; do - echo $(date +"%Y%m%d-%H%M%S.%N") "[$pr] $a" + log="[$pr] $a" + if [[ ${LOG_DATESTAMP} -ne 0 ]]; then + log="$(date +"%Y%m%d-%H%M%S.%N") ${log}" + fi + echo "${log}" done } # Log job control messages function log_jc { local msg="$1" - printf "[JOB-CONTROL] %s %s\n" "$(date)" "${msg}" + local log="[JOB-CONTROL] ${msg}" + + if [[ ${LOG_DATESTAMP} -ne 0 ]]; then + log="$(date +"%Y%m%d-%H%M%S.%N") ${log}" + fi + echo "${log}" } function job_cnt { @@ -158,15 +168,23 @@ for e in $DIB_ELEMENTS/*/test-elements/*; do TESTS+=("$element/$test_element") done +# +# Default values +# JOB_MAX_CNT=1 +LOG_DATESTAMP=0 -while getopts ":hlpj:" opt; do +# +# Parse args +# +while getopts ":hlj:t" opt; do case $opt in h) echo "run_functests.sh [-h] [-l] ..." echo " -h : show this help" echo " -l : list available tests" - echo " -p : run all tests in parallel" + echo " -j : parallel job count (default to 1)" + echo " -t : prefix log messages with timestamp" echo " : functional test to run" echo " Special test 'all' will run all tests" exit 0 @@ -184,6 +202,9 @@ while getopts ":hlpj:" opt; do JOB_MAX_CNT=${OPTARG} echo "Running parallel - using [${JOB_MAX_CNT}] jobs" ;; + t) + LOG_DATESTAMP=1 + ;; \?) echo "Invalid option: -$OPTARG" exit 1 diff --git a/tox.ini b/tox.ini index 10d870f9..d4746467 100644 --- a/tox.ini +++ b/tox.ini @@ -37,4 +37,4 @@ commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasen [flake8] ignore = E125,H202,H302,H803 -exclude = .venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,conf.py +exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,conf.py