sig_cloud/diskimage-builder
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/v2

Browse Source 
Change-Id: I0747a27560161ef869868967ab140b5d0be9d4da
This commit is contained in:
Gregory Haynes 2016-10-31 11:21:23 -07:00
commit 7cc65d3cbd
61 changed files with 660 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
bin/disk-image-create
View File

@ -109,6 +109,7 @@ function show_options () {
echo " -x -- turn on tracing (use -x -x for very detailed tracing)"
echo " -u -- uncompressed; do not compress the image - larger but faster"
echo " -c -- clear environment before starting work"
echo " --checksum -- generate MD5 and SHA256 checksum files for the created image"
echo " --image-size size -- image size in GB for the created image"
echo " --image-cache directory -- location for cached images(default ~/.cache/image-create)"
echo " --max-online-resize size -- max number of filesystem blocks to support when resizing."
@ -172,7 +173,7 @@ DIB_DEFAULT_INSTALLTYPE=${DIB_DEFAULT_INSTALLTYPE:-"source"}
MKFS_OPTS=""
ACI_MANIFEST=${ACI_MANIFEST:-}
DOCKER_TARGET=""
TEMP=`getopt -o a:ho:t:xucnp: -l no-tmpfs,offline,help,version,min-tmpfs:,image-size:,image-cache:,max-online-resize:,mkfs-options:,qemu-img-options:,ramdisk-element:,root-label:,install-type:,docker-target: -n $SCRIPTNAME -- "$@"`
TEMP=`getopt -o a:ho:t:xucnp: -l checksum,no-tmpfs,offline,help,version,min-tmpfs:,image-size:,image-cache:,max-online-resize:,mkfs-options:,qemu-img-options:,ramdisk-element:,root-label:,install-type:,docker-target: -n $SCRIPTNAME -- "$@"`
if [ $? -ne 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
# Note the quotes around `$TEMP': they are essential!
@ -190,7 +191,8 @@ while true ; do
-c) shift ; export CLEAR_ENV=1;;
-n) shift; export SKIP_BASE="1";;
-p) IFS="," read -a INSTALL_PACKAGES <<< "$2"; export INSTALL_PACKAGES ; shift 2 ;;
--image-size) DIB_IMAGE_SIZE=$2; shift 2;;
--checksum) shift; export DIB_CHECKSUM=1;;
--image-size) export DIB_IMAGE_SIZE=$2; shift 2;;
--image-cache) export DIB_IMAGE_CACHE=$2; shift 2;;
--max-online-resize) export MAX_ONLINE_RESIZE=$2; shift 2;;
--mkfs-options) MKFS_OPTS=$2; shift 2;;
@ -338,26 +340,28 @@ mv $TMP_BUILD_DIR/mnt $TMP_BUILD_DIR/built
# logs with du output below.
xtrace=$(set +o | grep xtrace)
# temp file for holding du output
du_output=${TMP_BUILD_DIR}/du_output.tmp
if [ -n "$DIB_IMAGE_SIZE" ]; then
du_size=$(echo "$DIB_IMAGE_SIZE" | awk '{printf("%d\n",$1 * 1024 *1024)}')
else
set +o xtrace
echo "Calculating image size (this may take a minute)..."
du_output=$(sudo du -a -c -x ${TMP_BUILD_DIR}/built)
sudo du -a -c -x ${TMP_BUILD_DIR}/built > ${du_output}
# the last line is the total size from "-c".
# scale this by 0.6 to create a slightly bigger image
du_size=$(echo "$du_output" | tail -n1 | cut -f1 | \
awk '{print int($1 / 0.6)}')
du_size=$(tail -n1 ${du_output} | cut -f1 | awk '{print int($1 / 0.6)}')
$xtrace
fi
if [[ "${DIB_SHOW_IMAGE_USAGE:-0}" != 0 ]]; then
set +o xtrace
if [ -z "$du_output" ]; then
du_output=$(sudo du -a -c -x ${TMP_BUILD_DIR}/built)
if [ ! -f "$du_output" ]; then
sudo du -a -c -x ${TMP_BUILD_DIR}/built > ${du_output}
fi
du_output_show="sort -nr |
du_output_show="sort -nr ${du_output} |
numfmt --to=iec-i --padding=7
--suffix=B --field=1 --from-unit=1024"
@ -376,7 +380,7 @@ if [[ "${DIB_SHOW_IMAGE_USAGE:-0}" != 0 ]]; then
echo "================="
fi
eval ${du_output_show} <<< "$du_output"
eval ${du_output_show}
echo
echo "===== end image size report ====="
@ -385,6 +389,8 @@ if [[ "${DIB_SHOW_IMAGE_USAGE:-0}" != 0 ]]; then
$xtrace
fi
rm -f ${du_output}
if [ "$FS_TYPE" = "ext4" ] ; then
# Very conservative to handle images being resized a lot
# We set journal size to 64M so our journal is large enough when we

2
bindep.txt Normal file
View File

@ -0,0 +1,2 @@
# This is a cross-platform list tracking distribution packages needed by tests;
# see http://docs.openstack.org/infra/bindep/ for additional information.

2
doc/source/conf.py
View File

@ -124,7 +124,7 @@ html_theme = 'default'
#html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
html_sidebars = { '**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html'], }
# Additional templates that should be rendered to pages, maps page names to
# template names.

20
doc/source/copyright.rst
View File

@ -1,20 +0,0 @@
Copyright
=========
Copyright 2012 Hewlett-Packard Development Company, L.P.
Copyright (c) 2012 NTT DOCOMO, INC.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.

5
doc/source/developer/developing_elements.rst
View File

@ -412,6 +412,11 @@ line to run it. If it should not be run as part of the default CI
run, you can submit a change with it added to ``DEFAULT_SKIP_TESTS``
in that file.
Running the functional tests is time consuming. Multiple parallel
jobs can be started by specifying ``-j <job count>``. Each of the
jobs uses a lot resources (CPU, disk space, RAM) - therefore the job
count must carefully be chosen.
python
""""""

32
doc/source/developer/index.rst
View File

@ -1,5 +1,16 @@
Developer Documentation
=======================
Developer Guide
===============
.. toctree::
:maxdepth: 1
design
components
invocation
caches
developing_elements
dib_lint
stable_interfaces
This documentation explains how to get started with creating your own
disk-image-builder elements as well as some high level concepts for element
@ -13,7 +24,7 @@ To get started developing with ``diskimage-builder``, install to a
$ mkdir dib
$ cd dib
$ virtualenv create env
$ virtualenv env
$ source env/bin/activate
$ git clone https://git.openstack.org/openstack/diskimage-builder
$ cd diskimage-builder
@ -24,13 +35,10 @@ and testing your changes. When you are done editing, use ``git
review`` to submit changes to the upstream gerrit.
.. toctree::
:maxdepth: 2
Finding Work
------------
design
components
invocation
caches
developing_elements
dib_lint
stable_interfaces
We maintain a list of low-hanging-fruit tags on launchpad:
* `https://bugs.launchpad.net/diskimage-builder/+bugs?field.tag=low-hanging-fruit`
<https://bugs.launchpad.net/diskimage-builder/+bugs?field.tag=low-hanging-fruit>

10
doc/source/index.rst
View File

@ -30,6 +30,15 @@ The code is available at:
<https://git.openstack.org/cgit/openstack/diskimage-builder/>`__
Issues
------
Issues are tracked on launchpad at:
* `https://bugs.launchpad.net/diskimage-builder/+bugs`
<https://bugs.launchpad.net/diskimage-builder/+bugs>
Communication
-------------
@ -46,4 +55,3 @@ Table of Contents
user_guide/index
developer/index
elements
copyright

10
doc/source/user_guide/installation.rst
View File

@ -55,3 +55,13 @@ Installing via pip is as simple as:
pip install diskimage-builder
Package Installation
--------------------
On Gentoo you can emerge diskimage-builder directly.
::
emerge app-emulation/diskimage-builder

2
elements/cloud-init/post-install.d/20-enable-cloud-init
View File

@ -10,6 +10,6 @@ set -o pipefail
if [[ "${DISTRO_NAME}" == "gentoo" ]]; then
rc-update add cloud-config default
rc-update add cloud-final default
rc-update add cloud-init-local default
rc-update add cloud-init-local boot
rc-update add cloud-init default
fi

12
elements/debian-minimal/README.rst
View File

@ -11,16 +11,16 @@ There are two ways to configure apt-sources:
and security repositories is the default. In this case you can
overwrite the two environment variables to adapt the behavior:
`DIB_DISTRIBUTION_MIRROR`: the mirror to use
default: http://httpredir.debian.org/debian
default: http://ftp.us.debian.org/debian
`DIB_DEBIAN_COMPONENTS`: (default) `main`
a comma separated list of components. For Debian this can be
e.g. `main,contrib,non-free`.
Note that the default Debian series is `unstable`, and the default
mirrors for Debian can be problematic for `unstable`. Because apt
does not handle changing Packages files well across multiple out of
sync mirrors, it is recommended that you choose a single mirror of
Debian, and pass it in via `DIB_DISTRIBUTION_MIRROR`.
Note it is not recommended to use http://httpredir.debian.org/ for
`DIB_DISTRIBUTION_MIRROR` due to how unreliable it is. Be sure to
select a mirror from the official mirror list:
https://www.debian.org/mirror/list
By default only `main` component is used. If
`DIB_DEBIAN_COMPONENTS` (comma separated) from the `debootstrap`

2
elements/debian-minimal/environment.d/10-debian-minimal.bash
View File

@ -1,6 +1,6 @@
export DISTRO_NAME=debian
export DIB_RELEASE=${DIB_RELEASE:-stable}
export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://httpredir.debian.org/debian}
export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://ftp.us.debian.org/debian}
export DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main}
export DIB_DEBIAN_COMPONENTS_WS=${DIB_DEBIAN_COMPONENTS//,/ }

2
elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces-udev.rules
View File

@ -1 +1 @@
SUBSYSTEM=="net", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}+="dhcp-interface@$name.service"
SUBSYSTEM=="net", KERNEL!="lo", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}+="dhcp-interface@$name.service"

1
elements/dhcp-all-interfaces/install.d/dhcp-interface@.service
View File

@ -11,6 +11,7 @@ User=root
ExecStartPre=/usr/local/sbin/dhcp-all-interfaces.sh %I
ExecStart=/sbin/ifup %I
RemainAfterExit=true
TimeoutStartSec=30s
[Install]
WantedBy=multi-user.target

1
elements/growroot/README.rst
View File

@ -9,3 +9,4 @@ This is only supported on:
* ubuntu trusty or later
* gentoo
* fedora & centos
* suse & opensuse

4
elements/growroot/pkg-map
View File

@ -8,6 +8,10 @@
"growpart": "cloud-utils",
"e2fsprogs": "e2fsprogs"
},
"suse": {
"growpart": "growpart",
"e2fsprogs": "e2fsprogs"
},
"gentoo": {
"growpart": "sys-fs/growpart",
"e2fsprogs": "sys-fs/e2fsprogs"

2
elements/ironic-agent/cleanup.d/99-ramdisk-create
View File

@ -11,6 +11,8 @@ set -o pipefail
[ -n "$TARGET_ROOT" ]
USER=${USER:-$(whoami)}
source $_LIB/img-functions
IMAGE_PATH=$(readlink -f $IMAGE_NAME)

28
elements/opensuse-minimal/README.rst Normal file
View File

@ -0,0 +1,28 @@
================
opensuse-minimal
================
This element will build a minimal openSUSE image. It requires 'zypper' to be
installed on the host.
These images should be considered experimental. There are curently only x86_64
images.
Environment Variables
---------------------
DIB_RELEASE
:Required: No
:Default: 42.1
:Description: Set the desired openSUSE release.
DIB_OPENSUSE_MIRROR:
:Required: No
:Default: http://download.opensuse.org
:Description: To use a specific openSUSE mirror, set this variable to the
mirror URL before running bin/disk-image-create. This URL
should point to the root directory as indicated in the
http://mirrors.opensuse.org/ webpage. You normally
don't want to change that since the default setting will
pick the mirror closest to you.
:Example: ``DIB_OPENSUSE_MIRROR=http://ftp.cc.uoc.gr/mirrors/linux/opensuse/opensuse/``

1
elements/opensuse-minimal/element-deps Normal file
View File

@ -0,0 +1 @@
zypper-minimal

1
elements/opensuse-minimal/element-provides Normal file
View File

@ -0,0 +1 @@
operating-system

20
elements/opensuse-minimal/environment.d/10-opensuse-distro-name.bash Normal file
View File

@ -0,0 +1,20 @@
export DISTRO_NAME=opensuse
export DIB_RELEASE=${DIB_RELEASE:-42.1}
export DIB_OPENSUSE_MIRROR=${DIB_OPENSUSE_MIRROR:-http://download.opensuse.org}
case ${DIB_RELEASE} in
# We are using "=>" as the assignment symbol since "@" "=" etc could be used in the URI itself.
# Remember, we can't export an array in bash so we use a string instead.
# Repo format: {name}=>{uri}
# Old openSUSE releases
13*)
ZYPPER_REPOS="update=>${DIB_OPENSUSE_MIRROR}/update/${DIB_RELEASE}/ "
ZYPPER_REPOS+="oss=>${DIB_OPENSUSE_MIRROR}/distribution/${DIB_RELEASE}/repo/oss/"
;;
# New Leap releases
42*)
ZYPPER_REPOS="update=>${DIB_OPENSUSE_MIRROR}/update/leap/${DIB_RELEASE}/oss/ "
ZYPPER_REPOS+="oss=>${DIB_OPENSUSE_MIRROR}/distribution/leap/${DIB_RELEASE}/repo/oss/"
;;
*) echo "Unsupported openSUSE release: ${DIB_RELEASE}"; exit 1 ;;
esac
export ZYPPER_REPOS

18
elements/opensuse/README.rst
View File

@ -13,6 +13,19 @@ For example, the images of openSUSE 13.2 can be found here:
These images should be considered experimental. There are curently only x86_64
images.
Environment Variables
---------------------
DIB_RELEASE
:Required: No
:Default: 13.1
:Description: Set the desired openSUSE release.
DIB_CLOUD_IMAGES
:Required: No
:Default: http://download.opensuse.org/repositories/Cloud:/Images:/(openSUSE|Leap)_${DIB_RELEASE}
:Description: Set the desired URL to fetch the images from.
Notes:
* There are very frequently new automated builds that include changes that
@ -21,8 +34,3 @@ Notes:
point to the latest image, but will frequently change its content. The versioned
one will never change content, but will frequently be deleted and replaced
by a newer build with a higher version-release number.
* Building with DIB\_EXTLINUX=1 doesn't work. It fails with:
/tmp/in\_target.d/finalise.d/51-bootloader: line 14: 16286 Segmentation fault
extlinux --install /boot/syslinux
(https://bugzilla.novell.com/show_bug.cgi?id=852856)

1
elements/opensuse/element-deps
View File

@ -1,5 +1,4 @@
cache-url
dib-run-parts
install-bin
package-installs
zypper

8
elements/opensuse/environment.d/10-opensuse-distro-name.bash
View File

@ -1 +1,9 @@
export DISTRO_NAME=opensuse
export DIB_RELEASE=${DIB_RELEASE:-13.1}
case ${DIB_RELEASE} in
# Old openSUSE releases
13*) export OPENSUSE_REPO_DIR=openSUSE_${DIB_RELEASE} ;;
# New Leap releases
42*) export OPENSUSE_REPO_DIR=openSUSE_Leap_${DIB_RELEASE} ;;
*) echo "Unsupported openSUSE release: ${DIB_RELEASE}"; exit 1 ;;
esac

2
elements/opensuse/pre-install.d/00-opensuse-setup
View File

@ -7,4 +7,4 @@ fi
set -eu
set -o pipefail
zypper ar -f http://download.opensuse.org/repositories/X11:/Bumblebee/openSUSE_13.1/X11:Bumblebee.repo
zypper ar -f http://download.opensuse.org/repositories/X11:/Bumblebee/${OPENSUSE_REPO_DIR}/X11:Bumblebee.repo

19
elements/opensuse/root.d/10-opensuse-cloud-image
View File

@ -18,11 +18,24 @@ if ! [ 'x86_64' = "$ARCH" ] ; then
exit 1
fi
DIB_RELEASE=${DIB_RELEASE:-13.1}
# Set some image defaults
case ${DIB_RELEASE} in
# Old openSUSE releases
13*)
OPENSUSE_IMAGE_BASEDIR=openSUSE
OPENSUSE_IMAGE_FILE=openSUSE-${DIB_RELEASE}-OS
;;
# New Leap releases
42*)
OPENSUSE_IMAGE_BASEDIR=Leap
OPENSUSE_IMAGE_FILE=openSUSE-Leap-${DIB_RELEASE}-OpenStack
;;
# We handle unknown cases in environment.d/10-opensuse-distro-name.bash
esac
# NOTE(toabctl): if something changes here on the buildservice side, please
# first ask in #opensuse-cloud on freenode before you change the format here!
DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-http://download.opensuse.org/repositories/Cloud:/Images:/openSUSE_${DIB_RELEASE}/images/}
BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-openSUSE-${DIB_RELEASE}-OS-rootfs.${ARCH}.tbz}
DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-http://download.opensuse.org/repositories/Cloud:/Images:/${OPENSUSE_IMAGE_BASEDIR}_${DIB_RELEASE}/images/}
BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-${OPENSUSE_IMAGE_FILE}-rootfs.${ARCH}.tbz}
SHA256SUMS_FILE=${SHA256SUMS_FILE:-${BASE_IMAGE_FILE}.sha256}
CACHED_FILE=$DIB_IMAGE_CACHE/$BASE_IMAGE_FILE

4
elements/redhat-common/bin/extract-image
View File

@ -59,7 +59,7 @@ function extract_image() {
qemu-img convert -f qcow2 -O raw $CACHED_IMAGE $RAW_FILE
ROOT_PARTITON=p$(sudo kpartx -l $RAW_FILE | awk "/loop[0-9]+p/"|wc -l)
ROOT_PARTITION=p$(sudo kpartx -l $RAW_FILE | awk "/loop[0-9]+p/"|wc -l)
sudo udevadm settle
# kpartx fails if no /dev/loop* exists, "losetup -f" prints first unused
@ -69,7 +69,7 @@ function extract_image() {
# XXX: Parsing stdout is dangerous, would like a better way to discover
# the device used for the image.
ROOT_LOOPDEV=$(sudo kpartx -av $RAW_FILE | \
awk "/loop[0-9]+$ROOT_PARTITON/ {print \$3}")
awk "/loop[0-9]+$ROOT_PARTITION/ {print \$3}")
# If running inside Docker, make our nodes manually, because udev will not be working.
if [ -f /.dockerenv ]; then
sudo dmsetup --noudevsync mknodes

4
elements/redhat-common/pre-install.d/15-remove-grub
View File

@ -23,7 +23,7 @@ fi
# XXX : it is not clear this is necessary for fedora/centos7 and it's
# install hooks. Investigation is required.
if rpm -q grub2; then
install-packages -e grub2
install-packages -e grub-pc
fi
# now configure things to re-install grub at the end. We don't want
@ -43,5 +43,5 @@ fi
# So we download the latest grub2 package and setup the install script
# to just install the single-package, which will be called later by
# vm/finalise.d/51-bootloader
install-packages -d /tmp/grub grub2
install-packages -d /tmp/grub grub-pc
echo "rpm -i /tmp/grub/*.rpm" > /tmp/grub/install

4
elements/rhel-common/pre-install.d/00-rhel-registration
View File

@ -97,6 +97,8 @@ case "${REG_METHOD:-}" in
echo "Attaching with options: $attach_opts"
subscription-manager attach $attach_opts
fi
echo "Disabling all previous repos"
subscription-manager repos --disable=\*
echo "Enabling repos: $repos"
subscription-manager $repos
;;
@ -108,6 +110,8 @@ case "${REG_METHOD:-}" in
rpm -Uvh "$REG_SAT_URL/pub/katello-ca-consumer-latest.noarch.rpm" || true
echo "Registering with options: $sanitized_opts"
subscription-manager register $opts
echo "Disabling all previous repos"
subscription-manager repos --disable=\*
echo "Enabling repos: $user_repos"
subscription-manager $repos
echo "Disabling satellite repo because it is no longer needed"

10
elements/runtime-ssh-host-keys/README.rst Normal file
View File

@ -0,0 +1,10 @@
=====================
runtime-ssh-host-keys
=====================
An element to generate SSH host keys on first boot.
Since ssh key generation is not yet common to all operating systems, we need to
create a DIB element to manage this. We force the removal of the SSH host keys,
then add init scripts to generate them on first boot.
This element currently supports Debian and Ubuntu (both systemd and upstart).

3
elements/simple-init/cleanup.d/90-remove-ssh-host-keys → elements/runtime-ssh-host-keys/cleanup.d/90-remove-ssh-host-keys
View File

@ -10,9 +10,6 @@ set -o pipefail
# in so that they are regenerated on first boot and
# are unique.
# TODO(greghaynes) This should be a thing we do for all images, not just
# simple-init.
if [ -d $TARGET_ROOT/etc/ssh ] ; then
sudo find $TARGET_ROOT/etc/ssh -name 'ssh_host*' -type f -delete
fi

1
elements/runtime-ssh-host-keys/element-deps Normal file
View File

@ -0,0 +1 @@
dib-init-system

22
elements/runtime-ssh-host-keys/init-scripts/systemd/ssh-keygen.service Normal file
View File

@ -0,0 +1,22 @@
[Unit]
Description=OpenSSH Server Key Generation
Before=ssh.service
ConditionPathExists=|!/etc/ssh/ssh_host_key
ConditionPathExists=|!/etc/ssh/ssh_host_key.pub
ConditionPathExists=|!/etc/ssh/ssh_host_rsa_key
ConditionPathExists=|!/etc/ssh/ssh_host_rsa_key.pub
ConditionPathExists=|!/etc/ssh/ssh_host_dsa_key
ConditionPathExists=|!/etc/ssh/ssh_host_dsa_key.pub
ConditionPathExists=|!/etc/ssh/ssh_host_ecdsa_key
ConditionPathExists=|!/etc/ssh/ssh_host_ecdsa_key.pub
ConditionPathExists=|!/etc/ssh/ssh_host_ed25519_key
ConditionPathExists=|!/etc/ssh/ssh_host_ed25519_key.pub
[Service]
ExecStart=/usr/bin/ssh-keygen -A
Type=oneshot
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target

8
elements/runtime-ssh-host-keys/init-scripts/upstart/ssh-keygen.conf Normal file
View File

@ -0,0 +1,8 @@
description "OpenSSH Server Key Generation"
start on starting ssh
console output
task
exec /usr/bin/ssh-keygen -A

1
elements/runtime-ssh-host-keys/package-installs.yaml Normal file
View File

@ -0,0 +1 @@
openssh-client:

10
elements/runtime-ssh-host-keys/pkg-map Normal file
View File

@ -0,0 +1,10 @@
{
"family": {
"redhat": {
"openssh-client": "openssh"
},
"gentoo": {
"openssh-client": ""
}
}
}

31
elements/runtime-ssh-host-keys/post-install.d/80-ssh-keygen Executable file
View File

@ -0,0 +1,31 @@
#!/bin/bash
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
case "$DIB_INIT_SYSTEM" in
upstart)
# nothing to do
exit 0
;;
systemd)
if [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then
# NOTE(pabelanger): Only support ubuntu / debian today.
systemctl enable ssh-keygen.service
else
# Since we are not enabling it, delete it.
rm /usr/lib/systemd/system/ssh-keygen.service
fi
;;
openrc)
# let dib-init-system's postinstall handle enabling init scripts
exit 0
;;
*)
echo "Unsupported init system"
exit 1
;;
esac

25
elements/simple-init/README.rst
View File

@ -33,3 +33,28 @@ not there.
Finally, glean will handle ssh-keypair-injection from config
drive if cloud-init is not installed.
Chosing glean installation source
---------------------------------
By default glean is installed using pip using the latest release on pypi.
It is also possible to install glean from a specified git repository
location. This is useful for debugging and testing new glean changes
for example. To do this you need to set these variables::
DIB_INSTALLTYPE_simple_init=repo
DIB_REPOLOCATION_glean=/path/to/glean/repo
DIB_REPOREF_glean=name_of_git_ref
For example to test glean change 364516 do::
git clone https://git.openstack.org/openstack-infra/glean /tmp/glean
cd /tmp/glean
git review -d 364516
git checkout -b my-test-ref
Then set your DIB env vars like this before running DIB::
DIB_INSTALLTYPE_simple_init=repo
DIB_REPOLOCATION_glean=/tmp/glean
DIB_REPOREF_glean=my-test-ref

2
elements/simple-init/element-deps
View File

@ -1,5 +1,5 @@
cloud-init-datasources
dib-init-system
install-types
pip-and-virtualenv
runtime-ssh-host-keys
source-repositories

3
elements/source-repositories/pkg-map
View File

@ -2,6 +2,9 @@
"family": {
"gentoo": {
"git": "dev-vcs/git"
},
"suse": {
"git": "git-core"
}
},
"default": {

2
elements/yum/bin/install-packages
View File

@ -14,7 +14,7 @@
# License for the specific language governing permissions and limitations
# under the License.
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
if [ ${DIB_DEBUG_TRACE:-0} -gt 1 ]; then
set -x
fi
set -eu

19
elements/zypper-minimal/README.rst Normal file
View File

@ -0,0 +1,19 @@
==============
zypper-minimal
==============
Base element for creating minimal SUSE-based images
This element is incomplete by itself so you probaby want to use it along
with the opensuse-minimal one. It requires 'zypper' to be installed on the
host.
Repositories
------------
This element expects the `ZYPPER_REPOS` variable to be exported by the
operating system element. This variable contains repository mappings in
the following format: `${repo_name}==>${repo_url}`. For example::
ZYPPER_REPOS="update=>http://download.opensuse.org/update/leap/42.1/oss/ "
ZYPPER_REPOS+="oss=>http://download.opensuse.org/distribution/leap/42.1/repo/oss/"
export ZYPPER_REPOS

3
elements/zypper-minimal/element-deps Normal file
View File

@ -0,0 +1,3 @@
dib-run-parts
package-installs
zypper

27
elements/zypper-minimal/install.d/15-zypper-fstab Executable file
View File

@ -0,0 +1,27 @@
#!/bin/bash
#
# Copyright 2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
cat << EOF > /etc/fstab
proc /proc proc nodev,noexec,nosuid 0 0
LABEL=${DIB_ROOT_LABEL} / ${FS_TYPE} errors=remount-ro 0 1
EOF

10
elements/zypper-minimal/package-installs.yaml Normal file
View File

@ -0,0 +1,10 @@
# kernel
linux-image-generic:
# And a few useful tools. Some are pulled
# as dependencies but that may change so lets
# be explicit.
bash:
lsb-release:
openssl:
sed:
sudo:

87
elements/zypper-minimal/root.d/08-zypper-chroot Executable file
View File

@ -0,0 +1,87 @@
#!/bin/bash
#
# Copyright 2016 SUSE Linux Products Gmb
# Copyright 2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# dib-lint: disable=safe_sudo
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
[ -n "${ZYPPER_REPOS}" ]
function cleanup() {
sudo umount $TMP_MOUNT_PATH/var/cache/zypp
}
trap cleanup EXIT
ZYPPER_TARGET_OPTS="--non-interactive --gpg-auto-import-keys --root $TARGET_ROOT"
ZYPPER_INSTALL_OPTS="--no-confirm --no-recommends"
for repo in ${ZYPPER_REPOS}; do
reponame=repo-${repo%%=>*}
repouri=${repo##*=>}
sudo zypper ${ZYPPER_TARGET_OPTS} addrepo --name ${reponame} --keep-packages ${repouri} ${reponame}
done
# Refresh it
sudo zypper ${ZYPPER_TARGET_OPTS} refresh
# It appears that zypper will clean up the repo's cache when it (re-)adds the
# repo so we need to add the cache now, once the repos are added. This is
# similar to what the zypper/50-zypper-cache script does
ZYPPER_CACHE_DIR=$DIB_IMAGE_CACHE/zypper
mkdir -p $ZYPPER_CACHE_DIR
sudo mkdir -p $TMP_MOUNT_PATH/var/cache/zypp
sudo mount --bind $ZYPPER_CACHE_DIR $TMP_MOUNT_PATH/var/cache/zypp
# Install filesystem, base and useful tools
sudo zypper ${ZYPPER_TARGET_OPTS} install ${ZYPPER_INSTALL_OPTS} filesystem
# Install basic components in order
sudo zypper ${ZYPPER_TARGET_OPTS} install ${ZYPPER_INSTALL_OPTS} -t pattern base
# Install a few useful tools
sudo zypper ${ZYPPER_TARGET_OPTS} install ${ZYPPER_INSTALL_OPTS} python zypper
# Put in a dummy /etc/resolv.conf over the temporary one we used
# to bootstrap. systemd has a bug/feature [1] that it will assume
# you want systemd-networkd as the network manager and create a
# broken symlink to /run/... if the base image doesn't have one.
# This broken link confuses things like dhclient.
# [1] https://bugzilla.redhat.com/show_bug.cgi?id=1197204
echo -e "# This file intentionally left blank\n" | \
sudo tee $TARGET_ROOT/etc/resolv.conf
# set the most reliable UTF-8 locale
echo -e 'LANG="en_US.UTF-8"' | \
sudo tee $TARGET_ROOT/etc/locale.conf
# default to UTC
sudo -E chroot $TARGET_ROOT ln -sf /usr/share/zoneinfo/UTC \
/etc/localtime
# RPM doesn't know whether files have been changed since install
# At this point though, we know for certain that we have changed no
# config files, so anything marked .rpmnew is just a bug.
for newfile in $(sudo find $TARGET_ROOT -type f -name '*rpmnew') ; do
sudo mv $newfile $(echo $newfile | sed 's/.rpmnew$//')
done
# Unmounting of the /var/cache/zypp is handled by the cleanup EXIT
# handler so there is nothing else to do here

4
elements/opensuse/bin/install-packages → elements/zypper/bin/install-packages
View File

@ -20,7 +20,7 @@ fi
set -eu
set -o pipefail
EXTRA_ARGS=""
EXTRA_ARGS="--no-recommends"
MAP_ELEMENT=""
ACTION=install
@ -47,7 +47,7 @@ eval set -- "$TEMP"
while true ; do
case "$1" in
-u) run_zypper dist-upgrade; exit 0;;
-u) run_zypper dist-upgrade --no-recommends; exit 0;;
-e) ACTION="remove"; shift;;
-d) EXTRA_ARGS="--download-only"; shift;;
-m) MAP_ELEMENT=$2; shift 2;;

0
elements/opensuse/bin/map-packages → elements/zypper/bin/map-packages
View File

66
elements/zypper/bin/map-services Executable file
View File

@ -0,0 +1,66 @@
#!/usr/bin/env python
# dib-lint: disable=indent
# Copyright 2012 Hewlett-Packard Development Company, L.P.
# Copyright 2014 SUSE, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import print_function
import os
import sys
# Manually maintained for brevity; consider making this compiled from
# distromatch or other rich data sources.
# Debian name on the left, openSUSE on the right.
service_map = {
# openstack mapping
'cinder-api': 'openstack-cinder-api',
'cinder-backup': 'openstack-cinder-backup',
'cinder-scheduler': 'openstack-cinder-scheduler',
'cinder-volume': 'openstack-cinder-volume',
'glance-api': 'openstack-glance-api',
'glance-reg': 'openstack-glance-registry',
'heat-api-cfn': 'openstack-heat-api-cfn',
'heat-api-cloudwatch': 'openstack-heat-api-cloudwatch',
'heat-api': 'openstack-heat-api',
'heat-engine': 'openstack-heat-engine',
'keystone': 'openstack-keystone',
'libvirt-bin': 'libvirtd',
'neutron-dhcp-agent': 'openstack-neutron-dhcp-agent',
'neutron-openvswitch-agent': 'openstack-neutron-openvswitch-agent',
'neutron-l3-agent': 'openstack-neutron-l3-agent',
'neutron-metadata-agent': 'openstack-neutron-metadata-agent',
'neutron-ovs-cleanup': 'openstack-neutron-ovs-cleanup',
'neutron-server': 'openstack-neutron',
'nova-api': 'openstack-nova-api',
'nova-cert': 'openstack-nova-cert',
'nova-compute': 'openstack-nova-compute',
'nova-conductor': 'openstack-nova-conductor',
'nova-consoleauth': 'openstack-nova-console',
'nova-baremetal-deploy-helper': 'openstack-nova-baremetal-deploy-helper',
'nova-novncproxy': 'openstack-nova-novncproxy',
'nova-scheduler': 'openstack-nova-scheduler',
}
print("WARNING: map-services has been deprecated. "
"Please use the svc-map element.", file=sys.stderr)
for arg in sys.argv[1:]:
# We need to support the service name being different when installing from
# source vs. packages. So, if the requested service file already exists,
# just use that.
if os.path.exists('/usr/lib/systemd/system/%s.service' % arg):
print(arg)
else:
print(service_map.get(arg, arg))
sys.exit(0)

1
elements/zypper/element-deps Normal file
View File

@ -0,0 +1 @@
install-bin

0
elements/opensuse/install.d/01-ccache-symlinks → elements/zypper/install.d/01-ccache-symlinks
View File

0
elements/opensuse/install.d/01-login-defs → elements/zypper/install.d/01-login-defs
View File

0
elements/opensuse/post-install.d/10-mkinitrd → elements/zypper/post-install.d/10-mkinitrd
View File

1
lib/common-defaults
View File

@ -34,6 +34,7 @@ fi
ARCH=${ARCH:-$_ARCH}
export ARCH
export DIB_CHECKSUM=${DIB_CHECKSUM:-0}
export DIB_NO_TMPFS=${DIB_NO_TMPFS:-0}
export DIB_MIN_TMPFS=${DIB_MIN_TMPFS:-2}
# Set via the CLI normally.

10
lib/common-functions
View File

@ -51,9 +51,19 @@ function finish_image () {
old_image="${1%.*}"-$(date +%Y.%m.%d-%H.%M.%S).${1##*.}
echo "Old image found. Renaming it to $old_image"
mv "$1" "$old_image"
if [ -f "$1.md5" ]; then
mv "$1.md5" "$old_image.md5"
fi
if [ -f "$1.sha256" ]; then
mv "$1.sha256" "$old_image.sha256"
fi
fi
mv $OUT_IMAGE_PATH $1
if [ "$DIB_CHECKSUM" == "1" ]; then
md5sum $1 > $1.md5
sha256sum $1 > $1.sha256
fi
echo "Image file $1 created..."
}

6
releasenotes/notes/opensuse-minimal-45267f5be1112c22.yaml Normal file
View File

@ -0,0 +1,6 @@
---
features:
- New zypper-minimal and opensuse-minimal elements to create basic
openSUSE images. These two new elements are also making use of the
existing zypper element which has been extended to include the
functionality previously present in the opensuse element.

6
releasenotes/notes/runtime-ssh-host-keys-7a2fc873cc90d33e.yaml Normal file
View File

@ -0,0 +1,6 @@
---
features:
- New element (runtime-ssh-host-keys) to manage SSH host keys at boot. Since
SSH host key generation is not standard across operating systems, add
support for both Debian and Ubuntu to handle it. While this is a new
element, simple-init has been updated to depend on it.

3
releasenotes/source/conf.py
View File

@ -235,3 +235,6 @@ texinfo_documents = [
# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False
# -- Options for Internationalization output ------------------------------
locale_dirs = ['locale/']

4
requirements.txt
View File

@ -4,7 +4,7 @@
Babel>=2.3.4 # BSD
dib-utils # Apache-2.0
pbr>=1.6 # Apache-2.0
PyYAML>=3.1.0 # MIT
PyYAML>=3.10.0 # MIT
flake8<2.6.0,>=2.5.4 # MIT
six>=1.9.0 # MIT
oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
oslosphinx>=4.7.0 # Apache-2.0

4
test-requirements.txt
View File

@ -7,8 +7,8 @@ oslotest>=1.10.0 # Apache-2.0
testrepository>=0.0.18 # Apache-2.0/BSD
# Doc requirements
sphinx!=1.3b1,<1.3,>=1.2.1 # BSD
oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
sphinx!=1.3b1,<1.4,>=1.2.1 # BSD
oslosphinx>=4.7.0 # Apache-2.0
# releasenotes
reno>=1.8.0 # Apache2

127
tests/run_functests.sh
View File

@ -25,21 +25,65 @@ DEFAULT_SKIP_TESTS=(
centos-minimal/build-succeeds
)
function log_with_prefix {
local pr=$1
while read a; do
echo $(date +"%Y%m%d-%H%M%S.%N") "[$pr] $a"
done
}
# Log job control messages
function log_jc {
local msg="$1"
printf "[JOB-CONTROL] %s %s\n" "$(date)" "${msg}"
}
function job_cnt {
running_jobs=$(jobs -p)
echo ${running_jobs} | wc -w
}
# This is needed, because the better 'wait -n' is
# available since bash 4.3 only.
function wait_minus_n {
if [ "${BASH_VERSINFO[0]}" -gt 4 \
-o "${BASH_VERSINFO[0]}" = 4 \
-a "${BASH_VERSINFO[1]}" -ge 3 ]; then
# Good way: wait on any job
wait -n
return $?
else
# Not that good way: wait on one specific job
# (others may be finished in the mean time)
local wait_for_pid=$(jobs -p | head -1)
wait ${wait_for_pid}
return $?
fi
}
# run_disk_element_test <test_element> <element>
# Run a disk-image-build .tar build of ELEMENT including any elements
# specified by TEST_ELEMENT
function run_disk_element_test() {
local test_element=$1
local element=$2
local dont_use_tmp=$3
local use_tmp_flag=""
local dest_dir=$(mktemp -d)
trap "rm -rf $dest_dir /tmp/dib-test-should-fail" EXIT
trap "rm -rf $dest_dir" EXIT
if [ "${dont_use_tmp}" = "yes" ]; then
use_tmp_flag="--no-tmpfs"
fi
if break="after-error" break_outside_target=1 \
break_cmd="cp \$TMP_MOUNT_PATH/tmp/dib-test-should-fail /tmp/ 2>&1 > /dev/null || true" \
break_cmd="cp -v \$TMP_MOUNT_PATH/tmp/dib-test-should-fail ${dest_dir} || true" \
DIB_SHOW_IMAGE_USAGE=1 \
ELEMENTS_PATH=$DIB_ELEMENTS:$DIB_ELEMENTS/$element/test-elements \
$DIB_CMD -x -t tar,qcow2 -o $dest_dir/image -n $element $test_element; then
$DIB_CMD -x -t tar,qcow2 ${use_tmp_flag} -o $dest_dir/image -n $element $test_element 2>&1 \
| log_with_prefix "${element}/${test_element}"; then
if ! [ -f "$dest_dir/image.qcow2" ]; then
echo "Error: qcow2 build failed for element: $element, test-element: $test_element."
@ -60,7 +104,7 @@ function run_disk_element_test() {
fi
fi
else
if [ -f "/tmp/dib-test-should-fail" ]; then
if [ -f "${dest_dir}/dib-test-should-fail" ]; then
echo "PASS: Element $element, test-element: $test_element"
else
echo "Error: Build failed for element: $element, test-element: $test_element."
@ -81,7 +125,8 @@ function run_ramdisk_element_test() {
local dest_dir=$(mktemp -d)
if ELEMENTS_PATH=$DIB_ELEMENTS/$element/test-elements \
$DIB_CMD -x -o $dest_dir/image $element $test_element; then
$DIB_CMD -x -o $dest_dir/image $element $test_element \
| log_with_prefix "${element}/${test_element}"; then
# TODO(dtantsur): test also kernel presence once we sort out its naming
# problem (vmlinuz vs kernel)
if ! [ -f "$dest_dir/image.initramfs" ]; then
@ -111,12 +156,15 @@ for e in $DIB_ELEMENTS/*/test-elements/*; do
TESTS+=("$element/$test_element")
done
while getopts ":hl" opt; do
JOB_MAX_CNT=1
while getopts ":hlpj:" opt; do
case $opt in
h)
echo "run_functests.sh [-h] [-l] <test> <test> ..."
echo " -h : show this help"
echo " -l : list available tests"
echo " -p : run all tests in parallel"
echo " <test> : functional test to run"
echo " Special test 'all' will run all tests"
exit 0
@ -130,6 +178,10 @@ while getopts ":hl" opt; do
echo
exit 0
;;
j)
JOB_MAX_CNT=${OPTARG}
echo "Running parallel - using [${JOB_MAX_CNT}] jobs"
;;
\?)
echo "Invalid option: -$OPTARG"
exit 1
@ -138,6 +190,15 @@ while getopts ":hl" opt; do
done
shift $((OPTIND-1))
DONT_USE_TMP="no"
if [ "${JOB_MAX_CNT}" -gt 1 ]; then
# switch off using tmp dir for image building
# (The mem check using the tmp dir is currently done
# based on the available memory - and not on the free.
# See #1618124 for more details)
DONT_USE_TMP="yes"
fi
# cull the list of tests to run into TESTS_TO_RUN
TESTS_TO_RUN=()
title=""
@ -173,7 +234,36 @@ for test in "${TESTS_TO_RUN[@]}"; do
done
echo "------"
function wait_and_exit_on_failure {
local pid=$1
wait ${pid}
result=$?
if [ "${result}" -ne 0 ]; then
exit ${result}
fi
return 0
}
EXIT_CODE=0
for test in "${TESTS_TO_RUN[@]}"; do
running_jobs_cnt=$(job_cnt)
log_jc "Number of running jobs [${running_jobs_cnt}] max jobs [${JOB_MAX_CNT}]"
if [ "${running_jobs_cnt}" -ge "${JOB_MAX_CNT}" ]; then
log_jc "Waiting for job to finish"
wait_minus_n
result=$?
if [ "${result}" -ne 0 ]; then
EXIT_CODE=1
# If a job fails, do not start any new ones.
break
fi
fi
log_jc "Starting new job"
# from above; each array value is element/test_element. split it
# back up
element=${test%/*}
@ -188,7 +278,30 @@ for test in "${TESTS_TO_RUN[@]}"; do
fi
echo "Running $test ($element_type)"
run_${element_type}_element_test $test_element $element
run_${element_type}_element_test $test_element $element ${DONT_USE_TMP} &
done
# Wait for the rest of the jobs
while true; do
running_jobs_cnt=$(job_cnt)
log_jc "Number of running jobs left [${running_jobs_cnt}]"
if [ "${running_jobs_cnt}" -eq 0 ]; then
break;
fi
wait_minus_n
result=$?
if [ "${result}" -ne 0 ]; then
EXIT_CODE=1
fi
done
if [ "${EXIT_CODE}" -eq 0 ]; then
echo "Tests passed!"
exit 0
else
echo "At least one test failed"
exit 1
fi