Build ramdisks in an image chroot.

Ramdisks are now built inside a chroot which is built by the normal
image build process. Doing so improves our independence of the
precise state of the build host.

This fixes bug 1194055.

Change-Id: Ibc254fbb9e7b404b5f38c1b35bcde8a4136e8e28

README.md

@@ -177,11 +177,14 @@ part of the process you need to customise:
 
 Ramdisk elements support the following files in their element directories:
 
-* binary-deps : executables required to be fed into the ramdisk. These need
+* binary-deps.d : text files listing executables required to be fed into the 
-  to be present in your $PATH.
+  ramdisk. These need to be present in $PATH in the build chroot (i.e. need to
+  be installed by your elements as described above).
 
-* init : a POSIX shell script fragment that will be appended to the default
+* init.d : POSIX shell script fragments that will be appended to the default
-  script executed as the ramdisk is booted (/init)
+  script executed as the ramdisk is booted (/init).
+
+* udev.d : udev rules files that will be copied into the ramdisk.
 
 Structure of an element
 -----------------------

bin/disk-image-create

@@ -30,6 +30,11 @@ SCRIPT_HOME=$(dirname $0)
 export _LIB=$(dirname $0)/../lib
 source $_LIB/die
 
+IS_RAMDISK=0
+if [ "$SCRIPTNAME" == "ramdisk-image-create" ]; then
+  IS_RAMDISK=1
+fi
+
 function show_options () {
     echo "Options:"
     echo "    -a i386|amd64|armhf -- set the architecture of the image"
@@ -37,9 +42,11 @@ function show_options () {
     echo "    -x -- turn on tracing"
     echo "    -u -- uncompressed; do not compress the image - larger but faster"
     echo "    -c -- clear environment before starting work"
+    echo "    --no-tmpfs -- do not use tmpfs to speed image build"
+    if [ "$IS_RAMDISK" == "0" ]; then
         echo "    -n skip the default inclusion of the 'base' element"
         echo "    -p package[,package,package] -- list of packages to install in the image"
-    echo "    --no-tmpfs -- do not use tmpfs to speed image build"
+    fi
     echo
     echo "ELEMENTS_PATH will allow you to specify multiple locations for the elements."
     exit 0
@@ -79,6 +86,11 @@ source $_LIB/img-defaults
 source $_LIB/common-functions
 source $_LIB/img-functions
 
+if [ "$IS_RAMDISK" == "1" ]; then
+  source $_LIB/ramdisk-defaults
+  source $_LIB/ramdisk-functions
+fi
+
 arg_to_elements "$@"
 
 IMAGE_NAME=${IMAGE_NAME%%\.${IMAGE_TYPE}}
@@ -124,6 +136,12 @@ sudo mv -t $TMP_BUILD_DIR/mnt ${TMP_BUILD_DIR}/built/*
 mount_proc_dev_sys
 run_d_in_target finalise
 finalise_base
+
 unmount_image
-compress_image
+
-save_image $IMAGE_NAME.$IMAGE_TYPE
+if [ "$IS_RAMDISK" == "0" ]; then
+  compress_image
+  save_image $IMAGE_NAME.$IMAGE_TYPE
+else
+  remove_image
+fi

bin/ramdisk-image-create

@@ -1,97 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2012 NTT DOCOMO, INC. 
-# Copyright 2013 Hewlett-Packard Development Company, L.P.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-set -e
-
-SCRIPTNAME=$(basename $0)
-SCRIPT_HOME=$(dirname $0)
-export _DIR=$(dirname $0)
-export _LIB=${_DIR}/../lib
-source $_LIB/die
-
-function show_options () {
-    echo "Options:"
-    echo "    -m PATH -- Path to find lib/modules. Default /"
-    echo "    -k VERSION -- Kernel version. Default $(uname -r)"
-    echo "    -h -- This help"
-    echo "    -o FILENAME -- Output file"
-    echo "    -x -- turn on tracing"
-    echo
-    echo "ELEMENTS_PATH will allow you to specify multiple locations for the elements."
-    exit 0
-}
-
-TEMP=$(getopt -o m:k:ho:x -n $SCRIPTNAME -- "$@")
-if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
-
-# Note the quotes around `$TEMP': they are essential!
-eval set -- "$TEMP"
-
-while true ; do
-    case "$1" in
-        -m) export MODULE_ROOT=$2; shift 2 ;;
-        -k) export KERNEL_VERSION=$2; shift 2 ;;
-        -o) export IMAGE_NAME=$2; shift 2 ;;
-        -h) show_options;;
-        -x) shift; set -x;;
-        --) shift ; break ;;
-        *) echo "Internal error!" ; exit 1 ;;
-    esac
-done
-
-source $_LIB/ramdisk-defaults
-source $_LIB/common-functions
-source $_LIB/ramdisk-functions
-
-arg_to_elements "$@"
-
-echo "Discovering binary dependencies"
-ensure_binaries
-
-INIT="$_DIR/../scripts/init"
-FUNCTIONS_D="$_DIR/../scripts/d"
-
-MODULE_DIR=$MODULE_ROOT/lib/modules/$KERNEL_VERSION
-FIRMWARE_DIR=$MODULE_ROOT/lib/firmware
-
-if [ ! -d "$MODULE_DIR" ]; then
-	echo "ERROR: kernel module directory not found at $MODULE_DIR"
-	exit 1
-fi
-
-LIB_UDEV=$LIB_UDEV_ROOT/lib/udev
-
-if [ ! -d "$LIB_UDEV" ]; then
-	echo "ERROR: udev directory not found at $LIB_UDEV"
-	exit 1
-fi
-
-mk_build_dir
-mkdir -p $TMP_BUILD_DIR/mnt
-TMP_MOUNT_PATH=$TMP_BUILD_DIR/mnt
-
-echo "working in $TMP_MOUNT_PATH"
-
-create_ramdisk_base
-populate_lib
-populate_busybox
-populate_init
-populate_udev
-finalise_image
-save_image $IMAGE_NAME
-

bin/ramdisk-image-create

@@ -0,0 +1 @@
+disk-image-create

elements/deploy/install.d/51-install-tgt

@@ -0,0 +1,2 @@
+#!/bin/sh
+install-packages tgt

elements/deploy/install.d/52-install-busybox

@@ -0,0 +1,2 @@
+#!/bin/sh
+install-packages busybox

elements/ramdisk/README.md

@@ -0,0 +1,14 @@
+This is the ramdisk element.
+
+Almost any user building a ramdisk will want to include this in their build,
+as it triggers many of the vital functionality from the basic diskimage-builder
+libraries (such as init script aggregation, busybox population, etc).
+
+An example of when one might want to use this toolchain to build a ramdisk would
+be the initial deployment of baremetal nodes in a TripleO setup. Various tools
+and scripts need to be injected into a ramdisk that will fetch and apply a
+machine image to local disks. That tooling/scripting customisation can be
+easily applied in a repeatable and automatable way, using this element.
+
+See the top-level README.me of the project, for more information about the
+mechanisms available to a ramdisk element.

elements/ramdisk/cleanup.d/99-extract-ramdisk-files

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+set -x
+
+cp $TMP_MOUNT_PATH/tmp/ramdisk $IMAGE_NAME.initramfs
+cp $TMP_MOUNT_PATH/tmp/kernel $IMAGE_NAME.kernel

elements/ramdisk/extra-data.d/01-inject-ramdisk-build-files

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+set -x
+
+export RAMDISK_BUILD_PATH="$TMP_MOUNT_PATH/tmp/ramdisk-build"
+mkdir -p $RAMDISK_BUILD_PATH
+
+for file in common-defaults common-functions ramdisk-defaults ramdisk-functions img-defaults img-functions ; do
+  cp $_LIB/$file $RAMDISK_BUILD_PATH
+done
+
+cp -r $(dirname $0)/scripts $RAMDISK_BUILD_PATH
+

elements/ramdisk/post-install.d/01-ensure-binaries

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Ensure that all binaries listed in ramdisk elements, exist
+
+set -e
+set -x
+
+export TARGET_DIR="/tmp/in_target.d/"
+
+if [ -z $TARGET_DIR ] ; then
+  echo "Target dir not specified"
+  exit 1
+fi
+
+if [ ! -d $TARGET_DIR ] ; then
+  echo "Unable to find specified directory in target: $TARGET_DIR"
+  bash
+  exit 1
+fi
+
+for _FILE in $(ls ${TARGET_DIR}/binary-deps.d/) ; do
+  _FILE="${TARGET_DIR}/binary-deps.d/${_FILE}"
+  if [ -a $_FILE ]; then
+    for _LINE in $(cat $_FILE) ; do
+      BINARY_DEPS="${BINARY_DEPS} $_LINE"
+    done
+  fi
+done
+
+for _BIN in $BINARY_DEPS ; do
+  _LOCATION=$(which "$_BIN" || echo "")
+  if [ -z "$_LOCATION" ]; then
+    echo "$_BIN is not found in PATH. Please ensure your elements install it"
+    exit 1
+  fi
+done
+
+if [ "$BINARY_DEPS" == "" ]; then
+  echo "No binary-deps found"
+else
+  DEPS_OUTPUT="/etc/dib_binary_deps"
+  echo "Creating binary_deps record at: ${DEPS_OUTPUT}"
+  echo "$BINARY_DEPS" >${DEPS_OUTPUT}
+fi

elements/ramdisk/post-install.d/99-build-ramdisk

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -e
+set -x
+
+_LIB="/tmp/ramdisk-build"
+
+source $_LIB/common-defaults
+source $_LIB/img-defaults
+source $_LIB/ramdisk-defaults
+
+source $_LIB/common-functions
+source $_LIB/img-functions
+source $_LIB/ramdisk-functions
+
+KERNEL_VERSION=${DIB_KERNEL_VERSION:-$(find_kernel_version)}
+MODULE_DIR=$MODULE_ROOT/lib/modules/$KERNEL_VERSION
+FIRMWARE_DIR=$MODULE_ROOT/lib/firmware
+LIB_UDEV=$LIB_UDEV_ROOT/lib/udev
+INIT="$_LIB/scripts/init"
+FUNCTIONS_D="$_LIB/scripts/d"
+
+mk_build_dir
+mkdir -p $TMP_BUILD_DIR/mnt
+TMP_MOUNT_PATH=$TMP_BUILD_DIR/mnt
+
+echo "working in $TMP_MOUNT_PATH"
+
+create_ramdisk_base
+populate_lib
+populate_busybox
+populate_init
+populate_udev
+finalise_image
+save_image /tmp/ramdisk
+cp /boot/vmlinuz-${KERNEL_VERSION} /tmp/kernel
+chmod o+r /tmp/kernel

lib/common-functions

@@ -180,6 +180,9 @@ function arg_to_elements() {
   if [ "$SKIP_BASE" != "1" ]; then
     IMAGE_ELEMENT="base $IMAGE_ELEMENT"
   fi
+  if [ "$IS_RAMDISK" == "1" ]; then
+    IMAGE_ELEMENT="ramdisk $IMAGE_ELEMENT"
+  fi
   echo "Building elements: $IMAGE_ELEMENT"
   echo "If prompted for sudo, install sudoers.d/img-build-sudoers into /etc/sudoers.d and restart the build."
 

lib/img-functions

@@ -113,6 +113,11 @@ function compress_image () {
     mv $TMP_IMAGE_PATH-new $TMP_IMAGE_PATH
 }
 
+function remove_image () {
+  echo "Removing $TMP_IMAGE_PATH"
+  rm $TMP_IMAGE_PATH
+}
+
 function do_extra_package_install () {
   # Install any packages that were requested with the -p command line option
   if [ "$INSTALL_PACKAGES" != "" ]; then

lib/ramdisk-defaults

@@ -18,8 +18,6 @@
 # options for ramdisk-image-create
 export DIB_NO_TMPFS=${DIB_NO_TMPFS:-1}
 source $_LIB/common-defaults
-KERNEL_VERSION=${KERNEL_VERSION:-$(uname -r)}
+MODULE_ROOT=${DIB_MODULE_ROOT:-""}
-MODULE_ROOT=${MODULE_ROOT:-""}
+LIB_UDEV_ROOT=${DIB_LIB_UDEV_ROOT:-""}
-LIB_UDEV_ROOT=${LIB_UDEV_ROOT:-""}
 BUSYBOX=${BUSYBOX:-$(which busybox)}
-IMAGE_NAME=${IMAGE_NAME:-"ramdisk"}

lib/ramdisk-functions

@@ -29,33 +29,6 @@ function cleanup () {
   rm -rf "$TMP_BUILD_DIR"
 }
 
-function ensure_binaries() {
-  BINARY_DEPS="${BUSYBOX}"
-  for _FLVR in ${IMAGE_ELEMENT} ; do
-    for dir in $(echo $ELEMENTS_PATH | tr ":" " ") ; do
-      [ -d $dir/$_FLVR ] || continue
-      _FILE="${dir}/${_FLVR}/binary-deps"
-      if [ -a $_FILE ]; then
-        for _LINE in $(cat $_FILE) ; do
-          BINARY_DEPS="${BINARY_DEPS} $_LINE"
-        done
-      fi
-      break
-    done
-  done
-
-  for _BIN in $BINARY_DEPS ; do
-    _LOCATION=$(which "$_BIN" || echo "")
-    if [ -z "$_LOCATION" ]; then
-      echo "$_BIN is not found in your PATH" 1>&2
-      echo "Please install it on your system"
-      exit 1
-    fi
-  done
-
-  export BINARY_DEPS
-}
-
 function create_ramdisk_base () {
   echo "Creating base system"
 
@@ -168,7 +141,7 @@ function populate_lib () {
 
   for i in "$BUSYBOX" bash lsmod modprobe udevadm \
            wget reboot shutdown $UDEVD $UDEV_FIRMWARE \
-           $BINARY_DEPS ; do
+           $(cat /etc/dib_binary_deps) ; do
     if "$BUSYBOX" --list | grep "^$i\$" >/dev/null; then
       continue
     fi
@@ -206,18 +179,16 @@ function populate_init () {
   done
 
   # Append /init with any element fragments that are present
-  for _FLVR in ${IMAGE_ELEMENT} ; do
+  TARGET_DIR="/tmp/in_target.d/"
-    for dir in $(echo $ELEMENTS_PATH | tr ":" " ") ; do
+  for _ELEMENT in $(ls $TARGET_DIR/init.d/) ; do
-      [ -d $dir/$_FLVR ] || continue
+    _FILE="${TARGET_DIR}/init.d/${_ELEMENT}"
-      _FILE="${dir}/${_FLVR}/init"
     if [ -a $_FILE ]; then
       cat >>$TMP_MOUNT_PATH/init <<EOF
-# init fragment from ${_FLVR}
+
+# init fragment from ${_ELEMENT}
 EOF
       cat <$_FILE >>$TMP_MOUNT_PATH/init
     fi
-      break
-    done
   done
 
   # Add our final steps to /init
@@ -232,16 +203,20 @@ function finalise_image () {
 function populate_udev () {
   echo "Installing udev rules"
 
-  for _FLVR in ${IMAGE_ELEMENT} ; do
+  TARGET_DIR="/tmp/in_target.d/"
-    for dir in $(echo $ELEMENTS_PATH | tr ":" " ") ; do
+  for _ELEMENT in $(ls $TARGET_DIR/udev.d/) ; do
-      [ -d $dir/$_FLVR ] || continue
+    _FILE="${TARGET_DIR}/udev.d/${_ELEMENT}"
-      _DIR="${dir}/${_FLVR}/udev"
+    if [ -a $_FILE ]; then
-      if [ -d $_DIR ]; then
+      cp ${_FILE} $TMP_MOUNT_PATH/lib/udev/rules.d/
-        find $_DIR -type f -exec cp -v {} $TMP_MOUNT_PATH/lib/udev/rules.d/ \;
     fi
-      break
-    done
   done
 }
 
-
+function find_kernel_version () {
+  _TMP=$(ls /boot/vmlinuz-* | sort | tail -1)
+  if [ "$_TMP" == "" ]; then
+    echo "Unable to find a suitable kernel" >>/dev/stderr
+    exit 1
+  fi
+  echo ${_TMP##/boot/vmlinuz-}
+}