Refactor: block-device handling (local loop)

Block device handling can be somewhat complex - especially
when taking things like md, lvm or encryption into account.

This patch factors out the creation and deletion of the local
loop image device handling into a python library.

The main propose of this patch is to implement the needed
infrastructure.  Based on this, more advanced functions can be added.
Example: (advanced) partitioning, LVM, handling different boot
scenarios (BIOS, UEFI, ...), possibility of handling multiple images
(local loop image, iSCSI, physical hard disk, ...), handling of
different filesystems for different partitions / LVs.

Change-Id: Ib626b36a00f8a5dc3dbde8df3e2619a2438eaaf1
Signed-off-by: Andreas Florath <andreas@florath.net>

bin/disk-image-create

@@ -190,7 +190,7 @@ while true ; do
         -c) shift ; export CLEAR_ENV=1;;
         -n) shift; export SKIP_BASE="1";;
         -p) IFS="," read -a INSTALL_PACKAGES <<< "$2"; export INSTALL_PACKAGES ; shift 2 ;;
-        --image-size) export DIB_IMAGE_SIZE=$2; shift 2;;
+        --image-size) DIB_IMAGE_SIZE=$2; shift 2;;
         --image-cache) export DIB_IMAGE_CACHE=$2; shift 2;;
         --max-online-resize) export MAX_ONLINE_RESIZE=$2; shift 2;;
         --mkfs-options) MKFS_OPTS=$2; shift 2;;
@@ -400,17 +400,79 @@ fi
 # Rounding down size so that is is a multiple of 64, works around a bug in
 # qemu-img that may occur when compressing raw images that aren't a multiple
 # of 64k. https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1180021
-du_size=$(echo "$du_size" | awk ' { if ($1 % 64 != 0) print $1 + 64 - ( $1 % 64); else print $1; } ')
+export DIB_IMAGE_SIZE=$(echo "$du_size" | awk ' { if ($1 % 64 != 0) print $1 + 64 - ( $1 % 64); else print $1; } ')
-truncate -s${du_size}K $TMP_IMAGE_PATH
 
 if [ -n "$MAX_ONLINE_RESIZE" ]; then
     MKFS_OPTS="-E resize=$MAX_ONLINE_RESIZE $MKFS_OPTS"
 fi
 
-LOOPDEV=$(sudo losetup --show -f $TMP_IMAGE_PATH)
+export TMP_IMAGE_DIR
-export EXTRA_UNMOUNT="detach_loopback $LOOPDEV"
+# Try the 'old fashioned' way calling the block device
-export IMAGE_BLOCK_DEVICE=$LOOPDEV
+# phase. If this gives no result, use the configuration based approach:
 eval_run_d block-device "IMAGE_BLOCK_DEVICE="
+if [ -z ${IMAGE_BLOCK_DEVICE} ] ; then
+    IMAGE_BLOCK_DEVICE=$(dib-block-device \
+                             --phase=create \
+                             --config="${DIB_BLOCK_DEVICE_CONFIG:-}" \
+                             --image-size="${DIB_IMAGE_SIZE}"KiB \
+                             --image-dir="${TMP_IMAGE_DIR}" \
+                             --build-dir="${TMP_BUILD_DIR}" )
+fi
+export IMAGE_BLOCK_DEVICE
+LOOPDEV=${IMAGE_BLOCK_DEVICE}
+export EXTRA_DETACH="detach_loopback $LOOPDEV"
+export EXTRA_UNMOUNT="dib-block-device --phase=umount \
+                      --build-dir=\"${TMP_BUILD_DIR}\""
+
+# Create the partitions and make them visible to the system
+
+# Create 2 partitions for PPC, one for PReP boot and other for root
+if [[ "$ARCH" =~ "ppc" ]] ; then
+    sudo parted -a optimal -s $IMAGE_BLOCK_DEVICE \
+        mklabel msdos \
+        mkpart primary 0 8cyl \
+        set 1 boot on \
+        set 1 prep on \
+        mkpart primary 9cyl 100%
+else
+    sudo parted -a optimal -s $IMAGE_BLOCK_DEVICE \
+        mklabel msdos \
+        mkpart primary 1MiB 100% \
+        set 1 boot on
+fi
+
+sudo partprobe $IMAGE_BLOCK_DEVICE
+
+# To ensure no race conditions exist from calling partprobe
+sudo udevadm settle
+
+# If the partition isn't under /dev/loop*p1, create it with kpartx
+DM=
+if [ ! -e "${IMAGE_BLOCK_DEVICE}p1" ]; then
+    DM=${IMAGE_BLOCK_DEVICE/#\/dev/\/dev\/mapper}
+    # If running inside Docker, make our nodes manually, because udev will not be working.
+    if [ -f /.dockerenv ]; then
+        # kpartx cannot run in sync mode in docker.
+        sudo kpartx -av $TMP_IMAGE_PATH
+        sudo dmsetup --noudevsync mknodes
+    else
+        sudo kpartx -asv $TMP_IMAGE_PATH
+    fi
+elif [[ "$ARCH" =~ "ppc" ]]; then
+    sudo kpartx -asv $TMP_IMAGE_PATH
+fi
+
+if [ -n "$DM" ]; then
+    export IMAGE_BLOCK_DEVICE=${DM}p1
+elif [[ "$ARCH" =~ "ppc" ]]; then
+    DM=${IMAGE_BLOCK_DEVICE/#\/dev/\/dev\/mapper}
+    export IMAGE_BLOCK_DEVICE=${DM}p2
+else
+    export IMAGE_BLOCK_DEVICE=${IMAGE_BLOCK_DEVICE}p1
+fi
+
+# End: Creation of the partitions
+
 sudo mkfs $MKFS_OPTS -t $FS_TYPE -L ${DIB_ROOT_LABEL} ${IMAGE_BLOCK_DEVICE}
 # Tuning the rootfs uuid works only for ext filesystems.
 if echo "$FS_TYPE" | grep -q "^ext"; then
@@ -457,7 +519,11 @@ fi
 
 # Unmount and cleanup the /mnt and /build subdirectories, to save
 # space before converting the image to some other format.
+export EXTRA_UNMOUNT=""
 unmount_image
+export TMP_IMAGE_PATH=$(dib-block-device \
+                            --phase=umount \
+                            --build-dir="${TMP_BUILD_DIR}" )
 cleanup_build_dir
 
 if [[ (! $IMAGE_ELEMENT =~ no-final-image) && "$IS_RAMDISK" == "0" ]]; then

diskimage_builder/block_device/__init__.py

@@ -0,0 +1,81 @@
+# Copyright 2016 Andreas Florath (andreas@florath.net)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+from diskimage_builder.block_device.blockdevice import BlockDevice
+from diskimage_builder import logging_config
+import logging
+
+
+def val_else_none(s):
+    return s if s is None or len(s) > 0 else None
+
+
+def generate_phase_doc():
+    phase_doc = ""
+    bdattrs = dir(BlockDevice)
+    for attr in bdattrs:
+        if attr.startswith("cmd_"):
+            phase_doc += "  '" + attr[4:] + "'\n"
+            method = getattr(BlockDevice, attr, None)
+            # The first line is the line that is used
+            phase_doc += "    " + method.__doc__.split("\n")[0] + "\n"
+    return phase_doc
+
+
+def main():
+    logging_config.setup()
+    logger = logging.getLogger(__name__)
+    phase_doc = generate_phase_doc()
+
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description="Create block device layer",
+        epilog="Available phases:\n" + phase_doc)
+    parser.add_argument('--phase', required=True,
+                        help="phase to execute")
+    parser.add_argument('--config', required=False,
+                        help="configuration for block device "
+                             "layer as JSON object")
+    parser.add_argument('--build-dir', required=True,
+                        help="path to temporary build dir")
+    parser.add_argument('--image-size', required=False,
+                        help="default image size")
+    parser.add_argument('--image-dir', required=False,
+                        help="default image directory")
+    args = parser.parse_args()
+
+    logger.info("phase [%s]" % args.phase)
+    logger.info("config [%s]" % args.config)
+    logger.info("build_dir [%s]" % args.build_dir)
+
+    bd = BlockDevice(val_else_none(args.config),
+                     val_else_none(args.build_dir),
+                     val_else_none(args.image_size),
+                     val_else_none(args.image_dir))
+
+    # Check if the method is available
+    method = getattr(bd, "cmd_" + args.phase, None)
+    if callable(method):
+        # If so: call it.
+        method()
+    else:
+        logger.error("phase [%s] does not exists" % args.phase)
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    main()

diskimage_builder/block_device/blockdevice.py

@@ -0,0 +1,119 @@
+# Copyright 2016 Andreas Florath (andreas@florath.net)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from diskimage_builder.block_device.level0 import Level0
+from diskimage_builder.block_device.utils import convert_to_utf8
+import json
+import logging
+import os
+import shutil
+
+
+logger = logging.getLogger(__name__)
+
+
+class BlockDevice(object):
+
+    # Currently there is only the need for a first element (which must
+    # be a list).
+    DefaultConfig = [
+        [["local_loop",
+          {"name": "rootdisk"}]]]
+    # The reason for the complex layout is, that for future layers
+    # there is a need to add additional lists, like:
+    #  DefaultConfig = [
+    #      [["local_loop",
+    #        {"name": "rootdisk"}]],
+    #      [["partitioning",
+    #        {"rootdisk": {
+    #            "label": "mbr",
+    #            "partitions":
+    #            [{"name": "rd-partition1",
+    #              "flags": ["boot"],
+    #              "size": "100%"}]}}]],
+    #      [["fs",
+    #        {"rd-partition1": {}}]]
+    #  ]
+
+    def __init__(self, block_device_config, build_dir,
+                 default_image_size, default_image_dir):
+        if block_device_config is None:
+            self.config = BlockDevice.DefaultConfig
+        else:
+            self.config = json.loads(block_device_config)
+        self.default_config = {
+            'image_size': default_image_size,
+            'image_dir': default_image_dir}
+        self.state_dir = os.path.join(build_dir,
+                                      "states/block-device")
+        self.state_json_file_name \
+            = os.path.join(self.state_dir, "state.json")
+
+    def write_state(self, result):
+        logger.debug("Write state [%s]" % self.state_json_file_name)
+        os.makedirs(self.state_dir)
+        with open(self.state_json_file_name, "w") as fd:
+            json.dump([self.config, self.default_config, result], fd)
+
+    def load_state(self):
+        with open(self.state_json_file_name, "r") as fd:
+            return convert_to_utf8(json.load(fd))
+
+    def cmd_create(self):
+        """Creates the block device"""
+
+        logger.info("create() called")
+        logger.debug("config [%s]" % self.config)
+        lvl0 = Level0(self.config[0], self.default_config, None)
+        result = lvl0.create()
+        logger.debug("Result level 0 [%s]" % result)
+
+        # To be compatible with the current implementation, echo the
+        # result to stdout.
+        print("%s" % result['rootdisk']['device'])
+
+        self.write_state(result)
+
+        logger.info("create() finished")
+        return 0
+
+    def cmd_umount(self):
+        """Unmounts the blockdevice and cleanup resources"""
+
+        logger.info("umount() called")
+        try:
+            os.stat(self.state_json_file_name)
+        except OSError:
+            logger.info("State already cleaned - no way to do anything here")
+            return 0
+
+        config, default_config, state = self.load_state()
+        logger.debug("Using config [%s]" % config)
+        logger.debug("Using default config [%s]" % default_config)
+        logger.debug("Using state [%s]" % state)
+
+        level0 = Level0(config[0], default_config, state)
+        result = level0.delete()
+
+        # If everything finished well, remove the results.
+        if result:
+            logger.info("Removing temporary dir [%s]" % self.state_dir)
+            shutil.rmtree(self.state_dir)
+
+        # To be compatible with the current implementation, echo the
+        # result to stdout.
+        print("%s" % state['rootdisk']['image'])
+
+        logger.info("umount() finished result [%d]" % result)
+        return 0 if result else 1

diskimage_builder/block_device/level0/__init__.py

@@ -0,0 +1,30 @@
+# Copyright 2016 Andreas Florath (andreas@florath.net)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from diskimage_builder.block_device.level0.localloop import LocalLoop
+from diskimage_builder.block_device.levelbase import LevelBase
+
+__all__ = [LocalLoop]
+
+
+class Level0(LevelBase):
+    """Block Device Level0: preparation of images
+
+    This is the class that handles level 0 block device setup:
+    creating the block device image and providing OS access to it.
+    """
+
+    def __init__(self, config, default_config, result):
+        LevelBase.__init__(self, 0, config, default_config, result,
+                           {LocalLoop.type_string: LocalLoop})

diskimage_builder/block_device/level0/localloop.py

@@ -0,0 +1,92 @@
+# Copyright 2016 Andreas Florath (andreas@florath.net)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from diskimage_builder.block_device.utils import parse_abs_size_spec
+import logging
+import os
+import subprocess
+import sys
+import time
+
+
+logger = logging.getLogger(__name__)
+
+
+class LocalLoop(object):
+    """Level0: Local loop image device handling.
+
+    This class handles local loop devices that can be used
+    for VM image installation.
+    """
+
+    type_string = "local_loop"
+
+    def __init__(self, config, default_config, result=None):
+        if 'size' in config:
+            self.size = parse_abs_size_spec(config['size'])
+            logger.debug("Image size [%s]" % self.size)
+        else:
+            self.size = parse_abs_size_spec(default_config['image_size'])
+            logger.debug("Using default image size [%s]" % self.size)
+        if 'directory' in config:
+            self.image_dir = config['directory']
+        else:
+            self.image_dir = default_config['image_dir']
+        self.name = config['name']
+        self.filename = os.path.join(self.image_dir, self.name + ".raw")
+        self.result = result
+        if self.result is not None:
+            self.block_device = self.result[self.name]['device']
+
+    def create(self):
+        logger.debug("[%s] Creating loop on [%s] with size [%d]" %
+                     (self.name, self.filename, self.size))
+
+        with open(self.filename, "w") as fd:
+            fd.seek(self.size - 1)
+            fd.write("\0")
+
+        logger.debug("Calling [sudo losetup --show -f %s]"
+                     % self.filename)
+        subp = subprocess.Popen(["sudo", "losetup", "--show", "-f",
+                                 self.filename], stdout=subprocess.PIPE)
+        rval = subp.wait()
+        if rval == 0:
+            # [:-1]: Cut of the newline
+            self.block_device = subp.stdout.read()[:-1]
+            logger.debug("New block device [%s]" % self.block_device)
+        else:
+            logger.error("losetup failed")
+            sys.exit(1)
+
+        return {self.name: {"device": self.block_device,
+                            "image": self.filename}}
+
+    def delete(self):
+        # loopback dev may be tied up a bit by udev events triggered
+        # by partition events
+        for try_cnt in range(10, 1, -1):
+            logger.debug("Delete loop [%s]" % self.block_device)
+            res = subprocess.call("sudo losetup -d %s" %
+                                  (self.block_device),
+                                  shell=True)
+            if res == 0:
+                return {self.name: True}
+            logger.debug("[%s] may be busy, sleeping [%d] more secs"
+                         % (self.block_device, try_cnt))
+            time.sleep(1)
+
+        logger.debug("Gave up trying to detach [%s]" %
+                     self.block_device)
+        return {self.name: False}

diskimage_builder/block_device/levelbase.py

@@ -0,0 +1,66 @@
+# Copyright 2016 Andreas Florath (andreas@florath.net)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import sys
+
+
+logger = logging.getLogger(__name__)
+
+
+class LevelBase(object):
+
+    def __init__(self, lvl, config, default_config, result, sub_modules):
+        self.level = lvl
+        self.config = config
+        self.default_config = default_config
+        self.result = result
+        self.sub_modules = sub_modules
+
+    def call_sub_modules(self, callback):
+        """Generic way calling submodules"""
+        result = {}
+        if self.result is not None:
+            result = self.result.copy()
+        for name, cfg in self.config:
+            if name in self.sub_modules:
+                logger.info("Calling sub module [%s]" % name)
+                sm = self.sub_modules[name](cfg, self.default_config,
+                                            self.result)
+                lres = callback(sm)
+                result.update(lres)
+            else:
+                logger.error("Unknown sub module [%s]" % name)
+                sys.exit(1)
+        return result
+
+    def create_cb(self, obj):
+        return obj.create()
+
+    def create(self):
+        """Create the configured block devices"""
+        logger.info("Starting to create level [%d] block devices" % self.level)
+        result = self.call_sub_modules(self.create_cb)
+        logger.info("Finished creating level [%d] block devices" % self.level)
+        return result
+
+    def delete_cb(self, obj):
+        return obj.delete()
+
+    def delete(self):
+        """Delete the configured block devices"""
+        logger.info("Starting to delete level [%d] block devices" % self.level)
+        res = self.call_sub_modules(self.delete_cb)
+        logger.info("Finished deleting level [%d] block devices" % self.level)
+        return all(p for p in res.values())

diskimage_builder/block_device/tests/test_utils.py

@@ -0,0 +1,29 @@
+# Copyright 2016 Andreas Florath (andreas@florath.net)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from diskimage_builder.block_device.utils import parse_abs_size_spec
+import testtools
+
+
+class TestLoggingConfig(testtools.TestCase):
+
+    def test_parse_size_spec(self):
+        map(lambda tspec:
+            self.assertEqual(parse_abs_size_spec(tspec[0]), tspec[1]),
+            [["20TiB", 20 * 1024**4],
+             ["1024KiB", 1024 * 1024],
+             ["1.2TB", 1.2 * 1000**4],
+             ["2.4T", 2.4 * 1000**4],
+             ["512B", 512],
+             ["364", 364]])

diskimage_builder/block_device/utils.py

@@ -0,0 +1,73 @@
+# Copyright 2016 Andreas Florath (andreas@florath.net)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+SIZE_SPECS = [
+    ["TiB", 1024**4],
+    ["GiB", 1024**3],
+    ["MiB", 1024**2],
+    ["KiB", 1024**1],
+    ["TB", 1000**4],
+    ["GB", 1000**3],
+    ["MB", 1000**2],
+    ["KB", 1000**1],
+    ["T", 1000**4],
+    ["G", 1000**3],
+    ["M", 1000**2],
+    ["K", 1000**1],
+    ["B", 1],
+    ["", 1],   # No unit -> size is given in bytes
+]
+
+
+def _split_size_spec(size_spec):
+    for spec_key, spec_value in SIZE_SPECS:
+        if len(spec_key) == 0:
+            return size_spec, spec_key
+        if size_spec.endswith(spec_key):
+            return size_spec[:-len(spec_key)], spec_key
+    raise RuntimeError("size_spec [%s] not known" % size_spec)
+
+
+def _get_unit_factor(unit_str):
+    for spec_key, spec_value in SIZE_SPECS:
+        if unit_str == spec_key:
+            return spec_value
+    raise RuntimeError("unit_str [%s] not known" % unit_str)
+
+
+def parse_abs_size_spec(size_spec):
+    size_cnt_str, size_unit_str = _split_size_spec(size_spec)
+    unit_factor = _get_unit_factor(size_unit_str)
+    return int(unit_factor * (
+        float(size_cnt_str) if len(size_cnt_str) > 0 else 1))
+
+
+def convert_to_utf8(jdata):
+    """Convert to UTF8.
+
+    The json parser returns unicode strings. Because in
+    some python implementations unicode strings are not
+    compatible with utf8 strings - especially when using
+    as keys in dictionaries - this function recursively
+    converts the json data.
+    """
+    if isinstance(jdata, unicode):
+        return jdata.encode('utf-8')
+    elif isinstance(jdata, dict):
+        return {convert_to_utf8(key): convert_to_utf8(value)
+                for key, value in jdata.iteritems()}
+    elif isinstance(jdata, list):
+        return [convert_to_utf8(je) for je in jdata]
+    else:
+        return jdata

doc/source/developer/developing_elements.rst

@@ -144,7 +144,7 @@ The phases are:
   operations here to only those necessary to affect the filesystem metadata and
   image itself. For most operations, ``post-install.d`` is preferred.
 
-  * runs: **in chroot** 
+  * runs: **in chroot**
 
 ``cleanup.d``
   Perform cleanup of the root filesystem content. For instance, temporary

doc/source/user_guide/building_an_image.rst

@@ -55,6 +55,87 @@ formats are:
  * docker
  * raw
 
+Disk Image Layout
+-----------------
+
+When generating a block image (e.g. qcow2 or raw), by default one
+image with one partition holding all files is created.
+
+The appropriate functionality to use multiple partitions and even LVMs
+is currently under development; therefore the possible configuration
+is currently limited, but will get more flexible as soon as all the
+functionality is implemented.
+
+The configuration is done by means of the environment variable
+`DIB_BLOCK_DEVICE_CONFIG`.  This variable must hold JSON structured
+configuration data.
+
+In future this will be a list of four elements, each describing one
+level of block device setup - but because currently only the lowest
+level is implemented, it contains only the configuration of the first
+level of block device setup
+
+The default is:
+
+::
+
+    DIB_BLOCK_DEVICE_CONFIG='[
+        [["local_loop",
+          {"name": "rootdisk"}]]]'
+
+In general each module is configured in the way, that the first
+element in the list is the name of the module (e.g.  `local_loop`)
+followed by a dictionary of parameters (here `{"name": "rootdisk"}`).
+
+Level 0
++++++++
+
+Module: Local Loop
+..................
+
+This module generates a local image file and uses the loop device to
+create a block device from it.  The symbolic name for this module is
+`local_loop`.
+
+Configuration options:
+
+name
+  (mandatory) The name of the image.  This is used as the name for the
+  image in the file system and also as a symbolic name to be able to
+  reference this image (e.g. to create a partition table on this
+  disk).
+
+size
+  (optional) The size of the disk. The size can be expressed using
+  unit names like TiB (1024^4 bytes) or GB (1000^3 bytes).
+  Examples: 2.5GiB, 12KB.
+  If the size is not specified here, the size as given to
+  disk-image-create (--image-size) or the automatically computed size
+  is used.
+
+directory
+  (optional) The directory where the image is created.
+
+Example:
+
+::
+
+   DIB_BLOCK_DEVICE_CONFIG='[
+        [["local_loop",
+          {"name": "rootdisk"}],
+         ["local_loop",
+          {"name": "datadisk",
+           "size": "7.5GiB",
+           "directory": "/var/tmp"}]]]'
+
+This creates two image files and uses the loop device to use them as
+block devices.  One image file called `rootdisk` is created with
+default size in the default temp directory.  The second image has the
+size of 7.5GiB and is created in the `/var/tmp` folder.
+
+Please note that due to current implementation restrictions it is only
+allowed to specify one local loop image.
+
 Filesystem Caveat
 -----------------
 

elements/vm/block-device.d/10-partition

@@ -1,56 +0,0 @@
-#!/bin/bash
-
-# dib-lint: disable=safe_sudo
-
-if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
-    set -x
-fi
-set -eu
-set -o pipefail
-
-[ -n "$IMAGE_BLOCK_DEVICE" ] || die "Image block device not set"
-
-# Create 2 partitions for PPC, one for PReP boot and other for root
-if [[ "$ARCH" =~ "ppc" ]] ; then
-    sudo parted -a optimal -s $IMAGE_BLOCK_DEVICE \
-        mklabel msdos \
-        mkpart primary 0 8cyl \
-        set 1 boot on \
-        set 1 prep on \
-        mkpart primary 9cyl 100%
-else
-    sudo parted -a optimal -s $IMAGE_BLOCK_DEVICE \
-        mklabel msdos \
-        mkpart primary 1MiB 100% \
-        set 1 boot on
-fi
-
-sudo partprobe $IMAGE_BLOCK_DEVICE
-
-# To ensure no race conditions exist from calling partprobe
-sudo udevadm settle
-
-# If the partition isn't under /dev/loop*p1, create it with kpartx
-DM=
-if [ ! -e "${IMAGE_BLOCK_DEVICE}p1" ]; then
-    DM=${IMAGE_BLOCK_DEVICE/#\/dev/\/dev\/mapper}
-    # If running inside Docker, make our nodes manually, because udev will not be working.
-    if [ -f /.dockerenv ]; then
-        # kpartx cannot run in sync mode in docker.
-        sudo kpartx -av $TMP_IMAGE_PATH
-        sudo dmsetup --noudevsync mknodes
-    else
-        sudo kpartx -asv $TMP_IMAGE_PATH
-    fi
-elif [[ "$ARCH" =~ "ppc" ]]; then
-    sudo kpartx -asv $TMP_IMAGE_PATH
-fi
-
-if [ -n "$DM" ]; then
-    echo "IMAGE_BLOCK_DEVICE=${DM}p1"
-elif [[ "$ARCH" =~ "ppc" ]]; then
-    DM=${IMAGE_BLOCK_DEVICE/#\/dev/\/dev\/mapper}
-    echo "IMAGE_BLOCK_DEVICE=${DM}p2"
-else
-    echo "IMAGE_BLOCK_DEVICE=${IMAGE_BLOCK_DEVICE}p1"
-fi

lib/common-functions

@@ -181,16 +181,7 @@ function detach_loopback() {
         sudo dmsetup --noudevsync remove $mapper_name
     fi
 
-    # loopback dev may be tied up a bit by udev events triggered by partition events
+    return 0
-    for try in $(seq 10 -1 1) ; do
-        if ! sudo losetup $loopdev || sudo losetup -d $loopdev ; then
-            return 0
-        fi
-        echo $loopdev may be busy, sleeping up to $try more seconds...
-        sleep 1
-    done
-    echo Gave up trying to detach $loopdev
-    return 1
 }
 
 function arg_to_elements() {

lib/img-functions

@@ -24,6 +24,9 @@ function unmount_image () {
     # unmount from the chroot
     # Don't use TMP_MOUNT_PATH here, it might not have been set.
     unmount_dir "$TMP_BUILD_DIR/mnt"
+    if [ -n "$EXTRA_DETACH" ]; then
+        $EXTRA_DETACH
+    fi
     if [ -n "$EXTRA_UNMOUNT" ]; then
         $EXTRA_UNMOUNT
     fi

setup.cfg

@@ -54,3 +54,7 @@ domain = diskimage_builder
 
 [wheel]
 universal = 1
+
+[entry_points]
+console_scripts =
+    dib-block-device = diskimage_builder.block_device:main

tox.ini

@@ -10,6 +10,7 @@ deps= -r{toxinidir}/requirements.txt
       -r{toxinidir}/test-requirements.txt
 commands=
     python setup.py testr --slowest --testr-args='{posargs}'
+passenv = ELEMENTS_PATH DIB_RELEASE DIB_DEBUG_TRACE DIB_DEV_USER_USERNAME DIB_DEV_USER_PWDLESS_SUDO DIB_DEV_USER_PASSWORD USER HOME http_proxy https_proxy
 
 [testenv:pep8]
 commands =