From 3d48a528c14f9038de4d00d9c7d08afda4830d8d Mon Sep 17 00:00:00 2001 From: Andreas Florath Date: Sat, 21 May 2016 21:32:35 +0200 Subject: [PATCH] Refactor: block-device handling (local loop) Block device handling can be somewhat complex - especially when taking things like md, lvm or encryption into account. This patch factors out the creation and deletion of the local loop image device handling into a python library. The main propose of this patch is to implement the needed infrastructure. Based on this, more advanced functions can be added. Example: (advanced) partitioning, LVM, handling different boot scenarios (BIOS, UEFI, ...), possibility of handling multiple images (local loop image, iSCSI, physical hard disk, ...), handling of different filesystems for different partitions / LVs. Change-Id: Ib626b36a00f8a5dc3dbde8df3e2619a2438eaaf1 Signed-off-by: Andreas Florath --- bin/disk-image-create | 78 +++++++++++- diskimage_builder/block_device/__init__.py | 81 ++++++++++++ diskimage_builder/block_device/blockdevice.py | 119 ++++++++++++++++++ .../block_device/level0/__init__.py | 30 +++++ .../block_device/level0/localloop.py | 92 ++++++++++++++ diskimage_builder/block_device/levelbase.py | 66 ++++++++++ .../block_device/tests/__init__.py | 0 .../block_device/tests/test_utils.py | 29 +++++ diskimage_builder/block_device/utils.py | 73 +++++++++++ doc/source/developer/developing_elements.rst | 2 +- doc/source/user_guide/building_an_image.rst | 81 ++++++++++++ elements/vm/block-device.d/10-partition | 56 --------- lib/common-functions | 11 +- lib/img-functions | 3 + setup.cfg | 4 + tox.ini | 1 + 16 files changed, 653 insertions(+), 73 deletions(-) create mode 100644 diskimage_builder/block_device/__init__.py create mode 100644 diskimage_builder/block_device/blockdevice.py create mode 100644 diskimage_builder/block_device/level0/__init__.py create mode 100644 diskimage_builder/block_device/level0/localloop.py create mode 100644 diskimage_builder/block_device/levelbase.py create mode 100644 diskimage_builder/block_device/tests/__init__.py create mode 100644 diskimage_builder/block_device/tests/test_utils.py create mode 100644 diskimage_builder/block_device/utils.py delete mode 100755 elements/vm/block-device.d/10-partition diff --git a/bin/disk-image-create b/bin/disk-image-create index acb50492..b03e302b 100755 --- a/bin/disk-image-create +++ b/bin/disk-image-create @@ -190,7 +190,7 @@ while true ; do -c) shift ; export CLEAR_ENV=1;; -n) shift; export SKIP_BASE="1";; -p) IFS="," read -a INSTALL_PACKAGES <<< "$2"; export INSTALL_PACKAGES ; shift 2 ;; - --image-size) export DIB_IMAGE_SIZE=$2; shift 2;; + --image-size) DIB_IMAGE_SIZE=$2; shift 2;; --image-cache) export DIB_IMAGE_CACHE=$2; shift 2;; --max-online-resize) export MAX_ONLINE_RESIZE=$2; shift 2;; --mkfs-options) MKFS_OPTS=$2; shift 2;; @@ -400,17 +400,79 @@ fi # Rounding down size so that is is a multiple of 64, works around a bug in # qemu-img that may occur when compressing raw images that aren't a multiple # of 64k. https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1180021 -du_size=$(echo "$du_size" | awk ' { if ($1 % 64 != 0) print $1 + 64 - ( $1 % 64); else print $1; } ') -truncate -s${du_size}K $TMP_IMAGE_PATH +export DIB_IMAGE_SIZE=$(echo "$du_size" | awk ' { if ($1 % 64 != 0) print $1 + 64 - ( $1 % 64); else print $1; } ') if [ -n "$MAX_ONLINE_RESIZE" ]; then MKFS_OPTS="-E resize=$MAX_ONLINE_RESIZE $MKFS_OPTS" fi -LOOPDEV=$(sudo losetup --show -f $TMP_IMAGE_PATH) -export EXTRA_UNMOUNT="detach_loopback $LOOPDEV" -export IMAGE_BLOCK_DEVICE=$LOOPDEV +export TMP_IMAGE_DIR +# Try the 'old fashioned' way calling the block device +# phase. If this gives no result, use the configuration based approach: eval_run_d block-device "IMAGE_BLOCK_DEVICE=" +if [ -z ${IMAGE_BLOCK_DEVICE} ] ; then + IMAGE_BLOCK_DEVICE=$(dib-block-device \ + --phase=create \ + --config="${DIB_BLOCK_DEVICE_CONFIG:-}" \ + --image-size="${DIB_IMAGE_SIZE}"KiB \ + --image-dir="${TMP_IMAGE_DIR}" \ + --build-dir="${TMP_BUILD_DIR}" ) +fi +export IMAGE_BLOCK_DEVICE +LOOPDEV=${IMAGE_BLOCK_DEVICE} +export EXTRA_DETACH="detach_loopback $LOOPDEV" +export EXTRA_UNMOUNT="dib-block-device --phase=umount \ + --build-dir=\"${TMP_BUILD_DIR}\"" + +# Create the partitions and make them visible to the system + +# Create 2 partitions for PPC, one for PReP boot and other for root +if [[ "$ARCH" =~ "ppc" ]] ; then + sudo parted -a optimal -s $IMAGE_BLOCK_DEVICE \ + mklabel msdos \ + mkpart primary 0 8cyl \ + set 1 boot on \ + set 1 prep on \ + mkpart primary 9cyl 100% +else + sudo parted -a optimal -s $IMAGE_BLOCK_DEVICE \ + mklabel msdos \ + mkpart primary 1MiB 100% \ + set 1 boot on +fi + +sudo partprobe $IMAGE_BLOCK_DEVICE + +# To ensure no race conditions exist from calling partprobe +sudo udevadm settle + +# If the partition isn't under /dev/loop*p1, create it with kpartx +DM= +if [ ! -e "${IMAGE_BLOCK_DEVICE}p1" ]; then + DM=${IMAGE_BLOCK_DEVICE/#\/dev/\/dev\/mapper} + # If running inside Docker, make our nodes manually, because udev will not be working. + if [ -f /.dockerenv ]; then + # kpartx cannot run in sync mode in docker. + sudo kpartx -av $TMP_IMAGE_PATH + sudo dmsetup --noudevsync mknodes + else + sudo kpartx -asv $TMP_IMAGE_PATH + fi +elif [[ "$ARCH" =~ "ppc" ]]; then + sudo kpartx -asv $TMP_IMAGE_PATH +fi + +if [ -n "$DM" ]; then + export IMAGE_BLOCK_DEVICE=${DM}p1 +elif [[ "$ARCH" =~ "ppc" ]]; then + DM=${IMAGE_BLOCK_DEVICE/#\/dev/\/dev\/mapper} + export IMAGE_BLOCK_DEVICE=${DM}p2 +else + export IMAGE_BLOCK_DEVICE=${IMAGE_BLOCK_DEVICE}p1 +fi + +# End: Creation of the partitions + sudo mkfs $MKFS_OPTS -t $FS_TYPE -L ${DIB_ROOT_LABEL} ${IMAGE_BLOCK_DEVICE} # Tuning the rootfs uuid works only for ext filesystems. if echo "$FS_TYPE" | grep -q "^ext"; then @@ -457,7 +519,11 @@ fi # Unmount and cleanup the /mnt and /build subdirectories, to save # space before converting the image to some other format. +export EXTRA_UNMOUNT="" unmount_image +export TMP_IMAGE_PATH=$(dib-block-device \ + --phase=umount \ + --build-dir="${TMP_BUILD_DIR}" ) cleanup_build_dir if [[ (! $IMAGE_ELEMENT =~ no-final-image) && "$IS_RAMDISK" == "0" ]]; then diff --git a/diskimage_builder/block_device/__init__.py b/diskimage_builder/block_device/__init__.py new file mode 100644 index 00000000..0fac8380 --- /dev/null +++ b/diskimage_builder/block_device/__init__.py @@ -0,0 +1,81 @@ +# Copyright 2016 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +from diskimage_builder.block_device.blockdevice import BlockDevice +from diskimage_builder import logging_config +import logging + + +def val_else_none(s): + return s if s is None or len(s) > 0 else None + + +def generate_phase_doc(): + phase_doc = "" + bdattrs = dir(BlockDevice) + for attr in bdattrs: + if attr.startswith("cmd_"): + phase_doc += " '" + attr[4:] + "'\n" + method = getattr(BlockDevice, attr, None) + # The first line is the line that is used + phase_doc += " " + method.__doc__.split("\n")[0] + "\n" + return phase_doc + + +def main(): + logging_config.setup() + logger = logging.getLogger(__name__) + phase_doc = generate_phase_doc() + + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="Create block device layer", + epilog="Available phases:\n" + phase_doc) + parser.add_argument('--phase', required=True, + help="phase to execute") + parser.add_argument('--config', required=False, + help="configuration for block device " + "layer as JSON object") + parser.add_argument('--build-dir', required=True, + help="path to temporary build dir") + parser.add_argument('--image-size', required=False, + help="default image size") + parser.add_argument('--image-dir', required=False, + help="default image directory") + args = parser.parse_args() + + logger.info("phase [%s]" % args.phase) + logger.info("config [%s]" % args.config) + logger.info("build_dir [%s]" % args.build_dir) + + bd = BlockDevice(val_else_none(args.config), + val_else_none(args.build_dir), + val_else_none(args.image_size), + val_else_none(args.image_dir)) + + # Check if the method is available + method = getattr(bd, "cmd_" + args.phase, None) + if callable(method): + # If so: call it. + method() + else: + logger.error("phase [%s] does not exists" % args.phase) + return 1 + + return 0 + + +if __name__ == "__main__": + main() diff --git a/diskimage_builder/block_device/blockdevice.py b/diskimage_builder/block_device/blockdevice.py new file mode 100644 index 00000000..6e9f5ef1 --- /dev/null +++ b/diskimage_builder/block_device/blockdevice.py @@ -0,0 +1,119 @@ +# Copyright 2016 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from diskimage_builder.block_device.level0 import Level0 +from diskimage_builder.block_device.utils import convert_to_utf8 +import json +import logging +import os +import shutil + + +logger = logging.getLogger(__name__) + + +class BlockDevice(object): + + # Currently there is only the need for a first element (which must + # be a list). + DefaultConfig = [ + [["local_loop", + {"name": "rootdisk"}]]] + # The reason for the complex layout is, that for future layers + # there is a need to add additional lists, like: + # DefaultConfig = [ + # [["local_loop", + # {"name": "rootdisk"}]], + # [["partitioning", + # {"rootdisk": { + # "label": "mbr", + # "partitions": + # [{"name": "rd-partition1", + # "flags": ["boot"], + # "size": "100%"}]}}]], + # [["fs", + # {"rd-partition1": {}}]] + # ] + + def __init__(self, block_device_config, build_dir, + default_image_size, default_image_dir): + if block_device_config is None: + self.config = BlockDevice.DefaultConfig + else: + self.config = json.loads(block_device_config) + self.default_config = { + 'image_size': default_image_size, + 'image_dir': default_image_dir} + self.state_dir = os.path.join(build_dir, + "states/block-device") + self.state_json_file_name \ + = os.path.join(self.state_dir, "state.json") + + def write_state(self, result): + logger.debug("Write state [%s]" % self.state_json_file_name) + os.makedirs(self.state_dir) + with open(self.state_json_file_name, "w") as fd: + json.dump([self.config, self.default_config, result], fd) + + def load_state(self): + with open(self.state_json_file_name, "r") as fd: + return convert_to_utf8(json.load(fd)) + + def cmd_create(self): + """Creates the block device""" + + logger.info("create() called") + logger.debug("config [%s]" % self.config) + lvl0 = Level0(self.config[0], self.default_config, None) + result = lvl0.create() + logger.debug("Result level 0 [%s]" % result) + + # To be compatible with the current implementation, echo the + # result to stdout. + print("%s" % result['rootdisk']['device']) + + self.write_state(result) + + logger.info("create() finished") + return 0 + + def cmd_umount(self): + """Unmounts the blockdevice and cleanup resources""" + + logger.info("umount() called") + try: + os.stat(self.state_json_file_name) + except OSError: + logger.info("State already cleaned - no way to do anything here") + return 0 + + config, default_config, state = self.load_state() + logger.debug("Using config [%s]" % config) + logger.debug("Using default config [%s]" % default_config) + logger.debug("Using state [%s]" % state) + + level0 = Level0(config[0], default_config, state) + result = level0.delete() + + # If everything finished well, remove the results. + if result: + logger.info("Removing temporary dir [%s]" % self.state_dir) + shutil.rmtree(self.state_dir) + + # To be compatible with the current implementation, echo the + # result to stdout. + print("%s" % state['rootdisk']['image']) + + logger.info("umount() finished result [%d]" % result) + return 0 if result else 1 diff --git a/diskimage_builder/block_device/level0/__init__.py b/diskimage_builder/block_device/level0/__init__.py new file mode 100644 index 00000000..58ec14aa --- /dev/null +++ b/diskimage_builder/block_device/level0/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2016 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from diskimage_builder.block_device.level0.localloop import LocalLoop +from diskimage_builder.block_device.levelbase import LevelBase + +__all__ = [LocalLoop] + + +class Level0(LevelBase): + """Block Device Level0: preparation of images + + This is the class that handles level 0 block device setup: + creating the block device image and providing OS access to it. + """ + + def __init__(self, config, default_config, result): + LevelBase.__init__(self, 0, config, default_config, result, + {LocalLoop.type_string: LocalLoop}) diff --git a/diskimage_builder/block_device/level0/localloop.py b/diskimage_builder/block_device/level0/localloop.py new file mode 100644 index 00000000..3131f17d --- /dev/null +++ b/diskimage_builder/block_device/level0/localloop.py @@ -0,0 +1,92 @@ +# Copyright 2016 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from diskimage_builder.block_device.utils import parse_abs_size_spec +import logging +import os +import subprocess +import sys +import time + + +logger = logging.getLogger(__name__) + + +class LocalLoop(object): + """Level0: Local loop image device handling. + + This class handles local loop devices that can be used + for VM image installation. + """ + + type_string = "local_loop" + + def __init__(self, config, default_config, result=None): + if 'size' in config: + self.size = parse_abs_size_spec(config['size']) + logger.debug("Image size [%s]" % self.size) + else: + self.size = parse_abs_size_spec(default_config['image_size']) + logger.debug("Using default image size [%s]" % self.size) + if 'directory' in config: + self.image_dir = config['directory'] + else: + self.image_dir = default_config['image_dir'] + self.name = config['name'] + self.filename = os.path.join(self.image_dir, self.name + ".raw") + self.result = result + if self.result is not None: + self.block_device = self.result[self.name]['device'] + + def create(self): + logger.debug("[%s] Creating loop on [%s] with size [%d]" % + (self.name, self.filename, self.size)) + + with open(self.filename, "w") as fd: + fd.seek(self.size - 1) + fd.write("\0") + + logger.debug("Calling [sudo losetup --show -f %s]" + % self.filename) + subp = subprocess.Popen(["sudo", "losetup", "--show", "-f", + self.filename], stdout=subprocess.PIPE) + rval = subp.wait() + if rval == 0: + # [:-1]: Cut of the newline + self.block_device = subp.stdout.read()[:-1] + logger.debug("New block device [%s]" % self.block_device) + else: + logger.error("losetup failed") + sys.exit(1) + + return {self.name: {"device": self.block_device, + "image": self.filename}} + + def delete(self): + # loopback dev may be tied up a bit by udev events triggered + # by partition events + for try_cnt in range(10, 1, -1): + logger.debug("Delete loop [%s]" % self.block_device) + res = subprocess.call("sudo losetup -d %s" % + (self.block_device), + shell=True) + if res == 0: + return {self.name: True} + logger.debug("[%s] may be busy, sleeping [%d] more secs" + % (self.block_device, try_cnt)) + time.sleep(1) + + logger.debug("Gave up trying to detach [%s]" % + self.block_device) + return {self.name: False} diff --git a/diskimage_builder/block_device/levelbase.py b/diskimage_builder/block_device/levelbase.py new file mode 100644 index 00000000..728dedd3 --- /dev/null +++ b/diskimage_builder/block_device/levelbase.py @@ -0,0 +1,66 @@ +# Copyright 2016 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import sys + + +logger = logging.getLogger(__name__) + + +class LevelBase(object): + + def __init__(self, lvl, config, default_config, result, sub_modules): + self.level = lvl + self.config = config + self.default_config = default_config + self.result = result + self.sub_modules = sub_modules + + def call_sub_modules(self, callback): + """Generic way calling submodules""" + result = {} + if self.result is not None: + result = self.result.copy() + for name, cfg in self.config: + if name in self.sub_modules: + logger.info("Calling sub module [%s]" % name) + sm = self.sub_modules[name](cfg, self.default_config, + self.result) + lres = callback(sm) + result.update(lres) + else: + logger.error("Unknown sub module [%s]" % name) + sys.exit(1) + return result + + def create_cb(self, obj): + return obj.create() + + def create(self): + """Create the configured block devices""" + logger.info("Starting to create level [%d] block devices" % self.level) + result = self.call_sub_modules(self.create_cb) + logger.info("Finished creating level [%d] block devices" % self.level) + return result + + def delete_cb(self, obj): + return obj.delete() + + def delete(self): + """Delete the configured block devices""" + logger.info("Starting to delete level [%d] block devices" % self.level) + res = self.call_sub_modules(self.delete_cb) + logger.info("Finished deleting level [%d] block devices" % self.level) + return all(p for p in res.values()) diff --git a/diskimage_builder/block_device/tests/__init__.py b/diskimage_builder/block_device/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/diskimage_builder/block_device/tests/test_utils.py b/diskimage_builder/block_device/tests/test_utils.py new file mode 100644 index 00000000..570c7265 --- /dev/null +++ b/diskimage_builder/block_device/tests/test_utils.py @@ -0,0 +1,29 @@ +# Copyright 2016 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from diskimage_builder.block_device.utils import parse_abs_size_spec +import testtools + + +class TestLoggingConfig(testtools.TestCase): + + def test_parse_size_spec(self): + map(lambda tspec: + self.assertEqual(parse_abs_size_spec(tspec[0]), tspec[1]), + [["20TiB", 20 * 1024**4], + ["1024KiB", 1024 * 1024], + ["1.2TB", 1.2 * 1000**4], + ["2.4T", 2.4 * 1000**4], + ["512B", 512], + ["364", 364]]) diff --git a/diskimage_builder/block_device/utils.py b/diskimage_builder/block_device/utils.py new file mode 100644 index 00000000..399092a6 --- /dev/null +++ b/diskimage_builder/block_device/utils.py @@ -0,0 +1,73 @@ +# Copyright 2016 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +SIZE_SPECS = [ + ["TiB", 1024**4], + ["GiB", 1024**3], + ["MiB", 1024**2], + ["KiB", 1024**1], + ["TB", 1000**4], + ["GB", 1000**3], + ["MB", 1000**2], + ["KB", 1000**1], + ["T", 1000**4], + ["G", 1000**3], + ["M", 1000**2], + ["K", 1000**1], + ["B", 1], + ["", 1], # No unit -> size is given in bytes +] + + +def _split_size_spec(size_spec): + for spec_key, spec_value in SIZE_SPECS: + if len(spec_key) == 0: + return size_spec, spec_key + if size_spec.endswith(spec_key): + return size_spec[:-len(spec_key)], spec_key + raise RuntimeError("size_spec [%s] not known" % size_spec) + + +def _get_unit_factor(unit_str): + for spec_key, spec_value in SIZE_SPECS: + if unit_str == spec_key: + return spec_value + raise RuntimeError("unit_str [%s] not known" % unit_str) + + +def parse_abs_size_spec(size_spec): + size_cnt_str, size_unit_str = _split_size_spec(size_spec) + unit_factor = _get_unit_factor(size_unit_str) + return int(unit_factor * ( + float(size_cnt_str) if len(size_cnt_str) > 0 else 1)) + + +def convert_to_utf8(jdata): + """Convert to UTF8. + + The json parser returns unicode strings. Because in + some python implementations unicode strings are not + compatible with utf8 strings - especially when using + as keys in dictionaries - this function recursively + converts the json data. + """ + if isinstance(jdata, unicode): + return jdata.encode('utf-8') + elif isinstance(jdata, dict): + return {convert_to_utf8(key): convert_to_utf8(value) + for key, value in jdata.iteritems()} + elif isinstance(jdata, list): + return [convert_to_utf8(je) for je in jdata] + else: + return jdata diff --git a/doc/source/developer/developing_elements.rst b/doc/source/developer/developing_elements.rst index 7caf1a94..c4d7d5b8 100644 --- a/doc/source/developer/developing_elements.rst +++ b/doc/source/developer/developing_elements.rst @@ -144,7 +144,7 @@ The phases are: operations here to only those necessary to affect the filesystem metadata and image itself. For most operations, ``post-install.d`` is preferred. - * runs: **in chroot** + * runs: **in chroot** ``cleanup.d`` Perform cleanup of the root filesystem content. For instance, temporary diff --git a/doc/source/user_guide/building_an_image.rst b/doc/source/user_guide/building_an_image.rst index 8137fd00..438af001 100644 --- a/doc/source/user_guide/building_an_image.rst +++ b/doc/source/user_guide/building_an_image.rst @@ -55,6 +55,87 @@ formats are: * docker * raw +Disk Image Layout +----------------- + +When generating a block image (e.g. qcow2 or raw), by default one +image with one partition holding all files is created. + +The appropriate functionality to use multiple partitions and even LVMs +is currently under development; therefore the possible configuration +is currently limited, but will get more flexible as soon as all the +functionality is implemented. + +The configuration is done by means of the environment variable +`DIB_BLOCK_DEVICE_CONFIG`. This variable must hold JSON structured +configuration data. + +In future this will be a list of four elements, each describing one +level of block device setup - but because currently only the lowest +level is implemented, it contains only the configuration of the first +level of block device setup + +The default is: + +:: + + DIB_BLOCK_DEVICE_CONFIG='[ + [["local_loop", + {"name": "rootdisk"}]]]' + +In general each module is configured in the way, that the first +element in the list is the name of the module (e.g. `local_loop`) +followed by a dictionary of parameters (here `{"name": "rootdisk"}`). + +Level 0 ++++++++ + +Module: Local Loop +.................. + +This module generates a local image file and uses the loop device to +create a block device from it. The symbolic name for this module is +`local_loop`. + +Configuration options: + +name + (mandatory) The name of the image. This is used as the name for the + image in the file system and also as a symbolic name to be able to + reference this image (e.g. to create a partition table on this + disk). + +size + (optional) The size of the disk. The size can be expressed using + unit names like TiB (1024^4 bytes) or GB (1000^3 bytes). + Examples: 2.5GiB, 12KB. + If the size is not specified here, the size as given to + disk-image-create (--image-size) or the automatically computed size + is used. + +directory + (optional) The directory where the image is created. + +Example: + +:: + + DIB_BLOCK_DEVICE_CONFIG='[ + [["local_loop", + {"name": "rootdisk"}], + ["local_loop", + {"name": "datadisk", + "size": "7.5GiB", + "directory": "/var/tmp"}]]]' + +This creates two image files and uses the loop device to use them as +block devices. One image file called `rootdisk` is created with +default size in the default temp directory. The second image has the +size of 7.5GiB and is created in the `/var/tmp` folder. + +Please note that due to current implementation restrictions it is only +allowed to specify one local loop image. + Filesystem Caveat ----------------- diff --git a/elements/vm/block-device.d/10-partition b/elements/vm/block-device.d/10-partition deleted file mode 100755 index 53ba25b0..00000000 --- a/elements/vm/block-device.d/10-partition +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -# dib-lint: disable=safe_sudo - -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -[ -n "$IMAGE_BLOCK_DEVICE" ] || die "Image block device not set" - -# Create 2 partitions for PPC, one for PReP boot and other for root -if [[ "$ARCH" =~ "ppc" ]] ; then - sudo parted -a optimal -s $IMAGE_BLOCK_DEVICE \ - mklabel msdos \ - mkpart primary 0 8cyl \ - set 1 boot on \ - set 1 prep on \ - mkpart primary 9cyl 100% -else - sudo parted -a optimal -s $IMAGE_BLOCK_DEVICE \ - mklabel msdos \ - mkpart primary 1MiB 100% \ - set 1 boot on -fi - -sudo partprobe $IMAGE_BLOCK_DEVICE - -# To ensure no race conditions exist from calling partprobe -sudo udevadm settle - -# If the partition isn't under /dev/loop*p1, create it with kpartx -DM= -if [ ! -e "${IMAGE_BLOCK_DEVICE}p1" ]; then - DM=${IMAGE_BLOCK_DEVICE/#\/dev/\/dev\/mapper} - # If running inside Docker, make our nodes manually, because udev will not be working. - if [ -f /.dockerenv ]; then - # kpartx cannot run in sync mode in docker. - sudo kpartx -av $TMP_IMAGE_PATH - sudo dmsetup --noudevsync mknodes - else - sudo kpartx -asv $TMP_IMAGE_PATH - fi -elif [[ "$ARCH" =~ "ppc" ]]; then - sudo kpartx -asv $TMP_IMAGE_PATH -fi - -if [ -n "$DM" ]; then - echo "IMAGE_BLOCK_DEVICE=${DM}p1" -elif [[ "$ARCH" =~ "ppc" ]]; then - DM=${IMAGE_BLOCK_DEVICE/#\/dev/\/dev\/mapper} - echo "IMAGE_BLOCK_DEVICE=${DM}p2" -else - echo "IMAGE_BLOCK_DEVICE=${IMAGE_BLOCK_DEVICE}p1" -fi diff --git a/lib/common-functions b/lib/common-functions index b919dedd..080aae69 100644 --- a/lib/common-functions +++ b/lib/common-functions @@ -181,16 +181,7 @@ function detach_loopback() { sudo dmsetup --noudevsync remove $mapper_name fi - # loopback dev may be tied up a bit by udev events triggered by partition events - for try in $(seq 10 -1 1) ; do - if ! sudo losetup $loopdev || sudo losetup -d $loopdev ; then - return 0 - fi - echo $loopdev may be busy, sleeping up to $try more seconds... - sleep 1 - done - echo Gave up trying to detach $loopdev - return 1 + return 0 } function arg_to_elements() { diff --git a/lib/img-functions b/lib/img-functions index f64a95a5..c83071f7 100644 --- a/lib/img-functions +++ b/lib/img-functions @@ -24,6 +24,9 @@ function unmount_image () { # unmount from the chroot # Don't use TMP_MOUNT_PATH here, it might not have been set. unmount_dir "$TMP_BUILD_DIR/mnt" + if [ -n "$EXTRA_DETACH" ]; then + $EXTRA_DETACH + fi if [ -n "$EXTRA_UNMOUNT" ]; then $EXTRA_UNMOUNT fi diff --git a/setup.cfg b/setup.cfg index ec576817..7839aeb0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,3 +54,7 @@ domain = diskimage_builder [wheel] universal = 1 + +[entry_points] +console_scripts = + dib-block-device = diskimage_builder.block_device:main diff --git a/tox.ini b/tox.ini index 8205d15b..10d870f9 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ deps= -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands= python setup.py testr --slowest --testr-args='{posargs}' +passenv = ELEMENTS_PATH DIB_RELEASE DIB_DEBUG_TRACE DIB_DEV_USER_USERNAME DIB_DEV_USER_PWDLESS_SUDO DIB_DEV_USER_PASSWORD USER HOME http_proxy https_proxy [testenv:pep8] commands =