From ec7f56c1b2d8aa385751f02a3fa82e5a13d20b9d Mon Sep 17 00:00:00 2001 From: Andreas Florath Date: Sat, 16 Jul 2016 22:16:13 +0200 Subject: [PATCH] Refactor: block-device handling (partitioning) During the creation of a disk image (e.g. for a VM), there is the need to create, setup, configure and afterwards detach some kind of storage where the newly installed OS can be copied to or directly installed in. This patch implements partitioning handling. Change-Id: I0ca6a4ae3a2684d473b44e5f332ee4225ee30f8c Signed-off-by: Andreas Florath --- diskimage_builder/block_device/__init__.py | 2 +- diskimage_builder/block_device/blockdevice.py | 173 ++++++--- .../block_device/blockdevicesetupexception.py | 17 + .../block_device/level0/__init__.py | 13 - .../block_device/level0/localloop.py | 107 ++++-- .../block_device/level1/__init__.py | 17 + diskimage_builder/block_device/level1/mbr.py | 360 ++++++++++++++++++ .../block_device/level1/partitioning.py | 184 +++++++++ diskimage_builder/block_device/levelbase.py | 66 ---- diskimage_builder/block_device/utils.py | 62 +-- .../bootloader/finalise.d/50-bootloader | 8 +- .../elements/partitioning-sfdisk/README.rst | 19 - .../block-device.d/10-partitioning-sfdisk | 46 --- .../environment.d/10-partitioning-sfdisk | 5 - .../50-partitioning-remove-bogus-udev-links | 12 - .../elements/vm/environment.d/10-partitioning | 15 + diskimage_builder/graph/__init__.py | 0 diskimage_builder/graph/digraph.py | 194 ++++++++++ diskimage_builder/lib/common-functions | 2 +- diskimage_builder/lib/disk-image-create | 71 ++-- diskimage_builder/lib/img-functions | 4 + .../tests/functional/__init__.py | 0 .../tests/functional/test_blockdevice.py | 87 +++++ .../tests/functional/test_blockdevice_mbr.py | 174 +++++++++ .../functional/test_blockdevice_utils.py | 49 +++ .../tests/functional/test_graph.py | 123 ++++++ .../tests/functional/test_graph_toposort.py | 69 ++++ diskimage_builder/utils.py | 20 + doc/source/user_guide/building_an_image.rst | 177 +++++++-- ...-device-partitioning-237249e7ed2bad26.yaml | 11 + tests/install_test_deps.sh | 1 + tests/run_functests.sh | 7 + tox.ini | 3 + 33 files changed, 1763 insertions(+), 335 deletions(-) create mode 100644 diskimage_builder/block_device/blockdevicesetupexception.py create mode 100644 diskimage_builder/block_device/level1/__init__.py create mode 100644 diskimage_builder/block_device/level1/mbr.py create mode 100644 diskimage_builder/block_device/level1/partitioning.py delete mode 100644 diskimage_builder/block_device/levelbase.py delete mode 100644 diskimage_builder/elements/partitioning-sfdisk/README.rst delete mode 100755 diskimage_builder/elements/partitioning-sfdisk/block-device.d/10-partitioning-sfdisk delete mode 100644 diskimage_builder/elements/partitioning-sfdisk/environment.d/10-partitioning-sfdisk delete mode 100755 diskimage_builder/elements/partitioning-sfdisk/finalise.d/50-partitioning-remove-bogus-udev-links create mode 100644 diskimage_builder/elements/vm/environment.d/10-partitioning create mode 100644 diskimage_builder/graph/__init__.py create mode 100644 diskimage_builder/graph/digraph.py create mode 100644 diskimage_builder/tests/functional/__init__.py create mode 100644 diskimage_builder/tests/functional/test_blockdevice.py create mode 100644 diskimage_builder/tests/functional/test_blockdevice_mbr.py create mode 100644 diskimage_builder/tests/functional/test_blockdevice_utils.py create mode 100644 diskimage_builder/tests/functional/test_graph.py create mode 100644 diskimage_builder/tests/functional/test_graph_toposort.py create mode 100644 diskimage_builder/utils.py create mode 100644 releasenotes/notes/block-device-partitioning-237249e7ed2bad26.yaml diff --git a/diskimage_builder/block_device/__init__.py b/diskimage_builder/block_device/__init__.py index 0fac8380..4a071509 100644 --- a/diskimage_builder/block_device/__init__.py +++ b/diskimage_builder/block_device/__init__.py @@ -69,7 +69,7 @@ def main(): method = getattr(bd, "cmd_" + args.phase, None) if callable(method): # If so: call it. - method() + return method() else: logger.error("phase [%s] does not exists" % args.phase) return 1 diff --git a/diskimage_builder/block_device/blockdevice.py b/diskimage_builder/block_device/blockdevice.py index 6e9f5ef1..97e01434 100644 --- a/diskimage_builder/block_device/blockdevice.py +++ b/diskimage_builder/block_device/blockdevice.py @@ -12,12 +12,18 @@ # License for the specific language governing permissions and limitations # under the License. -from diskimage_builder.block_device.level0 import Level0 -from diskimage_builder.block_device.utils import convert_to_utf8 +import codecs +from diskimage_builder.block_device.blockdevicesetupexception \ + import BlockDeviceSetupException +from diskimage_builder.block_device.level0 import LocalLoop +from diskimage_builder.block_device.level1 import Partitioning +from diskimage_builder.graph.digraph import Digraph import json import logging import os import shutil +import sys +import yaml logger = logging.getLogger(__name__) @@ -25,33 +31,36 @@ logger = logging.getLogger(__name__) class BlockDevice(object): - # Currently there is only the need for a first element (which must - # be a list). - DefaultConfig = [ - [["local_loop", - {"name": "rootdisk"}]]] - # The reason for the complex layout is, that for future layers - # there is a need to add additional lists, like: - # DefaultConfig = [ - # [["local_loop", - # {"name": "rootdisk"}]], - # [["partitioning", - # {"rootdisk": { - # "label": "mbr", - # "partitions": - # [{"name": "rd-partition1", - # "flags": ["boot"], - # "size": "100%"}]}}]], - # [["fs", - # {"rd-partition1": {}}]] - # ] + # Default configuration: + # one image, one partition, mounted under '/' + DefaultConfig = """ +local_loop: + name: image0 +""" + +# This is an example of the next level config +# mkfs: +# base: root_p1 +# type: ext4 +# mount_point: / + + # A dictionary to map sensible names to internal implementation. + cfg_type_map = { + 'local_loop': LocalLoop, + 'partitioning': Partitioning, + 'mkfs': 'not yet implemented', + } def __init__(self, block_device_config, build_dir, default_image_size, default_image_dir): + logger.debug("Creating BlockDevice object") + logger.debug("Config given [%s]" % block_device_config) + logger.debug("Build dir [%s]" % build_dir) if block_device_config is None: - self.config = BlockDevice.DefaultConfig - else: - self.config = json.loads(block_device_config) + block_device_config = BlockDevice.DefaultConfig + self.config = yaml.safe_load(block_device_config) + logger.debug("Using config [%s]" % self.config) + self.default_config = { 'image_size': default_image_size, 'image_dir': default_image_dir} @@ -67,53 +76,125 @@ class BlockDevice(object): json.dump([self.config, self.default_config, result], fd) def load_state(self): - with open(self.state_json_file_name, "r") as fd: - return convert_to_utf8(json.load(fd)) + with codecs.open(self.state_json_file_name, + encoding="utf-8", mode="r") as fd: + return json.load(fd) + + def create_graph(self, config, default_config): + # This is the directed graph of nodes: each parse method must + # add the appropriate nodes and edges. + dg = Digraph() + + for cfg_obj_name, cfg_obj_val in config.items(): + # As the first step the configured objects are created + # (if it exists) + if cfg_obj_name not in BlockDevice.cfg_type_map: + logger.error("Configured top level element [%s] " + "does not exists." % cfg_obj_name) + return 1 + cfg_obj = BlockDevice.cfg_type_map[cfg_obj_name]( + cfg_obj_val, default_config) + # At this point it is only possible to add the nodes: + # adding the edges needs all nodes first. + cfg_obj.insert_nodes(dg) + + # Now that all the nodes exists: add also the edges + for node in dg.get_iter_nodes_values(): + node.insert_edges(dg) + + call_order = dg.topological_sort() + logger.debug("Call order [%s]" % (list(call_order))) + return dg, call_order + + def create(self, result, rollback): + dg, call_order = self.create_graph(self.config, self.default_config) + for node in call_order: + node.create(result, rollback) def cmd_create(self): """Creates the block device""" logger.info("create() called") - logger.debug("config [%s]" % self.config) - lvl0 = Level0(self.config[0], self.default_config, None) - result = lvl0.create() - logger.debug("Result level 0 [%s]" % result) + logger.debug("Using config [%s]" % self.config) + + result = {} + rollback = [] + + try: + self.create(result, rollback) + except BlockDeviceSetupException as bdse: + logger.error("exception [%s]" % bdse) + for rollback_cb in reversed(rollback): + rollback_cb() + sys.exit(1) # To be compatible with the current implementation, echo the # result to stdout. - print("%s" % result['rootdisk']['device']) + # If there is no partition needed, pass back directly the + # image. + if 'root_p1' in result: + print("%s" % result['root_p1']['device']) + else: + print("%s" % result['image0']['device']) self.write_state(result) logger.info("create() finished") return 0 - def cmd_umount(self): - """Unmounts the blockdevice and cleanup resources""" - - logger.info("umount() called") + def _load_state(self): + logger.info("_load_state() called") try: os.stat(self.state_json_file_name) except OSError: logger.info("State already cleaned - no way to do anything here") - return 0 + return None, None, None config, default_config, state = self.load_state() logger.debug("Using config [%s]" % config) logger.debug("Using default config [%s]" % default_config) logger.debug("Using state [%s]" % state) - level0 = Level0(config[0], default_config, state) - result = level0.delete() + # Deleting must be done in reverse order + dg, call_order = self.create_graph(config, default_config) + reverse_order = reversed(call_order) + return dg, reverse_order, state - # If everything finished well, remove the results. - if result: - logger.info("Removing temporary dir [%s]" % self.state_dir) - shutil.rmtree(self.state_dir) + def cmd_umount(self): + """Unmounts the blockdevice and cleanup resources""" + + dg, reverse_order, state = self._load_state() + if dg is None: + return 0 + for node in reverse_order: + node.umount(state) # To be compatible with the current implementation, echo the # result to stdout. - print("%s" % state['rootdisk']['image']) + print("%s" % state['image0']['image']) - logger.info("umount() finished result [%d]" % result) - return 0 if result else 1 + return 0 + + def cmd_cleanup(self): + """Cleanup all remaining relicts - in good case""" + + dg, reverse_order, state = self._load_state() + for node in reverse_order: + node.cleanup(state) + + logger.info("Removing temporary dir [%s]" % self.state_dir) + shutil.rmtree(self.state_dir) + + return 0 + + def cmd_delete(self): + """Cleanup all remaining relicts - in case of an error""" + + dg, reverse_order, state = self._load_state() + for node in reverse_order: + node.delete(state) + + logger.info("Removing temporary dir [%s]" % self.state_dir) + shutil.rmtree(self.state_dir) + + return 0 diff --git a/diskimage_builder/block_device/blockdevicesetupexception.py b/diskimage_builder/block_device/blockdevicesetupexception.py new file mode 100644 index 00000000..b0e42356 --- /dev/null +++ b/diskimage_builder/block_device/blockdevicesetupexception.py @@ -0,0 +1,17 @@ +# Copyright 2016 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class BlockDeviceSetupException(Exception): + pass diff --git a/diskimage_builder/block_device/level0/__init__.py b/diskimage_builder/block_device/level0/__init__.py index 58ec14aa..dffb6254 100644 --- a/diskimage_builder/block_device/level0/__init__.py +++ b/diskimage_builder/block_device/level0/__init__.py @@ -13,18 +13,5 @@ # under the License. from diskimage_builder.block_device.level0.localloop import LocalLoop -from diskimage_builder.block_device.levelbase import LevelBase __all__ = [LocalLoop] - - -class Level0(LevelBase): - """Block Device Level0: preparation of images - - This is the class that handles level 0 block device setup: - creating the block device image and providing OS access to it. - """ - - def __init__(self, config, default_config, result): - LevelBase.__init__(self, 0, config, default_config, result, - {LocalLoop.type_string: LocalLoop}) diff --git a/diskimage_builder/block_device/level0/localloop.py b/diskimage_builder/block_device/level0/localloop.py index 3131f17d..7fd21b3d 100644 --- a/diskimage_builder/block_device/level0/localloop.py +++ b/diskimage_builder/block_device/level0/localloop.py @@ -12,18 +12,19 @@ # License for the specific language governing permissions and limitations # under the License. +from diskimage_builder.block_device.blockdevicesetupexception \ + import BlockDeviceSetupException from diskimage_builder.block_device.utils import parse_abs_size_spec +from diskimage_builder.graph.digraph import Digraph import logging import os import subprocess -import sys -import time logger = logging.getLogger(__name__) -class LocalLoop(object): +class LocalLoop(Digraph.Node): """Level0: Local loop image device handling. This class handles local loop devices that can be used @@ -32,7 +33,9 @@ class LocalLoop(object): type_string = "local_loop" - def __init__(self, config, default_config, result=None): + def __init__(self, config, default_config): + logger.debug("Creating LocalLoop object; config [%s] " + "default_config [%s]" % (config, default_config)) if 'size' in config: self.size = parse_abs_size_spec(config['size']) logger.debug("Image size [%s]" % self.size) @@ -44,49 +47,85 @@ class LocalLoop(object): else: self.image_dir = default_config['image_dir'] self.name = config['name'] + Digraph.Node.__init__(self, self.name) self.filename = os.path.join(self.image_dir, self.name + ".raw") - self.result = result - if self.result is not None: - self.block_device = self.result[self.name]['device'] - def create(self): - logger.debug("[%s] Creating loop on [%s] with size [%d]" % - (self.name, self.filename, self.size)) + def insert_nodes(self, dg): + dg.add_node(self) - with open(self.filename, "w") as fd: - fd.seek(self.size - 1) + def insert_edges(self, dg): + """Because this is created without base, there are no edges.""" + pass + + @staticmethod + def image_create(filename, size): + logger.info("Create image file [%s]" % filename) + with open(filename, "w") as fd: + fd.seek(size - 1) fd.write("\0") - logger.debug("Calling [sudo losetup --show -f %s]" - % self.filename) + @staticmethod + def _image_delete(filename): + logger.info("Remove image file [%s]" % filename) + os.remove(filename) + + @staticmethod + def _loopdev_attach(filename): + logger.info("loopdev attach") + logger.debug("Calling [sudo losetup --show -f %s]", filename) subp = subprocess.Popen(["sudo", "losetup", "--show", "-f", - self.filename], stdout=subprocess.PIPE) + filename], stdout=subprocess.PIPE) rval = subp.wait() if rval == 0: # [:-1]: Cut of the newline - self.block_device = subp.stdout.read()[:-1] - logger.debug("New block device [%s]" % self.block_device) + block_device = subp.stdout.read()[:-1].decode("utf-8") + logger.info("New block device [%s]" % block_device) + return block_device else: logger.error("losetup failed") - sys.exit(1) + raise BlockDeviceSetupException("losetup failed") - return {self.name: {"device": self.block_device, - "image": self.filename}} - - def delete(self): + @staticmethod + def _loopdev_detach(loopdev): + logger.info("loopdev detach") # loopback dev may be tied up a bit by udev events triggered # by partition events for try_cnt in range(10, 1, -1): - logger.debug("Delete loop [%s]" % self.block_device) - res = subprocess.call("sudo losetup -d %s" % - (self.block_device), - shell=True) - if res == 0: - return {self.name: True} - logger.debug("[%s] may be busy, sleeping [%d] more secs" - % (self.block_device, try_cnt)) - time.sleep(1) + logger.debug("Calling [sudo losetup -d %s]", loopdev) + subp = subprocess.Popen(["sudo", "losetup", "-d", + loopdev]) + rval = subp.wait() + if rval == 0: + logger.info("Successfully detached [%s]" % loopdev) + return 0 + else: + logger.error("loopdev detach failed") + # Do not raise an error - maybe other cleanup methods + # can at least do some more work. + logger.debug("Gave up trying to detach [%s]" % loopdev) + return rval - logger.debug("Gave up trying to detach [%s]" % - self.block_device) - return {self.name: False} + def create(self, result, rollback): + logger.debug("[%s] Creating loop on [%s] with size [%d]" % + (self.name, self.filename, self.size)) + + rollback.append(lambda: self._image_delete(self.filename)) + self.image_create(self.filename, self.size) + + block_device = self._loopdev_attach(self.filename) + rollback.append(lambda: self._loopdev_detach(block_device)) + + result[self.name] = {"device": block_device, + "image": self.filename} + logger.debug("Created loop name [%s] device [%s] image [%s]" + % (self.name, block_device, self.filename)) + return + + def umount(self, state): + self._loopdev_detach(state[self.name]['device']) + + def cleanup(self, state): + pass + + def delete(self, state): + self._image_delete(state[self.name]['image']) diff --git a/diskimage_builder/block_device/level1/__init__.py b/diskimage_builder/block_device/level1/__init__.py new file mode 100644 index 00000000..6c3b57a8 --- /dev/null +++ b/diskimage_builder/block_device/level1/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2016 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from diskimage_builder.block_device.level1.partitioning import Partitioning + +__all__ = [Partitioning] diff --git a/diskimage_builder/block_device/level1/mbr.py b/diskimage_builder/block_device/level1/mbr.py new file mode 100644 index 00000000..65b108b3 --- /dev/null +++ b/diskimage_builder/block_device/level1/mbr.py @@ -0,0 +1,360 @@ +# Copyright 2016 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import logging +import random +from struct import pack + + +logger = logging.getLogger(__name__) + + +# Details of the MBR object itself can be found in the inline +# documentation. +# +# General design and implementation remarks: +# o Because the whole GNU parted and co. (e.g. the python-parted that +# is based on GNU parted) cannot be used because of the license: +# everything falls under GPL2 (not LGPL2!) and therefore does not +# fit into the Apache License here. +# o It looks that there is no real alternative available (2016-06). +# o The interface of python-parted is not that simple to handle - and +# the initial try to use GNU (python-)parted was not that much +# easier and shorter than this approach. +# o When using tools (like fdisk or parted) they try to optimize the +# alignment of partitions based on the data found on the host +# system. These might be misleading and might lead to (very) poor +# performance. +# o These ready-to-use tools typically also change the CHS layout +# based on the disk size. In case that the disk is enlarged (which +# is a normal use case for golden images), the CHS layout of the +# disk changes for those tools (and is not longer correct). +# In the DIB implementation the CHS are chosen that way, that also +# for very small disks the maximum heads/cylinder and sectors/track +# is used: even if the disk size in increased, the CHS numbers will +# not change. +# o In the easy and straight forward way when only using one +# partition, exactly 40 bytes (!) must be written - and the biggest +# part of this data is fixed (same in all cases). +# +# Limitations and Incompatibilities +# o With the help of this class it is possible to create an +# arbitrarily number of extended partitions (tested with over 1000). +# o There are limitations and shortcomings in the OS and in tools +# handling these partitions. +# o Under Linux the loop device is able to handle a limited number of +# partitions. The module parameter max_loop can be set - the maximum +# number might vary depending on the distribution and kernel build. +# o Under Linux fdisk is able to handle 'only' 60 partitions. Only +# those are listed, can be changed or written. +# o Under Linux GNU parted can handle about 60 partitions. +# +# Be sure only to pass in the number of partitions that the host OS +# and target OS are able to handle. + +class MBR(object): + """MBR Disk / Partition Table Layout + + Primary partitions are created first - and must also be passed in + first. + The extended partition layout is done in the way, that there is + one entry in the MBR (the last) that uses the whole disk. + EBR (extended boot records) are used to describe the partitions + themselves. This has the advantage, that the same procedure can + be used for all partitions and arbitrarily many partitions can be + created in the same way (the EBR is placed as block 0 in each + partition itself). + In conjunction with a fixed and 'fits all' partition alignment the + major design focus is maximum performance for the installed image + (vs. minimal size). + Because of the chosen default alignment of 1MiB there will be + (1MiB - 512B) unused disk space for the MBR and also the same + size unused in every partition. + Assuming that 512 byte blocks are used, the resulting layout for + extended partitions looks like (blocks offset in extended + partition given): + 0: MBR - 2047 blocks unused + 2048: EBR for partition 1 - 2047 blocks unused + 4096: Start of data for partition 1 + ... + X: EBR for partition N - 2047 blocks unused + X+2048: Start of data for partition N + + Direct (native) writing of MBR, EBR (partition table) is + implemented - no other parititoning library or tools is used - + to be sure to get the correct CHS and alignment for a wide range + of host systems. + """ + + # Design & Implementation details: + # o A 'block' is a storage unit on disk. It is similar (equal) to a + # sector - but with LBA addressing. + # o It is assumed that a disk block has that number of bytes + bytes_per_sector = 512 + # o CHS is the 'good and very old way' specifying blocks. + # When passing around these numbers, they are also ordered like 'CHS': + # (cylinder, head, sector). + # o The computation from LBA to CHS is not unique (it is based + # on the 'real' (or assumed) number of heads/cylinder and + # sectors/track), these are the assumed numbers. Please note + # that these are also the maximum numbers: + heads_per_cylinder = 254 + sectors_per_track = 63 + max_cylinders = 1023 + # o There is the need for some offsets that are defined in the + # MBR/EBR domain. + MBR_offset_disk_id = 440 + MBR_offset_signature = 510 + MBR_offset_first_partition_table_entry = 446 + MBR_partition_type_extended_chs = 0x5 + MBR_partition_type_extended_lba = 0xF + MBR_signature = 0xAA55 + + def __init__(self, name, disk_size, alignment): + """Initialize a disk partitioning MBR object. + + The name is the (existing) name of the disk. + The disk_size is the (used) size of the disk. It must be a + proper multiple of the disk bytes per sector (currently 512) + """ + logger.info("Create MBR disk partitioning object") + + assert disk_size % MBR.bytes_per_sector == 0 + + self.disk_size = disk_size + self.disk_size_in_blocks \ + = self.disk_size // MBR.bytes_per_sector + self.alignment_blocks = alignment // MBR.bytes_per_sector + # Because the extended partitions are a chain of blocks, when + # creating a new partition, the reference in the already + # existing EBR must be updated. This holds a reference to the + # latest EBR. (A special case is the first: when it points to + # 0 (MBR) there is no need to update the reference.) + self.disk_block_last_ref = 0 + + self.name = name + self.partition_abs_start = None + self.partition_abs_next_free = None + # Start of partition number + self.partition_number = 0 + + self.primary_partitions_created = 0 + self.extended_partitions_created = 0 + + def __enter__(self): + # Open existing file for writing (r+) + self.image_fd = open(self.name, "r+b") + self.write_mbr() + self.write_mbr_signature(0) + self.partition_abs_start = self.align(1) + self.partition_abs_next_free \ + = self.partition_abs_start + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.image_fd.close() + + def lba2chs(self, lba): + """Converts a LBA block number to CHS + + If the LBA block number is bigger than the max (1023, 63, 254) + the maximum is returned. + """ + if lba > MBR.heads_per_cylinder * MBR.sectors_per_track \ + * MBR.max_cylinders: + return MBR.max_cylinders, MBR.heads_per_cylinder, \ + MBR.sectors_per_track + + cylinder = lba // (MBR.heads_per_cylinder * MBR.sectors_per_track) + head = (lba // MBR.sectors_per_track) % MBR.heads_per_cylinder + sector = (lba % MBR.sectors_per_track) + 1 + + logger.debug("Convert LBA to CHS [%d] -> [%d, %d, %d]" + % (lba, cylinder, head, sector)) + return cylinder, head, sector + + def encode_chs(self, cylinders, heads, sectors): + """Encodes a CHS triple into disk format""" + # Head - nothing to convert + assert heads <= MBR.heads_per_cylinder + eh = heads + + # Sector + assert sectors <= MBR.sectors_per_track + es = sectors + # top two bits are set in cylinder conversion + + # Cylinder + assert cylinders <= MBR.max_cylinders + ec = cylinders % 256 # lower part + hc = cylinders // 4 # extract top two bits and + es = es | hc # pass them into the top two bits of the sector + + logger.debug("Encode CHS to disk format [%d %d %d] " + "-> [%02x %02x %02x]" % (cylinders, heads, sectors, + eh, es, ec)) + return eh, es, ec + + def write_mbr(self): + """Write MBR + + This method writes the MBR to disk. It creates a random disk + id as well that it creates the extended partition (as + first partition) which uses the whole disk. + """ + disk_id = random.randint(0, 0xFFFFFFFF) + self.image_fd.seek(MBR.MBR_offset_disk_id) + self.image_fd.write(pack(" 0: + raise RuntimeError("All primary partitions must be " + "given first") + + if primaryflag: + return self.add_primary_partition(bootflag, size, ptype) + if self.extended_partitions_created == 0: + # When this is the first extended partition, the extended + # partition entry has to be written. + self.partition_abs_start = self.partition_abs_next_free + self.write_partition_entry( + False, 0, self.partition_number, + MBR.MBR_partition_type_extended_lba, + self.partition_abs_next_free, + self.disk_size_in_blocks - self.partition_abs_next_free) + self.partition_number = 4 + + return self.add_extended_partition(bootflag, size, ptype) + + def free(self): + """Returns the free (not yet partitioned) size""" + return self.disk_size \ + - (self.partition_abs_next_free + self.align(1)) \ + * MBR.bytes_per_sector diff --git a/diskimage_builder/block_device/level1/partitioning.py b/diskimage_builder/block_device/level1/partitioning.py new file mode 100644 index 00000000..36284af2 --- /dev/null +++ b/diskimage_builder/block_device/level1/partitioning.py @@ -0,0 +1,184 @@ +# Copyright 2016 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from diskimage_builder.block_device.blockdevicesetupexception \ + import BlockDeviceSetupException +from diskimage_builder.block_device.level1.mbr import MBR +from diskimage_builder.block_device.utils import parse_abs_size_spec +from diskimage_builder.block_device.utils import parse_rel_size_spec +from diskimage_builder.graph.digraph import Digraph +import logging + + +logger = logging.getLogger(__name__) + + +class Partition(Digraph.Node): + + def __init__(self, name, flags, size, ptype, base, partitioning): + Digraph.Node.__init__(self, name) + self.flags = flags + self.size = size + self.ptype = ptype + self.base = base + self.partitioning = partitioning + + def get_flags(self): + return self.flags + + def get_size(self): + return self.size + + def get_type(self): + return self.ptype + + def insert_edges(self, dg): + bnode = dg.find(self.base) + assert bnode is not None + dg.create_edge(bnode, self) + + def create(self, result, rollback): + self.partitioning.create(result, rollback) + + def umount(self, state): + """Partitioning does not need any umount task.""" + pass + + def cleanup(self, state): + """Partitioning does not need any cleanup.""" + pass + + def delete(self, state): + """Partitioning does not need any cleanup.""" + pass + + +class Partitioning(object): + + type_string = "partitioning" + + flag_boot = 1 + flag_primary = 2 + + def __init__(self, config, default_config): + logger.debug("Creating Partitioning object; config [%s]" % config) + # Because using multiple partitions of one base is done + # within one object, there is the need to store a flag if the + # creation of the partitions was already done. + self.already_created = False + + # Parameter check + if 'base' not in config: + self._config_error("Partitioning config needs 'base'") + self.base = config['base'] + + if 'label' not in config: + self._config_error("Partitioning config needs 'label'") + self.label = config['label'] + if self.label not in ("mbr", ): + self._config_error("Label must be 'mbr'") + + # It is VERY important to get the alignment correct. If this + # is not correct, the disk performance might be very poor. + # Example: In some tests a 'off by one' leads to a write + # performance of 30% compared to a correctly aligned + # partition. + # The problem for DIB is, that it cannot assume that the host + # system uses the same IO sizes as the target system, + # therefore here a fixed approach (as used in all modern + # systems with large disks) is used. The partitions are + # aligned to 1MiB (which are about 2048 times 512 bytes + # blocks) + self.align = 1024 * 1024 # 1MiB as default + if 'align' in config: + self.align = parse_abs_size_spec(config['align']) + + if 'partitions' not in config: + self._config_error("Partitioning config needs 'partitions'") + + self.partitions = {} + for part_cfg in config['partitions']: + if 'name' not in part_cfg: + self.config_error("Missing 'name' in partition config") + part_name = part_cfg['name'] + + flags = set() + if 'flags' in part_cfg: + for f in part_cfg['flags']: + if f == 'boot': + flags.add(Partitioning.flag_boot) + elif f == 'primary': + flags.add(Partitioning.flag_primary) + else: + self._config_error("Unknown flag [%s] in " + "partitioning for [%s]" + % (f, part_name)) + if 'size' not in part_cfg: + self._config_error("No 'size' in partition [%s]" + % part_name) + size = part_cfg['size'] + + ptype = int(part_cfg['type'], 16) if 'type' in part_cfg else 0x83 + + self.partitions[part_name] \ + = Partition(part_name, flags, size, ptype, self.base, self) + logger.debug(part_cfg) + + def _config_error(self, msg): + logger.error(msg) + raise BlockDeviceSetupException(msg) + + def _size_of_block_dev(self, dev): + with open(dev, "r") as fd: + fd.seek(0, 2) + return fd.tell() + + def insert_nodes(self, dg): + for _, part in self.partitions.items(): + dg.add_node(part) + + def create(self, result, rollback): + image_path = result[self.base]['image'] + device_path = result[self.base]['device'] + logger.info("Creating partition on [%s] [%s]" % + (self.base, image_path)) + + if self.already_created: + logger.info("Not creating the partitions a second time.") + return + + assert self.label == 'mbr' + + disk_size = self._size_of_block_dev(image_path) + with MBR(image_path, disk_size, self.align) as part_impl: + for part_name, part_cfg in self.partitions.items(): + part_bootflag = Partitioning.flag_boot \ + in part_cfg.get_flags() + part_primary = Partitioning.flag_primary \ + in part_cfg.get_flags() + part_size = part_cfg.get_size() + part_free = part_impl.free() + part_type = part_cfg.get_type() + logger.debug("Not partitioned space [%d]" % part_free) + part_size = parse_rel_size_spec(part_size, + part_free)[1] + part_no \ + = part_impl.add_partition(part_primary, part_bootflag, + part_size, part_type) + logger.debug("Create partition [%s] [%d]" % + (part_name, part_no)) + result[part_name] = {'device': device_path + "p%d" % part_no} + + self.already_created = True + return diff --git a/diskimage_builder/block_device/levelbase.py b/diskimage_builder/block_device/levelbase.py deleted file mode 100644 index 728dedd3..00000000 --- a/diskimage_builder/block_device/levelbase.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2016 Andreas Florath (andreas@florath.net) -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import sys - - -logger = logging.getLogger(__name__) - - -class LevelBase(object): - - def __init__(self, lvl, config, default_config, result, sub_modules): - self.level = lvl - self.config = config - self.default_config = default_config - self.result = result - self.sub_modules = sub_modules - - def call_sub_modules(self, callback): - """Generic way calling submodules""" - result = {} - if self.result is not None: - result = self.result.copy() - for name, cfg in self.config: - if name in self.sub_modules: - logger.info("Calling sub module [%s]" % name) - sm = self.sub_modules[name](cfg, self.default_config, - self.result) - lres = callback(sm) - result.update(lres) - else: - logger.error("Unknown sub module [%s]" % name) - sys.exit(1) - return result - - def create_cb(self, obj): - return obj.create() - - def create(self): - """Create the configured block devices""" - logger.info("Starting to create level [%d] block devices" % self.level) - result = self.call_sub_modules(self.create_cb) - logger.info("Finished creating level [%d] block devices" % self.level) - return result - - def delete_cb(self, obj): - return obj.delete() - - def delete(self): - """Delete the configured block devices""" - logger.info("Starting to delete level [%d] block devices" % self.level) - res = self.call_sub_modules(self.delete_cb) - logger.info("Finished deleting level [%d] block devices" % self.level) - return all(p for p in res.values()) diff --git a/diskimage_builder/block_device/utils.py b/diskimage_builder/block_device/utils.py index 399092a6..22c1895d 100644 --- a/diskimage_builder/block_device/utils.py +++ b/diskimage_builder/block_device/utils.py @@ -12,7 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. -SIZE_SPECS = [ +import re + + +SIZE_UNIT_SPECS = [ ["TiB", 1024**4], ["GiB", 1024**3], ["MiB", 1024**2], @@ -29,45 +32,52 @@ SIZE_SPECS = [ ["", 1], # No unit -> size is given in bytes ] +# Basic RE to check and split floats (without exponent) +# and a given unit specification (which must be non-numerical). +size_unit_spec_re = re.compile("^([\d\.]*) ?([a-zA-Z0-9_]*)$") -def _split_size_spec(size_spec): - for spec_key, spec_value in SIZE_SPECS: - if len(spec_key) == 0: - return size_spec, spec_key - if size_spec.endswith(spec_key): - return size_spec[:-len(spec_key)], spec_key - raise RuntimeError("size_spec [%s] not known" % size_spec) + +def _split_size_unit_spec(size_unit_spec): + """Helper function to split unit specification into parts. + + The first part is the numeric part - the second one is the unit. + """ + match = size_unit_spec_re.match(size_unit_spec) + if match is None: + raise RuntimeError("Invalid size unit spec [%s]" % size_unit_spec) + + return match.group(1), match.group(2) def _get_unit_factor(unit_str): - for spec_key, spec_value in SIZE_SPECS: + """Helper function to get the unit factor. + + The given unit_str needs to be a string of the + SIZE_UNIT_SPECS table. + If the unit is not found, a runtime error is raised. + """ + for spec_key, spec_value in SIZE_UNIT_SPECS: if unit_str == spec_key: return spec_value raise RuntimeError("unit_str [%s] not known" % unit_str) def parse_abs_size_spec(size_spec): - size_cnt_str, size_unit_str = _split_size_spec(size_spec) + size_cnt_str, size_unit_str = _split_size_unit_spec(size_spec) unit_factor = _get_unit_factor(size_unit_str) return int(unit_factor * ( float(size_cnt_str) if len(size_cnt_str) > 0 else 1)) -def convert_to_utf8(jdata): - """Convert to UTF8. +def parse_rel_size_spec(size_spec, abs_size): + """Parses size specifications - can be relative like 50% - The json parser returns unicode strings. Because in - some python implementations unicode strings are not - compatible with utf8 strings - especially when using - as keys in dictionaries - this function recursively - converts the json data. + In addition to the absolute parsing also a relative + parsing is done. If the size specification ends in '%', + then the relative size of the given 'abs_size' is returned. """ - if isinstance(jdata, unicode): - return jdata.encode('utf-8') - elif isinstance(jdata, dict): - return {convert_to_utf8(key): convert_to_utf8(value) - for key, value in jdata.iteritems()} - elif isinstance(jdata, list): - return [convert_to_utf8(je) for je in jdata] - else: - return jdata + if size_spec[-1] == '%': + percent = float(size_spec[:-1]) + return True, int(abs_size * percent / 100.0) + + return False, parse_abs_size_spec(size_spec) diff --git a/diskimage_builder/elements/bootloader/finalise.d/50-bootloader b/diskimage_builder/elements/bootloader/finalise.d/50-bootloader index b64f4e7d..d6e0d53d 100755 --- a/diskimage_builder/elements/bootloader/finalise.d/50-bootloader +++ b/diskimage_builder/elements/bootloader/finalise.d/50-bootloader @@ -9,14 +9,8 @@ fi set -eu set -o pipefail -# FIXME: -[ -n "$IMAGE_BLOCK_DEVICE" ] PART_DEV=$IMAGE_BLOCK_DEVICE -if [[ "$ARCH" =~ "ppc" ]]; then - BOOT_DEV=$(echo $IMAGE_BLOCK_DEVICE | sed -e 's#p2##')'p1' -else - BOOT_DEV=$(echo $IMAGE_BLOCK_DEVICE | sed -e 's#p1##' | sed -e 's#mapper/##') -fi +BOOT_DEV=$IMAGE_BLOCK_DEVICE_WITHOUT_PART function install_extlinux { install-packages -m bootloader extlinux diff --git a/diskimage_builder/elements/partitioning-sfdisk/README.rst b/diskimage_builder/elements/partitioning-sfdisk/README.rst deleted file mode 100644 index 8528e04d..00000000 --- a/diskimage_builder/elements/partitioning-sfdisk/README.rst +++ /dev/null @@ -1,19 +0,0 @@ -=================== -partitioning-sfdisk -=================== -Sets up a partitioned disk using sfdisk, according to user needs. - -Environment Variables ---------------------- -DIB_PARTITIONING_SFDISK_SCHEMA - : Required: Yes - : Default: 2048,,L * - 0 0; - 0 0; - 0 0; - : Description: A multi-line string specifying a disk schema in sectors. - : Example: ``DIB_PARTITIONING_SFDISK_SCHEMA=" - 2048,10000,L * - 10248,,L - 0 0; - " will create two partitions on disk, first one will be bootable. diff --git a/diskimage_builder/elements/partitioning-sfdisk/block-device.d/10-partitioning-sfdisk b/diskimage_builder/elements/partitioning-sfdisk/block-device.d/10-partitioning-sfdisk deleted file mode 100755 index 7ab23870..00000000 --- a/diskimage_builder/elements/partitioning-sfdisk/block-device.d/10-partitioning-sfdisk +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# dib-lint: disable=safe_sudo - -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -# sanity checks -[ -n "$IMAGE_BLOCK_DEVICE" ] || die "Image block device not set" - -# execute sfdisk with the given partitioning schema -sudo sfdisk -uS --force $IMAGE_BLOCK_DEVICE <