diff --git a/diskimage_builder/block_device/__init__.py b/diskimage_builder/block_device/__init__.py index 0fac8380..4a071509 100644 --- a/diskimage_builder/block_device/__init__.py +++ b/diskimage_builder/block_device/__init__.py @@ -69,7 +69,7 @@ def main(): method = getattr(bd, "cmd_" + args.phase, None) if callable(method): # If so: call it. - method() + return method() else: logger.error("phase [%s] does not exists" % args.phase) return 1 diff --git a/diskimage_builder/block_device/blockdevice.py b/diskimage_builder/block_device/blockdevice.py index 6e9f5ef1..97e01434 100644 --- a/diskimage_builder/block_device/blockdevice.py +++ b/diskimage_builder/block_device/blockdevice.py @@ -12,12 +12,18 @@ # License for the specific language governing permissions and limitations # under the License. -from diskimage_builder.block_device.level0 import Level0 -from diskimage_builder.block_device.utils import convert_to_utf8 +import codecs +from diskimage_builder.block_device.blockdevicesetupexception \ + import BlockDeviceSetupException +from diskimage_builder.block_device.level0 import LocalLoop +from diskimage_builder.block_device.level1 import Partitioning +from diskimage_builder.graph.digraph import Digraph import json import logging import os import shutil +import sys +import yaml logger = logging.getLogger(__name__) @@ -25,33 +31,36 @@ logger = logging.getLogger(__name__) class BlockDevice(object): - # Currently there is only the need for a first element (which must - # be a list). - DefaultConfig = [ - [["local_loop", - {"name": "rootdisk"}]]] - # The reason for the complex layout is, that for future layers - # there is a need to add additional lists, like: - # DefaultConfig = [ - # [["local_loop", - # {"name": "rootdisk"}]], - # [["partitioning", - # {"rootdisk": { - # "label": "mbr", - # "partitions": - # [{"name": "rd-partition1", - # "flags": ["boot"], - # "size": "100%"}]}}]], - # [["fs", - # {"rd-partition1": {}}]] - # ] + # Default configuration: + # one image, one partition, mounted under '/' + DefaultConfig = """ +local_loop: + name: image0 +""" + +# This is an example of the next level config +# mkfs: +# base: root_p1 +# type: ext4 +# mount_point: / + + # A dictionary to map sensible names to internal implementation. + cfg_type_map = { + 'local_loop': LocalLoop, + 'partitioning': Partitioning, + 'mkfs': 'not yet implemented', + } def __init__(self, block_device_config, build_dir, default_image_size, default_image_dir): + logger.debug("Creating BlockDevice object") + logger.debug("Config given [%s]" % block_device_config) + logger.debug("Build dir [%s]" % build_dir) if block_device_config is None: - self.config = BlockDevice.DefaultConfig - else: - self.config = json.loads(block_device_config) + block_device_config = BlockDevice.DefaultConfig + self.config = yaml.safe_load(block_device_config) + logger.debug("Using config [%s]" % self.config) + self.default_config = { 'image_size': default_image_size, 'image_dir': default_image_dir} @@ -67,53 +76,125 @@ class BlockDevice(object): json.dump([self.config, self.default_config, result], fd) def load_state(self): - with open(self.state_json_file_name, "r") as fd: - return convert_to_utf8(json.load(fd)) + with codecs.open(self.state_json_file_name, + encoding="utf-8", mode="r") as fd: + return json.load(fd) + + def create_graph(self, config, default_config): + # This is the directed graph of nodes: each parse method must + # add the appropriate nodes and edges. + dg = Digraph() + + for cfg_obj_name, cfg_obj_val in config.items(): + # As the first step the configured objects are created + # (if it exists) + if cfg_obj_name not in BlockDevice.cfg_type_map: + logger.error("Configured top level element [%s] " + "does not exists." % cfg_obj_name) + return 1 + cfg_obj = BlockDevice.cfg_type_map[cfg_obj_name]( + cfg_obj_val, default_config) + # At this point it is only possible to add the nodes: + # adding the edges needs all nodes first. + cfg_obj.insert_nodes(dg) + + # Now that all the nodes exists: add also the edges + for node in dg.get_iter_nodes_values(): + node.insert_edges(dg) + + call_order = dg.topological_sort() + logger.debug("Call order [%s]" % (list(call_order))) + return dg, call_order + + def create(self, result, rollback): + dg, call_order = self.create_graph(self.config, self.default_config) + for node in call_order: + node.create(result, rollback) def cmd_create(self): """Creates the block device""" logger.info("create() called") - logger.debug("config [%s]" % self.config) - lvl0 = Level0(self.config[0], self.default_config, None) - result = lvl0.create() - logger.debug("Result level 0 [%s]" % result) + logger.debug("Using config [%s]" % self.config) + + result = {} + rollback = [] + + try: + self.create(result, rollback) + except BlockDeviceSetupException as bdse: + logger.error("exception [%s]" % bdse) + for rollback_cb in reversed(rollback): + rollback_cb() + sys.exit(1) # To be compatible with the current implementation, echo the # result to stdout. - print("%s" % result['rootdisk']['device']) + # If there is no partition needed, pass back directly the + # image. + if 'root_p1' in result: + print("%s" % result['root_p1']['device']) + else: + print("%s" % result['image0']['device']) self.write_state(result) logger.info("create() finished") return 0 - def cmd_umount(self): - """Unmounts the blockdevice and cleanup resources""" - - logger.info("umount() called") + def _load_state(self): + logger.info("_load_state() called") try: os.stat(self.state_json_file_name) except OSError: logger.info("State already cleaned - no way to do anything here") - return 0 + return None, None, None config, default_config, state = self.load_state() logger.debug("Using config [%s]" % config) logger.debug("Using default config [%s]" % default_config) logger.debug("Using state [%s]" % state) - level0 = Level0(config[0], default_config, state) - result = level0.delete() + # Deleting must be done in reverse order + dg, call_order = self.create_graph(config, default_config) + reverse_order = reversed(call_order) + return dg, reverse_order, state - # If everything finished well, remove the results. - if result: - logger.info("Removing temporary dir [%s]" % self.state_dir) - shutil.rmtree(self.state_dir) + def cmd_umount(self): + """Unmounts the blockdevice and cleanup resources""" + + dg, reverse_order, state = self._load_state() + if dg is None: + return 0 + for node in reverse_order: + node.umount(state) # To be compatible with the current implementation, echo the # result to stdout. - print("%s" % state['rootdisk']['image']) + print("%s" % state['image0']['image']) - logger.info("umount() finished result [%d]" % result) - return 0 if result else 1 + return 0 + + def cmd_cleanup(self): + """Cleanup all remaining relicts - in good case""" + + dg, reverse_order, state = self._load_state() + for node in reverse_order: + node.cleanup(state) + + logger.info("Removing temporary dir [%s]" % self.state_dir) + shutil.rmtree(self.state_dir) + + return 0 + + def cmd_delete(self): + """Cleanup all remaining relicts - in case of an error""" + + dg, reverse_order, state = self._load_state() + for node in reverse_order: + node.delete(state) + + logger.info("Removing temporary dir [%s]" % self.state_dir) + shutil.rmtree(self.state_dir) + + return 0 diff --git a/diskimage_builder/block_device/blockdevicesetupexception.py b/diskimage_builder/block_device/blockdevicesetupexception.py new file mode 100644 index 00000000..b0e42356 --- /dev/null +++ b/diskimage_builder/block_device/blockdevicesetupexception.py @@ -0,0 +1,17 @@ +# Copyright 2016 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class BlockDeviceSetupException(Exception): + pass diff --git a/diskimage_builder/block_device/level0/__init__.py b/diskimage_builder/block_device/level0/__init__.py index 58ec14aa..dffb6254 100644 --- a/diskimage_builder/block_device/level0/__init__.py +++ b/diskimage_builder/block_device/level0/__init__.py @@ -13,18 +13,5 @@ # under the License. from diskimage_builder.block_device.level0.localloop import LocalLoop -from diskimage_builder.block_device.levelbase import LevelBase __all__ = [LocalLoop] - - -class Level0(LevelBase): - """Block Device Level0: preparation of images - - This is the class that handles level 0 block device setup: - creating the block device image and providing OS access to it. - """ - - def __init__(self, config, default_config, result): - LevelBase.__init__(self, 0, config, default_config, result, - {LocalLoop.type_string: LocalLoop}) diff --git a/diskimage_builder/block_device/level0/localloop.py b/diskimage_builder/block_device/level0/localloop.py index 3131f17d..7fd21b3d 100644 --- a/diskimage_builder/block_device/level0/localloop.py +++ b/diskimage_builder/block_device/level0/localloop.py @@ -12,18 +12,19 @@ # License for the specific language governing permissions and limitations # under the License. +from diskimage_builder.block_device.blockdevicesetupexception \ + import BlockDeviceSetupException from diskimage_builder.block_device.utils import parse_abs_size_spec +from diskimage_builder.graph.digraph import Digraph import logging import os import subprocess -import sys -import time logger = logging.getLogger(__name__) -class LocalLoop(object): +class LocalLoop(Digraph.Node): """Level0: Local loop image device handling. This class handles local loop devices that can be used @@ -32,7 +33,9 @@ class LocalLoop(object): type_string = "local_loop" - def __init__(self, config, default_config, result=None): + def __init__(self, config, default_config): + logger.debug("Creating LocalLoop object; config [%s] " + "default_config [%s]" % (config, default_config)) if 'size' in config: self.size = parse_abs_size_spec(config['size']) logger.debug("Image size [%s]" % self.size) @@ -44,49 +47,85 @@ class LocalLoop(object): else: self.image_dir = default_config['image_dir'] self.name = config['name'] + Digraph.Node.__init__(self, self.name) self.filename = os.path.join(self.image_dir, self.name + ".raw") - self.result = result - if self.result is not None: - self.block_device = self.result[self.name]['device'] - def create(self): - logger.debug("[%s] Creating loop on [%s] with size [%d]" % - (self.name, self.filename, self.size)) + def insert_nodes(self, dg): + dg.add_node(self) - with open(self.filename, "w") as fd: - fd.seek(self.size - 1) + def insert_edges(self, dg): + """Because this is created without base, there are no edges.""" + pass + + @staticmethod + def image_create(filename, size): + logger.info("Create image file [%s]" % filename) + with open(filename, "w") as fd: + fd.seek(size - 1) fd.write("\0") - logger.debug("Calling [sudo losetup --show -f %s]" - % self.filename) + @staticmethod + def _image_delete(filename): + logger.info("Remove image file [%s]" % filename) + os.remove(filename) + + @staticmethod + def _loopdev_attach(filename): + logger.info("loopdev attach") + logger.debug("Calling [sudo losetup --show -f %s]", filename) subp = subprocess.Popen(["sudo", "losetup", "--show", "-f", - self.filename], stdout=subprocess.PIPE) + filename], stdout=subprocess.PIPE) rval = subp.wait() if rval == 0: # [:-1]: Cut of the newline - self.block_device = subp.stdout.read()[:-1] - logger.debug("New block device [%s]" % self.block_device) + block_device = subp.stdout.read()[:-1].decode("utf-8") + logger.info("New block device [%s]" % block_device) + return block_device else: logger.error("losetup failed") - sys.exit(1) + raise BlockDeviceSetupException("losetup failed") - return {self.name: {"device": self.block_device, - "image": self.filename}} - - def delete(self): + @staticmethod + def _loopdev_detach(loopdev): + logger.info("loopdev detach") # loopback dev may be tied up a bit by udev events triggered # by partition events for try_cnt in range(10, 1, -1): - logger.debug("Delete loop [%s]" % self.block_device) - res = subprocess.call("sudo losetup -d %s" % - (self.block_device), - shell=True) - if res == 0: - return {self.name: True} - logger.debug("[%s] may be busy, sleeping [%d] more secs" - % (self.block_device, try_cnt)) - time.sleep(1) + logger.debug("Calling [sudo losetup -d %s]", loopdev) + subp = subprocess.Popen(["sudo", "losetup", "-d", + loopdev]) + rval = subp.wait() + if rval == 0: + logger.info("Successfully detached [%s]" % loopdev) + return 0 + else: + logger.error("loopdev detach failed") + # Do not raise an error - maybe other cleanup methods + # can at least do some more work. + logger.debug("Gave up trying to detach [%s]" % loopdev) + return rval - logger.debug("Gave up trying to detach [%s]" % - self.block_device) - return {self.name: False} + def create(self, result, rollback): + logger.debug("[%s] Creating loop on [%s] with size [%d]" % + (self.name, self.filename, self.size)) + + rollback.append(lambda: self._image_delete(self.filename)) + self.image_create(self.filename, self.size) + + block_device = self._loopdev_attach(self.filename) + rollback.append(lambda: self._loopdev_detach(block_device)) + + result[self.name] = {"device": block_device, + "image": self.filename} + logger.debug("Created loop name [%s] device [%s] image [%s]" + % (self.name, block_device, self.filename)) + return + + def umount(self, state): + self._loopdev_detach(state[self.name]['device']) + + def cleanup(self, state): + pass + + def delete(self, state): + self._image_delete(state[self.name]['image']) diff --git a/diskimage_builder/block_device/level1/__init__.py b/diskimage_builder/block_device/level1/__init__.py new file mode 100644 index 00000000..6c3b57a8 --- /dev/null +++ b/diskimage_builder/block_device/level1/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2016 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from diskimage_builder.block_device.level1.partitioning import Partitioning + +__all__ = [Partitioning] diff --git a/diskimage_builder/block_device/level1/mbr.py b/diskimage_builder/block_device/level1/mbr.py new file mode 100644 index 00000000..65b108b3 --- /dev/null +++ b/diskimage_builder/block_device/level1/mbr.py @@ -0,0 +1,360 @@ +# Copyright 2016 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import logging +import random +from struct import pack + + +logger = logging.getLogger(__name__) + + +# Details of the MBR object itself can be found in the inline +# documentation. +# +# General design and implementation remarks: +# o Because the whole GNU parted and co. (e.g. the python-parted that +# is based on GNU parted) cannot be used because of the license: +# everything falls under GPL2 (not LGPL2!) and therefore does not +# fit into the Apache License here. +# o It looks that there is no real alternative available (2016-06). +# o The interface of python-parted is not that simple to handle - and +# the initial try to use GNU (python-)parted was not that much +# easier and shorter than this approach. +# o When using tools (like fdisk or parted) they try to optimize the +# alignment of partitions based on the data found on the host +# system. These might be misleading and might lead to (very) poor +# performance. +# o These ready-to-use tools typically also change the CHS layout +# based on the disk size. In case that the disk is enlarged (which +# is a normal use case for golden images), the CHS layout of the +# disk changes for those tools (and is not longer correct). +# In the DIB implementation the CHS are chosen that way, that also +# for very small disks the maximum heads/cylinder and sectors/track +# is used: even if the disk size in increased, the CHS numbers will +# not change. +# o In the easy and straight forward way when only using one +# partition, exactly 40 bytes (!) must be written - and the biggest +# part of this data is fixed (same in all cases). +# +# Limitations and Incompatibilities +# o With the help of this class it is possible to create an +# arbitrarily number of extended partitions (tested with over 1000). +# o There are limitations and shortcomings in the OS and in tools +# handling these partitions. +# o Under Linux the loop device is able to handle a limited number of +# partitions. The module parameter max_loop can be set - the maximum +# number might vary depending on the distribution and kernel build. +# o Under Linux fdisk is able to handle 'only' 60 partitions. Only +# those are listed, can be changed or written. +# o Under Linux GNU parted can handle about 60 partitions. +# +# Be sure only to pass in the number of partitions that the host OS +# and target OS are able to handle. + +class MBR(object): + """MBR Disk / Partition Table Layout + + Primary partitions are created first - and must also be passed in + first. + The extended partition layout is done in the way, that there is + one entry in the MBR (the last) that uses the whole disk. + EBR (extended boot records) are used to describe the partitions + themselves. This has the advantage, that the same procedure can + be used for all partitions and arbitrarily many partitions can be + created in the same way (the EBR is placed as block 0 in each + partition itself). + In conjunction with a fixed and 'fits all' partition alignment the + major design focus is maximum performance for the installed image + (vs. minimal size). + Because of the chosen default alignment of 1MiB there will be + (1MiB - 512B) unused disk space for the MBR and also the same + size unused in every partition. + Assuming that 512 byte blocks are used, the resulting layout for + extended partitions looks like (blocks offset in extended + partition given): + 0: MBR - 2047 blocks unused + 2048: EBR for partition 1 - 2047 blocks unused + 4096: Start of data for partition 1 + ... + X: EBR for partition N - 2047 blocks unused + X+2048: Start of data for partition N + + Direct (native) writing of MBR, EBR (partition table) is + implemented - no other parititoning library or tools is used - + to be sure to get the correct CHS and alignment for a wide range + of host systems. + """ + + # Design & Implementation details: + # o A 'block' is a storage unit on disk. It is similar (equal) to a + # sector - but with LBA addressing. + # o It is assumed that a disk block has that number of bytes + bytes_per_sector = 512 + # o CHS is the 'good and very old way' specifying blocks. + # When passing around these numbers, they are also ordered like 'CHS': + # (cylinder, head, sector). + # o The computation from LBA to CHS is not unique (it is based + # on the 'real' (or assumed) number of heads/cylinder and + # sectors/track), these are the assumed numbers. Please note + # that these are also the maximum numbers: + heads_per_cylinder = 254 + sectors_per_track = 63 + max_cylinders = 1023 + # o There is the need for some offsets that are defined in the + # MBR/EBR domain. + MBR_offset_disk_id = 440 + MBR_offset_signature = 510 + MBR_offset_first_partition_table_entry = 446 + MBR_partition_type_extended_chs = 0x5 + MBR_partition_type_extended_lba = 0xF + MBR_signature = 0xAA55 + + def __init__(self, name, disk_size, alignment): + """Initialize a disk partitioning MBR object. + + The name is the (existing) name of the disk. + The disk_size is the (used) size of the disk. It must be a + proper multiple of the disk bytes per sector (currently 512) + """ + logger.info("Create MBR disk partitioning object") + + assert disk_size % MBR.bytes_per_sector == 0 + + self.disk_size = disk_size + self.disk_size_in_blocks \ + = self.disk_size // MBR.bytes_per_sector + self.alignment_blocks = alignment // MBR.bytes_per_sector + # Because the extended partitions are a chain of blocks, when + # creating a new partition, the reference in the already + # existing EBR must be updated. This holds a reference to the + # latest EBR. (A special case is the first: when it points to + # 0 (MBR) there is no need to update the reference.) + self.disk_block_last_ref = 0 + + self.name = name + self.partition_abs_start = None + self.partition_abs_next_free = None + # Start of partition number + self.partition_number = 0 + + self.primary_partitions_created = 0 + self.extended_partitions_created = 0 + + def __enter__(self): + # Open existing file for writing (r+) + self.image_fd = open(self.name, "r+b") + self.write_mbr() + self.write_mbr_signature(0) + self.partition_abs_start = self.align(1) + self.partition_abs_next_free \ + = self.partition_abs_start + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.image_fd.close() + + def lba2chs(self, lba): + """Converts a LBA block number to CHS + + If the LBA block number is bigger than the max (1023, 63, 254) + the maximum is returned. + """ + if lba > MBR.heads_per_cylinder * MBR.sectors_per_track \ + * MBR.max_cylinders: + return MBR.max_cylinders, MBR.heads_per_cylinder, \ + MBR.sectors_per_track + + cylinder = lba // (MBR.heads_per_cylinder * MBR.sectors_per_track) + head = (lba // MBR.sectors_per_track) % MBR.heads_per_cylinder + sector = (lba % MBR.sectors_per_track) + 1 + + logger.debug("Convert LBA to CHS [%d] -> [%d, %d, %d]" + % (lba, cylinder, head, sector)) + return cylinder, head, sector + + def encode_chs(self, cylinders, heads, sectors): + """Encodes a CHS triple into disk format""" + # Head - nothing to convert + assert heads <= MBR.heads_per_cylinder + eh = heads + + # Sector + assert sectors <= MBR.sectors_per_track + es = sectors + # top two bits are set in cylinder conversion + + # Cylinder + assert cylinders <= MBR.max_cylinders + ec = cylinders % 256 # lower part + hc = cylinders // 4 # extract top two bits and + es = es | hc # pass them into the top two bits of the sector + + logger.debug("Encode CHS to disk format [%d %d %d] " + "-> [%02x %02x %02x]" % (cylinders, heads, sectors, + eh, es, ec)) + return eh, es, ec + + def write_mbr(self): + """Write MBR + + This method writes the MBR to disk. It creates a random disk + id as well that it creates the extended partition (as + first partition) which uses the whole disk. + """ + disk_id = random.randint(0, 0xFFFFFFFF) + self.image_fd.seek(MBR.MBR_offset_disk_id) + self.image_fd.write(pack(" 0: + raise RuntimeError("All primary partitions must be " + "given first") + + if primaryflag: + return self.add_primary_partition(bootflag, size, ptype) + if self.extended_partitions_created == 0: + # When this is the first extended partition, the extended + # partition entry has to be written. + self.partition_abs_start = self.partition_abs_next_free + self.write_partition_entry( + False, 0, self.partition_number, + MBR.MBR_partition_type_extended_lba, + self.partition_abs_next_free, + self.disk_size_in_blocks - self.partition_abs_next_free) + self.partition_number = 4 + + return self.add_extended_partition(bootflag, size, ptype) + + def free(self): + """Returns the free (not yet partitioned) size""" + return self.disk_size \ + - (self.partition_abs_next_free + self.align(1)) \ + * MBR.bytes_per_sector diff --git a/diskimage_builder/block_device/level1/partitioning.py b/diskimage_builder/block_device/level1/partitioning.py new file mode 100644 index 00000000..36284af2 --- /dev/null +++ b/diskimage_builder/block_device/level1/partitioning.py @@ -0,0 +1,184 @@ +# Copyright 2016 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from diskimage_builder.block_device.blockdevicesetupexception \ + import BlockDeviceSetupException +from diskimage_builder.block_device.level1.mbr import MBR +from diskimage_builder.block_device.utils import parse_abs_size_spec +from diskimage_builder.block_device.utils import parse_rel_size_spec +from diskimage_builder.graph.digraph import Digraph +import logging + + +logger = logging.getLogger(__name__) + + +class Partition(Digraph.Node): + + def __init__(self, name, flags, size, ptype, base, partitioning): + Digraph.Node.__init__(self, name) + self.flags = flags + self.size = size + self.ptype = ptype + self.base = base + self.partitioning = partitioning + + def get_flags(self): + return self.flags + + def get_size(self): + return self.size + + def get_type(self): + return self.ptype + + def insert_edges(self, dg): + bnode = dg.find(self.base) + assert bnode is not None + dg.create_edge(bnode, self) + + def create(self, result, rollback): + self.partitioning.create(result, rollback) + + def umount(self, state): + """Partitioning does not need any umount task.""" + pass + + def cleanup(self, state): + """Partitioning does not need any cleanup.""" + pass + + def delete(self, state): + """Partitioning does not need any cleanup.""" + pass + + +class Partitioning(object): + + type_string = "partitioning" + + flag_boot = 1 + flag_primary = 2 + + def __init__(self, config, default_config): + logger.debug("Creating Partitioning object; config [%s]" % config) + # Because using multiple partitions of one base is done + # within one object, there is the need to store a flag if the + # creation of the partitions was already done. + self.already_created = False + + # Parameter check + if 'base' not in config: + self._config_error("Partitioning config needs 'base'") + self.base = config['base'] + + if 'label' not in config: + self._config_error("Partitioning config needs 'label'") + self.label = config['label'] + if self.label not in ("mbr", ): + self._config_error("Label must be 'mbr'") + + # It is VERY important to get the alignment correct. If this + # is not correct, the disk performance might be very poor. + # Example: In some tests a 'off by one' leads to a write + # performance of 30% compared to a correctly aligned + # partition. + # The problem for DIB is, that it cannot assume that the host + # system uses the same IO sizes as the target system, + # therefore here a fixed approach (as used in all modern + # systems with large disks) is used. The partitions are + # aligned to 1MiB (which are about 2048 times 512 bytes + # blocks) + self.align = 1024 * 1024 # 1MiB as default + if 'align' in config: + self.align = parse_abs_size_spec(config['align']) + + if 'partitions' not in config: + self._config_error("Partitioning config needs 'partitions'") + + self.partitions = {} + for part_cfg in config['partitions']: + if 'name' not in part_cfg: + self.config_error("Missing 'name' in partition config") + part_name = part_cfg['name'] + + flags = set() + if 'flags' in part_cfg: + for f in part_cfg['flags']: + if f == 'boot': + flags.add(Partitioning.flag_boot) + elif f == 'primary': + flags.add(Partitioning.flag_primary) + else: + self._config_error("Unknown flag [%s] in " + "partitioning for [%s]" + % (f, part_name)) + if 'size' not in part_cfg: + self._config_error("No 'size' in partition [%s]" + % part_name) + size = part_cfg['size'] + + ptype = int(part_cfg['type'], 16) if 'type' in part_cfg else 0x83 + + self.partitions[part_name] \ + = Partition(part_name, flags, size, ptype, self.base, self) + logger.debug(part_cfg) + + def _config_error(self, msg): + logger.error(msg) + raise BlockDeviceSetupException(msg) + + def _size_of_block_dev(self, dev): + with open(dev, "r") as fd: + fd.seek(0, 2) + return fd.tell() + + def insert_nodes(self, dg): + for _, part in self.partitions.items(): + dg.add_node(part) + + def create(self, result, rollback): + image_path = result[self.base]['image'] + device_path = result[self.base]['device'] + logger.info("Creating partition on [%s] [%s]" % + (self.base, image_path)) + + if self.already_created: + logger.info("Not creating the partitions a second time.") + return + + assert self.label == 'mbr' + + disk_size = self._size_of_block_dev(image_path) + with MBR(image_path, disk_size, self.align) as part_impl: + for part_name, part_cfg in self.partitions.items(): + part_bootflag = Partitioning.flag_boot \ + in part_cfg.get_flags() + part_primary = Partitioning.flag_primary \ + in part_cfg.get_flags() + part_size = part_cfg.get_size() + part_free = part_impl.free() + part_type = part_cfg.get_type() + logger.debug("Not partitioned space [%d]" % part_free) + part_size = parse_rel_size_spec(part_size, + part_free)[1] + part_no \ + = part_impl.add_partition(part_primary, part_bootflag, + part_size, part_type) + logger.debug("Create partition [%s] [%d]" % + (part_name, part_no)) + result[part_name] = {'device': device_path + "p%d" % part_no} + + self.already_created = True + return diff --git a/diskimage_builder/block_device/levelbase.py b/diskimage_builder/block_device/levelbase.py deleted file mode 100644 index 728dedd3..00000000 --- a/diskimage_builder/block_device/levelbase.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2016 Andreas Florath (andreas@florath.net) -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -import sys - - -logger = logging.getLogger(__name__) - - -class LevelBase(object): - - def __init__(self, lvl, config, default_config, result, sub_modules): - self.level = lvl - self.config = config - self.default_config = default_config - self.result = result - self.sub_modules = sub_modules - - def call_sub_modules(self, callback): - """Generic way calling submodules""" - result = {} - if self.result is not None: - result = self.result.copy() - for name, cfg in self.config: - if name in self.sub_modules: - logger.info("Calling sub module [%s]" % name) - sm = self.sub_modules[name](cfg, self.default_config, - self.result) - lres = callback(sm) - result.update(lres) - else: - logger.error("Unknown sub module [%s]" % name) - sys.exit(1) - return result - - def create_cb(self, obj): - return obj.create() - - def create(self): - """Create the configured block devices""" - logger.info("Starting to create level [%d] block devices" % self.level) - result = self.call_sub_modules(self.create_cb) - logger.info("Finished creating level [%d] block devices" % self.level) - return result - - def delete_cb(self, obj): - return obj.delete() - - def delete(self): - """Delete the configured block devices""" - logger.info("Starting to delete level [%d] block devices" % self.level) - res = self.call_sub_modules(self.delete_cb) - logger.info("Finished deleting level [%d] block devices" % self.level) - return all(p for p in res.values()) diff --git a/diskimage_builder/block_device/utils.py b/diskimage_builder/block_device/utils.py index 399092a6..22c1895d 100644 --- a/diskimage_builder/block_device/utils.py +++ b/diskimage_builder/block_device/utils.py @@ -12,7 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. -SIZE_SPECS = [ +import re + + +SIZE_UNIT_SPECS = [ ["TiB", 1024**4], ["GiB", 1024**3], ["MiB", 1024**2], @@ -29,45 +32,52 @@ SIZE_SPECS = [ ["", 1], # No unit -> size is given in bytes ] +# Basic RE to check and split floats (without exponent) +# and a given unit specification (which must be non-numerical). +size_unit_spec_re = re.compile("^([\d\.]*) ?([a-zA-Z0-9_]*)$") -def _split_size_spec(size_spec): - for spec_key, spec_value in SIZE_SPECS: - if len(spec_key) == 0: - return size_spec, spec_key - if size_spec.endswith(spec_key): - return size_spec[:-len(spec_key)], spec_key - raise RuntimeError("size_spec [%s] not known" % size_spec) + +def _split_size_unit_spec(size_unit_spec): + """Helper function to split unit specification into parts. + + The first part is the numeric part - the second one is the unit. + """ + match = size_unit_spec_re.match(size_unit_spec) + if match is None: + raise RuntimeError("Invalid size unit spec [%s]" % size_unit_spec) + + return match.group(1), match.group(2) def _get_unit_factor(unit_str): - for spec_key, spec_value in SIZE_SPECS: + """Helper function to get the unit factor. + + The given unit_str needs to be a string of the + SIZE_UNIT_SPECS table. + If the unit is not found, a runtime error is raised. + """ + for spec_key, spec_value in SIZE_UNIT_SPECS: if unit_str == spec_key: return spec_value raise RuntimeError("unit_str [%s] not known" % unit_str) def parse_abs_size_spec(size_spec): - size_cnt_str, size_unit_str = _split_size_spec(size_spec) + size_cnt_str, size_unit_str = _split_size_unit_spec(size_spec) unit_factor = _get_unit_factor(size_unit_str) return int(unit_factor * ( float(size_cnt_str) if len(size_cnt_str) > 0 else 1)) -def convert_to_utf8(jdata): - """Convert to UTF8. +def parse_rel_size_spec(size_spec, abs_size): + """Parses size specifications - can be relative like 50% - The json parser returns unicode strings. Because in - some python implementations unicode strings are not - compatible with utf8 strings - especially when using - as keys in dictionaries - this function recursively - converts the json data. + In addition to the absolute parsing also a relative + parsing is done. If the size specification ends in '%', + then the relative size of the given 'abs_size' is returned. """ - if isinstance(jdata, unicode): - return jdata.encode('utf-8') - elif isinstance(jdata, dict): - return {convert_to_utf8(key): convert_to_utf8(value) - for key, value in jdata.iteritems()} - elif isinstance(jdata, list): - return [convert_to_utf8(je) for je in jdata] - else: - return jdata + if size_spec[-1] == '%': + percent = float(size_spec[:-1]) + return True, int(abs_size * percent / 100.0) + + return False, parse_abs_size_spec(size_spec) diff --git a/diskimage_builder/elements/bootloader/finalise.d/50-bootloader b/diskimage_builder/elements/bootloader/finalise.d/50-bootloader index b64f4e7d..d6e0d53d 100755 --- a/diskimage_builder/elements/bootloader/finalise.d/50-bootloader +++ b/diskimage_builder/elements/bootloader/finalise.d/50-bootloader @@ -9,14 +9,8 @@ fi set -eu set -o pipefail -# FIXME: -[ -n "$IMAGE_BLOCK_DEVICE" ] PART_DEV=$IMAGE_BLOCK_DEVICE -if [[ "$ARCH" =~ "ppc" ]]; then - BOOT_DEV=$(echo $IMAGE_BLOCK_DEVICE | sed -e 's#p2##')'p1' -else - BOOT_DEV=$(echo $IMAGE_BLOCK_DEVICE | sed -e 's#p1##' | sed -e 's#mapper/##') -fi +BOOT_DEV=$IMAGE_BLOCK_DEVICE_WITHOUT_PART function install_extlinux { install-packages -m bootloader extlinux diff --git a/diskimage_builder/elements/partitioning-sfdisk/README.rst b/diskimage_builder/elements/partitioning-sfdisk/README.rst deleted file mode 100644 index 8528e04d..00000000 --- a/diskimage_builder/elements/partitioning-sfdisk/README.rst +++ /dev/null @@ -1,19 +0,0 @@ -=================== -partitioning-sfdisk -=================== -Sets up a partitioned disk using sfdisk, according to user needs. - -Environment Variables ---------------------- -DIB_PARTITIONING_SFDISK_SCHEMA - : Required: Yes - : Default: 2048,,L * - 0 0; - 0 0; - 0 0; - : Description: A multi-line string specifying a disk schema in sectors. - : Example: ``DIB_PARTITIONING_SFDISK_SCHEMA=" - 2048,10000,L * - 10248,,L - 0 0; - " will create two partitions on disk, first one will be bootable. diff --git a/diskimage_builder/elements/partitioning-sfdisk/block-device.d/10-partitioning-sfdisk b/diskimage_builder/elements/partitioning-sfdisk/block-device.d/10-partitioning-sfdisk deleted file mode 100755 index 7ab23870..00000000 --- a/diskimage_builder/elements/partitioning-sfdisk/block-device.d/10-partitioning-sfdisk +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# dib-lint: disable=safe_sudo - -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -# sanity checks -[ -n "$IMAGE_BLOCK_DEVICE" ] || die "Image block device not set" - -# execute sfdisk with the given partitioning schema -sudo sfdisk -uS --force $IMAGE_BLOCK_DEVICE <