GPT partitioning support

This adds support for a GPT label type to the partitioning code.  This
is relatively straight-forward translation of the partition config
into a sgparted command-line and subsequent call.

A unit test is added based on a working GPT/EFI configuration and the
fedora-minimal functional test is updated to build a single-partition
GPT based using the new block-device-gpt override element.  See notes
in the sample configuration files about partition requirements and
types.

Documentation has been updated.

Co-Authored-By: Marcin Juszkiewicz <marcin.juszkiewicz@linaro.org>
Change-Id: I6b819a8071389e7e4eb4874ff7750bd192695ff2

diskimage_builder/block_device/level1/partition.py

@@ -33,6 +33,12 @@ class PartitionNode(NodeBase):
         self.partitioning = parent
         self.prev_partition = prev_partition
 
+        # filter out some MBR only options for clarity
+        if self.partitioning.label == 'gpt':
+            if 'flags' in config and 'primary' in config['flags']:
+                raise BlockDeviceSetupException(
+                    "Primary flag not supported for GPT partitions")
+
         self.flags = set()
         if 'flags' in config:
             for f in config['flags']:
@@ -47,7 +53,10 @@ class PartitionNode(NodeBase):
             raise BlockDeviceSetupException("No size in partition" % self.name)
         self.size = config['size']
 
-        self.ptype = int(config['type'], 16) if 'type' in config else 0x83
+        if self.partitioning.label == 'gpt':
+            self.ptype = str(config['type']) if 'type' in config else '8300'
+        elif self.partitioning.label == 'mbr':
+            self.ptype = int(config['type'], 16) if 'type' in config else 83
 
     def get_flags(self):
         return self.flags

diskimage_builder/block_device/level1/partitioning.py

@@ -58,8 +58,8 @@ class Partitioning(PluginBase):
             raise BlockDeviceSetupException(
                 "Partitioning config needs 'label'")
         self.label = config['label']
-        if self.label not in ("mbr", ):
+        if self.label not in ("mbr", "gpt"):
-            raise BlockDeviceSetupException("Label must be 'mbr'")
+            raise BlockDeviceSetupException("Label must be 'mbr' or 'gpt'")
 
         # It is VERY important to get the alignment correct. If this
         # is not correct, the disk performance might be very poor.
@@ -93,29 +93,9 @@ class Partitioning(PluginBase):
             fd.seek(0, 2)
             return fd.tell()
 
-    # not this is NOT a node and this is not called directly!  The
+    def _create_mbr(self):
-    # create() calls in the partition nodes this plugin has
+        """Create partitions with MBR"""
-    # created are calling back into this.
+        with MBR(self.image_path, self.disk_size, self.align) as part_impl:
-    def create(self):
-        # This is a bit of a hack.  Each of the partitions is actually
-        # in the graph, so for every partition we get a create() call
-        # as the walk happens.  But we only need to create the
-        # partition table once...
-        if self.already_created:
-            logger.info("Not creating the partitions a second time.")
-            return
-        self.already_created = True
-
-        # the raw file on disk
-        image_path = self.state['blockdev'][self.base]['image']
-        # the /dev/loopX device of the parent
-        device_path = self.state['blockdev'][self.base]['device']
-        logger.info("Creating partition on [%s] [%s]", self.base, image_path)
-
-        assert self.label == 'mbr'
-
-        disk_size = self._size_of_block_dev(image_path)
-        with MBR(image_path, disk_size, self.align) as part_impl:
             for part_cfg in self.partitions:
                 part_name = part_cfg.get_name()
                 part_bootflag = PartitionNode.flag_boot \
@@ -137,24 +117,100 @@ class Partitioning(PluginBase):
                 # We're going to mount all partitions with kpartx
                 # below once we're done.  So the device this partition
                 # will be seen at becomes "/dev/mapper/loop0pX"
-                assert device_path[:5] == "/dev/"
+                assert self.device_path[:5] == "/dev/"
                 partition_device_name = "/dev/mapper/%sp%d" % \
-                                        (device_path[5:], part_no)
+                                        (self.device_path[5:], part_no)
                 self.state['blockdev'][part_name] \
                     = {'device': partition_device_name}
 
+    def _create_gpt(self):
+        """Create partitions with GPT"""
+
+        cmd = ['sgdisk', self.image_path]
+
+        # This padding gives us a little room for rounding so we don't
+        # go over the end of the disk
+        disk_free = self.disk_size - (2048 * 1024)
+        pnum = 1
+
+        for p in self.partitions:
+            args = {}
+            args['pnum'] = pnum
+            args['name'] = '"%s"' % p.get_name()
+            args['type'] = '%s' % p.get_type()
+
+            # convert from a relative/string size to bytes
+            size = parse_rel_size_spec(p.get_size(), disk_free)[1]
+
+            # We keep track in bytes, but specify things to sgdisk in
+            # megabytes so it can align on sensible boundaries. And
+            # create partitions right after previous so no need to
+            # calculate start/end - just size.
+            assert size <= disk_free
+            args['size'] = size // (1024 * 1024)
+
+            new_cmd = ("-n {pnum}:0:+{size}M -t {pnum}:{type} "
+                       "-c {pnum}:{name}".format(**args))
+            cmd.extend(new_cmd.strip().split(' '))
+
+            # Fill the state; we mount all partitions with kpartx
+            # below once we're done.  So the device this partition
+            # will be seen at becomes "/dev/mapper/loop0pX"
+            assert self.device_path[:5] == "/dev/"
+            device_name = "/dev/mapper/%sp%d" % (self.device_path[5:], pnum)
+            self.state['blockdev'][p.get_name()] \
+                = {'device': device_name}
+
+            disk_free = disk_free - size
+            pnum = pnum + 1
+            logger.debug("Partition %s added, %s remaining in disk",
+                         pnum, disk_free)
+
+        logger.debug("cmd: %s", ' '.join(cmd))
+        exec_sudo(cmd)
+
+    # not this is NOT a node and this is not called directly!  The
+    # create() calls in the partition nodes this plugin has
+    # created are calling back into this.
+    def create(self):
+        # This is a bit of a hack.  Each of the partitions is actually
+        # in the graph, so for every partition we get a create() call
+        # as the walk happens.  But we only need to create the
+        # partition table once...
+        if self.already_created:
+            logger.info("Not creating the partitions a second time.")
+            return
+        self.already_created = True
+
+        # the raw file on disk
+        self.image_path = self.state['blockdev'][self.base]['image']
+        # the /dev/loopX device of the parent
+        self.device_path = self.state['blockdev'][self.base]['device']
+        # underlying size
+        self.disk_size = self._size_of_block_dev(self.image_path)
+
+        logger.info("Creating partition on [%s] [%s]",
+                    self.base, self.image_path)
+
+        assert self.label in ('mbr', 'gpt')
+
+        if self.label == 'mbr':
+            self._create_mbr()
+        elif self.label == 'gpt':
+            self._create_gpt()
+
         # "saftey sync" to make sure the partitions are written
         exec_sudo(["sync"])
 
         # now all the partitions are created, get device-mapper to
         # mount them
         if not os.path.exists("/.dockerenv"):
-            exec_sudo(["kpartx", "-avs", device_path])
+            exec_sudo(["kpartx", "-avs", self.device_path])
         else:
             # If running inside Docker, make our nodes manually,
             # because udev will not be working. kpartx cannot run in
             # sync mode in docker.
-            exec_sudo(["kpartx", "-av", device_path])
+            exec_sudo(["kpartx", "-av", self.device_path])
             exec_sudo(["dmsetup", "--noudevsync", "mknodes"])
 
         return

diskimage_builder/block_device/tests/config/gpt_efi.yaml

@@ -0,0 +1,32 @@
+# A sample config that has GPT/bios and EFI boot partitions
+
+- local_loop:
+    name: image0
+
+- partitioning:
+    base: image0
+    label: gpt
+    partitions:
+      - name: ESP
+        type: 'EF00'
+        size: 8MiB
+        mkfs:
+          type: vfat
+          mount:
+            mount_point: /boot/efi
+            fstab:
+              options: "defaults"
+              fsck-passno: 1
+      - name: BSP
+        type: 'EF02'
+        size: 8MiB
+      - name: root
+        type: '8300'
+        size: 100%
+        mkfs:
+          type: ext4
+          mount:
+            mount_point: /
+            fstab:
+              options: "defaults"
+              fsck-passno: 1

diskimage_builder/block_device/tests/test_gpt.py

@@ -0,0 +1,85 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import fixtures
+import logging
+import mock
+import os
+
+import diskimage_builder.block_device.tests.test_config as tc
+
+from diskimage_builder.block_device.blockdevice import BlockDeviceState
+from diskimage_builder.block_device.config import config_tree_to_graph
+from diskimage_builder.block_device.config import create_graph
+from diskimage_builder.block_device.level0.localloop import image_create
+from diskimage_builder.block_device.level1.partition import PartitionNode
+
+logger = logging.getLogger(__name__)
+
+
+class TestGPT(tc.TestGraphGeneration):
+
+    @mock.patch('diskimage_builder.block_device.level1.partitioning.exec_sudo')
+    def test_gpt_efi(self, mock_exec_sudo):
+        # Test the command-sequence for a GPT/EFI partition setup
+        tree = self.load_config_file('gpt_efi.yaml')
+        config = config_tree_to_graph(tree)
+
+        state = BlockDeviceState()
+
+        graph, call_order = create_graph(config, self.fake_default_config,
+                                         state)
+
+        # Create a fake temp backing file (we check the size of it,
+        # etc).
+        # TODO(ianw): exec_sudo is generically mocked out, thus the
+        # actual creation is mocked out ... but we could do this
+        # without root and use parted to create the partitions on this
+        # for slightly better testing.  An exercise for another day...
+        self.tmp_dir = fixtures.TempDir()
+        self.useFixture(self.tmp_dir)
+        self.image_path = os.path.join(self.tmp_dir.path, "image.raw")
+        # should be sparse...
+        image_create(self.image_path, 1024 * 1024 * 1024)
+        logger.debug("Temp image in %s", self.image_path)
+
+        # Fake state for the loopback device
+        state['blockdev'] = {}
+        state['blockdev']['image0'] = {}
+        state['blockdev']['image0']['image'] = self.image_path
+        state['blockdev']['image0']['device'] = "/dev/loopX"
+
+        for node in call_order:
+            if isinstance(node, PartitionNode):
+                node.create()
+
+        # check the parted call looks right
+        parted_cmd = ('sgdisk %s '
+                      '-n 1:0:+8M -t 1:EF00 -c 1:"ESP" '
+                      '-n 2:0:+8M -t 2:EF02 -c 2:"BSP" '
+                      '-n 3:0:+1006M -t 3:8300 -c 3:"root"'
+                      % self.image_path)
+        cmd_sequence = [
+            mock.call(parted_cmd.split(' ')),
+            mock.call(['sync']),
+            mock.call(['kpartx', '-avs', '/dev/loopX'])
+        ]
+        self.assertEqual(mock_exec_sudo.call_count, len(cmd_sequence))
+        mock_exec_sudo.assert_has_calls(cmd_sequence)
+
+        # Check two new partitions appear in state correctly
+        self.assertDictEqual(state['blockdev']['ESP'],
+                             {'device': '/dev/mapper/loopXp1'})
+        self.assertDictEqual(state['blockdev']['BSP'],
+                             {'device': '/dev/mapper/loopXp2'})
+        self.assertDictEqual(state['blockdev']['root'],
+                             {'device': '/dev/mapper/loopXp3'})

diskimage_builder/elements/block-device-gpt/README.rst

@@ -0,0 +1,11 @@
+================
+Block Device GPT
+================
+
+This is an override for the default block-device configuration
+provided in the ``vm`` element to get a GPT based single-partition
+disk, rather than the default MBR.
+
+Note this provides the extra `BIOS boot partition
+<https://en.wikipedia.org/wiki/BIOS_boot_partition>`__ as required for
+non-EFI boot environments.

diskimage_builder/elements/block-device-gpt/block-device-default.yaml

@@ -0,0 +1,22 @@
+# Default single partition loopback using a GPT based partition table
+
+- local_loop:
+    name: image0
+
+- partitioning:
+    base: image0
+    label: gpt
+    partitions:
+      - name: BSP
+        type: 'EF02'
+        size: 8MiB
+      - name: root
+        flags: [ boot ]
+        size: 100%
+        mkfs:
+          type: ext4
+          mount:
+            mount_point: /
+            fstab:
+              options: "defaults"
+              fsck-passno: 1

diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/element-deps

@@ -1 +1,3 @@
-openstack-ci-mirrors
+block-device-gpt
+openstack-ci-mirrors
+vm

doc/source/user_guide/building_an_image.rst

@@ -75,7 +75,8 @@ There are currently two defaults:
 
 The user can overwrite the default handling by setting the environment
 variable `DIB_BLOCK_DEVICE_CONFIG`.  This variable must hold YAML
-structured configuration data.
+structured configuration data or be a ``file://`` URL reference to a
+on-disk configuration file.
 
 The default when using the `vm` element is:
 
@@ -247,8 +248,8 @@ encrypted, ...) and create partition information in it.
 
 The symbolic name for this module is `partitioning`.
 
-Currently the only supported partitioning layout is Master Boot Record
+MBR
-`MBR`.
+***
 
 It is possible to create primary or logical partitions or a mix of
 them. The numbering of the primary partitions will start at 1,
@@ -267,19 +268,27 @@ partitions.
 Partitions are created in the order they are configured.  Primary
 partitions - if needed - must be first in the list.
 
+GPT
+***
+
+GPT partitioning requires the ``sgdisk`` tool to be available.
+
+Options
+*******
+
 There are the following key / value pairs to define one partition
 table:
 
 base
-   (mandatory) The base device where to create the partitions in.
+   (mandatory) The base device to create the partitions in.
 
 label
-   (mandatory) Possible values: 'mbr'
+   (mandatory) Possible values: 'mbr', 'gpt'
-   This uses the Master Boot Record (MBR) layout for the disk.
+   Configure use of either the Master Boot Record (MBR) or GUID
-   (There are currently plans to add GPT later on.)
+   Partition Table (GPT) formats
 
 align
-   (optional - default value '1MiB')
+   (optional - default value '1MiB'; MBR only)
    Set the alignment of the partition.  This must be a multiple of the
    block size (i.e. 512 bytes).  The default of 1MiB (~ 2048 * 512
    bytes blocks) is the default for modern systems and known to
@@ -308,9 +317,9 @@ flags
    (optional) List of flags for the partition. Default: empty.
    Possible values:
 
-   boot
+   boot (MBR only)
       Sets the boot flag for the partition
-   primary
+   primary (MBR only)
       Partition should be a primary partition. If not set a logical
       partition will be created.
 
@@ -321,10 +330,15 @@ size
    based on the remaining free space.
 
 type (optional)
-   The partition type stored in the MBR partition table entry. The
+   The partition type stored in the MBR or GPT partition table entry.
-   default value is '0x83' (Linux Default partition). Any valid one
+
+   For MBR the default value is '0x83' (Linux Default partition). Any valid one
    byte hexadecimal value may be specified here.
 
+   For GPT the default value is '8300' (Linux Default partition). Any valid two
+   byte hexadecimal value may be specified here. Due to ``sgdisk`` leading '0x'
+   should not be used.
+
 Example:
 
 .. code-block:: yaml
@@ -350,12 +364,28 @@ Example:
         - name: data2
           size: 100%
 
+  - partitioning:
+      base: gpt_image
+      label: gpt
+      partitions:
+        - name: ESP
+          type: EF00
+          size: 16MiB
+        - name: data1
+          size: 1GiB
+        - name: lvmdata
+          type: 8E00
+          size: 100%
+
 On the `image0` two partitions are created.  The size of the first is
 1GiB, the second uses the remaining free space.  On the `data_image`
-three partitions are created: all are about 1/3 of the disk size.
+three partitions are created: all are about 1/3 of the disk size. On
+the `gpt_image` three partitions are created: 16MiB one for EFI
+bootloader, 1GiB Linux filesystem one and rest of disk will be used
+for LVM partition.
 
-Module: Lvm
+Module: LVM
-···········
+...........
 
 This module generates volumes on existing block devices. This means that it is
 possible to take any previous created partition, and create volumes information

releasenotes/notes/bootloader-gpt-d1047f81f3a0631b.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    GPT support is added to the bootloader; see documentation for
+    configuration examples.  This should be considered a technology
+    preview; there may be minor behaviour modifications as we enable
+    UEFI and support across more architectures.

tests/install_test_deps.sh

@@ -9,6 +9,8 @@ sudo apt-get install -y --force-yes \
         bzip2 \
         debootstrap \
         docker.io \
+        dosfstools \
+        gdisk \
         inetutils-ping \
         lsb-release \
         kpartx \
@@ -22,6 +24,8 @@ sudo apt-get install -y --force-yes \
         dpkg \
         debootstrap \
         docker \
+        dosfstools \
+        gdisk \
         kpartx \
         util-linux \
         qemu-img \
@@ -30,6 +34,8 @@ sudo apt-get install -y --force-yes \
         bzip2 \
         debootstrap \
         docker \
+        dosfstools \
+        gdisk \
         kpartx \
         util-linux \
         python-pyliblzma \
@@ -40,6 +46,8 @@ sudo apt-get install -y --force-yes \
         app-emulation/qemu \
         dev-python/pyyaml \
         sys-block/parted \
+        sys-apps/gptfdisk \
         sys-fs/multipath-tools \
+        sys-fs/dosfstools \
         qemu-img \
         yum-utils