sig_cloud/diskimage-builder
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge "GPT partitioning support"

Browse Source
This commit is contained in:
Zuul 2018-03-16 08:14:26 +00:00 committed by Gerrit Code Review
commit c60a20b59d
10 changed files with 308 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
diskimage_builder/block_device/level1/partition.py
View File

@ -33,6 +33,12 @@ class PartitionNode(NodeBase):
self.partitioning = parent
self.prev_partition = prev_partition
# filter out some MBR only options for clarity
if self.partitioning.label == 'gpt':
if 'flags' in config and 'primary' in config['flags']:
raise BlockDeviceSetupException(
"Primary flag not supported for GPT partitions")
self.flags = set()
if 'flags' in config:
for f in config['flags']:
@ -47,7 +53,10 @@ class PartitionNode(NodeBase):
raise BlockDeviceSetupException("No size in partition" % self.name)
self.size = config['size']
self.ptype = int(config['type'], 16) if 'type' in config else 0x83
if self.partitioning.label == 'gpt':
self.ptype = str(config['type']) if 'type' in config else '8300'
elif self.partitioning.label == 'mbr':
self.ptype = int(config['type'], 16) if 'type' in config else 83
def get_flags(self):
return self.flags

114
diskimage_builder/block_device/level1/partitioning.py
View File

@ -58,8 +58,8 @@ class Partitioning(PluginBase):
raise BlockDeviceSetupException(
"Partitioning config needs 'label'")
self.label = config['label']
if self.label not in ("mbr", ):
raise BlockDeviceSetupException("Label must be 'mbr'")
if self.label not in ("mbr", "gpt"):
raise BlockDeviceSetupException("Label must be 'mbr' or 'gpt'")
# It is VERY important to get the alignment correct. If this
# is not correct, the disk performance might be very poor.
@ -93,29 +93,9 @@ class Partitioning(PluginBase):
fd.seek(0, 2)
return fd.tell()
# not this is NOT a node and this is not called directly! The
# create() calls in the partition nodes this plugin has
# created are calling back into this.
def create(self):
# This is a bit of a hack. Each of the partitions is actually
# in the graph, so for every partition we get a create() call
# as the walk happens. But we only need to create the
# partition table once...
if self.already_created:
logger.info("Not creating the partitions a second time.")
return
self.already_created = True
# the raw file on disk
image_path = self.state['blockdev'][self.base]['image']
# the /dev/loopX device of the parent
device_path = self.state['blockdev'][self.base]['device']
logger.info("Creating partition on [%s] [%s]", self.base, image_path)
assert self.label == 'mbr'
disk_size = self._size_of_block_dev(image_path)
with MBR(image_path, disk_size, self.align) as part_impl:
def _create_mbr(self):
"""Create partitions with MBR"""
with MBR(self.image_path, self.disk_size, self.align) as part_impl:
for part_cfg in self.partitions:
part_name = part_cfg.get_name()
part_bootflag = PartitionNode.flag_boot \
@ -137,24 +117,100 @@ class Partitioning(PluginBase):
# We're going to mount all partitions with kpartx
# below once we're done. So the device this partition
# will be seen at becomes "/dev/mapper/loop0pX"
assert device_path[:5] == "/dev/"
assert self.device_path[:5] == "/dev/"
partition_device_name = "/dev/mapper/%sp%d" % \
(device_path[5:], part_no)
(self.device_path[5:], part_no)
self.state['blockdev'][part_name] \
= {'device': partition_device_name}
def _create_gpt(self):
"""Create partitions with GPT"""
cmd = ['sgdisk', self.image_path]
# This padding gives us a little room for rounding so we don't
# go over the end of the disk
disk_free = self.disk_size - (2048 * 1024)
pnum = 1
for p in self.partitions:
args = {}
args['pnum'] = pnum
args['name'] = '"%s"' % p.get_name()
args['type'] = '%s' % p.get_type()
# convert from a relative/string size to bytes
size = parse_rel_size_spec(p.get_size(), disk_free)[1]
# We keep track in bytes, but specify things to sgdisk in
# megabytes so it can align on sensible boundaries. And
# create partitions right after previous so no need to
# calculate start/end - just size.
assert size <= disk_free
args['size'] = size // (1024 * 1024)
new_cmd = ("-n {pnum}:0:+{size}M -t {pnum}:{type} "
"-c {pnum}:{name}".format(**args))
cmd.extend(new_cmd.strip().split(' '))
# Fill the state; we mount all partitions with kpartx
# below once we're done. So the device this partition
# will be seen at becomes "/dev/mapper/loop0pX"
assert self.device_path[:5] == "/dev/"
device_name = "/dev/mapper/%sp%d" % (self.device_path[5:], pnum)
self.state['blockdev'][p.get_name()] \
= {'device': device_name}
disk_free = disk_free - size
pnum = pnum + 1
logger.debug("Partition %s added, %s remaining in disk",
pnum, disk_free)
logger.debug("cmd: %s", ' '.join(cmd))
exec_sudo(cmd)
# not this is NOT a node and this is not called directly! The
# create() calls in the partition nodes this plugin has
# created are calling back into this.
def create(self):
# This is a bit of a hack. Each of the partitions is actually
# in the graph, so for every partition we get a create() call
# as the walk happens. But we only need to create the
# partition table once...
if self.already_created:
logger.info("Not creating the partitions a second time.")
return
self.already_created = True
# the raw file on disk
self.image_path = self.state['blockdev'][self.base]['image']
# the /dev/loopX device of the parent
self.device_path = self.state['blockdev'][self.base]['device']
# underlying size
self.disk_size = self._size_of_block_dev(self.image_path)
logger.info("Creating partition on [%s] [%s]",
self.base, self.image_path)
assert self.label in ('mbr', 'gpt')
if self.label == 'mbr':
self._create_mbr()
elif self.label == 'gpt':
self._create_gpt()
# "saftey sync" to make sure the partitions are written
exec_sudo(["sync"])
# now all the partitions are created, get device-mapper to
# mount them
if not os.path.exists("/.dockerenv"):
exec_sudo(["kpartx", "-avs", device_path])
exec_sudo(["kpartx", "-avs", self.device_path])
else:
# If running inside Docker, make our nodes manually,
# because udev will not be working. kpartx cannot run in
# sync mode in docker.
exec_sudo(["kpartx", "-av", device_path])
exec_sudo(["kpartx", "-av", self.device_path])
exec_sudo(["dmsetup", "--noudevsync", "mknodes"])
return

32
diskimage_builder/block_device/tests/config/gpt_efi.yaml Normal file
View File

@ -0,0 +1,32 @@
# A sample config that has GPT/bios and EFI boot partitions
- local_loop:
name: image0
- partitioning:
base: image0
label: gpt
partitions:
- name: ESP
type: 'EF00'
size: 8MiB
mkfs:
type: vfat
mount:
mount_point: /boot/efi
fstab:
options: "defaults"
fsck-passno: 1
- name: BSP
type: 'EF02'
size: 8MiB
- name: root
type: '8300'
size: 100%
mkfs:
type: ext4
mount:
mount_point: /
fstab:
options: "defaults"
fsck-passno: 1

85
diskimage_builder/block_device/tests/test_gpt.py Normal file
View File

@ -0,0 +1,85 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import fixtures
import logging
import mock
import os
import diskimage_builder.block_device.tests.test_config as tc
from diskimage_builder.block_device.blockdevice import BlockDeviceState
from diskimage_builder.block_device.config import config_tree_to_graph
from diskimage_builder.block_device.config import create_graph
from diskimage_builder.block_device.level0.localloop import image_create
from diskimage_builder.block_device.level1.partition import PartitionNode
logger = logging.getLogger(__name__)
class TestGPT(tc.TestGraphGeneration):
@mock.patch('diskimage_builder.block_device.level1.partitioning.exec_sudo')
def test_gpt_efi(self, mock_exec_sudo):
# Test the command-sequence for a GPT/EFI partition setup
tree = self.load_config_file('gpt_efi.yaml')
config = config_tree_to_graph(tree)
state = BlockDeviceState()
graph, call_order = create_graph(config, self.fake_default_config,
state)
# Create a fake temp backing file (we check the size of it,
# etc).
# TODO(ianw): exec_sudo is generically mocked out, thus the
# actual creation is mocked out ... but we could do this
# without root and use parted to create the partitions on this
# for slightly better testing. An exercise for another day...
self.tmp_dir = fixtures.TempDir()
self.useFixture(self.tmp_dir)
self.image_path = os.path.join(self.tmp_dir.path, "image.raw")
# should be sparse...
image_create(self.image_path, 1024 * 1024 * 1024)
logger.debug("Temp image in %s", self.image_path)
# Fake state for the loopback device
state['blockdev'] = {}
state['blockdev']['image0'] = {}
state['blockdev']['image0']['image'] = self.image_path
state['blockdev']['image0']['device'] = "/dev/loopX"
for node in call_order:
if isinstance(node, PartitionNode):
node.create()
# check the parted call looks right
parted_cmd = ('sgdisk %s '
'-n 1:0:+8M -t 1:EF00 -c 1:"ESP" '
'-n 2:0:+8M -t 2:EF02 -c 2:"BSP" '
'-n 3:0:+1006M -t 3:8300 -c 3:"root"'
% self.image_path)
cmd_sequence = [
mock.call(parted_cmd.split(' ')),
mock.call(['sync']),
mock.call(['kpartx', '-avs', '/dev/loopX'])
]
self.assertEqual(mock_exec_sudo.call_count, len(cmd_sequence))
mock_exec_sudo.assert_has_calls(cmd_sequence)
# Check two new partitions appear in state correctly
self.assertDictEqual(state['blockdev']['ESP'],
{'device': '/dev/mapper/loopXp1'})
self.assertDictEqual(state['blockdev']['BSP'],
{'device': '/dev/mapper/loopXp2'})
self.assertDictEqual(state['blockdev']['root'],
{'device': '/dev/mapper/loopXp3'})

11
diskimage_builder/elements/block-device-gpt/README.rst Normal file
View File

@ -0,0 +1,11 @@
================
Block Device GPT
================
This is an override for the default block-device configuration
provided in the ``vm`` element to get a GPT based single-partition
disk, rather than the default MBR.
Note this provides the extra `BIOS boot partition
<https://en.wikipedia.org/wiki/BIOS_boot_partition>`__ as required for
non-EFI boot environments.

22
diskimage_builder/elements/block-device-gpt/block-device-default.yaml Normal file
View File

@ -0,0 +1,22 @@
# Default single partition loopback using a GPT based partition table
- local_loop:
name: image0
- partitioning:
base: image0
label: gpt
partitions:
- name: BSP
type: 'EF02'
size: 8MiB
- name: root
flags: [ boot ]
size: 100%
mkfs:
type: ext4
mount:
mount_point: /
fstab:
options: "defaults"
fsck-passno: 1

2
diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/element-deps
View File

@ -1 +1,3 @@
block-device-gpt
openstack-ci-mirrors
vm

60
doc/source/user_guide/building_an_image.rst
View File

@ -75,7 +75,8 @@ There are currently two defaults:
The user can overwrite the default handling by setting the environment
variable `DIB_BLOCK_DEVICE_CONFIG`. This variable must hold YAML
structured configuration data.
structured configuration data or be a ``file://`` URL reference to a
on-disk configuration file.
The default when using the `vm` element is:
@ -247,8 +248,8 @@ encrypted, ...) and create partition information in it.
The symbolic name for this module is `partitioning`.
Currently the only supported partitioning layout is Master Boot Record
`MBR`.
MBR
***
It is possible to create primary or logical partitions or a mix of
them. The numbering of the primary partitions will start at 1,
@ -267,19 +268,27 @@ partitions.
Partitions are created in the order they are configured. Primary
partitions - if needed - must be first in the list.
GPT
***
GPT partitioning requires the ``sgdisk`` tool to be available.
Options
*******
There are the following key / value pairs to define one partition
table:
base
(mandatory) The base device where to create the partitions in.
(mandatory) The base device to create the partitions in.
label
(mandatory) Possible values: 'mbr'
This uses the Master Boot Record (MBR) layout for the disk.
(There are currently plans to add GPT later on.)
(mandatory) Possible values: 'mbr', 'gpt'
Configure use of either the Master Boot Record (MBR) or GUID
Partition Table (GPT) formats
align
(optional - default value '1MiB')
(optional - default value '1MiB'; MBR only)
Set the alignment of the partition. This must be a multiple of the
block size (i.e. 512 bytes). The default of 1MiB (~ 2048 * 512
bytes blocks) is the default for modern systems and known to
@ -308,9 +317,9 @@ flags
(optional) List of flags for the partition. Default: empty.
Possible values:
boot
boot (MBR only)
Sets the boot flag for the partition
primary
primary (MBR only)
Partition should be a primary partition. If not set a logical
partition will be created.
@ -321,10 +330,15 @@ size
based on the remaining free space.
type (optional)
The partition type stored in the MBR partition table entry. The
default value is '0x83' (Linux Default partition). Any valid one
The partition type stored in the MBR or GPT partition table entry.
For MBR the default value is '0x83' (Linux Default partition). Any valid one
byte hexadecimal value may be specified here.
For GPT the default value is '8300' (Linux Default partition). Any valid two
byte hexadecimal value may be specified here. Due to ``sgdisk`` leading '0x'
should not be used.
Example:
.. code-block:: yaml
@ -350,12 +364,28 @@ Example:
- name: data2
size: 100%
- partitioning:
base: gpt_image
label: gpt
partitions:
- name: ESP
type: EF00
size: 16MiB
- name: data1
size: 1GiB
- name: lvmdata
type: 8E00
size: 100%
On the `image0` two partitions are created. The size of the first is
1GiB, the second uses the remaining free space. On the `data_image`
three partitions are created: all are about 1/3 of the disk size.
three partitions are created: all are about 1/3 of the disk size. On
the `gpt_image` three partitions are created: 16MiB one for EFI
bootloader, 1GiB Linux filesystem one and rest of disk will be used
for LVM partition.
Module: Lvm
···········
Module: LVM
...........
This module generates volumes on existing block devices. This means that it is
possible to take any previous created partition, and create volumes information

7
releasenotes/notes/bootloader-gpt-d1047f81f3a0631b.yaml Normal file
View File

@ -0,0 +1,7 @@
---
features:
- |
GPT support is added to the bootloader; see documentation for
configuration examples. This should be considered a technology
preview; there may be minor behaviour modifications as we enable
UEFI and support across more architectures.

8
tests/install_test_deps.sh
View File

@ -9,6 +9,8 @@ sudo apt-get install -y --force-yes \
bzip2 \
debootstrap \
docker.io \
dosfstools \
gdisk \
inetutils-ping \
lsb-release \
kpartx \
@ -22,6 +24,8 @@ sudo apt-get install -y --force-yes \
dpkg \
debootstrap \
docker \
dosfstools \
gdisk \
kpartx \
util-linux \
qemu-img \
@ -30,6 +34,8 @@ sudo apt-get install -y --force-yes \
bzip2 \
debootstrap \
docker \
dosfstools \
gdisk \
kpartx \
util-linux \
python-pyliblzma \
@ -40,6 +46,8 @@ sudo apt-get install -y --force-yes \
app-emulation/qemu \
dev-python/pyyaml \
sys-block/parted \
sys-apps/gptfdisk \
sys-fs/multipath-tools \
sys-fs/dosfstools \
qemu-img \
yum-utils