From e4e23897a13a3f3b9d28cc8d288990ab0fcc5b92 Mon Sep 17 00:00:00 2001 From: Andreas Florath Date: Sun, 29 Jan 2017 23:52:40 +0000 Subject: [PATCH] Refactor: block-device filesystem creation, mount and fstab This patch finalizes the block device refactoring. It moves the three remaining levels (filesystem creation, mount and fstab handling) into the new python module. Now it is possible to use any number of disk images, any number of partitions and used them mounted to different directories. Notes: * unmount_dir : modified to only unmount the subdirs mounted by mount_proc_sys_dev(). dib-block-device unmounts $TMP_MOUNT_PATH/mnt (see I85e01f3898d3c043071de5fad82307cb091a64a9) Change-Id: I592c0b1329409307197460cfa8fd69798013f1f8 Signed-off-by: Andreas Florath Closes-Bug: #1664924 --- diskimage_builder/block_device/blockdevice.py | 81 +++++++- .../block_device/level1/partitioning.py | 3 + .../level2/__init__.py} | 16 +- diskimage_builder/block_device/level2/mkfs.py | 186 ++++++++++++++++++ .../level3/__init__.py} | 16 +- .../block_device/level3/mount.py | 161 +++++++++++++++ .../block_device/level4/__init__.py | 17 ++ .../block_device/level4/fstab.py | 82 ++++++++ .../install.d/15-cleanup-debootstrap | 5 - .../post-install.d/05-fstab-rootfs-label | 12 -- .../elements/vm/block-device-default.yaml | 18 +- .../elements/vm/block-device-ppc.yaml | 6 + .../elements/vm/environment.d/10-partitioning | 14 -- .../zypper/post-install.d/10-mkinitrd | 6 +- diskimage_builder/lib/common-functions | 4 +- diskimage_builder/lib/disk-image-create | 65 ++---- doc/source/user_guide/building_an_image.rst | 171 ++++++++++++++-- ...ice-mkfs-mount-fstab-42d7efe28fc2df04.yaml | 4 + setup.cfg | 3 + 19 files changed, 739 insertions(+), 131 deletions(-) rename diskimage_builder/{elements/zypper-minimal/install.d/15-zypper-fstab => block_device/level2/__init__.py} (65%) mode change 100755 => 100644 create mode 100644 diskimage_builder/block_device/level2/mkfs.py rename diskimage_builder/{elements/yum-minimal/install.d/15-base-fstab => block_device/level3/__init__.py} (64%) mode change 100755 => 100644 create mode 100644 diskimage_builder/block_device/level3/mount.py create mode 100644 diskimage_builder/block_device/level4/__init__.py create mode 100644 diskimage_builder/block_device/level4/fstab.py delete mode 100755 diskimage_builder/elements/rpm-distro/post-install.d/05-fstab-rootfs-label delete mode 100644 diskimage_builder/elements/vm/environment.d/10-partitioning create mode 100644 releasenotes/notes/block-device-mkfs-mount-fstab-42d7efe28fc2df04.yaml diff --git a/diskimage_builder/block_device/blockdevice.py b/diskimage_builder/block_device/blockdevice.py index daa1ddd2..9ff49182 100644 --- a/diskimage_builder/block_device/blockdevice.py +++ b/diskimage_builder/block_device/blockdevice.py @@ -22,6 +22,7 @@ import yaml from stevedore import extension +from diskimage_builder.block_device.utils import exec_sudo from diskimage_builder.graph.digraph import Digraph @@ -55,6 +56,8 @@ class BlockDevice(object): After this call it is possible to copy / install all the needed files into the appropriate directories. + cmd_writefstab: creates the (complete) fstab for the system. + cmd_umount: unmount and detaches all directories and used many resources. After this call the used (e.g.) images are still available for further handling, e.g. converting from raw in @@ -248,6 +251,28 @@ class BlockDevice(object): logger.info("Wrote final block device config to [%s]" % self.config_json_file_name) + def _config_get_mount(self, path): + for entry in self.config: + for k, v in entry.items(): + if k == 'mount' and v['mount_point'] == path: + return v + assert False + + def _config_get_all_mount_points(self): + rvec = [] + for entry in self.config: + for k, v in entry.items(): + if k == 'mount': + rvec.append(v['mount_point']) + return rvec + + def _config_get_mkfs(self, name): + for entry in self.config: + for k, v in entry.items(): + if k == 'mkfs' and v['name'] == name: + return v + assert False + def cmd_getval(self, symbol): """Retrieve value from block device level @@ -256,9 +281,25 @@ class BlockDevice(object): (non python) access to internal configuration. Arguments: - :symbol: The symbol to find + :param symbol: the symbol to get """ logger.info("Getting value for [%s]" % symbol) + if symbol == "root-label": + root_mount = self._config_get_mount("/") + root_fs = self._config_get_mkfs(root_mount['base']) + logger.debug("root-label [%s]" % root_fs['label']) + print("%s" % root_fs['label']) + return 0 + if symbol == "root-fstype": + root_mount = self._config_get_mount("/") + root_fs = self._config_get_mkfs(root_mount['base']) + logger.debug("root-fstype [%s]" % root_fs['type']) + print("%s" % root_fs['type']) + return 0 + if symbol == 'mount-points': + mount_points = self._config_get_all_mount_points() + print("%s" % " ".join(mount_points)) + return 0 if symbol == 'image-block-partition': # If there is no partition needed, pass back directly the # image. @@ -270,9 +311,47 @@ class BlockDevice(object): if symbol == 'image-path': print("%s" % self.state['blockdev']['image0']['image']) return 0 + logger.error("Invalid symbol [%s] for getval" % symbol) return 1 + def cmd_writefstab(self): + """Creates the fstab""" + logger.info("Creating fstab") + + tmp_fstab = os.path.join(self.state_dir, "fstab") + with open(tmp_fstab, "wt") as fstab_fd: + # This gives the order in which this must be mounted + for mp in self.state['mount_order']: + logger.debug("Writing fstab entry for [%s]" % mp) + fs_base = self.state['mount'][mp]['base'] + fs_name = self.state['mount'][mp]['name'] + fs_val = self.state['filesys'][fs_base] + if 'label' in fs_val: + diskid = "LABEL=%s" % fs_val['label'] + else: + diskid = "UUID=%s" % fs_val['uuid'] + + # If there is no fstab entry - do not write anything + if 'fstab' not in self.state: + continue + if fs_name not in self.state['fstab']: + continue + + options = self.state['fstab'][fs_name]['options'] + dump_freq = self.state['fstab'][fs_name]['dump-freq'] + fsck_passno = self.state['fstab'][fs_name]['fsck-passno'] + + fstab_fd.write("%s %s %s %s %s %s\n" + % (diskid, mp, fs_val['fstype'], + options, dump_freq, fsck_passno)) + + target_etc_dir = os.path.join(self.params['build-dir'], 'built', 'etc') + exec_sudo(['mkdir', '-p', target_etc_dir]) + exec_sudo(['cp', tmp_fstab, os.path.join(target_etc_dir, "fstab")]) + + return 0 + def cmd_create(self): """Creates the block device""" diff --git a/diskimage_builder/block_device/level1/partitioning.py b/diskimage_builder/block_device/level1/partitioning.py index 3b1cd54e..937379cb 100644 --- a/diskimage_builder/block_device/level1/partitioning.py +++ b/diskimage_builder/block_device/level1/partitioning.py @@ -85,6 +85,9 @@ class Partition(Digraph.Node): def get_type(self): return self.ptype + def get_name(self): + return self.name + def insert_edges(self, dg): bnode = dg.find(self.base) assert bnode is not None diff --git a/diskimage_builder/elements/zypper-minimal/install.d/15-zypper-fstab b/diskimage_builder/block_device/level2/__init__.py old mode 100755 new mode 100644 similarity index 65% rename from diskimage_builder/elements/zypper-minimal/install.d/15-zypper-fstab rename to diskimage_builder/block_device/level2/__init__.py index cffbe1df..25456900 --- a/diskimage_builder/elements/zypper-minimal/install.d/15-zypper-fstab +++ b/diskimage_builder/block_device/level2/__init__.py @@ -1,6 +1,4 @@ -#!/bin/bash -# -# Copyright 2015 Hewlett-Packard Development Company, L.P. +# Copyright 2017 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -13,15 +11,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -# -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail +from diskimage_builder.block_device.level2.mkfs import Mkfs -cat << EOF > /etc/fstab -proc /proc proc nodev,noexec,nosuid 0 0 -LABEL=${DIB_ROOT_LABEL} / ${FS_TYPE} errors=remount-ro 0 1 -EOF +__all__ = [Mkfs] diff --git a/diskimage_builder/block_device/level2/mkfs.py b/diskimage_builder/block_device/level2/mkfs.py new file mode 100644 index 00000000..fe27dda8 --- /dev/null +++ b/diskimage_builder/block_device/level2/mkfs.py @@ -0,0 +1,186 @@ +# Copyright 2017 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import uuid + +from diskimage_builder.block_device.blockdevice \ + import BlockDeviceSetupException +from diskimage_builder.block_device.tree_config import TreeConfig +from diskimage_builder.block_device.utils import exec_sudo +from diskimage_builder.graph.digraph import Digraph + + +logger = logging.getLogger(__name__) + + +# There is the need that filesystem labels are unique: +# if not the boot and / or mount (with LABEL=) might fail. +file_system_labels = set() + +# There is the need to check the length of the label of +# the filesystem. The maximum length depends on the used filesystem. +# This map provides information about the maximum label length. +file_system_max_label_length = { + "ext2": 16, + "ext3": 16, + "ext4": 16, + "xfs": 12, + "vfat": 11 +} + + +class Filesystem(Digraph.Node): + + def _config_error(self, msg): + logger.error(msg) + raise BlockDeviceSetupException(msg) + + def __init__(self, config): + logger.debug("Create filesystem object; config [%s]" % config) + # Parameter check (mandatory) + for pname in ['base', 'name', 'type']: + if pname not in config: + self._config_error("Mkfs config needs [%s]" % pname) + setattr(self, pname, config[pname]) + + # Parameter check (optional) + for pname in ['label', 'opts', 'uuid']: + setattr(self, pname, + config[pname] if pname in config else None) + + if self.label is None: + self.label = self.name + + # Historic reasons - this will hopefully vanish in one of + # the next major releases + if self.label == "cloudimg-rootfs" and self.type == "xfs": + logger.warning("Default label [cloudimg-rootfs] too long for xfs " + "file system - using [img-rootfs] instead") + self.label = "img-rootfs" + + if self.label in file_system_labels: + self._config_error( + "File system label [%s] used more than once" % + self.label) + file_system_labels.add(self.label) + + if self.type in file_system_max_label_length: + if file_system_max_label_length[self.type] < \ + len(self.label): + self._config_error( + "Label [%s] too long for filesystem [%s]: " + "maximum length [%d] provided length [%d]" % + (self.label, self.type, + file_system_max_label_length[self.type], + len(self.label))) + else: + logger.warning("Length of label [%s] cannot be checked for " + "filesystem [%s]: unknown max length" % + (self.label, self.type)) + logger.warning("Continue - but this might lead to an error") + + if self.opts is not None: + self.opts = self.opts.strip().split(' ') + + if self.uuid is None: + self.uuid = str(uuid.uuid4()) + + Digraph.Node.__init__(self, self.name) + + logger.debug("Filesystem created [%s]" % self) + + def __repr__(self): + return "" \ + % (self.base, self.name, self.type) + + def insert_edges(self, dg): + logger.debug("Insert edge [%s]" % self) + bnode = dg.find(self.base) + assert bnode is not None + dg.create_edge(bnode, self) + + def create(self, result, rollback): + logger.info("create called; result [%s]" % result) + + cmd = ["mkfs"] + + cmd.extend(['-t', self.type]) + if self.opts: + cmd.extend(self.opts) + cmd.extend(["-L", self.label]) + + if self.type in ('ext2', 'ext3', 'ext4'): + cmd.extend(['-U', self.uuid]) + elif self.type == 'xfs': + cmd.extend(['-m', "uuid=%s" % self.uuid]) + else: + logger.warning("UUID will not be written for fs type [%s]" + % self.type) + + if self.type in ('ext2', 'ext3', 'ext4', 'xfs'): + cmd.append('-q') + + if 'blockdev' not in result: + result['blockdev'] = {} + device = result['blockdev'][self.base]['device'] + cmd.append(device) + + logger.debug("Creating fs command [%s]" % (cmd)) + exec_sudo(cmd) + + if 'filesys' not in result: + result['filesys'] = {} + result['filesys'][self.name] \ + = {'uuid': self.uuid, 'label': self.label, + 'fstype': self.type, 'opts': self.opts, + 'device': device} + + def umount(self, state): + """Mkfs does not need any umount.""" + pass + + def cleanup(self, state): + """Mkfs does not need any cleanup.""" + pass + + def delete(self, state): + """Mkfs does not need any delete.""" + pass + + +class Mkfs(object): + """Module for creating file systems + + This block device module handles creating different file + systems. + """ + + type_string = "mkfs" + tree_config = TreeConfig("mkfs") + + def __init__(self, config, default_config): + logger.debug("Create Mkfs object; config [%s]" % config) + logger.debug("default_config [%s]" % default_config) + self.config = config + self.default_config = default_config + self.filesystems = {} + + fs = Filesystem(self.config) + self.filesystems[fs.get_name()] = fs + + def insert_nodes(self, dg): + for _, fs in self.filesystems.items(): + logger.debug("Insert node [%s]" % fs) + dg.add_node(fs) diff --git a/diskimage_builder/elements/yum-minimal/install.d/15-base-fstab b/diskimage_builder/block_device/level3/__init__.py old mode 100755 new mode 100644 similarity index 64% rename from diskimage_builder/elements/yum-minimal/install.d/15-base-fstab rename to diskimage_builder/block_device/level3/__init__.py index a766a9d8..5fa3fe4b --- a/diskimage_builder/elements/yum-minimal/install.d/15-base-fstab +++ b/diskimage_builder/block_device/level3/__init__.py @@ -1,6 +1,4 @@ -#!/bin/bash -# -# Copyright 2015 Hewlett-Packard Development Company, L.P. +# Copyright 2017 Andreas Florath (andreas@florath.net) # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -13,15 +11,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -# -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail +from diskimage_builder.block_device.level3.mount import Mount -cat << EOF | tee /etc/fstab > /dev/null -proc /proc proc nodev,noexec,nosuid 0 0 -LABEL=${DIB_ROOT_LABEL} / ${FS_TYPE} errors=remount-ro 0 1 -EOF +__all__ = [Mount] diff --git a/diskimage_builder/block_device/level3/mount.py b/diskimage_builder/block_device/level3/mount.py new file mode 100644 index 00000000..b247b1f8 --- /dev/null +++ b/diskimage_builder/block_device/level3/mount.py @@ -0,0 +1,161 @@ +# Copyright 2017 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os + +from diskimage_builder.block_device.blockdevice \ + import BlockDeviceSetupException +from diskimage_builder.block_device.tree_config import TreeConfig +from diskimage_builder.block_device.utils import exec_sudo +from diskimage_builder.block_device.utils import sort_mount_points +from diskimage_builder.graph.digraph import Digraph + + +logger = logging.getLogger(__name__) + + +# There is the need to collect all mount points to be able to +# sort them in a sensible way. +mount_points = {} +# The order of mounting and unmounting is important. +sorted_mount_points = None + + +class MountPoint(Digraph.Node): + + @staticmethod + def _config_error(msg): + logger.error(msg) + raise BlockDeviceSetupException(msg) + + def __init__(self, mount_base, config): + # Parameter check + self.mount_base = mount_base + for pname in ['base', 'name', 'mount_point']: + if pname not in config: + self._config_error("MountPoint config needs [%s]" % pname) + setattr(self, pname, config[pname]) + Digraph.Node.__init__(self, self.name) + logger.debug("MountPoint created [%s]" % self) + + def __repr__(self): + return "" \ + % (self.base, self.name, self.mount_point) + + def insert_node(self, dg): + global mount_points + if self.mount_point in mount_points: + self._config_error("Mount point [%s] specified more than once" + % self.mount_point) + logger.debug("Insert node [%s]" % self) + mount_points[self.mount_point] = self + dg.add_node(self) + + def insert_edges(self, dg): + """Insert all edges + + After inserting all the nodes, the order of the mounting and + umounting can be computed. There is the need to mount + mount-points that contain other mount-points first. + Example: '/var' must be mounted before '/var/log'. If not the + second is not used for files at all. + + The dependency edge is created in all cases from the base + element (typically a mkfs) and, if this is not the 'first' + mount-point, also depend on the mount point before. This + ensures that during mounting (and umounting) the correct + order is used. + """ + logger.debug("Insert edge [%s]" % self) + global mount_points + global sorted_mount_points + if sorted_mount_points is None: + logger.debug("Mount points [%s]" % mount_points) + sorted_mount_points = sort_mount_points(mount_points.keys()) + logger.info("Sorted mount points [%s]" % (sorted_mount_points)) + + # Look for the occurance in the list + mpi = sorted_mount_points.index(self.mount_point) + if mpi > 0: + # If not the first: add also the dependency + dg.create_edge(mount_points[sorted_mount_points[mpi - 1]], self) + + bnode = dg.find(self.base) + assert bnode is not None + dg.create_edge(bnode, self) + + def create(self, result, rollback): + logger.debug("mount called [%s]" % self.mount_point) + logger.debug("result [%s]" % result) + rel_mp = self.mount_point if self.mount_point[0] != '/' \ + else self.mount_point[1:] + mount_point = os.path.join(self.mount_base, rel_mp) + if not os.path.exists(mount_point): + # Need to sudo this because of permissions in the new + # file system tree. + exec_sudo(['mkdir', '-p', mount_point]) + logger.info("Mounting [%s] to [%s]" % (self.name, mount_point)) + exec_sudo(["mount", result['filesys'][self.base]['device'], + mount_point]) + + if 'mount' not in result: + result['mount'] = {} + result['mount'][self.mount_point] \ + = {'name': self.name, 'base': self.base, 'path': mount_point} + + if 'mount_order' not in result: + result['mount_order'] = [] + result['mount_order'].append(self.mount_point) + + def umount(self, state): + logger.info("Called for [%s]" % self.name) + exec_sudo(["umount", state['mount'][self.mount_point]['path']]) + + def cleanup(self, state): + """Mount does not need any cleanup.""" + pass + + def delete(self, state): + self.umount(state) + + +class Mount(object): + + type_string = "mount" + tree_config = TreeConfig("mount") + + def _config_error(self, msg): + logger.error(msg) + raise BlockDeviceSetupException(msg) + + def __init__(self, config, params): + logger.debug("Mounting object; config [%s]" % config) + self.config = config + self.params = params + + if 'mount-base' not in self.params: + MountPoint._config_error("Mount default config needs 'mount-base'") + self.mount_base = self.params['mount-base'] + + self.mount_points = {} + + mp = MountPoint(self.mount_base, self.config) + self.mount_points[mp.get_name()] = mp + + def insert_nodes(self, dg): + global sorted_mount_points + assert sorted_mount_points is None + for _, mp in self.mount_points.items(): + mp.insert_node(dg) diff --git a/diskimage_builder/block_device/level4/__init__.py b/diskimage_builder/block_device/level4/__init__.py new file mode 100644 index 00000000..568d86b0 --- /dev/null +++ b/diskimage_builder/block_device/level4/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2017 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from diskimage_builder.block_device.level4.fstab import Fstab + +__all__ = [Fstab] diff --git a/diskimage_builder/block_device/level4/fstab.py b/diskimage_builder/block_device/level4/fstab.py new file mode 100644 index 00000000..dc91ebc7 --- /dev/null +++ b/diskimage_builder/block_device/level4/fstab.py @@ -0,0 +1,82 @@ +# Copyright 2017 Andreas Florath (andreas@florath.net) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +from diskimage_builder.block_device.blockdevice \ + import BlockDeviceSetupException +from diskimage_builder.block_device.tree_config import TreeConfig +from diskimage_builder.graph.digraph import Digraph + + +logger = logging.getLogger(__name__) + + +class Fstab(Digraph.Node): + + type_string = "fstab" + tree_config = TreeConfig("fstab") + + def _config_error(self, msg): + logger.error(msg) + raise BlockDeviceSetupException(msg) + + def __init__(self, config, params): + logger.debug("Fstab object; config [%s]" % config) + self.config = config + self.params = params + self.name = self.config['name'] + self.base = self.config['base'] + Digraph.Node.__init__(self, self.name) + + self.options = self.config.get('options', 'defaults') + self.dump_freq = self.config.get('dump-freq', 0) + self.fsck_passno = self.config.get('fsck-passno', 2) + + def insert_nodes(self, dg): + logger.debug("Insert node") + dg.add_node(self) + + def insert_edges(self, dg): + logger.debug("Insert edge [%s]" % self) + bnode = dg.find(self.base) + assert bnode is not None + dg.create_edge(bnode, self) + + def create(self, result, rollback): + logger.debug("fstab create called [%s]" % self.name) + logger.debug("result [%s]" % result) + + if 'fstab' not in result: + result['fstab'] = {} + + result['fstab'][self.base] = { + 'name': self.name, + 'base': self.base, + 'options': self.options, + 'dump-freq': self.dump_freq, + 'fsck-passno': self.fsck_passno + } + + def umount(self, state): + """Fstab does not need any umount task.""" + pass + + def cleanup(self, state): + """Fstab does not need any cleanup.""" + pass + + def delete(self, state): + """Fstab does not need any cleanup.""" + pass diff --git a/diskimage_builder/elements/debootstrap/install.d/15-cleanup-debootstrap b/diskimage_builder/elements/debootstrap/install.d/15-cleanup-debootstrap index a256612b..3059a05a 100755 --- a/diskimage_builder/elements/debootstrap/install.d/15-cleanup-debootstrap +++ b/diskimage_builder/elements/debootstrap/install.d/15-cleanup-debootstrap @@ -23,8 +23,3 @@ set -o pipefail install -d -m 0755 -o root -g root /etc/sudoers.d echo 'blacklist pcspkr' > /etc/modprobe.d/blacklist.conf - -cat << EOF | tee /etc/fstab > /dev/null -proc /proc proc nodev,noexec,nosuid 0 0 -LABEL=${DIB_ROOT_LABEL} / ${FS_TYPE} errors=remount-ro 0 1 -EOF diff --git a/diskimage_builder/elements/rpm-distro/post-install.d/05-fstab-rootfs-label b/diskimage_builder/elements/rpm-distro/post-install.d/05-fstab-rootfs-label deleted file mode 100755 index bf3e1595..00000000 --- a/diskimage_builder/elements/rpm-distro/post-install.d/05-fstab-rootfs-label +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -# Fedora 18 sets up for root to have a label of "_/" -# Fedora 19 sets up for root to have a UUID -# This regex will catch both -sed -i "s%.*\s\/\s%LABEL=${DIB_ROOT_LABEL} / %" /etc/fstab diff --git a/diskimage_builder/elements/vm/block-device-default.yaml b/diskimage_builder/elements/vm/block-device-default.yaml index 85b41a3b..31cfdd55 100644 --- a/diskimage_builder/elements/vm/block-device-default.yaml +++ b/diskimage_builder/elements/vm/block-device-default.yaml @@ -4,9 +4,15 @@ name: image0 - partitioning: - base: image0 - label: mbr - partitions: - - name: root - flags: [ boot, primary ] - size: 100% + base: image0 + label: mbr + partitions: + - name: root + flags: [ boot, primary ] + size: 100% + mkfs: + mount: + mount_point: / + fstab: + options: "defaults" + fsck-passno: 1 diff --git a/diskimage_builder/elements/vm/block-device-ppc.yaml b/diskimage_builder/elements/vm/block-device-ppc.yaml index 2b3a78dd..cf3a26b9 100644 --- a/diskimage_builder/elements/vm/block-device-ppc.yaml +++ b/diskimage_builder/elements/vm/block-device-ppc.yaml @@ -28,3 +28,9 @@ - name: root flags: [ primary ] size: 100% + mkfs: + mount: + mount_point: / + fstab: + options: "defaults" + fsck-passno: 1 diff --git a/diskimage_builder/elements/vm/environment.d/10-partitioning b/diskimage_builder/elements/vm/environment.d/10-partitioning deleted file mode 100644 index 7aeea5a3..00000000 --- a/diskimage_builder/elements/vm/environment.d/10-partitioning +++ /dev/null @@ -1,14 +0,0 @@ -_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -if [[ "$ARCH" =~ "ppc" ]] ; then - DIB_BLOCK_DEVICE_DEFAULT_CONFIG="$(cat $_DIR/../block-device-ppc.yaml)" -else - DIB_BLOCK_DEVICE_DEFAULT_CONFIG="$(cat $_DIR/../block-device-default.yaml)" -fi - -DIB_BLOCK_DEVICE_CONFIG=${DIB_BLOCK_DEVICE_CONFIG:-${DIB_BLOCK_DEVICE_DEFAULT_CONFIG}} -export DIB_BLOCK_DEVICE_CONFIG - -# Local variables: -# mode: sh -# End: diff --git a/diskimage_builder/elements/zypper/post-install.d/10-mkinitrd b/diskimage_builder/elements/zypper/post-install.d/10-mkinitrd index 6162655f..6d572d6b 100755 --- a/diskimage_builder/elements/zypper/post-install.d/10-mkinitrd +++ b/diskimage_builder/elements/zypper/post-install.d/10-mkinitrd @@ -12,10 +12,12 @@ set -o pipefail # that will actually be used for the final image. This is likely something # different than what the chroot is currently on (which might currently be a # tmpfs even). -echo "rootfstype=$FS_TYPE" > /etc/sysconfig/initrd +echo "rootfstype=${DIB_ROOT_FSTYPE}" > /etc/sysconfig/initrd +# ToDo: This it not really clear to me. +# Let's see which error occurs. # openSuse mkinitrd requires a valid root device be in fstab. -sed -i 's/vda1/sda1/' /etc/fstab +##sed -i 's/vda1/sda1/' /etc/fstab mkinitrd -A -B # And cleanup again diff --git a/diskimage_builder/lib/common-functions b/diskimage_builder/lib/common-functions index e1153a4b..7faee8f2 100644 --- a/diskimage_builder/lib/common-functions +++ b/diskimage_builder/lib/common-functions @@ -353,7 +353,9 @@ function unmount_dir { # /proc/mounts is the real path real_dir=$(readlink -e $dir) - mnts=$(awk '{print $2}' < /proc/mounts | grep "^$real_dir" | sort -r) + # note the "/" on real_dir ... we are just looking for things + # mounted *underneath* this directory. + mnts=$(awk '{print $2}' < /proc/mounts | grep "^$real_dir/" | sort -r) for m in $mnts; do echo "Unmount $m" sudo umount -fl $m || true diff --git a/diskimage_builder/lib/disk-image-create b/diskimage_builder/lib/disk-image-create index 2fe6a039..3b76af66 100644 --- a/diskimage_builder/lib/disk-image-create +++ b/diskimage_builder/lib/disk-image-create @@ -115,7 +115,7 @@ DIB_DEBUG_TRACE=${DIB_DEBUG_TRACE:-0} INSTALL_PACKAGES="" IMAGE_TYPES=("qcow2") COMPRESS_IMAGE="true" -export DIB_ROOT_LABEL="" +ROOT_LABEL="" DIB_DEFAULT_INSTALLTYPE=${DIB_DEFAULT_INSTALLTYPE:-"source"} MKFS_OPTS="" ACI_MANIFEST=${ACI_MANIFEST:-} @@ -147,7 +147,7 @@ while true ; do --no-tmpfs) shift; export DIB_NO_TMPFS=1;; --offline) shift; export DIB_OFFLINE=1;; --qemu-img-options) QEMU_IMG_OPTIONS=$2; shift 2;; - --root-label) export DIB_ROOT_LABEL=$2; shift 2;; + --root-label) ROOT_LABEL=$2; shift 2;; --ramdisk-element) RAMDISK_ELEMENT=$2; shift 2;; --install-type) DIB_DEFAULT_INSTALLTYPE=$2; shift 2;; --docker-target) export DOCKER_TARGET=$2; shift 2 ;; @@ -250,26 +250,6 @@ if [[ -z "$(which fstrim)" ]]; then exit 1 fi -# NOTE: Tuning the rootfs uuid works only for ext filesystems. -# Rely on the below environment variable only for ext filesystems. -export DIB_IMAGE_ROOT_FS_UUID=$(uuidgen -r) -if echo "$FS_TYPE" | grep -q "^ext" && [ -z "${DIB_IMAGE_ROOT_FS_UUID}" ]; then - echo "ext filesystem detected but no DIB_IMAGE_ROOT_FS_UUID found." - echo "Is the uuidgen utility installed on your system?" - exit 1 -fi - -# FS_TYPE isn't available until after we source img-defaults -if [ -z "$DIB_ROOT_LABEL" ]; then - # NOTE(bnemec): XFS has a limit of 12 characters for filesystem labels - # Not changing the default for other filesystems to maintain backwards compatibility - if [ "$FS_TYPE" = "xfs" ]; then - DIB_ROOT_LABEL="img-rootfs" - else - DIB_ROOT_LABEL="cloudimg-rootfs" - fi -fi - # xattr support cannot be relied upon with tmpfs builds # some kernels supoprt it, some don't if [[ -n "${GENTOO_PROFILE}" ]]; then @@ -294,13 +274,22 @@ cat >${DIB_BLOCK_DEVICE_PARAMS_YAML} <> ${DIB_BLOCK_DEVICE_PARAMS_YAML} + if [ -n "${MKFS_OPTS}" ] ; then + echo "root-fs-opts: '${MKFS_OPTS}'" >> ${DIB_BLOCK_DEVICE_PARAMS_YAML} + fi + # After changeing the parameters, there is the need to # re-run dib-block-device init because some value might # change based on the new set parameters. @@ -431,6 +411,9 @@ if [ -z ${IMAGE_BLOCK_DEVICE} ] ; then # It's called 'DEVICE' but it's the partition. IMAGE_BLOCK_DEVICE=$(dib-block-device getval image-block-partition) + + # Write the fstab + dib-block-device writefstab fi export IMAGE_BLOCK_DEVICE LOOPDEV=${IMAGE_BLOCK_DEVICE} @@ -442,14 +425,6 @@ export IMAGE_BLOCK_DEVICE_WITHOUT_PART export EXTRA_DETACH="detach_loopback ${IMAGE_BLOCK_DEVICE_WITHOUT_PART}" export EXTRA_UNMOUNT="dib-block-device cleanup" -sudo mkfs -t $FS_TYPE $MKFS_OPTS -L ${DIB_ROOT_LABEL} ${IMAGE_BLOCK_DEVICE} -# Tuning the rootfs uuid works only for ext filesystems. -if echo "$FS_TYPE" | grep -q "^ext"; then - sudo tune2fs -U ${DIB_IMAGE_ROOT_FS_UUID} ${IMAGE_BLOCK_DEVICE} -fi -mkdir $TMP_BUILD_DIR/mnt -sudo mount ${IMAGE_BLOCK_DEVICE} $TMP_BUILD_DIR/mnt - # 'mv' is not usable here - especially when a top level directory # has the same name as a mount point of a partition. If so, 'mv' # will complain: diff --git a/doc/source/user_guide/building_an_image.rst b/doc/source/user_guide/building_an_image.rst index 383e123d..e7c8e5d4 100644 --- a/doc/source/user_guide/building_an_image.rst +++ b/doc/source/user_guide/building_an_image.rst @@ -83,7 +83,7 @@ The default when using the `vm` element is: DIB_BLOCK_DEVICE_CONFIG=' - local_loop: - name: image0 + name: image0 - partitioning: base: image0 @@ -92,6 +92,12 @@ The default when using the `vm` element is: - name: root flags: [ boot, primary ] size: 100% + mkfs: + mount: + mount_point: / + fstab: + options: "defaults" + fsck-passno: 1' The default when not using the `vm` element is: @@ -100,6 +106,13 @@ The default when not using the `vm` element is: DIB_BLOCK_DEVICE_CONFIG=' - local_loop: name: image0 + mkfs: + name: mkfs_root + mount: + mount_point: / + fstab: + options: "defaults" + fsck-passno: 1' There are a lot of different options for the different levels. The following sections describe each level in detail. @@ -162,23 +175,21 @@ Tree and digraph notations can be mixed as needed in a configuration. Limitations +++++++++++ -The appropriate functionality to use multiple partitions and even LVMs -is currently under development; therefore the possible configuration -is currently limited, but will get more flexible as soon as all the -functionality is implemented. -In future this will be a list of some elements, each describing one -part of block device setup - but because currently only `local_loop` -and `partitioning` are implemented, it contains only the configuration -of these steps. +There are a couple of new modules planned, but not yet implemented, +like LVM, MD, encryption, ... -Currently it is possible to create multiple local loop devices, but -all but the `image0` will be not useable (are deleted during the -build process). +To provide an interface towards the existing elements, there are +currently three fixed keys used - which are not configurable: + +* `root-label`: this is the label of the block device that is mounted at + `/`. +* `image-block-partition`: if there is a block device with the name + `root` this is used else the block device with the name `image0` is + used. +* `image-path`: the path of the image that contains the root file + system is taken from the `image0`. -Currently only one partitions is used for the image. The name of this -partition must be `root`. Other partitions are created but not -used. Level 0 +++++++ @@ -213,7 +224,6 @@ Example: .. code-block:: yaml -:: local_loop: name: image0 @@ -227,8 +237,6 @@ block devices. One image file called `image0` is created with default size in the default temp directory. The second image has the size of 7.5GiB and is created in the `/var/tmp` folder. -Please note that due to current implementation restrictions it is only -allowed to specify one local loop image. Level 1 +++++++ @@ -278,7 +286,7 @@ align Set the alignment of the partition. This must be a multiple of the block size (i.e. 512 bytes). The default of 1MiB (~ 2048 * 512 bytes blocks) is the default for modern systems and known to - perform well on a wide range of targets [6]. For each partition + perform well on a wide range of targets. For each partition there might be some space that is not used - which is `align` - 512 bytes. For the default of 1MiB exactly 1048064 bytes (= 1 MiB - 512 byte) are not used in the partition itself. Please note that @@ -344,6 +352,130 @@ On the `image0` two partitions are created. The size of the first is 1GiB, the second uses the remaining free space. On the `data_image` three partitions are created: all are about 1/3 of the disk size. + +Level 2 ++++++++ + +Module: Mkfs +............ + +This module creates file systems on the block device given as `base`. +The following key / value pairs can be given: + +base + (mandatory) The name of the block device where the filesystem will + be created on. + +name + (mandatory) The name of the partition. This can be used to + reference (e.g. mounting) the filesystem. + +type + (mandatory) The type of the filesystem, like `ext4` or `xfs`. + +label + (optional - defaults to the name) + The label of the filesystem. This can be used e.g. by grub or in + the fstab. + +opts + (optional - defaults to empty list) + Options that will passed to the mkfs command. + +uuid + (optional - no default / not used if not givem) + The UUID of the filesystem. Not all file systems might + support this. Currently there is support for `ext2`, `ext3`, + `ext4` and `xfs`. + +Example: + +.. code-block:: yaml + + - mkfs: + name: mkfs_root + base: root + type: ext4 + label: cloudimage-root + uuid: b733f302-0336-49c0-85f2-38ca109e8bdb + opts: "-i 16384" + + +Level 3 ++++++++ + +Module: Mount +............. + +This module mounts a filesystem. The options are: + +base + (mandatory) The name of the filesystem that will be mounted. + +name + (mandatory) The name of the mount point. This can be used for + reference the mount (e.g. creating the fstab). + +mount_point + (mandatory) The mount point of the filesystem. + +There is no need to list the mount points in the correct order: an +algorithm will automatically detect the mount order. + +Example: + +.. code-block:: yaml + + - mount: + name: root_mnt + base: mkfs_root + mount_point: / + + +Level 4 ++++++++ + +Module: fstab +............. + +This module creates fstab entries. The following options exists. For +details please consult the fstab man page. + +base + (mandatory) The name of the mount point that will be written to + fstab. + +name + (mandatory) The name of the fstab entry. This can be used later on + as reference - and is currently unused. + +options + (optional, defaults to `default`) + Special mount options can be given. This is used as the fourth + field in the fstab entry. + +dump-freq + (optional, defaults to 0 - don't dump) + This is passed to dump to determine which filesystem should be + dumped. This is used as the fifth field in the fstab entry. + +fsck-passno + (optional, defaults to 2) + Determines the order to run fsck. Please note that this should be + set to 1 for the root file system. This is used as the sixth field + in the fstab entry. + +Example: + +.. code-block:: yaml + + - fstab: + name: var_log_fstab + base: var_log_mnt + options: nodev,nosuid + dump-freq: 2 + + Filesystem Caveat ----------------- @@ -381,3 +513,4 @@ creates ramdisk. If tmpfs is not used, you will need enough room in /tmp to store two uncompressed cloud images. If tmpfs is used, you would still need /tmp space for one uncompressed cloud image and about 20% of that image for working files. + diff --git a/releasenotes/notes/block-device-mkfs-mount-fstab-42d7efe28fc2df04.yaml b/releasenotes/notes/block-device-mkfs-mount-fstab-42d7efe28fc2df04.yaml new file mode 100644 index 00000000..4d8fed0d --- /dev/null +++ b/releasenotes/notes/block-device-mkfs-mount-fstab-42d7efe28fc2df04.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds mkfs, mount and fstab to the block device layer. diff --git a/setup.cfg b/setup.cfg index f9daac21..c5e5485e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -59,3 +59,6 @@ console_scripts = diskimage_builder.block_device.plugin = local_loop = diskimage_builder.block_device.level0.localloop:LocalLoop partitioning = diskimage_builder.block_device.level1.partitioning:Partitioning + mkfs = diskimage_builder.block_device.level2.mkfs:Mkfs + mount = diskimage_builder.block_device.level3.mount:Mount + fstab = diskimage_builder.block_device.level4.fstab:Fstab