Refactor: block-device filesystem creation, mount and fstab

This patch finalizes the block device refactoring.  It moves the three
remaining levels (filesystem creation, mount and fstab handling) into
the new python module.

Now it is possible to use any number of disk images, any number of
partitions and used them mounted to different directories.

Notes:

 * unmount_dir : modified to only unmount the subdirs mounted by
   mount_proc_sys_dev().  dib-block-device unmounts
   $TMP_MOUNT_PATH/mnt (see I85e01f3898d3c043071de5fad82307cb091a64a9)

Change-Id: I592c0b1329409307197460cfa8fd69798013f1f8
Signed-off-by: Andreas Florath <andreas@florath.net>
Closes-Bug: #1664924

diskimage_builder/block_device/blockdevice.py

@@ -22,6 +22,7 @@ import yaml
 
 from stevedore import extension
 
+from diskimage_builder.block_device.utils import exec_sudo
 from diskimage_builder.graph.digraph import Digraph
 
 
@@ -55,6 +56,8 @@ class BlockDevice(object):
        After this call it is possible to copy / install all the needed
        files into the appropriate directories.
 
+    cmd_writefstab: creates the (complete) fstab for the system.
+
     cmd_umount: unmount and detaches all directories and used many
        resources. After this call the used (e.g.) images are still
        available for further handling, e.g. converting from raw in
@@ -248,6 +251,28 @@ class BlockDevice(object):
         logger.info("Wrote final block device config to [%s]"
                     % self.config_json_file_name)
 
+    def _config_get_mount(self, path):
+        for entry in self.config:
+            for k, v in entry.items():
+                if k == 'mount' and v['mount_point'] == path:
+                    return v
+        assert False
+
+    def _config_get_all_mount_points(self):
+        rvec = []
+        for entry in self.config:
+            for k, v in entry.items():
+                if k == 'mount':
+                    rvec.append(v['mount_point'])
+        return rvec
+
+    def _config_get_mkfs(self, name):
+        for entry in self.config:
+            for k, v in entry.items():
+                if k == 'mkfs' and v['name'] == name:
+                    return v
+        assert False
+
     def cmd_getval(self, symbol):
         """Retrieve value from block device level
 
@@ -256,9 +281,25 @@ class BlockDevice(object):
         (non python) access to internal configuration.
 
         Arguments:
-        :symbol: The symbol to find
+        :param symbol: the symbol to get
         """
         logger.info("Getting value for [%s]" % symbol)
+        if symbol == "root-label":
+            root_mount = self._config_get_mount("/")
+            root_fs = self._config_get_mkfs(root_mount['base'])
+            logger.debug("root-label [%s]" % root_fs['label'])
+            print("%s" % root_fs['label'])
+            return 0
+        if symbol == "root-fstype":
+            root_mount = self._config_get_mount("/")
+            root_fs = self._config_get_mkfs(root_mount['base'])
+            logger.debug("root-fstype [%s]" % root_fs['type'])
+            print("%s" % root_fs['type'])
+            return 0
+        if symbol == 'mount-points':
+            mount_points = self._config_get_all_mount_points()
+            print("%s" % " ".join(mount_points))
+            return 0
         if symbol == 'image-block-partition':
             # If there is no partition needed, pass back directly the
             # image.
@@ -270,9 +311,47 @@ class BlockDevice(object):
         if symbol == 'image-path':
             print("%s" % self.state['blockdev']['image0']['image'])
             return 0
+
         logger.error("Invalid symbol [%s] for getval" % symbol)
         return 1
 
+    def cmd_writefstab(self):
+        """Creates the fstab"""
+        logger.info("Creating fstab")
+
+        tmp_fstab = os.path.join(self.state_dir, "fstab")
+        with open(tmp_fstab, "wt") as fstab_fd:
+            # This gives the order in which this must be mounted
+            for mp in self.state['mount_order']:
+                logger.debug("Writing fstab entry for [%s]" % mp)
+                fs_base = self.state['mount'][mp]['base']
+                fs_name = self.state['mount'][mp]['name']
+                fs_val = self.state['filesys'][fs_base]
+                if 'label' in fs_val:
+                    diskid = "LABEL=%s" % fs_val['label']
+                else:
+                    diskid = "UUID=%s" % fs_val['uuid']
+
+                # If there is no fstab entry - do not write anything
+                if 'fstab' not in self.state:
+                    continue
+                if fs_name not in self.state['fstab']:
+                    continue
+
+                options = self.state['fstab'][fs_name]['options']
+                dump_freq = self.state['fstab'][fs_name]['dump-freq']
+                fsck_passno = self.state['fstab'][fs_name]['fsck-passno']
+
+                fstab_fd.write("%s %s %s %s %s %s\n"
+                               % (diskid, mp, fs_val['fstype'],
+                                  options, dump_freq, fsck_passno))
+
+        target_etc_dir = os.path.join(self.params['build-dir'], 'built', 'etc')
+        exec_sudo(['mkdir', '-p', target_etc_dir])
+        exec_sudo(['cp', tmp_fstab, os.path.join(target_etc_dir, "fstab")])
+
+        return 0
+
     def cmd_create(self):
         """Creates the block device"""
 

diskimage_builder/block_device/level1/partitioning.py

@@ -85,6 +85,9 @@ class Partition(Digraph.Node):
     def get_type(self):
         return self.ptype
 
+    def get_name(self):
+        return self.name
+
     def insert_edges(self, dg):
         bnode = dg.find(self.base)
         assert bnode is not None

diskimage_builder/block_device/level2/__init__.py

@@ -1,6 +1,4 @@
-#!/bin/bash
-#
-# Copyright 2015 Hewlett-Packard Development Company, L.P.
+# Copyright 2017 Andreas Florath (andreas@florath.net)
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -13,15 +11,7 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
-#
 
-if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
-    set -x
-fi
-set -eu
-set -o pipefail
+from diskimage_builder.block_device.level2.mkfs import Mkfs
 
-cat << EOF > /etc/fstab
-proc /proc proc nodev,noexec,nosuid 0 0
-LABEL=${DIB_ROOT_LABEL} / ${FS_TYPE} errors=remount-ro 0 1
-EOF
+__all__ = [Mkfs]

diskimage_builder/block_device/level2/mkfs.py

@@ -0,0 +1,186 @@
+# Copyright 2017 Andreas Florath (andreas@florath.net)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import uuid
+
+from diskimage_builder.block_device.blockdevice \
+    import BlockDeviceSetupException
+from diskimage_builder.block_device.tree_config import TreeConfig
+from diskimage_builder.block_device.utils import exec_sudo
+from diskimage_builder.graph.digraph import Digraph
+
+
+logger = logging.getLogger(__name__)
+
+
+# There is the need that filesystem labels are unique:
+# if not the boot and / or mount (with LABEL=) might fail.
+file_system_labels = set()
+
+# There is the need to check the length of the label of
+# the filesystem.  The maximum length depends on the used filesystem.
+# This map provides information about the maximum label length.
+file_system_max_label_length = {
+    "ext2": 16,
+    "ext3": 16,
+    "ext4": 16,
+    "xfs": 12,
+    "vfat": 11
+}
+
+
+class Filesystem(Digraph.Node):
+
+    def _config_error(self, msg):
+        logger.error(msg)
+        raise BlockDeviceSetupException(msg)
+
+    def __init__(self, config):
+        logger.debug("Create filesystem object; config [%s]" % config)
+        # Parameter check (mandatory)
+        for pname in ['base', 'name', 'type']:
+            if pname not in config:
+                self._config_error("Mkfs config needs [%s]" % pname)
+            setattr(self, pname, config[pname])
+
+        # Parameter check (optional)
+        for pname in ['label', 'opts', 'uuid']:
+            setattr(self, pname,
+                    config[pname] if pname in config else None)
+
+        if self.label is None:
+            self.label = self.name
+
+        # Historic reasons - this will hopefully vanish in one of
+        # the next major releases
+        if self.label == "cloudimg-rootfs" and self.type == "xfs":
+            logger.warning("Default label [cloudimg-rootfs] too long for xfs "
+                           "file system - using [img-rootfs] instead")
+            self.label = "img-rootfs"
+
+        if self.label in file_system_labels:
+            self._config_error(
+                "File system label [%s] used more than once" %
+                self.label)
+        file_system_labels.add(self.label)
+
+        if self.type in file_system_max_label_length:
+            if file_system_max_label_length[self.type] < \
+               len(self.label):
+                self._config_error(
+                    "Label [%s] too long for filesystem [%s]: "
+                    "maximum length [%d] provided length [%d]" %
+                    (self.label, self.type,
+                     file_system_max_label_length[self.type],
+                     len(self.label)))
+        else:
+            logger.warning("Length of label [%s] cannot be checked for "
+                           "filesystem [%s]: unknown max length" %
+                           (self.label, self.type))
+            logger.warning("Continue - but this might lead to an error")
+
+        if self.opts is not None:
+            self.opts = self.opts.strip().split(' ')
+
+        if self.uuid is None:
+            self.uuid = str(uuid.uuid4())
+
+        Digraph.Node.__init__(self, self.name)
+
+        logger.debug("Filesystem created [%s]" % self)
+
+    def __repr__(self):
+        return "<Filesystem base [%s] name [%s] type [%s]>" \
+            % (self.base, self.name, self.type)
+
+    def insert_edges(self, dg):
+        logger.debug("Insert edge [%s]" % self)
+        bnode = dg.find(self.base)
+        assert bnode is not None
+        dg.create_edge(bnode, self)
+
+    def create(self, result, rollback):
+        logger.info("create called; result [%s]" % result)
+
+        cmd = ["mkfs"]
+
+        cmd.extend(['-t', self.type])
+        if self.opts:
+            cmd.extend(self.opts)
+        cmd.extend(["-L", self.label])
+
+        if self.type in ('ext2', 'ext3', 'ext4'):
+            cmd.extend(['-U', self.uuid])
+        elif self.type == 'xfs':
+            cmd.extend(['-m', "uuid=%s" % self.uuid])
+        else:
+            logger.warning("UUID will not be written for fs type [%s]"
+                           % self.type)
+
+        if self.type in ('ext2', 'ext3', 'ext4', 'xfs'):
+            cmd.append('-q')
+
+        if 'blockdev' not in result:
+            result['blockdev'] = {}
+        device = result['blockdev'][self.base]['device']
+        cmd.append(device)
+
+        logger.debug("Creating fs command [%s]" % (cmd))
+        exec_sudo(cmd)
+
+        if 'filesys' not in result:
+            result['filesys'] = {}
+        result['filesys'][self.name] \
+            = {'uuid': self.uuid, 'label': self.label,
+               'fstype': self.type, 'opts': self.opts,
+               'device': device}
+
+    def umount(self, state):
+        """Mkfs does not need any umount."""
+        pass
+
+    def cleanup(self, state):
+        """Mkfs does not need any cleanup."""
+        pass
+
+    def delete(self, state):
+        """Mkfs does not need any delete."""
+        pass
+
+
+class Mkfs(object):
+    """Module for creating file systems
+
+    This block device module handles creating different file
+    systems.
+    """
+
+    type_string = "mkfs"
+    tree_config = TreeConfig("mkfs")
+
+    def __init__(self, config, default_config):
+        logger.debug("Create Mkfs object; config [%s]" % config)
+        logger.debug("default_config [%s]" % default_config)
+        self.config = config
+        self.default_config = default_config
+        self.filesystems = {}
+
+        fs = Filesystem(self.config)
+        self.filesystems[fs.get_name()] = fs
+
+    def insert_nodes(self, dg):
+        for _, fs in self.filesystems.items():
+            logger.debug("Insert node [%s]" % fs)
+            dg.add_node(fs)

diskimage_builder/block_device/level3/__init__.py

@@ -1,6 +1,4 @@
-#!/bin/bash
-#
-# Copyright 2015 Hewlett-Packard Development Company, L.P.
+# Copyright 2017 Andreas Florath (andreas@florath.net)
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -13,15 +11,7 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
-#
 
-if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
-    set -x
-fi
-set -eu
-set -o pipefail
+from diskimage_builder.block_device.level3.mount import Mount
 
-cat << EOF | tee /etc/fstab > /dev/null
-proc /proc proc nodev,noexec,nosuid 0 0
-LABEL=${DIB_ROOT_LABEL} / ${FS_TYPE} errors=remount-ro 0 1
-EOF
+__all__ = [Mount]

diskimage_builder/block_device/level3/mount.py

@@ -0,0 +1,161 @@
+# Copyright 2017 Andreas Florath (andreas@florath.net)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import os
+
+from diskimage_builder.block_device.blockdevice \
+    import BlockDeviceSetupException
+from diskimage_builder.block_device.tree_config import TreeConfig
+from diskimage_builder.block_device.utils import exec_sudo
+from diskimage_builder.block_device.utils import sort_mount_points
+from diskimage_builder.graph.digraph import Digraph
+
+
+logger = logging.getLogger(__name__)
+
+
+# There is the need to collect all mount points to be able to
+# sort them in a sensible way.
+mount_points = {}
+# The order of mounting and unmounting is important.
+sorted_mount_points = None
+
+
+class MountPoint(Digraph.Node):
+
+    @staticmethod
+    def _config_error(msg):
+        logger.error(msg)
+        raise BlockDeviceSetupException(msg)
+
+    def __init__(self, mount_base, config):
+        # Parameter check
+        self.mount_base = mount_base
+        for pname in ['base', 'name', 'mount_point']:
+            if pname not in config:
+                self._config_error("MountPoint config needs [%s]" % pname)
+            setattr(self, pname, config[pname])
+        Digraph.Node.__init__(self, self.name)
+        logger.debug("MountPoint created [%s]" % self)
+
+    def __repr__(self):
+        return "<MountPoint base [%s] name [%s] mount_point [%s]>" \
+            % (self.base, self.name, self.mount_point)
+
+    def insert_node(self, dg):
+        global mount_points
+        if self.mount_point in mount_points:
+            self._config_error("Mount point [%s] specified more than once"
+                               % self.mount_point)
+        logger.debug("Insert node [%s]" % self)
+        mount_points[self.mount_point] = self
+        dg.add_node(self)
+
+    def insert_edges(self, dg):
+        """Insert all edges
+
+        After inserting all the nodes, the order of the mounting and
+        umounting can be computed.  There is the need to mount
+        mount-points that contain other mount-points first.
+        Example: '/var' must be mounted before '/var/log'. If not the
+        second is not used for files at all.
+
+        The dependency edge is created in all cases from the base
+        element (typically a mkfs) and, if this is not the 'first'
+        mount-point, also depend on the mount point before.  This
+        ensures that during mounting (and umounting) the correct
+        order is used.
+        """
+        logger.debug("Insert edge [%s]" % self)
+        global mount_points
+        global sorted_mount_points
+        if sorted_mount_points is None:
+            logger.debug("Mount points [%s]" % mount_points)
+            sorted_mount_points = sort_mount_points(mount_points.keys())
+            logger.info("Sorted mount points [%s]" % (sorted_mount_points))
+
+        # Look for the occurance in the list
+        mpi = sorted_mount_points.index(self.mount_point)
+        if mpi > 0:
+            # If not the first: add also the dependency
+            dg.create_edge(mount_points[sorted_mount_points[mpi - 1]], self)
+
+        bnode = dg.find(self.base)
+        assert bnode is not None
+        dg.create_edge(bnode, self)
+
+    def create(self, result, rollback):
+        logger.debug("mount called [%s]" % self.mount_point)
+        logger.debug("result [%s]" % result)
+        rel_mp = self.mount_point if self.mount_point[0] != '/' \
+                 else self.mount_point[1:]
+        mount_point = os.path.join(self.mount_base, rel_mp)
+        if not os.path.exists(mount_point):
+            # Need to sudo this because of permissions in the new
+            # file system tree.
+            exec_sudo(['mkdir', '-p', mount_point])
+        logger.info("Mounting [%s] to [%s]" % (self.name, mount_point))
+        exec_sudo(["mount", result['filesys'][self.base]['device'],
+                   mount_point])
+
+        if 'mount' not in result:
+            result['mount'] = {}
+        result['mount'][self.mount_point] \
+            = {'name': self.name, 'base': self.base, 'path': mount_point}
+
+        if 'mount_order' not in result:
+            result['mount_order'] = []
+        result['mount_order'].append(self.mount_point)
+
+    def umount(self, state):
+        logger.info("Called for [%s]" % self.name)
+        exec_sudo(["umount", state['mount'][self.mount_point]['path']])
+
+    def cleanup(self, state):
+        """Mount does not need any cleanup."""
+        pass
+
+    def delete(self, state):
+        self.umount(state)
+
+
+class Mount(object):
+
+    type_string = "mount"
+    tree_config = TreeConfig("mount")
+
+    def _config_error(self, msg):
+        logger.error(msg)
+        raise BlockDeviceSetupException(msg)
+
+    def __init__(self, config, params):
+        logger.debug("Mounting object; config [%s]" % config)
+        self.config = config
+        self.params = params
+
+        if 'mount-base' not in self.params:
+            MountPoint._config_error("Mount default config needs 'mount-base'")
+        self.mount_base = self.params['mount-base']
+
+        self.mount_points = {}
+
+        mp = MountPoint(self.mount_base, self.config)
+        self.mount_points[mp.get_name()] = mp
+
+    def insert_nodes(self, dg):
+        global sorted_mount_points
+        assert sorted_mount_points is None
+        for _, mp in self.mount_points.items():
+            mp.insert_node(dg)

diskimage_builder/block_device/level4/__init__.py

@@ -0,0 +1,17 @@
+# Copyright 2017 Andreas Florath (andreas@florath.net)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from diskimage_builder.block_device.level4.fstab import Fstab
+
+__all__ = [Fstab]

diskimage_builder/block_device/level4/fstab.py

@@ -0,0 +1,82 @@
+# Copyright 2017 Andreas Florath (andreas@florath.net)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+
+from diskimage_builder.block_device.blockdevice \
+    import BlockDeviceSetupException
+from diskimage_builder.block_device.tree_config import TreeConfig
+from diskimage_builder.graph.digraph import Digraph
+
+
+logger = logging.getLogger(__name__)
+
+
+class Fstab(Digraph.Node):
+
+    type_string = "fstab"
+    tree_config = TreeConfig("fstab")
+
+    def _config_error(self, msg):
+        logger.error(msg)
+        raise BlockDeviceSetupException(msg)
+
+    def __init__(self, config, params):
+        logger.debug("Fstab object; config [%s]" % config)
+        self.config = config
+        self.params = params
+        self.name = self.config['name']
+        self.base = self.config['base']
+        Digraph.Node.__init__(self, self.name)
+
+        self.options = self.config.get('options', 'defaults')
+        self.dump_freq = self.config.get('dump-freq', 0)
+        self.fsck_passno = self.config.get('fsck-passno', 2)
+
+    def insert_nodes(self, dg):
+        logger.debug("Insert node")
+        dg.add_node(self)
+
+    def insert_edges(self, dg):
+        logger.debug("Insert edge [%s]" % self)
+        bnode = dg.find(self.base)
+        assert bnode is not None
+        dg.create_edge(bnode, self)
+
+    def create(self, result, rollback):
+        logger.debug("fstab create called [%s]" % self.name)
+        logger.debug("result [%s]" % result)
+
+        if 'fstab' not in result:
+            result['fstab'] = {}
+
+        result['fstab'][self.base] = {
+            'name': self.name,
+            'base': self.base,
+            'options': self.options,
+            'dump-freq': self.dump_freq,
+            'fsck-passno': self.fsck_passno
+        }
+
+    def umount(self, state):
+        """Fstab does not need any umount task."""
+        pass
+
+    def cleanup(self, state):
+        """Fstab does not need any cleanup."""
+        pass
+
+    def delete(self, state):
+        """Fstab does not need any cleanup."""
+        pass

diskimage_builder/elements/debootstrap/install.d/15-cleanup-debootstrap

@@ -23,8 +23,3 @@ set -o pipefail
 
 install -d -m 0755 -o root -g root /etc/sudoers.d
 echo 'blacklist pcspkr' > /etc/modprobe.d/blacklist.conf
-
-cat << EOF | tee /etc/fstab > /dev/null
-proc /proc proc nodev,noexec,nosuid 0 0
-LABEL=${DIB_ROOT_LABEL} / ${FS_TYPE} errors=remount-ro 0 1
-EOF

diskimage_builder/elements/rpm-distro/post-install.d/05-fstab-rootfs-label

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
-    set -x
-fi
-set -eu
-set -o pipefail
-
-# Fedora 18 sets up for root to have a label of "_/"
-# Fedora 19 sets up for root to have a UUID
-# This regex will catch both
-sed -i "s%.*\s\/\s%LABEL=${DIB_ROOT_LABEL} / %" /etc/fstab

diskimage_builder/elements/vm/block-device-default.yaml

@@ -10,3 +10,9 @@
       - name: root
         flags: [ boot, primary ]
         size: 100%
+        mkfs:
+          mount:
+            mount_point: /
+            fstab:
+              options: "defaults"
+              fsck-passno: 1

diskimage_builder/elements/vm/block-device-ppc.yaml

@@ -28,3 +28,9 @@
       - name: root
         flags: [ primary ]
         size: 100%
+        mkfs:
+          mount:
+            mount_point: /
+            fstab:
+              options: "defaults"
+              fsck-passno: 1

diskimage_builder/elements/vm/environment.d/10-partitioning

@@ -1,14 +0,0 @@
-_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-if [[ "$ARCH" =~ "ppc" ]] ; then
-    DIB_BLOCK_DEVICE_DEFAULT_CONFIG="$(cat $_DIR/../block-device-ppc.yaml)"
-else
-    DIB_BLOCK_DEVICE_DEFAULT_CONFIG="$(cat $_DIR/../block-device-default.yaml)"
-fi
-
-DIB_BLOCK_DEVICE_CONFIG=${DIB_BLOCK_DEVICE_CONFIG:-${DIB_BLOCK_DEVICE_DEFAULT_CONFIG}}
-export DIB_BLOCK_DEVICE_CONFIG
-
-# Local variables:
-# mode: sh
-# End:

diskimage_builder/elements/zypper/post-install.d/10-mkinitrd

@@ -12,10 +12,12 @@ set -o pipefail
 # that will actually be used for the final image. This is likely something
 # different than what the chroot is currently on (which might currently be a
 # tmpfs even).
-echo "rootfstype=$FS_TYPE" > /etc/sysconfig/initrd
+echo "rootfstype=${DIB_ROOT_FSTYPE}" > /etc/sysconfig/initrd
 
+# ToDo: This it not really clear to me.
+#       Let's see which error occurs.
 # openSuse mkinitrd requires a valid root device be in fstab.
-sed -i 's/vda1/sda1/' /etc/fstab
+##sed -i 's/vda1/sda1/' /etc/fstab
 mkinitrd -A -B
 
 # And cleanup again

diskimage_builder/lib/common-functions

@@ -353,7 +353,9 @@ function unmount_dir {
     # /proc/mounts is the real path
     real_dir=$(readlink -e $dir)
 
-    mnts=$(awk '{print $2}' < /proc/mounts | grep "^$real_dir" | sort -r)
+    # note the "/" on real_dir ... we are just looking for things
+    # mounted *underneath* this directory.
+    mnts=$(awk '{print $2}' < /proc/mounts | grep "^$real_dir/" | sort -r)
     for m in $mnts; do
         echo "Unmount $m"
         sudo umount -fl $m || true

diskimage_builder/lib/disk-image-create

@@ -115,7 +115,7 @@ DIB_DEBUG_TRACE=${DIB_DEBUG_TRACE:-0}
 INSTALL_PACKAGES=""
 IMAGE_TYPES=("qcow2")
 COMPRESS_IMAGE="true"
-export DIB_ROOT_LABEL=""
+ROOT_LABEL=""
 DIB_DEFAULT_INSTALLTYPE=${DIB_DEFAULT_INSTALLTYPE:-"source"}
 MKFS_OPTS=""
 ACI_MANIFEST=${ACI_MANIFEST:-}
@@ -147,7 +147,7 @@ while true ; do
         --no-tmpfs) shift; export DIB_NO_TMPFS=1;;
         --offline) shift; export DIB_OFFLINE=1;;
         --qemu-img-options) QEMU_IMG_OPTIONS=$2; shift 2;;
-        --root-label) export DIB_ROOT_LABEL=$2; shift 2;;
+        --root-label) ROOT_LABEL=$2; shift 2;;
         --ramdisk-element) RAMDISK_ELEMENT=$2; shift 2;;
         --install-type) DIB_DEFAULT_INSTALLTYPE=$2; shift 2;;
         --docker-target) export DOCKER_TARGET=$2; shift 2 ;;
@@ -250,26 +250,6 @@ if [[ -z "$(which fstrim)" ]]; then
     exit 1
 fi
 
-# NOTE: Tuning the rootfs uuid works only for ext filesystems.
-# Rely on the below environment variable only for ext filesystems.
-export DIB_IMAGE_ROOT_FS_UUID=$(uuidgen -r)
-if echo "$FS_TYPE" | grep -q "^ext" && [ -z "${DIB_IMAGE_ROOT_FS_UUID}" ]; then
-    echo "ext filesystem detected but no DIB_IMAGE_ROOT_FS_UUID found."
-    echo "Is the uuidgen utility installed on your system?"
-    exit 1
-fi
-
-# FS_TYPE isn't available until after we source img-defaults
-if [ -z "$DIB_ROOT_LABEL" ]; then
-    # NOTE(bnemec): XFS has a limit of 12 characters for filesystem labels
-    # Not changing the default for other filesystems to maintain backwards compatibility
-    if [ "$FS_TYPE" = "xfs" ]; then
-        DIB_ROOT_LABEL="img-rootfs"
-    else
-        DIB_ROOT_LABEL="cloudimg-rootfs"
-    fi
-fi
-
 # xattr support cannot be relied upon with tmpfs builds
 # some kernels supoprt it, some don't
 if [[ -n "${GENTOO_PROFILE}" ]]; then
@@ -294,13 +274,22 @@ cat >${DIB_BLOCK_DEVICE_PARAMS_YAML} <<EOF
 config: ${BLOCK_DEVICE_CONFIG_YAML}
 image-dir: ${TMP_IMAGE_DIR}
 root-fs-type: ${FS_TYPE}
-root-label: ${DIB_ROOT_LABEL}
+root-label: ${ROOT_LABEL}
 mount-base: ${TMP_BUILD_DIR}/mnt
 build-dir: ${TMP_BUILD_DIR}
 EOF
 
 dib-block-device init
 
+# Need to get the real root label because it can be overwritten
+# by the BLOCK_DEVICE_CONFIG.
+DIB_ROOT_LABEL=$(dib-block-device getval root-label)
+export DIB_ROOT_LABEL
+
+# Need to get the real fs type for the root filesystem
+DIB_ROOT_FSTYPE=$(dib-block-device getval root-fstype)
+export DIB_ROOT_FSTYPE
+
 create_base
 # This variable needs to be propagated into the chroot
 mkdir -p $TMP_HOOKS_PATH/environment.d
@@ -377,7 +366,7 @@ fi
 
 rm -f ${du_output}
 
-if [ "$FS_TYPE" = "ext4" ] ; then
+if [ "$DIB_ROOT_FSTYPE" = "ext4" ] ; then
   # Very conservative to handle images being resized a lot
   # We set journal size to 64M so our journal is large enough when we
   # perform an FS resize.
@@ -403,24 +392,15 @@ export TMP_IMAGE_DIR
 # phase. If this gives no result, use the configuration based approach:
 eval_run_d block-device "IMAGE_BLOCK_DEVICE="
 
-# Because there is currently no generic way of passing in variables
-# from elements to the main, get here the well known config of a well
-# known element.
-# (This is only temporary and will go away when the complete block
-# device handling including file system handling and mounting is
-# implemented using python.)
-if [[ $IMAGE_ELEMENT =~ vm ]]; then
-    for EPATH in $(echo ${ELEMENTS_PATH} | tr ":" " "); do
-        PART_CFG_PATH=${EPATH}/vm/environment.d/10-partitioning
-        [ -e ${PART_CFG_PATH} ] && source ${PART_CFG_PATH}
-    done
-fi
-
 if [ -z ${IMAGE_BLOCK_DEVICE} ] ; then
     # For compatibily reasons in addition to the YAML configuration
     # there is the need to handle the old environment variables.
     echo "image-size: ${DIB_IMAGE_SIZE}KiB" >> ${DIB_BLOCK_DEVICE_PARAMS_YAML}
 
+    if [ -n "${MKFS_OPTS}" ] ; then
+        echo "root-fs-opts: '${MKFS_OPTS}'" >> ${DIB_BLOCK_DEVICE_PARAMS_YAML}
+    fi
+
     # After changeing the parameters, there is the need to
     # re-run dib-block-device init because some value might
     # change based on the new set parameters.
@@ -431,6 +411,9 @@ if [ -z ${IMAGE_BLOCK_DEVICE} ] ; then
 
     # It's called 'DEVICE' but it's the partition.
     IMAGE_BLOCK_DEVICE=$(dib-block-device getval image-block-partition)
+
+    # Write the fstab
+    dib-block-device writefstab
 fi
 export IMAGE_BLOCK_DEVICE
 LOOPDEV=${IMAGE_BLOCK_DEVICE}
@@ -442,14 +425,6 @@ export IMAGE_BLOCK_DEVICE_WITHOUT_PART
 export EXTRA_DETACH="detach_loopback ${IMAGE_BLOCK_DEVICE_WITHOUT_PART}"
 export EXTRA_UNMOUNT="dib-block-device cleanup"
 
-sudo mkfs -t $FS_TYPE $MKFS_OPTS -L ${DIB_ROOT_LABEL} ${IMAGE_BLOCK_DEVICE}
-# Tuning the rootfs uuid works only for ext filesystems.
-if echo "$FS_TYPE" | grep -q "^ext"; then
-    sudo tune2fs -U ${DIB_IMAGE_ROOT_FS_UUID} ${IMAGE_BLOCK_DEVICE}
-fi
-mkdir $TMP_BUILD_DIR/mnt
-sudo mount ${IMAGE_BLOCK_DEVICE} $TMP_BUILD_DIR/mnt
-
 # 'mv' is not usable here - especially when a top level directory
 # has the same name as a mount point of a partition.  If so, 'mv'
 # will complain:

doc/source/user_guide/building_an_image.rst

@@ -92,6 +92,12 @@ The default when using the `vm` element is:
             - name: root
               flags: [ boot, primary ]
               size: 100%
+              mkfs:
+                mount:
+                  mount_point: /
+                  fstab:
+                    options: "defaults"
+                    fsck-passno: 1'
 
 The default when not using the `vm` element is:
 
@@ -100,6 +106,13 @@ The default when not using the `vm` element is:
     DIB_BLOCK_DEVICE_CONFIG='
       - local_loop:
           name: image0
+          mkfs:
+            name: mkfs_root
+            mount:
+              mount_point: /
+              fstab:
+                options: "defaults"
+                fsck-passno: 1'
 
 There are a lot of different options for the different levels.  The
 following sections describe each level in detail.
@@ -162,23 +175,21 @@ Tree and digraph notations can be mixed as needed in a configuration.
 
 Limitations
 +++++++++++
-The appropriate functionality to use multiple partitions and even LVMs
-is currently under development; therefore the possible configuration
-is currently limited, but will get more flexible as soon as all the
-functionality is implemented.
 
-In future this will be a list of some elements, each describing one
-part of block device setup - but because currently only `local_loop`
-and `partitioning` are implemented, it contains only the configuration
-of these steps.
+There are a couple of new modules planned, but not yet implemented,
+like LVM, MD, encryption, ...
 
-Currently it is possible to create multiple local loop devices, but
-all but the `image0` will be not useable (are deleted during the
-build process).
+To provide an interface towards the existing elements, there are
+currently three fixed keys used - which are not configurable:
+
+* `root-label`: this is the label of the block device that is mounted at
+  `/`.
+* `image-block-partition`: if there is a block device with the name
+  `root` this is used else the block device with the name `image0` is
+  used.
+* `image-path`: the path of the image that contains the root file
+  system is taken from the `image0`.
 
-Currently only one partitions is used for the image.  The name of this
-partition must be `root`.  Other partitions are created but not
-used.
 
 Level 0
 +++++++
@@ -213,7 +224,6 @@ Example:
 
 .. code-block:: yaml
 
-::
         local_loop:
           name: image0
 
@@ -227,8 +237,6 @@ block devices.  One image file called `image0` is created with
 default size in the default temp directory.  The second image has the
 size of 7.5GiB and is created in the `/var/tmp` folder.
 
-Please note that due to current implementation restrictions it is only
-allowed to specify one local loop image.
 
 Level 1
 +++++++
@@ -278,7 +286,7 @@ align
    Set the alignment of the partition.  This must be a multiple of the
    block size (i.e. 512 bytes).  The default of 1MiB (~ 2048 * 512
    bytes blocks) is the default for modern systems and known to
-   perform well on a wide range of targets [6].  For each partition
+   perform well on a wide range of targets.  For each partition
    there might be some space that is not used - which is `align` - 512
    bytes.  For the default of 1MiB exactly 1048064 bytes (= 1 MiB -
    512 byte) are not used in the partition itself.  Please note that
@@ -344,6 +352,130 @@ On the `image0` two partitions are created.  The size of the first is
 1GiB, the second uses the remaining free space.  On the `data_image`
 three partitions are created: all are about 1/3 of the disk size.
 
+
+Level 2
++++++++
+
+Module: Mkfs
+............
+
+This module creates file systems on the block device given as `base`.
+The following key / value pairs can be given:
+
+base
+   (mandatory) The name of the block device where the filesystem will
+   be created on.
+
+name
+   (mandatory) The name of the partition.  This can be used to
+   reference (e.g. mounting) the filesystem.
+
+type
+   (mandatory) The type of the filesystem, like `ext4` or `xfs`.
+
+label
+   (optional - defaults to the name)
+   The label of the filesystem.  This can be used e.g. by grub or in
+   the fstab.
+
+opts
+   (optional - defaults to empty list)
+   Options that will passed to the mkfs command.
+
+uuid
+   (optional - no default / not used if not givem)
+   The UUID of the filesystem.  Not all file systems might
+   support this.  Currently there is support for `ext2`, `ext3`,
+   `ext4` and `xfs`.
+
+Example:
+
+.. code-block:: yaml
+
+   - mkfs:
+       name: mkfs_root
+       base: root
+       type: ext4
+       label: cloudimage-root
+       uuid: b733f302-0336-49c0-85f2-38ca109e8bdb
+       opts: "-i 16384"
+
+
+Level 3
++++++++
+
+Module: Mount
+.............
+
+This module mounts a filesystem.  The options are:
+
+base
+   (mandatory) The name of the filesystem that will be mounted.
+
+name
+   (mandatory) The name of the mount point.  This can be used for
+   reference the mount (e.g. creating the fstab).
+
+mount_point
+   (mandatory) The mount point of the filesystem.
+
+There is no need to list the mount points in the correct order: an
+algorithm will automatically detect the mount order.
+
+Example:
+
+.. code-block:: yaml
+
+   - mount:
+       name: root_mnt
+       base: mkfs_root
+       mount_point: /
+
+
+Level 4
++++++++
+
+Module: fstab
+.............
+
+This module creates fstab entries.  The following options exists.  For
+details please consult the fstab man page.
+
+base
+   (mandatory) The name of the mount point that will be written to
+   fstab.
+
+name
+   (mandatory) The name of the fstab entry.  This can be used later on
+   as reference - and is currently unused.
+
+options
+   (optional, defaults to `default`)
+   Special mount options can be given.  This is used as the fourth
+   field in the fstab entry.
+
+dump-freq
+   (optional, defaults to 0 - don't dump)
+   This is passed to dump to determine which filesystem should be
+   dumped. This is used as the fifth field in the fstab entry.
+
+fsck-passno
+   (optional, defaults to 2)
+   Determines the order to run fsck.  Please note that this should be
+   set to 1 for the root file system. This is used as the sixth field
+   in the fstab entry.
+
+Example:
+
+.. code-block:: yaml
+
+   - fstab:
+       name: var_log_fstab
+       base: var_log_mnt
+       options: nodev,nosuid
+       dump-freq: 2
+
+
 Filesystem Caveat
 -----------------
 
@@ -381,3 +513,4 @@ creates ramdisk.
 If tmpfs is not used, you will need enough room in /tmp to store two
 uncompressed cloud images. If tmpfs is used, you would still need /tmp space
 for one uncompressed cloud image and about 20% of that image for working files.
+

releasenotes/notes/block-device-mkfs-mount-fstab-42d7efe28fc2df04.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Adds mkfs, mount and fstab to the block device layer.

setup.cfg

@@ -59,3 +59,6 @@ console_scripts =
 diskimage_builder.block_device.plugin =
     local_loop = diskimage_builder.block_device.level0.localloop:LocalLoop
     partitioning = diskimage_builder.block_device.level1.partitioning:Partitioning
+    mkfs = diskimage_builder.block_device.level2.mkfs:Mkfs
+    mount = diskimage_builder.block_device.level3.mount:Mount
+    fstab = diskimage_builder.block_device.level4.fstab:Fstab