Refactor block_device: passing command line parameters

The original approach was to pass each and every command
line parameter to the block device.  While the block device
functionality gets extended, this is not any longer practical.

Instead of passing in all the parameters separately this patch
collects these in a YAML file that is passed in to the block device
layer.

Change-Id: I9d07593a01441b62632234468ac25a982cf1a9f0
Signed-off-by: Andreas Florath <andreas@florath.net>

diskimage_builder/block_device/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2016 Andreas Florath (andreas@florath.net)
+# Copyright 2016-2017 Andreas Florath (andreas@florath.net)
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -45,25 +45,18 @@ def main():
         epilog="Available phases:\n" + phase_doc)
     parser.add_argument('--phase', required=True,
                         help="phase to execute")
-    parser.add_argument('--config', required=False,
+    parser.add_argument('--params', required=True,
-                        help="configuration for block device "
+                        help="parameters for block device handling")
-                             "layer as JSON object")
+    parser.add_argument('--symbol', required=False,
-    parser.add_argument('--build-dir', required=True,
+                        help="symbol to query for getval")
-                        help="path to temporary build dir")
-    parser.add_argument('--image-size', required=False,
-                        help="default image size")
-    parser.add_argument('--image-dir', required=False,
-                        help="default image directory")
     args = parser.parse_args()
 
     logger.info("phase [%s]" % args.phase)
-    logger.info("config [%s]" % args.config)
+    logger.info("params [%s]" % args.params)
-    logger.info("build_dir [%s]" % args.build_dir)
+    if args.symbol:
+        logger.info("symbol [%s]" % args.symbol)
 
-    bd = BlockDevice(val_else_none(args.config),
+    bd = BlockDevice(args)
-                     val_else_none(args.build_dir),
-                     val_else_none(args.image_size),
-                     val_else_none(args.image_dir))
 
     # Check if the method is available
     method = getattr(bd, "cmd_" + args.phase, None)
@@ -76,6 +69,5 @@ def main():
 
     return 0
 
-
 if __name__ == "__main__":
     main()

diskimage_builder/block_device/blockdevice.py

@@ -1,4 +1,4 @@
-# Copyright 2016 Andreas Florath (andreas@florath.net)
+# Copyright 2016-2017 Andreas Florath (andreas@florath.net)
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -36,6 +36,10 @@ class BlockDevice(object):
 
     A typical call sequence:
 
+    cmd_init: initialized the block device level config.  After this
+       call it is possible to e.g. query information from the (partially
+       automatic generated) internal state like root-label.
+
     cmd_create: creates all the different aspects of the block
        device. When this call is successful, the complete block level
        device is set up, filesystems are created and are mounted at
@@ -62,6 +66,13 @@ class BlockDevice(object):
 
     In a script this should be called in the following way:
 
+    dib-block-device --phase=init ...
+    # From that point the database can be queried, like
+    ROOT_LABEL=$(dib-block-device --phase=getval --symbol=root-label ...)
+
+    Please note that currently the dib-block-device executable can
+    only be used outside the chroot.
+
     dib-block-device --phase=create ...
     trap "dib-block-device --phase=delete ..." EXIT
     # copy / install files
@@ -71,50 +82,74 @@ class BlockDevice(object):
     trap - EXIT
     """
 
-    # Default configuration:
+    def _merge_into_config(self):
-    # one image, one partition, mounted under '/'
+        """Merge old (default) config into new
-    DefaultConfig = """
-- local_loop:
-    name: image0
-"""
 
-# This is an example of the next level config
+        There is the need to be compatible using some old environment
-# mkfs:
+        variables.  This is done in the way, that if there is no
-#  base: root
+        explicit value given, these values are inserted into the current
-#  type: ext4
+        configuration.
-#  mount_point: /
+        """
+        for entry in self.config:
+            for k, v in entry.items():
+                if k == 'mkfs':
+                    if 'name' not in v:
+                        continue
+                    if v['name'] != 'mkfs_root':
+                        continue
+                    if 'type' not in v \
+                       and 'root-fs-type' in self.params:
+                        v['type'] = self.params['root-fs-type']
+                    if 'opts' not in v \
+                       and 'root-fs-opts' in self.params:
+                        v['opts'] = self.params['root-fs-opts']
+                    if 'label' not in v \
+                       and 'root-label' in self.params:
+                        if self.params['root-label'] is not None:
+                            v['label'] = self.params['root-label']
+                        else:
+                            v['label'] = "cloudimg-rootfs"
 
-    def __init__(self, block_device_config, build_dir,
+    @staticmethod
-                 default_image_size, default_image_dir):
+    def _load_json(file_name):
+        if os.path.exists(file_name):
+            with codecs.open(file_name, encoding="utf-8", mode="r") as fd:
+                return json.load(fd)
+        return None
+
+    def __init__(self, args):
         logger.debug("Creating BlockDevice object")
-        logger.debug("Config given [%s]" % block_device_config)
+        logger.debug("Param file [%s]" % args.params)
-        logger.debug("Build dir [%s]" % build_dir)
+        self.args = args
-        if block_device_config is None:
-            block_device_config = BlockDevice.DefaultConfig
-        self.config = yaml.safe_load(block_device_config)
-        logger.debug("Using config [%s]" % self.config)
 
-        self.default_config = {
+        with open(self.args.params) as param_fd:
-            'image_size': default_image_size,
+            self.params = yaml.safe_load(param_fd)
-            'image_dir': default_image_dir}
+        logger.debug("Params [%s]" % self.params)
-        self.state_dir = os.path.join(build_dir,
+
-                                      "states/block-device")
+        self.state_dir = os.path.join(
+            self.params['build-dir'], "states/block-device")
         self.state_json_file_name \
             = os.path.join(self.state_dir, "state.json")
         self.plugin_manager = extension.ExtensionManager(
             namespace='diskimage_builder.block_device.plugin',
             invoke_on_load=False)
+        self.config_json_file_name \
+            = os.path.join(self.state_dir, "config.json")
 
-    def write_state(self, result):
+        self.config = self._load_json(self.config_json_file_name)
+        self.state = self._load_json(self.state_json_file_name)
+        logger.debug("Using state [%s]", self.state)
+
+        # This needs to exists for the state and config files
+        try:
+            os.makedirs(self.state_dir)
+        except OSError:
+            pass
+
+    def write_state(self, state):
         logger.debug("Write state [%s]" % self.state_json_file_name)
-        os.makedirs(self.state_dir)
         with open(self.state_json_file_name, "w") as fd:
-            json.dump([self.config, self.default_config, result], fd)
+            json.dump(state, fd)
-
-    def load_state(self):
-        with codecs.open(self.state_json_file_name,
-                         encoding="utf-8", mode="r") as fd:
-            return json.load(fd)
 
     def create_graph(self, config, default_config):
         # This is the directed graph of nodes: each parse method must
@@ -152,10 +187,28 @@ class BlockDevice(object):
         return dg, call_order
 
     def create(self, result, rollback):
-        dg, call_order = self.create_graph(self.config, self.default_config)
+        dg, call_order = self.create_graph(self.config, self.params)
         for node in call_order:
             node.create(result, rollback)
 
+    def cmd_init(self):
+        """Initialize block device setup
+
+        This initializes the block device setup layer. One major task
+        is to parse and check the configuration, write it down for
+        later examiniation and execution.
+        """
+        with open(self.params['config'], "rt") as config_fd:
+            self.config = yaml.safe_load(config_fd)
+        logger.debug("Config before merge [%s]" % self.config)
+        self._merge_into_config()
+        logger.debug("Final config [%s]" % self.config)
+        # Write the final config
+        with open(self.config_json_file_name, "wt") as fd:
+            json.dump(self.config, fd)
+        logger.info("Wrote final block device config to [%s]"
+                    % self.config_json_file_name)
+
     def cmd_create(self):
         """Creates the block device"""
 
@@ -187,47 +240,33 @@ class BlockDevice(object):
         logger.info("create() finished")
         return 0
 
-    def _load_state(self):
-        logger.info("_load_state() called")
-        try:
-            os.stat(self.state_json_file_name)
-        except OSError:
-            logger.info("State already cleaned - no way to do anything here")
-            return None, None, None
-
-        config, default_config, state = self.load_state()
-        logger.debug("Using config [%s]" % config)
-        logger.debug("Using default config [%s]" % default_config)
-        logger.debug("Using state [%s]" % state)
-
-        # Deleting must be done in reverse order
-        dg, call_order = self.create_graph(config, default_config)
-        reverse_order = reversed(call_order)
-        return dg, reverse_order, state
-
     def cmd_umount(self):
         """Unmounts the blockdevice and cleanup resources"""
 
-        dg, reverse_order, state = self._load_state()
+        dg, call_order = self.create_graph(self.config, self.params)
+        reverse_order = reversed(call_order)
         if dg is None:
             return 0
         for node in reverse_order:
-            node.umount(state)
+            node.umount(self.state)
 
         # To be compatible with the current implementation, echo the
         # result to stdout.
-        print("%s" % state['image0']['image'])
+        print("%s" % self.state['image0']['image'])
 
         return 0
 
     def cmd_cleanup(self):
         """Cleanup all remaining relicts - in good case"""
 
-        dg, reverse_order, state = self._load_state()
+        # Deleting must be done in reverse order
+        dg, call_order = self.create_graph(self.config, self.params)
+        reverse_order = reversed(call_order)
+
         if dg is None:
             return 0
         for node in reverse_order:
-            node.cleanup(state)
+            node.cleanup(self.state)
 
         logger.info("Removing temporary dir [%s]" % self.state_dir)
         shutil.rmtree(self.state_dir)
@@ -237,11 +276,14 @@ class BlockDevice(object):
     def cmd_delete(self):
         """Cleanup all remaining relicts - in case of an error"""
 
-        dg, reverse_order, state = self._load_state()
+        # Deleting must be done in reverse order
+        dg, call_order = self.create_graph(self.config, self.params)
+        reverse_order = reversed(call_order)
+
         if dg is None:
             return 0
         for node in reverse_order:
-            node.delete(state)
+            node.delete(self.state)
 
         logger.info("Removing temporary dir [%s]" % self.state_dir)
         shutil.rmtree(self.state_dir)

diskimage_builder/block_device/level0/localloop.py

@@ -39,12 +39,12 @@ class LocalLoop(NodePluginBase):
             self.size = parse_abs_size_spec(config['size'])
             logger.debug("Image size [%s]" % self.size)
         else:
-            self.size = parse_abs_size_spec(default_config['image_size'])
+            self.size = parse_abs_size_spec(default_config['image-size'])
             logger.debug("Using default image size [%s]" % self.size)
         if 'directory' in config:
             self.image_dir = config['directory']
         else:
-            self.image_dir = default_config['image_dir']
+            self.image_dir = default_config['image-dir']
         self.name = config['name']
         Digraph.Node.__init__(self, self.name)
         self.filename = os.path.join(self.image_dir, self.name + ".raw")

diskimage_builder/lib/common-functions

@@ -359,3 +359,51 @@ function unmount_dir {
         sudo umount -fl $m || true
     done
 }
+
+# Create YAML config file for the block device layer
+# The order here is: use the one the user provides - if there is
+# none provided, fall back to the possible one element which
+# defines a fallback configuration.
+# Parameters:
+#   - name of the to be created config file
+function block_device_create_config_file {
+    local CONFIG_YAML="$1"
+
+    # User setting overwrites this
+    if [[ ${DIB_BLOCK_DEVICE_CONFIG:-} == file://* ]]; then
+        cp $(echo ${DIB_BLOCK_DEVICE_CONFIG} | cut -c 8-) ${CONFIG_YAML}
+        return
+    fi
+    if [ -n "${DIB_BLOCK_DEVICE_CONFIG:-}" ]; then
+        printf "%s" "${DIB_BLOCK_DEVICE_CONFIG}" >${CONFIG_YAML}
+        return
+    fi
+
+    eval declare -A image_elements=($(get_image_element_array))
+    for i in ${!image_elements[@]}; do
+        local BLOCK_DEVICE_CFG_PATH=${image_elements[$i]}/block-device-${ARCH}.yaml
+        if [ -e ${BLOCK_DEVICE_CFG_PATH} ]; then
+            cp ${BLOCK_DEVICE_CFG_PATH} ${CONFIG_YAML}
+        else
+            BLOCK_DEVICE_CFG_PATH=${image_elements[$i]}/block-device-default.yaml
+            if [ -e ${BLOCK_DEVICE_CFG_PATH} ]; then
+                cp ${BLOCK_DEVICE_CFG_PATH} ${CONFIG_YAML}
+            fi
+        fi
+    done
+
+    # If no config is there (until now) use the default config
+    if [ ! -e ${CONFIG_YAML} ]; then
+        cat >${CONFIG_YAML} <<EOF
+- local_loop:
+    name: image0
+    mkfs:
+      name: mkfs_root
+      mount:
+        mount_point: /
+        fstab:
+          options: "defaults"
+          fsck-passno: 1
+EOF
+    fi
+}

diskimage_builder/lib/disk-image-create

@@ -276,6 +276,28 @@ if [[ -n "${GENTOO_PROFILE}" ]]; then
 fi
 
 mk_build_dir
+
+# Create the YAML file with the final and raw configuration for
+# the block device layer.
+mkdir -p ${TMP_BUILD_DIR}/block-device
+BLOCK_DEVICE_CONFIG_YAML=${TMP_BUILD_DIR}/block-device/config.yaml
+block_device_create_config_file "${BLOCK_DEVICE_CONFIG_YAML}"
+
+# Write out the parameter file
+DIB_BLOCK_DEVICE_PARAMS_YAML=${TMP_BUILD_DIR}/block-device/params.yaml
+export DIB_BLOCK_DEVICE_PARAMS_YAML
+cat >${DIB_BLOCK_DEVICE_PARAMS_YAML} <<EOF
+config: ${BLOCK_DEVICE_CONFIG_YAML}
+image-dir: ${TMP_IMAGE_DIR}
+root-fs-type: ${FS_TYPE}
+root-label: ${DIB_ROOT_LABEL}
+mount-base: ${TMP_BUILD_DIR}/mnt
+build-dir: ${TMP_BUILD_DIR}
+EOF
+
+dib-block-device --phase=init \
+                 --params="${DIB_BLOCK_DEVICE_PARAMS_YAML}"
+
 create_base
 # This variable needs to be propagated into the chroot
 mkdir -p $TMP_HOOKS_PATH/environment.d
@@ -397,12 +419,19 @@ fi
 DIB_BLOCK_DEVICE_SCRIPT=$(which dib-block-device)
 
 if [ -z ${IMAGE_BLOCK_DEVICE} ] ; then
-    IMAGE_BLOCK_DEVICE=$(${DIB_BLOCK_DEVICE_SCRIPT} \
+    # For compatibily reasons in addition to the YAML configuration
-                             --phase=create \
+    # there is the need to handle the old environment variables.
-                             --config="${DIB_BLOCK_DEVICE_CONFIG:-}" \
+    echo "image-size: ${DIB_IMAGE_SIZE}KiB" >> ${DIB_BLOCK_DEVICE_PARAMS_YAML}
-                             --image-size="${DIB_IMAGE_SIZE}"KiB \
+
-                             --image-dir="${TMP_IMAGE_DIR}" \
+    # After changeing the parameters, there is the need to
-                             --build-dir="${TMP_BUILD_DIR}" )
+    # re-run dib-block-device init because some value might
+    # change based on the new set parameters.
+    dib-block-device --phase=init \
+                     --params="${DIB_BLOCK_DEVICE_PARAMS_YAML}"
+
+    # values to dib-block-device: using the YAML config and
+    IMAGE_BLOCK_DEVICE=$(dib-block-device --phase=create \
+                                          --params="${DIB_BLOCK_DEVICE_PARAMS_YAML}")
 fi
 export IMAGE_BLOCK_DEVICE
 LOOPDEV=${IMAGE_BLOCK_DEVICE}
@@ -413,7 +442,7 @@ export IMAGE_BLOCK_DEVICE_WITHOUT_PART
 
 export EXTRA_DETACH="detach_loopback ${IMAGE_BLOCK_DEVICE_WITHOUT_PART}"
 export EXTRA_UNMOUNT="dib-block-device --phase=cleanup \
-                      --build-dir=\"${TMP_BUILD_DIR}\""
+                      --params=\"${DIB_BLOCK_DEVICE_PARAMS_YAML}\""
 
 sudo mkfs -t $FS_TYPE $MKFS_OPTS -L ${DIB_ROOT_LABEL} ${IMAGE_BLOCK_DEVICE}
 # Tuning the rootfs uuid works only for ext filesystems.
@@ -467,13 +496,12 @@ fi
 # space before converting the image to some other format.
 export EXTRA_UNMOUNT=""
 unmount_image
-export TMP_IMAGE_PATH=$(${DIB_BLOCK_DEVICE_SCRIPT} \
+TMP_IMAGE_PATH=$(dib-block-device --phase=umount \
-                            --phase=umount \
+                                  --params="${DIB_BLOCK_DEVICE_PARAMS_YAML}")
-                            --build-dir="${TMP_BUILD_DIR}" )
+export TMP_IMAGE_PATH
 
-${DIB_BLOCK_DEVICE_SCRIPT} \
+dib-block-device --phase=cleanup \
-    --phase=cleanup \
+                 --params="${DIB_BLOCK_DEVICE_PARAMS_YAML}"
-    --build-dir="${TMP_BUILD_DIR}"
 
 cleanup_build_dir