sig_cloud/diskimage-builder
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor block_device: passing command line parameters

Browse Source 
The original approach was to pass each and every command
line parameter to the block device.  While the block device
functionality gets extended, this is not any longer practical.

Instead of passing in all the parameters separately this patch
collects these in a YAML file that is passed in to the block device
layer.

Change-Id: I9d07593a01441b62632234468ac25a982cf1a9f0
Signed-off-by: Andreas Florath <andreas@florath.net>
This commit is contained in:
Andreas Florath 2017-04-22 07:49:22 +00:00
parent 1ce16a987b
commit 803d40b0c6
5 changed files with 201 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
diskimage_builder/block_device/__init__.py
View File

@ -1,4 +1,4 @@
# Copyright 2016 Andreas Florath (andreas@florath.net)
# Copyright 2016-2017 Andreas Florath (andreas@florath.net)
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -45,25 +45,18 @@ def main():
epilog="Available phases:\n" + phase_doc)
parser.add_argument('--phase', required=True,
help="phase to execute")
parser.add_argument('--config', required=False,
help="configuration for block device "
"layer as JSON object")
parser.add_argument('--build-dir', required=True,
help="path to temporary build dir")
parser.add_argument('--image-size', required=False,
help="default image size")
parser.add_argument('--image-dir', required=False,
help="default image directory")
parser.add_argument('--params', required=True,
help="parameters for block device handling")
parser.add_argument('--symbol', required=False,
help="symbol to query for getval")
args = parser.parse_args()
logger.info("phase [%s]" % args.phase)
logger.info("config [%s]" % args.config)
logger.info("build_dir [%s]" % args.build_dir)
logger.info("params [%s]" % args.params)
if args.symbol:
logger.info("symbol [%s]" % args.symbol)
bd = BlockDevice(val_else_none(args.config),
val_else_none(args.build_dir),
val_else_none(args.image_size),
val_else_none(args.image_dir))
bd = BlockDevice(args)
# Check if the method is available
method = getattr(bd, "cmd_" + args.phase, None)
@ -76,6 +69,5 @@ def main():
return 0
if __name__ == "__main__":
main()

160
diskimage_builder/block_device/blockdevice.py
View File

@ -1,4 +1,4 @@
# Copyright 2016 Andreas Florath (andreas@florath.net)
# Copyright 2016-2017 Andreas Florath (andreas@florath.net)
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -36,6 +36,10 @@ class BlockDevice(object):
A typical call sequence:
cmd_init: initialized the block device level config. After this
call it is possible to e.g. query information from the (partially
automatic generated) internal state like root-label.
cmd_create: creates all the different aspects of the block
device. When this call is successful, the complete block level
device is set up, filesystems are created and are mounted at
@ -62,6 +66,13 @@ class BlockDevice(object):
In a script this should be called in the following way:
dib-block-device --phase=init ...
# From that point the database can be queried, like
ROOT_LABEL=$(dib-block-device --phase=getval --symbol=root-label ...)
Please note that currently the dib-block-device executable can
only be used outside the chroot.
dib-block-device --phase=create ...
trap "dib-block-device --phase=delete ..." EXIT
# copy / install files
@ -71,50 +82,74 @@ class BlockDevice(object):
trap - EXIT
"""
# Default configuration:
# one image, one partition, mounted under '/'
DefaultConfig = """
- local_loop:
name: image0
"""
def _merge_into_config(self):
"""Merge old (default) config into new
# This is an example of the next level config
# mkfs:
# base: root
# type: ext4
# mount_point: /
There is the need to be compatible using some old environment
variables. This is done in the way, that if there is no
explicit value given, these values are inserted into the current
configuration.
"""
for entry in self.config:
for k, v in entry.items():
if k == 'mkfs':
if 'name' not in v:
continue
if v['name'] != 'mkfs_root':
continue
if 'type' not in v \
and 'root-fs-type' in self.params:
v['type'] = self.params['root-fs-type']
if 'opts' not in v \
and 'root-fs-opts' in self.params:
v['opts'] = self.params['root-fs-opts']
if 'label' not in v \
and 'root-label' in self.params:
if self.params['root-label'] is not None:
v['label'] = self.params['root-label']
else:
v['label'] = "cloudimg-rootfs"
def __init__(self, block_device_config, build_dir,
default_image_size, default_image_dir):
@staticmethod
def _load_json(file_name):
if os.path.exists(file_name):
with codecs.open(file_name, encoding="utf-8", mode="r") as fd:
return json.load(fd)
return None
def __init__(self, args):
logger.debug("Creating BlockDevice object")
logger.debug("Config given [%s]" % block_device_config)
logger.debug("Build dir [%s]" % build_dir)
if block_device_config is None:
block_device_config = BlockDevice.DefaultConfig
self.config = yaml.safe_load(block_device_config)
logger.debug("Using config [%s]" % self.config)
logger.debug("Param file [%s]" % args.params)
self.args = args
self.default_config = {
'image_size': default_image_size,
'image_dir': default_image_dir}
self.state_dir = os.path.join(build_dir,
"states/block-device")
with open(self.args.params) as param_fd:
self.params = yaml.safe_load(param_fd)
logger.debug("Params [%s]" % self.params)
self.state_dir = os.path.join(
self.params['build-dir'], "states/block-device")
self.state_json_file_name \
= os.path.join(self.state_dir, "state.json")
self.plugin_manager = extension.ExtensionManager(
namespace='diskimage_builder.block_device.plugin',
invoke_on_load=False)
self.config_json_file_name \
= os.path.join(self.state_dir, "config.json")
def write_state(self, result):
logger.debug("Write state [%s]" % self.state_json_file_name)
self.config = self._load_json(self.config_json_file_name)
self.state = self._load_json(self.state_json_file_name)
logger.debug("Using state [%s]", self.state)
# This needs to exists for the state and config files
try:
os.makedirs(self.state_dir)
with open(self.state_json_file_name, "w") as fd:
json.dump([self.config, self.default_config, result], fd)
except OSError:
pass
def load_state(self):
with codecs.open(self.state_json_file_name,
encoding="utf-8", mode="r") as fd:
return json.load(fd)
def write_state(self, state):
logger.debug("Write state [%s]" % self.state_json_file_name)
with open(self.state_json_file_name, "w") as fd:
json.dump(state, fd)
def create_graph(self, config, default_config):
# This is the directed graph of nodes: each parse method must
@ -152,10 +187,28 @@ class BlockDevice(object):
return dg, call_order
def create(self, result, rollback):
dg, call_order = self.create_graph(self.config, self.default_config)
dg, call_order = self.create_graph(self.config, self.params)
for node in call_order:
node.create(result, rollback)
def cmd_init(self):
"""Initialize block device setup
This initializes the block device setup layer. One major task
is to parse and check the configuration, write it down for
later examiniation and execution.
"""
with open(self.params['config'], "rt") as config_fd:
self.config = yaml.safe_load(config_fd)
logger.debug("Config before merge [%s]" % self.config)
self._merge_into_config()
logger.debug("Final config [%s]" % self.config)
# Write the final config
with open(self.config_json_file_name, "wt") as fd:
json.dump(self.config, fd)
logger.info("Wrote final block device config to [%s]"
% self.config_json_file_name)
def cmd_create(self):
"""Creates the block device"""
@ -187,47 +240,33 @@ class BlockDevice(object):
logger.info("create() finished")
return 0
def _load_state(self):
logger.info("_load_state() called")
try:
os.stat(self.state_json_file_name)
except OSError:
logger.info("State already cleaned - no way to do anything here")
return None, None, None
config, default_config, state = self.load_state()
logger.debug("Using config [%s]" % config)
logger.debug("Using default config [%s]" % default_config)
logger.debug("Using state [%s]" % state)
# Deleting must be done in reverse order
dg, call_order = self.create_graph(config, default_config)
reverse_order = reversed(call_order)
return dg, reverse_order, state
def cmd_umount(self):
"""Unmounts the blockdevice and cleanup resources"""
dg, reverse_order, state = self._load_state()
dg, call_order = self.create_graph(self.config, self.params)
reverse_order = reversed(call_order)
if dg is None:
return 0
for node in reverse_order:
node.umount(state)
node.umount(self.state)
# To be compatible with the current implementation, echo the
# result to stdout.
print("%s" % state['image0']['image'])
print("%s" % self.state['image0']['image'])
return 0
def cmd_cleanup(self):
"""Cleanup all remaining relicts - in good case"""
dg, reverse_order, state = self._load_state()
# Deleting must be done in reverse order
dg, call_order = self.create_graph(self.config, self.params)
reverse_order = reversed(call_order)
if dg is None:
return 0
for node in reverse_order:
node.cleanup(state)
node.cleanup(self.state)
logger.info("Removing temporary dir [%s]" % self.state_dir)
shutil.rmtree(self.state_dir)
@ -237,11 +276,14 @@ class BlockDevice(object):
def cmd_delete(self):
"""Cleanup all remaining relicts - in case of an error"""
dg, reverse_order, state = self._load_state()
# Deleting must be done in reverse order
dg, call_order = self.create_graph(self.config, self.params)
reverse_order = reversed(call_order)
if dg is None:
return 0
for node in reverse_order:
node.delete(state)
node.delete(self.state)
logger.info("Removing temporary dir [%s]" % self.state_dir)
shutil.rmtree(self.state_dir)

4
diskimage_builder/block_device/level0/localloop.py
View File

@ -39,12 +39,12 @@ class LocalLoop(NodePluginBase):
self.size = parse_abs_size_spec(config['size'])
logger.debug("Image size [%s]" % self.size)
else:
self.size = parse_abs_size_spec(default_config['image_size'])
self.size = parse_abs_size_spec(default_config['image-size'])
logger.debug("Using default image size [%s]" % self.size)
if 'directory' in config:
self.image_dir = config['directory']
else:
self.image_dir = default_config['image_dir']
self.image_dir = default_config['image-dir']
self.name = config['name']
Digraph.Node.__init__(self, self.name)
self.filename = os.path.join(self.image_dir, self.name + ".raw")

48
diskimage_builder/lib/common-functions
View File

@ -359,3 +359,51 @@ function unmount_dir {
sudo umount -fl $m || true
done
}
# Create YAML config file for the block device layer
# The order here is: use the one the user provides - if there is
# none provided, fall back to the possible one element which
# defines a fallback configuration.
# Parameters:
# - name of the to be created config file
function block_device_create_config_file {
local CONFIG_YAML="$1"
# User setting overwrites this
if [[ ${DIB_BLOCK_DEVICE_CONFIG:-} == file://* ]]; then
cp $(echo ${DIB_BLOCK_DEVICE_CONFIG} | cut -c 8-) ${CONFIG_YAML}
return
fi
if [ -n "${DIB_BLOCK_DEVICE_CONFIG:-}" ]; then
printf "%s" "${DIB_BLOCK_DEVICE_CONFIG}" >${CONFIG_YAML}
return
fi
eval declare -A image_elements=($(get_image_element_array))
for i in ${!image_elements[@]}; do
local BLOCK_DEVICE_CFG_PATH=${image_elements[$i]}/block-device-${ARCH}.yaml
if [ -e ${BLOCK_DEVICE_CFG_PATH} ]; then
cp ${BLOCK_DEVICE_CFG_PATH} ${CONFIG_YAML}
else
BLOCK_DEVICE_CFG_PATH=${image_elements[$i]}/block-device-default.yaml
if [ -e ${BLOCK_DEVICE_CFG_PATH} ]; then
cp ${BLOCK_DEVICE_CFG_PATH} ${CONFIG_YAML}
fi
fi
done
# If no config is there (until now) use the default config
if [ ! -e ${CONFIG_YAML} ]; then
cat >${CONFIG_YAML} <<EOF
- local_loop:
name: image0
mkfs:
name: mkfs_root
mount:
mount_point: /
fstab:
options: "defaults"
fsck-passno: 1
EOF
fi
}

54
diskimage_builder/lib/disk-image-create
View File

@ -276,6 +276,28 @@ if [[ -n "${GENTOO_PROFILE}" ]]; then
fi
mk_build_dir
# Create the YAML file with the final and raw configuration for
# the block device layer.
mkdir -p ${TMP_BUILD_DIR}/block-device
BLOCK_DEVICE_CONFIG_YAML=${TMP_BUILD_DIR}/block-device/config.yaml
block_device_create_config_file "${BLOCK_DEVICE_CONFIG_YAML}"
# Write out the parameter file
DIB_BLOCK_DEVICE_PARAMS_YAML=${TMP_BUILD_DIR}/block-device/params.yaml
export DIB_BLOCK_DEVICE_PARAMS_YAML
cat >${DIB_BLOCK_DEVICE_PARAMS_YAML} <<EOF
config: ${BLOCK_DEVICE_CONFIG_YAML}
image-dir: ${TMP_IMAGE_DIR}
root-fs-type: ${FS_TYPE}
root-label: ${DIB_ROOT_LABEL}
mount-base: ${TMP_BUILD_DIR}/mnt
build-dir: ${TMP_BUILD_DIR}
EOF
dib-block-device --phase=init \
--params="${DIB_BLOCK_DEVICE_PARAMS_YAML}"
create_base
# This variable needs to be propagated into the chroot
mkdir -p $TMP_HOOKS_PATH/environment.d
@ -397,12 +419,19 @@ fi
DIB_BLOCK_DEVICE_SCRIPT=$(which dib-block-device)
if [ -z ${IMAGE_BLOCK_DEVICE} ] ; then
IMAGE_BLOCK_DEVICE=$(${DIB_BLOCK_DEVICE_SCRIPT} \
--phase=create \
--config="${DIB_BLOCK_DEVICE_CONFIG:-}" \
--image-size="${DIB_IMAGE_SIZE}"KiB \
--image-dir="${TMP_IMAGE_DIR}" \
--build-dir="${TMP_BUILD_DIR}" )
# For compatibily reasons in addition to the YAML configuration
# there is the need to handle the old environment variables.
echo "image-size: ${DIB_IMAGE_SIZE}KiB" >> ${DIB_BLOCK_DEVICE_PARAMS_YAML}
# After changeing the parameters, there is the need to
# re-run dib-block-device init because some value might
# change based on the new set parameters.
dib-block-device --phase=init \
--params="${DIB_BLOCK_DEVICE_PARAMS_YAML}"
# values to dib-block-device: using the YAML config and
IMAGE_BLOCK_DEVICE=$(dib-block-device --phase=create \
--params="${DIB_BLOCK_DEVICE_PARAMS_YAML}")
fi
export IMAGE_BLOCK_DEVICE
LOOPDEV=${IMAGE_BLOCK_DEVICE}
@ -413,7 +442,7 @@ export IMAGE_BLOCK_DEVICE_WITHOUT_PART
export EXTRA_DETACH="detach_loopback ${IMAGE_BLOCK_DEVICE_WITHOUT_PART}"
export EXTRA_UNMOUNT="dib-block-device --phase=cleanup \
--build-dir=\"${TMP_BUILD_DIR}\""
--params=\"${DIB_BLOCK_DEVICE_PARAMS_YAML}\""
sudo mkfs -t $FS_TYPE $MKFS_OPTS -L ${DIB_ROOT_LABEL} ${IMAGE_BLOCK_DEVICE}
# Tuning the rootfs uuid works only for ext filesystems.
@ -467,13 +496,12 @@ fi
# space before converting the image to some other format.
export EXTRA_UNMOUNT=""
unmount_image
export TMP_IMAGE_PATH=$(${DIB_BLOCK_DEVICE_SCRIPT} \
--phase=umount \
--build-dir="${TMP_BUILD_DIR}" )
TMP_IMAGE_PATH=$(dib-block-device --phase=umount \
--params="${DIB_BLOCK_DEVICE_PARAMS_YAML}")
export TMP_IMAGE_PATH
${DIB_BLOCK_DEVICE_SCRIPT} \
--phase=cleanup \
--build-dir="${TMP_BUILD_DIR}"
dib-block-device --phase=cleanup \
--params="${DIB_BLOCK_DEVICE_PARAMS_YAML}"
cleanup_build_dir