From deb832d685e1193e977b4385810abc6d48e011fd Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Thu, 25 May 2017 11:28:13 +1000 Subject: [PATCH] Create and use plugin/node abstract classes This completes the transitions started in Ic5a61365ef0132476b11bdbf1dd96885e91c3cb6 The new file plugin.py is the place to start with this change. The abstract base classes PluginBase and NodeBase are heavily documented. NodeBase essentially replaces Digraph.Node The changes in level?/*.py make no functional changes, but are just refactoring to implement the plugin and node classes consistently. Additionally we have added asserts during parsing & generation to ensure plugins are implemented PluginBase, and get_nodes() is always returning NodeBase objects for the graph. Change-Id: Ie648e9224749491260dea65d7e8b8151a6824b9c --- diskimage_builder/block_device/blockdevice.py | 16 +- .../block_device/level0/__init__.py | 17 -- .../block_device/level0/localloop.py | 22 +- .../block_device/level1/__init__.py | 18 -- .../block_device/level1/partition.py | 27 +-- .../block_device/level1/partitioning.py | 30 ++- .../block_device/level2/__init__.py | 17 -- diskimage_builder/block_device/level2/mkfs.py | 39 +-- .../block_device/level3/__init__.py | 17 -- .../block_device/level3/mount.py | 38 ++- .../block_device/level4/__init__.py | 17 -- .../block_device/level4/fstab.py | 42 ++-- diskimage_builder/block_device/plugin.py | 205 ++++++++++++++++ .../block_device/tests/test_mount_order.py | 4 +- diskimage_builder/graph/__init__.py | 0 diskimage_builder/graph/digraph.py | 228 ------------------ .../tests/functional/test_blockdevice_mbr.py | 3 +- .../tests/functional/test_graph.py | 143 ----------- .../tests/functional/test_graph_toposort.py | 116 --------- 19 files changed, 298 insertions(+), 701 deletions(-) create mode 100644 diskimage_builder/block_device/plugin.py delete mode 100644 diskimage_builder/graph/__init__.py delete mode 100644 diskimage_builder/graph/digraph.py delete mode 100644 diskimage_builder/tests/functional/test_graph.py delete mode 100644 diskimage_builder/tests/functional/test_graph_toposort.py diff --git a/diskimage_builder/block_device/blockdevice.py b/diskimage_builder/block_device/blockdevice.py index 8d2cd514..456f4bc8 100644 --- a/diskimage_builder/block_device/blockdevice.py +++ b/diskimage_builder/block_device/blockdevice.py @@ -28,6 +28,8 @@ from diskimage_builder.block_device.config import \ config_tree_to_graph from diskimage_builder.block_device.exception import \ BlockDeviceSetupException +from diskimage_builder.block_device.plugin import NodeBase +from diskimage_builder.block_device.plugin import PluginBase from diskimage_builder.block_device.utils import exec_sudo @@ -189,20 +191,24 @@ class BlockDevice(object): # Instantiate a "plugin" object, passing it the # configuration entry + # XXX would a "factory" pattern for plugins, where we make + # a method call on an object stevedore has instantiated be + # better here? if cfg_obj_name not in self.plugin_manager: raise BlockDeviceSetupException( ("Config element [%s] is not implemented" % cfg_obj_name)) - cfg_obj = self.plugin_manager[cfg_obj_name].plugin( - cfg_obj_val, default_config) + plugin = self.plugin_manager[cfg_obj_name].plugin + assert issubclass(plugin, PluginBase) + cfg_obj = plugin(cfg_obj_val, default_config) # Ask the plugin for the nodes it would like to insert # into the graph. Some plugins, such as partitioning, # return multiple nodes from one config entry. nodes = cfg_obj.get_nodes() + assert isinstance(nodes, list) for node in nodes: - # would only be missing if a plugin was way out of - # line and didn't put it in... - assert node.name + # plugins should return nodes... + assert isinstance(node, NodeBase) # ensure node names are unique. networkx by default # just appends the attribute to the node dict for # existing nodes, which is not what we want. diff --git a/diskimage_builder/block_device/level0/__init__.py b/diskimage_builder/block_device/level0/__init__.py index dffb6254..e69de29b 100644 --- a/diskimage_builder/block_device/level0/__init__.py +++ b/diskimage_builder/block_device/level0/__init__.py @@ -1,17 +0,0 @@ -# Copyright 2016 Andreas Florath (andreas@florath.net) -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from diskimage_builder.block_device.level0.localloop import LocalLoop - -__all__ = [LocalLoop] diff --git a/diskimage_builder/block_device/level0/localloop.py b/diskimage_builder/block_device/level0/localloop.py index 2fe6c12c..e7aa670d 100644 --- a/diskimage_builder/block_device/level0/localloop.py +++ b/diskimage_builder/block_device/level0/localloop.py @@ -18,14 +18,15 @@ import subprocess from diskimage_builder.block_device.exception import \ BlockDeviceSetupException +from diskimage_builder.block_device.plugin import NodeBase +from diskimage_builder.block_device.plugin import PluginBase from diskimage_builder.block_device.utils import parse_abs_size_spec -from diskimage_builder.graph.digraph import Digraph logger = logging.getLogger(__name__) -class LocalLoop(Digraph.Node): +class LocalLoopNode(NodeBase): """Level0: Local loop image device handling. This class handles local loop devices that can be used @@ -34,6 +35,7 @@ class LocalLoop(Digraph.Node): def __init__(self, config, default_config): logger.debug("Creating LocalLoop object; config [%s] " "default_config [%s]" % (config, default_config)) + super(LocalLoopNode, self).__init__(config['name']) if 'size' in config: self.size = parse_abs_size_spec(config['size']) logger.debug("Image size [%s]" % self.size) @@ -44,18 +46,12 @@ class LocalLoop(Digraph.Node): self.image_dir = config['directory'] else: self.image_dir = default_config['image-dir'] - self.name = config['name'] - Digraph.Node.__init__(self, self.name) self.filename = os.path.join(self.image_dir, self.name + ".raw") def get_edges(self): """Because this is created without base, there are no edges.""" return ([], []) - def get_nodes(self): - """Returns nodes for adding to the graph""" - return [self] - @staticmethod def image_create(filename, size): logger.info("Create image file [%s]" % filename) @@ -131,3 +127,13 @@ class LocalLoop(Digraph.Node): def delete(self, state): self._image_delete(state['blockdev'][self.name]['image']) + + +class LocalLoop(PluginBase): + + def __init__(self, config, defaults): + super(PluginBase, self).__init__() + self.node = LocalLoopNode(config, defaults) + + def get_nodes(self): + return [self.node] diff --git a/diskimage_builder/block_device/level1/__init__.py b/diskimage_builder/block_device/level1/__init__.py index 0e31c834..e69de29b 100644 --- a/diskimage_builder/block_device/level1/__init__.py +++ b/diskimage_builder/block_device/level1/__init__.py @@ -1,18 +0,0 @@ -# Copyright -# 2016-2017 Andreas Florath (andreas@florath.net) -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from diskimage_builder.block_device.level1.partitioning import Partitioning - -__all__ = [Partitioning] diff --git a/diskimage_builder/block_device/level1/partition.py b/diskimage_builder/block_device/level1/partition.py index 50c96047..b77b730d 100644 --- a/diskimage_builder/block_device/level1/partition.py +++ b/diskimage_builder/block_device/level1/partition.py @@ -14,26 +14,20 @@ import logging from diskimage_builder.block_device.exception import \ BlockDeviceSetupException -from diskimage_builder.graph.digraph import Digraph +from diskimage_builder.block_device.plugin import NodeBase logger = logging.getLogger(__name__) -class Partition(Digraph.Node): - - type_string = "partitions" +class PartitionNode(NodeBase): flag_boot = 1 flag_primary = 2 def __init__(self, config, parent, prev_partition): - if 'name' not in config: - raise BlockDeviceSetupException( - "Missing 'name' in partition config: %s" % config) - self.name = config['name'] - Digraph.Node.__init__(self, self.name) + super(PartitionNode, self).__init__(config['name']) self.base = config['base'] self.partitioning = parent @@ -64,9 +58,6 @@ class Partition(Digraph.Node): def get_type(self): return self.ptype - def get_name(self): - return self.name - def get_edges(self): edge_from = [self.base] edge_to = [] @@ -76,15 +67,3 @@ class Partition(Digraph.Node): def create(self, result, rollback): self.partitioning.create(result, rollback) - - def umount(self, state): - """Partition does not need any umount task.""" - pass - - def cleanup(self, state): - """Partition does not need any cleanup.""" - pass - - def delete(self, state): - """Partition does not need any cleanup.""" - pass diff --git a/diskimage_builder/block_device/level1/partitioning.py b/diskimage_builder/block_device/level1/partitioning.py index 7b04344b..58294546 100644 --- a/diskimage_builder/block_device/level1/partitioning.py +++ b/diskimage_builder/block_device/level1/partitioning.py @@ -20,21 +20,22 @@ from subprocess import CalledProcessError from diskimage_builder.block_device.exception import \ BlockDeviceSetupException from diskimage_builder.block_device.level1.mbr import MBR -from diskimage_builder.block_device.level1.partition import \ - Partition +from diskimage_builder.block_device.level1.partition import PartitionNode +from diskimage_builder.block_device.plugin import PluginBase from diskimage_builder.block_device.utils import exec_sudo from diskimage_builder.block_device.utils import parse_abs_size_spec from diskimage_builder.block_device.utils import parse_rel_size_spec -from diskimage_builder.graph.digraph import Digraph logger = logging.getLogger(__name__) -class Partitioning(Digraph.Node): +class Partitioning(PluginBase): def __init__(self, config, default_config): logger.debug("Creating Partitioning object; config [%s]" % config) + super(Partitioning, self).__init__() + # Because using multiple partitions of one base is done # within one object, there is the need to store a flag if the # creation of the partitions was already done. @@ -75,19 +76,19 @@ class Partitioning(Digraph.Node): prev_partition = None for part_cfg in config['partitions']: - np = Partition(part_cfg, self, prev_partition) + np = PartitionNode(part_cfg, self, prev_partition) self.partitions.append(np) prev_partition = np + def get_nodes(self): + # return the list of partitions + return self.partitions + def _size_of_block_dev(self, dev): with open(dev, "r") as fd: fd.seek(0, 2) return fd.tell() - def get_nodes(self): - # We just add partitions - return self.partitions - def _all_part_devices_exist(self, expected_part_devices): for part_device in expected_part_devices: logger.debug("Checking if partition device [%s] exists" % @@ -127,11 +128,18 @@ class Partitioning(Digraph.Node): exec_sudo(["kpartx", "-avs", device_path]) def create(self, result, rollback): + # not this is NOT a node and this is not called directly! The + # create() calls in the partition nodes this plugin has + # created are calling back into this. image_path = result['blockdev'][self.base]['image'] device_path = result['blockdev'][self.base]['device'] logger.info("Creating partition on [%s] [%s]" % (self.base, image_path)) + # This is a bit of a hack. Each of the partitions is actually + # in the graph, so for every partition we get a create() call + # as the walk happens. But we only need to create the + # partition table once... if self.already_created: logger.info("Not creating the partitions a second time.") return @@ -143,9 +151,9 @@ class Partitioning(Digraph.Node): with MBR(image_path, disk_size, self.align) as part_impl: for part_cfg in self.partitions: part_name = part_cfg.get_name() - part_bootflag = Partition.flag_boot \ + part_bootflag = PartitionNode.flag_boot \ in part_cfg.get_flags() - part_primary = Partition.flag_primary \ + part_primary = PartitionNode.flag_primary \ in part_cfg.get_flags() part_size = part_cfg.get_size() part_free = part_impl.free() diff --git a/diskimage_builder/block_device/level2/__init__.py b/diskimage_builder/block_device/level2/__init__.py index 25456900..e69de29b 100644 --- a/diskimage_builder/block_device/level2/__init__.py +++ b/diskimage_builder/block_device/level2/__init__.py @@ -1,17 +0,0 @@ -# Copyright 2017 Andreas Florath (andreas@florath.net) -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from diskimage_builder.block_device.level2.mkfs import Mkfs - -__all__ = [Mkfs] diff --git a/diskimage_builder/block_device/level2/mkfs.py b/diskimage_builder/block_device/level2/mkfs.py index 71aef546..4541ad83 100644 --- a/diskimage_builder/block_device/level2/mkfs.py +++ b/diskimage_builder/block_device/level2/mkfs.py @@ -17,8 +17,9 @@ import uuid from diskimage_builder.block_device.exception \ import BlockDeviceSetupException +from diskimage_builder.block_device.plugin import NodeBase +from diskimage_builder.block_device.plugin import PluginBase from diskimage_builder.block_device.utils import exec_sudo -from diskimage_builder.graph.digraph import Digraph logger = logging.getLogger(__name__) @@ -40,12 +41,14 @@ file_system_max_label_length = { } -class Filesystem(Digraph.Node): +class FilesystemNode(NodeBase): def __init__(self, config): logger.debug("Create filesystem object; config [%s]" % config) + super(FilesystemNode, self).__init__(config['name']) + # Parameter check (mandatory) - for pname in ['base', 'name', 'type']: + for pname in ['base', 'type']: if pname not in config: raise BlockDeviceSetupException( "Mkfs config needs [%s]" % pname) @@ -92,8 +95,6 @@ class Filesystem(Digraph.Node): if self.uuid is None: self.uuid = str(uuid.uuid4()) - Digraph.Node.__init__(self, self.name) - logger.debug("Filesystem created [%s]" % self) def get_edges(self): @@ -137,36 +138,18 @@ class Filesystem(Digraph.Node): 'fstype': self.type, 'opts': self.opts, 'device': device} - def umount(self, state): - """Mkfs does not need any umount.""" - pass - def cleanup(self, state): - """Mkfs does not need any cleanup.""" - pass - - def delete(self, state): - """Mkfs does not need any delete.""" - pass - - -class Mkfs(object): - """Module for creating file systems +class Mkfs(PluginBase): + """Create a file system This block device module handles creating different file systems. """ - type_string = "mkfs" - - def __init__(self, config, default_config): - logger.debug("Create Mkfs object; config [%s]" % config) - logger.debug("default_config [%s]" % default_config) - self.config = config - self.default_config = default_config + def __init__(self, config, defaults): + super(Mkfs, self).__init__() self.filesystems = {} - - fs = Filesystem(self.config) + fs = FilesystemNode(config) self.filesystems[fs.get_name()] = fs def get_nodes(self): diff --git a/diskimage_builder/block_device/level3/__init__.py b/diskimage_builder/block_device/level3/__init__.py index 5fa3fe4b..e69de29b 100644 --- a/diskimage_builder/block_device/level3/__init__.py +++ b/diskimage_builder/block_device/level3/__init__.py @@ -1,17 +0,0 @@ -# Copyright 2017 Andreas Florath (andreas@florath.net) -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from diskimage_builder.block_device.level3.mount import Mount - -__all__ = [Mount] diff --git a/diskimage_builder/block_device/level3/mount.py b/diskimage_builder/block_device/level3/mount.py index f8ef8314..6b111688 100644 --- a/diskimage_builder/block_device/level3/mount.py +++ b/diskimage_builder/block_device/level3/mount.py @@ -17,9 +17,10 @@ import os from diskimage_builder.block_device.exception \ import BlockDeviceSetupException +from diskimage_builder.block_device.plugin import NodeBase +from diskimage_builder.block_device.plugin import PluginBase from diskimage_builder.block_device.utils import exec_sudo from diskimage_builder.block_device.utils import sort_mount_points -from diskimage_builder.graph.digraph import Digraph logger = logging.getLogger(__name__) @@ -32,17 +33,18 @@ mount_points = {} sorted_mount_points = None -class MountPoint(Digraph.Node): +class MountPointNode(NodeBase): def __init__(self, mount_base, config): + super(MountPointNode, self).__init__(config['name']) + # Parameter check self.mount_base = mount_base - for pname in ['base', 'name', 'mount_point']: + for pname in ['base', 'mount_point']: if pname not in config: raise BlockDeviceSetupException( "MountPoint config needs [%s]" % pname) setattr(self, pname, config[pname]) - Digraph.Node.__init__(self, self.name) logger.debug("MountPoint created [%s]" % self) def get_node(self): @@ -116,30 +118,22 @@ class MountPoint(Digraph.Node): logger.info("Called for [%s]" % self.name) exec_sudo(["umount", state['mount'][self.mount_point]['path']]) - def cleanup(self, state): - """Mount does not need any cleanup.""" - pass - def delete(self, state): self.umount(state) -class Mount(object): - - type_string = "mount" - - def __init__(self, config, params): - logger.debug("Mounting object; config [%s]" % config) - self.config = config - self.params = params - - if 'mount-base' not in self.params: - raise BlockDeviceSetupException( - "Mount default config needs 'mount-base'") - self.mount_base = self.params['mount-base'] +class Mount(PluginBase): + def __init__(self, config, defaults): + super(Mount, self).__init__() self.mount_points = {} - mp = MountPoint(self.mount_base, self.config) + + if 'mount-base' not in defaults: + raise BlockDeviceSetupException( + "Mount default config needs 'mount-base'") + self.mount_base = defaults['mount-base'] + + mp = MountPointNode(self.mount_base, config) self.mount_points[mp.get_name()] = mp def get_nodes(self): diff --git a/diskimage_builder/block_device/level4/__init__.py b/diskimage_builder/block_device/level4/__init__.py index 568d86b0..e69de29b 100644 --- a/diskimage_builder/block_device/level4/__init__.py +++ b/diskimage_builder/block_device/level4/__init__.py @@ -1,17 +0,0 @@ -# Copyright 2017 Andreas Florath (andreas@florath.net) -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from diskimage_builder.block_device.level4.fstab import Fstab - -__all__ = [Fstab] diff --git a/diskimage_builder/block_device/level4/fstab.py b/diskimage_builder/block_device/level4/fstab.py index b0883984..97adf299 100644 --- a/diskimage_builder/block_device/level4/fstab.py +++ b/diskimage_builder/block_device/level4/fstab.py @@ -14,30 +14,20 @@ import logging -from diskimage_builder.graph.digraph import Digraph +from diskimage_builder.block_device.plugin import NodeBase +from diskimage_builder.block_device.plugin import PluginBase logger = logging.getLogger(__name__) -class Fstab(Digraph.Node): - - type_string = "fstab" - +class FstabNode(NodeBase): def __init__(self, config, params): - logger.debug("Fstab object; config [%s]" % config) - self.config = config - self.params = params - self.name = self.config['name'] - self.base = self.config['base'] - Digraph.Node.__init__(self, self.name) - - self.options = self.config.get('options', 'defaults') - self.dump_freq = self.config.get('dump-freq', 0) - self.fsck_passno = self.config.get('fsck-passno', 2) - - def get_nodes(self): - return [self] + super(FstabNode, self).__init__(config['name']) + self.base = config['base'] + self.options = config.get('options', 'defaults') + self.dump_freq = config.get('dump-freq', 0) + self.fsck_passno = config.get('fsck-passno', 2) def get_edges(self): edge_from = [self.base] @@ -59,14 +49,12 @@ class Fstab(Digraph.Node): 'fsck-passno': self.fsck_passno } - def umount(self, state): - """Fstab does not need any umount task.""" - pass - def cleanup(self, state): - """Fstab does not need any cleanup.""" - pass +class Fstab(PluginBase): + def __init__(self, config, defaults): + super(Fstab, self).__init__() - def delete(self, state): - """Fstab does not need any cleanup.""" - pass + self.node = FstabNode(config, defaults) + + def get_nodes(self): + return [self.node] diff --git a/diskimage_builder/block_device/plugin.py b/diskimage_builder/block_device/plugin.py new file mode 100644 index 00000000..7a0ffa52 --- /dev/null +++ b/diskimage_builder/block_device/plugin.py @@ -0,0 +1,205 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import six + +# +# Plugins convert configuration entries into graph nodes ready for +# processing. This defines the abstract classes for both. +# + + +@six.add_metaclass(abc.ABCMeta) +class NodeBase(object): + """A configuration node entry + + This is the main driver class for dib-block-device operation. + + The final operations graph is composed of instantiations of this + class. The graph undergoes a topological sort (i.e. is linearised + in dependency order) and each node has :func:`create` called in + order to perform its operations. + + Every node has a unique string ``name``. This is its key in the + graph and used for edge relationships. Implementations must + ensure they initalize it; e.g. + + .. code-block:: python + + class FooNode(NodeBase): + def __init__(name, arg1, ...): + super(FooNode, self).__init__(name) + + """ + def __init__(self, name): + self.name = name + + def get_name(self): + return self.name + + @abc.abstractmethod + def get_edges(self): + """Return the dependencies/edges for this node + + This function will be called after all nodes are created (this + is because some plugins need to know the global state of all + nodes to decide their dependencies). + + This function returns a tuple with two lists + + * ``edges_from`` : a list of node names that point to us + * ``edges_to`` : a list of node names we point to + + In most cases, node creation will have saved a single parent + that was given in the ``base`` parameter of the configuration. + A usual return might look like: + + .. code-block:: python + + def get_edges(self): + return ( [self.base], [] ) + + Some nodes (``level0``) don't have a base, however + """ + return + + @abc.abstractmethod + def create(self, results, rollback): + """Main creation driver + + This is the main driver function. After the graph is + linearised, each node has it's :func:`create` function called. + + Arguments: + + :param results: A shared dictionary of prior results. This + dictionary is passed by reference to each call, meaning any + entries inserted will be available to subsequent :func:`create` + calls of following nodes. + + :param rollback: A shared list of functions to be called in + the failure case. Nodes should only append to this list. + On failure, the callbacks will be processed in reverse + order. + + :raises Exception: A failure should raise an exception. This + will initiate the rollback + + :return: None + """ + return + + def umount(self, state): + """Umount actions + + Actions to taken when ``dib-block-device umount`` is called + + :param state: the current state dictionary. This is the + `results` dictionary from :func:`create` before this call is + made. + :return: None + """ + return + + def cleanup(self, state): + """Cleanup actions + + Actions to taken when ``dib-block-device cleanup`` is called. + This is the cleanup path in the *success* case. The nodes are + called in the reverse order to :func:`create` + + :param state: the current state dictionary. This is the + `results` dictionary from :func:`create` before this call is + made. + :return: None + """ + return + + def delete(self, state): + """Cleanup actions + + Actions to taken when ``dib-block-device delete`` is called. + This is the cleanup path in case of a reported external + *failure*. The nodes are called in the reverse order to + :func:`create` + + :param state: the current state dictionary. This is the + `results` dictionary from :func:`create` before this call is + made. + :return: None + """ + return + + +@six.add_metaclass(abc.ABCMeta) +class PluginBase(object): + """The base plugin object + + This is the base plugin object. Plugins are an instantiation of + this class. There should be an entry-point (see setup.cfg) + defined under ``diskimage_builder.block_device.plugin`` for each + plugin, e.g. + + foo = diskimage_builder.block_device.levelX.foo:Foo + + A configuration entry in the graph config that matches this entry + point will create an instance of this class, e.g. + + .. code-block:: yaml + + foo: + name: foo_node + base: parent_node + argument_a: bar + argument_b: baz + + The ``__init__`` function will be passed two arguments: + + ``config`` + The full configuration dictionary for the entry. + A unique ``name`` entry can be assumed. In most cases + a ``base`` entry will be present giving the parent node + (see :func:`NodeBase.get_edges`). + ``defaults`` + The global defaults dictionary (see ``--params``) + + ``get_nodes()`` should return the node object(s) created by the + config for insertion into the final configuration graph. In the + simplest case, this is probably a single node created during + instantiation. e.g. + + .. code-block:: python + + class Foo(PluginBase): + + def __init__(self, config, defaults): + super(Foo, self).__init__() + self.node = FooNode(config.name, ...) + + def get_nodes(self): + return [self.node] + + + Some plugins require more, however. + """ + def __init__(self): + pass + + @abc.abstractmethod + def get_nodes(self): + """Return nodes created by the plugin + + :returns: a list of :class:`.NodeBase` objects for insertion + into the graph + """ + return diff --git a/diskimage_builder/block_device/tests/test_mount_order.py b/diskimage_builder/block_device/tests/test_mount_order.py index e589cf2f..8c948e43 100644 --- a/diskimage_builder/block_device/tests/test_mount_order.py +++ b/diskimage_builder/block_device/tests/test_mount_order.py @@ -15,7 +15,7 @@ import mock import diskimage_builder.block_device.tests.test_config as tc -from diskimage_builder.block_device.level3.mount import MountPoint +from diskimage_builder.block_device.level3.mount import MountPointNode logger = logging.getLogger(__name__) @@ -42,7 +42,7 @@ class TestMountOrder(tc.TestGraphGeneration): rollback = [] for node in call_order: - if isinstance(node, MountPoint): + if isinstance(node, MountPointNode): # XXX: do we even need to create? We could test the # sudo arguments from the mock in the below asserts # too diff --git a/diskimage_builder/graph/__init__.py b/diskimage_builder/graph/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/diskimage_builder/graph/digraph.py b/diskimage_builder/graph/digraph.py deleted file mode 100644 index 5b7758d9..00000000 --- a/diskimage_builder/graph/digraph.py +++ /dev/null @@ -1,228 +0,0 @@ -# Copyright 2016 Andreas Florath (andreas@florath.net) -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -import bisect - - -class Digraph(object): - """Implements a directed graph. - - Each node of the digraph must have a unique name. - """ - - class Edge(object): - """Directed graph edge. - - The digraph has weighted edges. This class holds the weight and - a reference to the node. - """ - - def __init__(self, node, weight): - self.__node = node - self.__weight = weight - - def __eq__(self, other): - return self.__weight == other.get_weight() \ - and self.__node == other.get_node() - - def __lt__(self, other): - return self.__weight < other.get_weight() - - def get_node(self): - """Return the (pointed to) node""" - return self.__node - - def get_weight(self): - """Return the edge's weight""" - return self.__weight - - class Node(object): - """Directed graph node. - - This holds the incoming and outgoing edges as well as the - nodes' name. - """ - - def __init__(self, name): - """Initializes a node. - - Incoming and outgoing are lists of nodes. Typically one - direction is provided and the other can be automatically - computed. - """ - self.__name = name - self.__incoming = [] - self.__outgoing = [] - - def __repr__(self): - return "" % self.__name - - def get_name(self): - """Returns the name of the node.""" - return self.__name - - def add_incoming(self, node, weight): - """Add node to the incoming list.""" - bisect.insort(self.__incoming, Digraph.Edge(node, weight)) - - def add_outgoing(self, node, weight): - """Add node to the outgoing list.""" - bisect.insort(self.__outgoing, Digraph.Edge(node, weight)) - - def get_iter_outgoing(self): - """Return an iterator over the outgoing nodes.""" - - return iter([x.get_node() for x in self.__outgoing]) - - def has_incoming(self): - """Returns True if the node has incoming edges""" - return self.__incoming - - @staticmethod - def __as_named_list(inlist): - """Return given list as list of names.""" - - return [x.get_node().get_name() for x in inlist] - - def get_outgoing_as_named_list(self): - """Return the names of all outgoing nodes as a list.""" - - return self.__as_named_list(self.__outgoing) - - def __init__(self): - """Create a empty digraph.""" - self._named_nodes = {} - - def create_from_dict(self, init_dgraph, node_gen_func=Node): - """Creates a new digraph based on the given information.""" - - # First run: create all nodes - for node_name in init_dgraph: - # Create the node and put it into the object list of all - # nodes and into the local dictionary of named nodes. - named_node = node_gen_func(node_name) - self.add_node(named_node) - - # Second run: run through all nodes and create the edges. - for node_name, outs in init_dgraph.items(): - node_from = self.find(node_name) - for onode in outs: - node_to = self.find(onode) - if node_to is None: - raise RuntimeError("Node '%s' is referenced " - "but not specified" % onode) - self.create_edge(node_from, node_to) - - def add_node(self, anode): - """Adds a new node to the graph. - - Checks if the node with the same name already exists. - """ - assert issubclass(anode.__class__, Digraph.Node) - - for node in self._named_nodes.values(): - if node.get_name() == anode.get_name(): - raise RuntimeError("Node with name [%s] already " - "exists" % node.get_name()) - self._named_nodes[anode.get_name()] = anode - - def create_edge(self, anode, bnode, weight=0): - """Creates an edge from a to b - both must be nodes.""" - - assert issubclass(anode.__class__, Digraph.Node) - assert issubclass(bnode.__class__, Digraph.Node) - assert anode.get_name() in self._named_nodes.keys() - assert anode == self._named_nodes[anode.get_name()] - assert bnode.get_name() in self._named_nodes.keys() - assert bnode == self._named_nodes[bnode.get_name()] - anode.add_outgoing(bnode, weight) - bnode.add_incoming(anode, weight) - - def get_iter_nodes_values(self): - """Returns the nodes dict to the values. - - Note: it is not possible to change things with the help of the - result of this function. - """ - return iter(self._named_nodes.values()) - - def find(self, name): - """Get the node with the given name. - - Return None if not available. - """ - if name not in self._named_nodes: - return None - - return self._named_nodes[name] - - def as_dict(self): - """Outputs this digraph and create a dictionary.""" - - # Start with an empty dictionary - rval = {} - for node in self._named_nodes.values(): - rval[node.get_name()] = node.get_outgoing_as_named_list() - return rval - - def topological_sort(self): - """Digraph topological search. - - This algorithm is based upon a depth first search with - 'making' some special nodes. - The result is the topological sorted list of nodes. - """ - - # List of topological sorted nodes - tsort = [] - # List of nodes already visited. - # (This is held here - local to the algorithm - to not modify the - # nodes themselves.) - visited = [] - - def visit(node): - """Recursive deep first search function.""" - - if node not in visited: - visited.append(node) - for onode in node.get_iter_outgoing(): - visit(onode) - tsort.insert(0, node) - - # The 'main' function of the topological sort - for node in self.get_iter_nodes_values(): - if node.has_incoming(): - continue - visit(node) - - return tsort - - -# Utility functions - -def digraph_create_from_dict(init_dgraph, node_gen_func=Digraph.Node): - """Creates a new digraph based on the given information.""" - - digraph = Digraph() - digraph.create_from_dict(init_dgraph, node_gen_func) - return digraph - - -def node_list_to_node_name_list(node_list): - """Converts a node list into a list of the corresponding node names.""" - - node_name_list = [] - for node in node_list: - node_name_list.append(node.get_name()) - return node_name_list diff --git a/diskimage_builder/tests/functional/test_blockdevice_mbr.py b/diskimage_builder/tests/functional/test_blockdevice_mbr.py index be6a7077..b8df5d51 100644 --- a/diskimage_builder/tests/functional/test_blockdevice_mbr.py +++ b/diskimage_builder/tests/functional/test_blockdevice_mbr.py @@ -19,7 +19,8 @@ import subprocess import tempfile import testtools -from diskimage_builder.block_device.level0.localloop import LocalLoop +from diskimage_builder.block_device.level0.localloop \ + import LocalLoopNode as LocalLoop from diskimage_builder.block_device.level1.mbr import MBR diff --git a/diskimage_builder/tests/functional/test_graph.py b/diskimage_builder/tests/functional/test_graph.py deleted file mode 100644 index 8066cdbe..00000000 --- a/diskimage_builder/tests/functional/test_graph.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright 2016 Andreas Florath (andreas@florath.net) -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from diskimage_builder.graph.digraph import Digraph -from diskimage_builder.graph.digraph import digraph_create_from_dict -import testtools - - -class TestDigraph(testtools.TestCase): - - def test_constructor_001(self): - """Test conversion from dictionary to graph and back (two nodes)""" - - d = {"A": ["B"], "B": []} - dg = digraph_create_from_dict(d) - e = dg.as_dict() - self.assertEqual(d["A"], list(e["A"])) - - def test_constructor_002(self): - """Test conversion from dictionary to graph and back (zero nodes)""" - - d = {} - dg = digraph_create_from_dict(d) - e = dg.as_dict() - self.assertEqual(d, e) - - def test_constructor_003(self): - """Test conversion from dictionary to graph and back (one node)""" - - d = {"A": []} - dg = digraph_create_from_dict(d) - e = dg.as_dict() - self.assertEqual(d["A"], list(e["A"])) - - def test_constructor_004(self): - """Test conversion from dictionary to graph and back (one node)""" - - d = {"A": ["A"]} - dg = digraph_create_from_dict(d) - e = dg.as_dict() - self.assertEqual(d["A"], list(e["A"])) - - def test_constructor_005(self): - """Test conversion: error: pointed node does not exists""" - - d = {"A": ["B"]} - try: - d = digraph_create_from_dict(d) - self.assertTrue(False) - except RuntimeError: - pass - - def test_constructor_006(self): - """Test conversion from dictionary: two node circle""" - - d = {"A": ["B"], "B": ["A"]} - dg = digraph_create_from_dict(d) - e = dg.as_dict() - self.assertEqual(d["A"], list(e["A"])) - self.assertEqual(d["B"], list(e["B"])) - - def test_constructor_007(self): - """Test conversion from dictionary: more complex graph""" - - d = {"A": ["B"], "B": ["A", "D", "C"], "C": ["A", "D"], - "D": ["D"]} - dg = digraph_create_from_dict(d) - e = dg.as_dict() - self.assertEqual(d['A'], list(e['A'])) - self.assertEqual(set(d['B']), set(e['B'])) - self.assertEqual(set(d['C']), set(e['C'])) - self.assertEqual(d['D'], list(e['D'])) - - def test_find_01(self): - """Digraph find with element available""" - - d = {"A": ["B"], "B": ["A", "C", "D"], "C": ["A", "D"], - "D": ["D"]} - dg = digraph_create_from_dict(d) - n = dg.find("A") - self.assertEqual("A", n.get_name(),) - - def test_find_02(self): - """Digraph find with element not available""" - - d = {"A": ["B"], "B": ["A", "C", "D"], "C": ["A", "D"], - "D": ["D"]} - dg = digraph_create_from_dict(d) - n = dg.find("Z") - self.assertIsNone(n) - - def test_get_named_node_01(self): - """Digraph get named node with map available""" - - d = {"A": ["B"], "B": ["A", "C", "D"], "C": ["A", "D"], - "D": ["D"]} - dg = digraph_create_from_dict(d) - n = dg.find("A") - self.assertEqual("A", n.get_name()) - - def test_add_node_01(self): - """Digraph add node with two times same name""" - - dg = Digraph() - n1 = Digraph.Node("myname") - n2 = Digraph.Node("myname") - dg.add_node(n1) - try: - dg.add_node(n2) - self.assertTrue(False) - except RuntimeError: - pass - - def test_iter_outgoing_weight_01(self): - """Tests iter_outgoing in a graph with weights""" - - digraph = Digraph() - node0 = Digraph.Node("R") - digraph.add_node(node0) - node1 = Digraph.Node("A") - digraph.add_node(node1) - node2 = Digraph.Node("B") - digraph.add_node(node2) - node3 = Digraph.Node("C") - digraph.add_node(node3) - - digraph.create_edge(node0, node1, 1) - digraph.create_edge(node0, node2, 2) - digraph.create_edge(node0, node3, 3) - - self.assertEqual([node1, node2, node3], - list(node0.get_iter_outgoing())) diff --git a/diskimage_builder/tests/functional/test_graph_toposort.py b/diskimage_builder/tests/functional/test_graph_toposort.py deleted file mode 100644 index 4d30cdb9..00000000 --- a/diskimage_builder/tests/functional/test_graph_toposort.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2016 Andreas Florath (andreas@florath.net) -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import testtools - -from diskimage_builder.graph.digraph import Digraph -from diskimage_builder.graph.digraph import digraph_create_from_dict -from diskimage_builder.graph.digraph import node_list_to_node_name_list - - -class TestTopologicalSearch(testtools.TestCase): - - def test_tsort_001(self): - """Simple three node digraph""" - - dg = digraph_create_from_dict( - {"A": ["B", "C"], "B": ["C"], "C": []}) - tsort = dg.topological_sort() - tnames = node_list_to_node_name_list(tsort) - self.assertEqual(tnames, ['A', 'B', 'C'], "incorrect") - - def test_tsort_002(self): - """Zero node digraph""" - - dg = digraph_create_from_dict({}) - tsort = dg.topological_sort() - tnames = node_list_to_node_name_list(tsort) - self.assertEqual(tnames, [], "incorrect") - - def test_tsort_003(self): - """One node digraph""" - - dg = digraph_create_from_dict({"A": []}) - tsort = dg.topological_sort() - tnames = node_list_to_node_name_list(tsort) - self.assertEqual(tnames, ["A"], "incorrect") - - def test_tsort_004(self): - """More complex digraph""" - - dg = digraph_create_from_dict( - {"A": ["B", "C"], "B": ["C", "E"], "C": ["D", "E"], - "D": ["E"], "E": []}) - tsort = dg.topological_sort() - tnames = node_list_to_node_name_list(tsort) - self.assertEqual(tnames, ['A', 'B', 'C', 'D', 'E'], "incorrect") - - def test_tsort_005(self): - """Digraph with two components""" - - dg = digraph_create_from_dict({"A": ["B", "C"], "B": ["C"], "C": [], - "D": ["E"], "E": []}) - tsort = dg.topological_sort() - tnames = node_list_to_node_name_list(tsort) - # Because of two components, there exist a couple of different - # possibilities - but these are all the requirements that have - # to be fulfilled to be a correct topological sort: - self.assertTrue(tnames.index('A') < tnames.index('B')) - self.assertTrue(tnames.index('B') < tnames.index('C')) - self.assertTrue(tnames.index('D') < tnames.index('E')) - - def test_tsort_006(self): - """Complex digraph with weights""" - - digraph = Digraph() - node0 = Digraph.Node("R") - digraph.add_node(node0) - node1 = Digraph.Node("A") - digraph.add_node(node1) - node2 = Digraph.Node("B") - digraph.add_node(node2) - node3 = Digraph.Node("C") - digraph.add_node(node3) - node4 = Digraph.Node("B1") - digraph.add_node(node4) - node5 = Digraph.Node("B2") - digraph.add_node(node5) - node6 = Digraph.Node("B3") - digraph.add_node(node6) - - digraph.create_edge(node0, node1, 1) - digraph.create_edge(node0, node2, 2) - digraph.create_edge(node0, node3, 3) - - digraph.create_edge(node2, node4, 7) - digraph.create_edge(node2, node5, 14) - digraph.create_edge(node2, node6, 21) - - tsort = digraph.topological_sort() - tnames = node_list_to_node_name_list(tsort) - - # Also here: many possible solutions - self.assertTrue(tnames.index('R') < tnames.index('A')) - self.assertTrue(tnames.index('R') < tnames.index('B')) - self.assertTrue(tnames.index('R') < tnames.index('C')) - self.assertTrue(tnames.index('B') < tnames.index('B1')) - self.assertTrue(tnames.index('B') < tnames.index('B2')) - self.assertTrue(tnames.index('B') < tnames.index('B3')) - - # In addition in the weighted graph the following - # must also hold: - self.assertTrue(tnames.index('B') < tnames.index('A')) - self.assertTrue(tnames.index('C') < tnames.index('B')) - self.assertTrue(tnames.index('B2') < tnames.index('B1')) - self.assertTrue(tnames.index('B3') < tnames.index('B2'))