Semi-automatic doc generation of element dependency

Currently there is no description of dependencies in the generated
documentation of the elements: therefore a user of an element does not
know which other elements are automatically included and e.g. which
configuration options are available. In addition there are some
copy&pastes of parts of the README.rst scattered thought different
Ubuntu and Debian specific elements.

This patch adds a semi-automatic generation of dependency information
of all elements.  Nevertheless these are not automatically included.
The author of the element's README.rst can decide if and where the
dependency information should appear and can use the descriptor

.. element_deps::

for this.

This patch adds the dependency information for some Debian and
Ubuntu patches - and creates the base for later removing the
duplicated parts.

A call is added to element_dependencies._find_all_elements() to
populate reverse dependencies for Element objects.

(This is a reworking of I31d2b6050b6c46fefe37378698e9a330025db430 for
the feature/v2 branch)

Change-Id: Iebb83916fed71565071246baa550849eef40560b

diskimage_builder/element_dependencies.py

@@ -68,6 +68,24 @@ class Element(object):
             else:
                 raise
 
+    def _make_rdeps(self, all_elements):
+        """Make a list of reverse dependencies (who depends on us).
+
+           Only valid after _find_all_elements()
+
+           Arguments:
+           :param all_elements: dict as returned by _find_all_elements()
+
+           :return: nothing, but elements will have r_depends var
+        """
+
+        # note; deliberatly left out of __init__ so that accidental
+        # access without init raises error
+        self.r_depends = []
+        for name, element in all_elements.items():
+            if self.name in element.depends:
+                self.r_depends.append(element.name)
+
     def __init__(self, name, path):
         """A new element
 
@@ -208,6 +226,11 @@ def _find_all_elements(paths=None):
 
             all_elements[name] = new_element
 
+    # Now we have all the elements, make a call on each element to
+    # store it's reverse dependencies
+    for name, element in all_elements.items():
+        element._make_rdeps(all_elements)
+
     return all_elements
 
 

diskimage_builder/elements/debian-minimal/README.rst

@@ -72,3 +72,5 @@ Because there is not a one-to-one mapping of `ARCH` to a kernel package, if
 you are building an image for ARM on debian, you need to specify which kernel
 you want in the environment variable `DIB_ARM_KERNEL`. For instance, if you want
 the `linux-image-mx5` package installed, set `DIB_ARM_KERNEL` to `mx5`.
+
+.. element_deps::

diskimage_builder/elements/debian-systemd/README.rst

@@ -3,3 +3,5 @@ debian-systemd
 ==============
 You may want to use `systemd` instead of the classic sysv init system.
 In this case, include this element in your element list.
+
+.. element_deps::

diskimage_builder/elements/debian-upstart/README.rst

@@ -4,3 +4,5 @@ debian-upstart
 By default Debian will use sysvinit for booting. If you want to experiment
 with Upstart, or have need of it due to a need for upstart jobs, this
 element will build the image with upstart as the init system.
+
+.. element_deps::

diskimage_builder/elements/debian/README.rst

@@ -40,3 +40,5 @@ Because there is not a one-to-one mapping of `ARCH` to a kernel package, if
 you are building an image for ARM on debian, you need to specify which kernel
 you want in the environment variable `DIB_ARM_KERNEL`. For instance, if you want
 the `linux-image-mx5` package installed, set `DIB_ARM_KERNEL` to `mx5`.
+
+.. element_deps::

diskimage_builder/elements/debootstrap/README.rst

@@ -44,3 +44,5 @@ Because there is not a one-to-one mapping of `ARCH` to a kernel package, if
 you are building an image for ARM on debian, you need to specify which kernel
 you want in the environment variable `DIB_ARM_KERNEL`. For instance, if you want
 the `linux-image-mx5` package installed, set `DIB_ARM_KERNEL` to `mx5`.
+
+.. element_deps::

diskimage_builder/elements/dpkg/README.rst

@@ -38,3 +38,4 @@ DIB_DISABLE_APT_CLEANUP
     at the end of a dib run.
   :Example: ``DIB_DISABLE_APT_CLEANUP=1`` will disable cleanup.
 
+.. element_deps::

diskimage_builder/elements/ubuntu-core/README.rst

@@ -11,3 +11,5 @@ Overrides:
    environment variable DIB\_RELEASE
  * To use different mirrors rather than the default of archive.ubuntu.com and
    security.ubuntu.com, use the environment variable DIB\_DISTRIBUTION\_MIRROR
+
+.. element_deps::

diskimage_builder/elements/ubuntu-minimal/README.rst

@@ -30,3 +30,5 @@ it will override `DIB_APT_KEYRING` if that is used as well.
 For further information about `DIB_DEBIAN_DEBOOTSTRAP_SCRIPT`,
 `DIB_DEBIAN_USE_DEBOOTSTRAP_CACHE` and `DIB_DEBOOTSTRAP_EXTRA_ARGS`
 please consult "README.rst" of the debootstrap element.
+
+.. element_deps::

diskimage_builder/elements/ubuntu/README.rst

@@ -13,3 +13,5 @@ Overrides:
    variable.
  * To use different mirrors rather than the default of archive.ubuntu.com and
    security.ubuntu.com, use the environment variable DIB\_DISTRIBUTION\_MIRROR
+
+.. element_deps::

doc/lib/element_deps.py

@@ -0,0 +1,68 @@
+#
+# Copyright 2017 Andreas Florath (andreas@florath.net)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Scan the element directory, looks for element  dependencies and
+# writes them out when the directive
+#   .. element_deps::
+# is used.
+# This was developed only for internal use and must be called
+# from the source top directory.
+
+from diskimage_builder.element_dependencies import _find_all_elements
+from diskimage_builder.paths import get_path
+
+from docutils.parsers.rst import Directive
+import os
+
+all_elements = _find_all_elements(get_path("elements"))
+
+
+def append_dependency_list(lines, title, element, deps):
+    if not deps:
+        return
+    lines.append(title)
+    lines.append("+" * len(title))
+    for dep in deps:
+        lines.append("* :doc:`../%s/README`\n" % dep)
+
+
+class ElementDepsDirective(Directive):
+
+    # this enables content in the directive
+    has_content = True
+
+    def run(self):
+        source = self.state_machine.input_lines.source(
+            self.lineno - self.state_machine.input_offset - 1)
+        # Extract the element from the source attribute of the document
+        element_name = os.path.basename(os.path.dirname(
+            self.state_machine.document.attributes['source']))
+
+        lines = ["Element Dependencies",
+                 "--------------------"]
+
+        # This should not fail -- sphinx would be finding an element
+        # that dib doesn't know about?
+        element = all_elements[element_name]
+
+        append_dependency_list(lines, "Uses", element, element.depends)
+        append_dependency_list(lines, "Used by", element, element.r_depends)
+
+        self.state_machine.insert_input(lines, source)
+        return []
+
+
+def setup(app):
+    app.add_directive('element_deps', ElementDepsDirective)

doc/source/conf.py

@@ -4,7 +4,8 @@ import os
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['oslosphinx']
+sys.path.append(os.path.abspath('../lib'))
+extensions = ['oslosphinx', 'element_deps']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

releasenotes/notes/doc-auto-element-dependency-cb7488c5bb7301a4.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Create sphinx directive 'element_deps' that automatically
+    generates dependencies in the element documentation.

tox.ini

@@ -37,4 +37,4 @@ commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasen
 
 [flake8]
 ignore = E125,E126,E127,H202,H803
-exclude =  .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,conf.py
+exclude =  .venv,.git,.tox,dist,doc/build,doc/source,*lib/python*,*egg,build,conf.py