Move elements & lib relative to diskimage_builder package

Currently we have all our elements and library files in a top-level
directory and install them into
<root>/share/diskimage-builder/[elements|lib] (where root is either /
or the root of a virtualenv).

The problem with this is that editable/development installs (pip -e)
do *not* install data_files.  Thus we have no canonical location to
look for elements -- leading to the various odd things we do such as a
whole bunch of guessing at the top of disk-image-create and having a
special test-loader in tests/test_elements.py so we can run python
unit tests on those elements that have it.

data_files is really the wrong thing to use for what are essentially
assets of the program.  data_files install works well for things like
config-files, init.d files or dropping documentation files.

By moving the elements under the diskimage_builder package, we always
know where they are relative to where we import from.  In fact,
pkg_resources has an api for this which we wrap in the new
diskimage_builder/paths.py helper [1].

We use this helper to find the correct path in the couple of places we
need to find the base-elements dir, and for the paths to import the
library shell functions.

Elements such as svc-map and pkg-map include python unit-tests, which
we do not need tests/test_elements.py to special-case load any more.
They just get found automatically by the normal subunit loader.

I have a follow-on change (I69ca3d26fede0506a6353c077c69f735c8d84d28)
to move disk-image-create to a regular python entry-point.

Unfortunately, this has to move to work with setuptools.  You'd think
a symlink under diskimage_builder/[elements|lib] would work, but it
doesn't.

[1] this API handles stuff like getting files out of .zip archive
modules, which we don't do.  Essentially for us it's returning
__file__.

Change-Id: I5e3e3c97f385b1a4ff2031a161a55b231895df5b

bin/dib-lint

@@ -21,6 +21,8 @@
 set -eu
 set -o pipefail
 
+ELEMENTS_DIR=diskimage_builder/elements
+
 parse_exclusions() {
     # Per-file exclusions
     # Example: # dib-lint: disable=sete setpipefail
@@ -63,7 +65,7 @@ echo "Running dib-lint in $(pwd)"
 rc=0
 TMPDIR=$(mktemp -d /tmp/tmp.XXXXXXXXXX)
 trap "rm -rf $TMPDIR" EXIT
-for i in $(find elements -type f   \
+for i in $(find $ELEMENTS_DIR -type f   \
                 -not -name \*~     \
                 -not -name \#\*\#  \
                 -not -name \*.orig \
@@ -180,7 +182,7 @@ done
 
 echo "Checking indents..."
 
-for i in $(find elements -type f -and -name '*.rst' -or -type f -executable); do
+for i in $(find $ELEMENTS_DIR -type f -and -name '*.rst' -or -type f -executable); do
     # Check for tab indentation
     if ! excluded tabindent; then
         if grep -q $'^ *\t' ${i}; then
@@ -196,14 +198,14 @@ for i in $(find elements -type f -and -name '*.rst' -or -type f -executable); do
 done
 
 if ! excluded mddocs; then
-    md_docs=$(find elements -name '*.md')
+    md_docs=$(find $ELEMENTS_DIR -name '*.md')
     if [ -n "$md_docs" ]; then
         error ".md docs found: $md_docs"
     fi
 fi
 
 echo "Checking YAML parsing..."
-for i in $(find elements -type f -name '*.yaml'); do
+for i in $(find $ELEMENTS_DIR -type f -name '*.yaml'); do
     echo "Parsing  $i"
     py_check="
 import yaml
@@ -218,7 +220,7 @@ except yaml.parser.ParserError:
     fi
 done
 echo "Checking pkg-map files..."
-for i in $(find elements -type f \
+for i in $(find $ELEMENTS_DIR -type f \
                 -name 'pkg-map' -a \! -executable); do
     echo "Parsing  $i"
     py_check="

bin/disk-image-create

@@ -26,69 +26,14 @@ export DIB_ENV=$(export | grep ' DIB_.*=')
 
 SCRIPTNAME=$(basename $0)
 
-# this is a bit tricky to handle the case of being installed with "pip
+export _LIB=$(python -c '
-# -e" (i.e. setuptools develop mode) and a regular install
+import diskimage_builder.paths
-#
+diskimage_builder.paths.get_path("lib")')
-# When installed normally, the scripts are installed into /usr/bin/
-# and the other bits specified in "data_files" into
-# /usr/share/diskimage-builder/[elements|lib|scripts] (if you're in a
-# virtualenv, modulo all this with the right prefix-paths)
-#
-# When installed with -e, the scripts will still be installed into
-# /usr/bin, but the data_files will *not* be installed.  Because the
-# "diskimage_builder" python module will be linked to the source repo
-# (that's the idea of develop mode) what we can do is assume the
-# following:
-#
-# - if the python module directory has a "bin" directory, then it must
-#   be the source repo and hence we have been installed via develop
-#   mode.  Thus setup ourselves to use the scripts from the source
-#   repo.
-#
-# - otherwise, try to find libraires and elements have been installed
-#   into the system paths via a "normal" pip install
-#
-# - lastly, we might be running completely uninstalled.
-#   XXX : this might cause problems at some point; we might need to "cd"
-#         so python can find things, or use pip -e.
-#
-# This means if you want to develop your elements, then "pip -e" is
-# the way to go ... your disk-image-create runs will be referencing
-# the scripts from the editable source repo.  But note that changes to
-# anything in bin/ will *not* be applied; those files have been
-# statically copied in during install.  You'll need to iterate with
-# another run of "pip install -e" if you're actually working on those
-# bin/* scripts.
 
-export SCRIPT_HOME=$(dirname $(readlink -f $0))
+_BASE_ELEMENT_DIR=$(python -c '
+import diskimage_builder.paths
+diskimage_builder.paths.get_path("elements")')
 
-_DIB_PYTHON_INSTALL=$(python -c '
-
-import inspect
-import os
-import sys
-
-# this can fail if we are being run with pwd outside the source
-# directory *and* have not been installed
-try:
-    import diskimage_builder
-except ImportError:
-    sys.exit(0)
-
-print(os.path.dirname(inspect.getfile(diskimage_builder)))')
-
-if [ -n "$_DIB_PYTHON_INSTALL"  -a -d $_DIB_PYTHON_INSTALL/../bin ]; then
-    # we have been installed with "pip -e"
-    export SCRIPT_HOME=$_DIB_PYTHON_INSTALL/../bin
-    export _PREFIX=$SCRIPT_HOME/..
-elif [ -d $SCRIPT_HOME/../share/diskimage-builder ]; then
-    # we have been installed in /usr
-    export _PREFIX=$SCRIPT_HOME/../share/diskimage-builder
-else
-    # we have not been installed in any way
-    export _PREFIX=$SCRIPT_HOME/..
-fi
-export _LIB=$_PREFIX/lib
 source $_LIB/die
 
 IS_RAMDISK=0