package-installs: provide for skip from env var

Provide a "when" option that provides for not installing packages
based on a = or != match on an environment variable.

Unit tests are added.

Change-Id: Ifa824dccaff69fd447f45d54cb4a3083bcabdd86

.testr.conf

@@ -3,7 +3,7 @@ test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
              OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
              OS_LOG_CAPTURE=${OS_LOG_CAPTURE:-1} \
              OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
-             OS_DEBUG=${OS_DEBUG:-0} \
+             OS_DEBUG=${OS_DEBUG:-1} \
              ${PYTHON:-python} -m subunit.run discover . $LISTOPT $IDOPTION
 test_id_option=--load-list $IDFILE
 test_list_option=--list

diskimage_builder/elements/package-installs/README.rst

@@ -30,6 +30,10 @@ example ``package-installs.yaml``
     dib_python_version: 2
   python3-dev:
     dib_python_version: 3
+  package-a:
+    when: DIB_USE_PACKAGE_A = 1
+  package-b:
+    when: DIB_USE_PACKAGE_A != 1
 
 example package-installs.json
 
@@ -62,6 +66,26 @@ architectures the package should be excluded from.  Either ``arch`` or
 ``not-arch`` can be given for one package - not both.  See
 documentation about the ARCH variable for more information.
 
+The ``when`` property is a simple ``=`` or ``!=`` match on a value in
+an environment variable.  If the given environment variable matches
+the operation and value, the package is installed.  If the variable is
+not available in the environment, an exception is raised (thus
+defaults will likely need to be provided in ``environment.d`` files or
+similar for flags used here).  For example, to install an extra
+package when a feature is enabled::
+
+  package:
+    when: DIB_FEATURE_FLAG=1
+
+To install ``package`` when ``DIB_FEATURE_FLAG=0`` but
+``other_package`` when ``DIB_FEATURE_FLAG=1`` (i.e. toggle between two
+packages), you can use something like::
+
+  package:
+    when: DIB_FEATURE_FLAG=0
+  other_package:
+    when: DIB_FEATURE_FLAG!=0
+
 DEPRECATED: Adding a file under your elements pre-install.d, install.d, or
 post-install.d directories called package-installs-<element-name> will cause
 the list of packages in that file to be installed at the beginning of the

diskimage_builder/elements/package-installs/bin/package-installs-squash

@@ -20,6 +20,7 @@ import functools
 import json
 import logging
 import os
+import re
 import sys
 import yaml
 
@@ -60,11 +61,54 @@ def _valid_for_arch(pkg_name, arch, not_arch):
     return not _is_arch_in_list(not_arch)
 
 
-def collect_data(data, filename, element_name):
+def _when(statement):
-    try:
+    '''evaulate a when: statement
-        objs = json.load(open(filename))
+
-    except ValueError:
+    Evaluate statements of the form
-        objs = yaml.safe_load(open(filename))
+
+     when: ENVIRONMENT_VARIABLE[!]=value
+
+    Returns True if the package should be installed, False otherwise
+
+    If the ENVIRONMENT_VARIABLE is unset, raises an error
+
+    '''
+    # No statement means install
+    if statement is None:
+        return True
+
+    # FOO =  BAR
+    # var op val
+    match = re.match(
+        r"(?P<var>[\w]+)(\s*)(?P<op>=|!=)(\s*)(?P<val>.*)", statement)
+    if not match:
+        print("Malformed when line: <%s>" % statement)
+        sys.exit(1)
+    match = match.groupdict()
+    var = match['var']
+    op = match['op']
+    val = match['val']
+
+    if var not in os.environ:
+        raise RuntimeError("The variable <%s> is not set" % var)
+
+    logger.debug("when eval %s%s%s against <%s>" %
+                 (var, op, val, os.environ[var]))
+
+    if op == '=':
+        if val == os.environ[var]:
+            return True
+    elif op == '!=':
+        if val != os.environ[var]:
+            return True
+    else:
+        print("Malformed when op: %s" % op)
+        sys.exit(1)
+
+    return False
+
+
+def collect_data(data, objs, element_name):
     for pkg_name, params in objs.items():
         if not params:
             params = {}
@@ -85,6 +129,12 @@ def collect_data(data, filename, element_name):
         valid_dib_python_version = (dib_py_version == '' or
                                     dib_py_version == dib_py_version_env)
 
+        # True means install, false skip
+        if _when(params.get('when', None)) is False:
+            logger.debug("Skipped due to when: %s/%s" %
+                         (element_name, pkg_name))
+            continue
+
         if valid_installtype and valid_arch and valid_dib_python_version:
             data[phase][install].append((pkg_name, element_name))
 
@@ -126,7 +176,12 @@ def main():
             if not os.path.exists(target_file):
                 continue
             logger.info("Squashing install file: %s" % target_file)
-            final_dict = collect_data(final_dict, target_file, element_name)
+            try:
+                objs = json.load(open(target_file))
+            except ValueError:
+                objs = yaml.safe_load(open(target_file))
+
+            final_dict = collect_data(final_dict, objs, element_name)
 
     logger.debug("final_dict -> %s" % final_dict)
 

diskimage_builder/elements/package-installs/tests/test_package_squash.py

@@ -0,0 +1,140 @@
+# Copyright 2018 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+import collections
+import functools
+import imp
+import mock
+import os
+
+from oslotest import base
+from testtools.matchers import Mismatch
+
+installs_squash_src = (os.path.dirname(os.path.realpath(__file__)) +
+                       '/../bin/package-installs-squash')
+installs_squash = imp.load_source('installs_squash', installs_squash_src)
+
+
+class IsMatchingInstallList(object):
+
+    def __init__(self, expected):
+        self.expected = expected
+
+    def match(self, actual):
+        for phase, ops in self.expected.items():
+            if phase not in actual:
+                # missing the phase
+                return Mismatch(
+                    "Phase %d does not exist in %s" % (phase, actual))
+            for op, pkgs in ops.items():
+                if op not in actual[phase]:
+                    # missing op (install/uninstall)
+                    return Mismatch(
+                        "Operation %s does not exist in %s" % (op, ops))
+                # on py2 these can be out of order, we just want a match
+                expected_phase_ops = sorted(self.expected[phase][op])
+                actual_phase_ops = sorted(actual[phase][op])
+                if expected_phase_ops != actual_phase_ops:
+                    return Mismatch(
+                        "Operation list %s does not match expected  %s" %
+                        (actual[phase][op], self.expected[phase][op]))
+
+
+class TestPackageInstall(base.BaseTestCase):
+    def setUp(self):
+        super(TestPackageInstall, self).setUp()
+        self.final_dict = collections.defaultdict(
+            functools.partial(collections.defaultdict, list))
+
+    def test_simple(self):
+        '''Test a basic package install'''
+        objs = {
+            'test_package': ''
+        }
+
+        result = installs_squash.collect_data(
+            self.final_dict, objs, 'test_element')
+
+        expected = {
+            'install.d': {
+                'install': [('test_package', 'test_element')]
+            }
+        }
+
+        self.assertThat(result, IsMatchingInstallList(expected))
+
+    @mock.patch.object(os, 'environ', dict(ARCH='arm64', **os.environ))
+    def test_arch(self):
+        '''Exercise the arch and not-arch flags'''
+        objs = {
+            'test_package': '',
+            'test_arm64_package': {
+                'arch': 'arm64'
+            },
+            'do_not_install': {
+                'not-arch': 'arm64'
+            }
+        }
+
+        result = installs_squash.collect_data(
+            self.final_dict, objs, 'test_element')
+
+        expected = {
+            'install.d': {
+                'install': [('test_package', 'test_element'),
+                            ('test_arm64_package', 'test_element')]
+            }
+        }
+
+        self.assertThat(result, IsMatchingInstallList(expected))
+
+    @mock.patch.object(os, 'environ', dict(DIB_FEATURE='1', **os.environ))
+    def test_skip_when(self):
+        '''Exercise the when flag'''
+        objs = {
+            'skipped_package': {
+                'when': 'DIB_FEATURE=0'
+            },
+            'not_skipped_package': {
+                'when': 'DIB_FEATURE=1'
+            },
+            'not_equal_package': {
+                'when': 'DIB_FEATURE!=0'
+            },
+            'not_equal_skipped_package': {
+                'when': 'DIB_FEATURE!=1'
+            },
+        }
+
+        result = installs_squash.collect_data(
+            self.final_dict, objs, 'test_element')
+
+        expected = {
+            'install.d': {
+                'install': [('not_skipped_package', 'test_element'),
+                            ('not_equal_package', 'test_element')]
+            }
+        }
+
+        self.assertThat(result, IsMatchingInstallList(expected))
+
+    def test_skip_no_var(self):
+        '''Exercise the skip_when missing variable failure case'''
+        objs = {
+            'package': {
+                'when': 'MISSING_VAR=1'
+            },
+        }
+
+        self.assertRaises(RuntimeError, installs_squash.collect_data,
+                          self.final_dict, objs, 'test_element')

releasenotes/notes/skip-packages-env-c97e7b4820f9bfda.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    The `package-installs` element now supports skipping installation
+    of packages based on an environment variable specified in the
+    config file.  See the `package-installs` element documentation for
+    full details.