From c52c383f1bbbff137b5f8324ddbf62c2416c940f Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Wed, 21 Nov 2018 10:20:34 +1100 Subject: [PATCH] package-installs: provide for skip from env var Provide a "when" option that provides for not installing packages based on a = or != match on an environment variable. Unit tests are added. Change-Id: Ifa824dccaff69fd447f45d54cb4a3083bcabdd86 --- .testr.conf | 2 +- .../elements/package-installs/README.rst | 24 +++ .../elements/package-installs/__init__.py | 0 .../bin/package-installs-squash | 67 ++++++++- .../package-installs/tests/__init__.py | 0 .../tests/test_package_squash.py | 140 ++++++++++++++++++ .../skip-packages-env-c97e7b4820f9bfda.yaml | 7 + 7 files changed, 233 insertions(+), 7 deletions(-) create mode 100644 diskimage_builder/elements/package-installs/__init__.py create mode 100644 diskimage_builder/elements/package-installs/tests/__init__.py create mode 100644 diskimage_builder/elements/package-installs/tests/test_package_squash.py create mode 100644 releasenotes/notes/skip-packages-env-c97e7b4820f9bfda.yaml diff --git a/.testr.conf b/.testr.conf index d7b8b32f..f2d50751 100644 --- a/.testr.conf +++ b/.testr.conf @@ -3,7 +3,7 @@ test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ OS_LOG_CAPTURE=${OS_LOG_CAPTURE:-1} \ OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ - OS_DEBUG=${OS_DEBUG:-0} \ + OS_DEBUG=${OS_DEBUG:-1} \ ${PYTHON:-python} -m subunit.run discover . $LISTOPT $IDOPTION test_id_option=--load-list $IDFILE test_list_option=--list diff --git a/diskimage_builder/elements/package-installs/README.rst b/diskimage_builder/elements/package-installs/README.rst index 9b453c7c..329d9c48 100644 --- a/diskimage_builder/elements/package-installs/README.rst +++ b/diskimage_builder/elements/package-installs/README.rst @@ -30,6 +30,10 @@ example ``package-installs.yaml`` dib_python_version: 2 python3-dev: dib_python_version: 3 + package-a: + when: DIB_USE_PACKAGE_A = 1 + package-b: + when: DIB_USE_PACKAGE_A != 1 example package-installs.json @@ -62,6 +66,26 @@ architectures the package should be excluded from. Either ``arch`` or ``not-arch`` can be given for one package - not both. See documentation about the ARCH variable for more information. +The ``when`` property is a simple ``=`` or ``!=`` match on a value in +an environment variable. If the given environment variable matches +the operation and value, the package is installed. If the variable is +not available in the environment, an exception is raised (thus +defaults will likely need to be provided in ``environment.d`` files or +similar for flags used here). For example, to install an extra +package when a feature is enabled:: + + package: + when: DIB_FEATURE_FLAG=1 + +To install ``package`` when ``DIB_FEATURE_FLAG=0`` but +``other_package`` when ``DIB_FEATURE_FLAG=1`` (i.e. toggle between two +packages), you can use something like:: + + package: + when: DIB_FEATURE_FLAG=0 + other_package: + when: DIB_FEATURE_FLAG!=0 + DEPRECATED: Adding a file under your elements pre-install.d, install.d, or post-install.d directories called package-installs- will cause the list of packages in that file to be installed at the beginning of the diff --git a/diskimage_builder/elements/package-installs/__init__.py b/diskimage_builder/elements/package-installs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/diskimage_builder/elements/package-installs/bin/package-installs-squash b/diskimage_builder/elements/package-installs/bin/package-installs-squash index ee142f35..36f1ddc1 100755 --- a/diskimage_builder/elements/package-installs/bin/package-installs-squash +++ b/diskimage_builder/elements/package-installs/bin/package-installs-squash @@ -20,6 +20,7 @@ import functools import json import logging import os +import re import sys import yaml @@ -60,11 +61,54 @@ def _valid_for_arch(pkg_name, arch, not_arch): return not _is_arch_in_list(not_arch) -def collect_data(data, filename, element_name): - try: - objs = json.load(open(filename)) - except ValueError: - objs = yaml.safe_load(open(filename)) +def _when(statement): + '''evaulate a when: statement + + Evaluate statements of the form + + when: ENVIRONMENT_VARIABLE[!]=value + + Returns True if the package should be installed, False otherwise + + If the ENVIRONMENT_VARIABLE is unset, raises an error + + ''' + # No statement means install + if statement is None: + return True + + # FOO = BAR + # var op val + match = re.match( + r"(?P[\w]+)(\s*)(?P=|!=)(\s*)(?P.*)", statement) + if not match: + print("Malformed when line: <%s>" % statement) + sys.exit(1) + match = match.groupdict() + var = match['var'] + op = match['op'] + val = match['val'] + + if var not in os.environ: + raise RuntimeError("The variable <%s> is not set" % var) + + logger.debug("when eval %s%s%s against <%s>" % + (var, op, val, os.environ[var])) + + if op == '=': + if val == os.environ[var]: + return True + elif op == '!=': + if val != os.environ[var]: + return True + else: + print("Malformed when op: %s" % op) + sys.exit(1) + + return False + + +def collect_data(data, objs, element_name): for pkg_name, params in objs.items(): if not params: params = {} @@ -85,6 +129,12 @@ def collect_data(data, filename, element_name): valid_dib_python_version = (dib_py_version == '' or dib_py_version == dib_py_version_env) + # True means install, false skip + if _when(params.get('when', None)) is False: + logger.debug("Skipped due to when: %s/%s" % + (element_name, pkg_name)) + continue + if valid_installtype and valid_arch and valid_dib_python_version: data[phase][install].append((pkg_name, element_name)) @@ -126,7 +176,12 @@ def main(): if not os.path.exists(target_file): continue logger.info("Squashing install file: %s" % target_file) - final_dict = collect_data(final_dict, target_file, element_name) + try: + objs = json.load(open(target_file)) + except ValueError: + objs = yaml.safe_load(open(target_file)) + + final_dict = collect_data(final_dict, objs, element_name) logger.debug("final_dict -> %s" % final_dict) diff --git a/diskimage_builder/elements/package-installs/tests/__init__.py b/diskimage_builder/elements/package-installs/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/diskimage_builder/elements/package-installs/tests/test_package_squash.py b/diskimage_builder/elements/package-installs/tests/test_package_squash.py new file mode 100644 index 00000000..de7779e3 --- /dev/null +++ b/diskimage_builder/elements/package-installs/tests/test_package_squash.py @@ -0,0 +1,140 @@ +# Copyright 2018 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import collections +import functools +import imp +import mock +import os + +from oslotest import base +from testtools.matchers import Mismatch + +installs_squash_src = (os.path.dirname(os.path.realpath(__file__)) + + '/../bin/package-installs-squash') +installs_squash = imp.load_source('installs_squash', installs_squash_src) + + +class IsMatchingInstallList(object): + + def __init__(self, expected): + self.expected = expected + + def match(self, actual): + for phase, ops in self.expected.items(): + if phase not in actual: + # missing the phase + return Mismatch( + "Phase %d does not exist in %s" % (phase, actual)) + for op, pkgs in ops.items(): + if op not in actual[phase]: + # missing op (install/uninstall) + return Mismatch( + "Operation %s does not exist in %s" % (op, ops)) + # on py2 these can be out of order, we just want a match + expected_phase_ops = sorted(self.expected[phase][op]) + actual_phase_ops = sorted(actual[phase][op]) + if expected_phase_ops != actual_phase_ops: + return Mismatch( + "Operation list %s does not match expected %s" % + (actual[phase][op], self.expected[phase][op])) + + +class TestPackageInstall(base.BaseTestCase): + def setUp(self): + super(TestPackageInstall, self).setUp() + self.final_dict = collections.defaultdict( + functools.partial(collections.defaultdict, list)) + + def test_simple(self): + '''Test a basic package install''' + objs = { + 'test_package': '' + } + + result = installs_squash.collect_data( + self.final_dict, objs, 'test_element') + + expected = { + 'install.d': { + 'install': [('test_package', 'test_element')] + } + } + + self.assertThat(result, IsMatchingInstallList(expected)) + + @mock.patch.object(os, 'environ', dict(ARCH='arm64', **os.environ)) + def test_arch(self): + '''Exercise the arch and not-arch flags''' + objs = { + 'test_package': '', + 'test_arm64_package': { + 'arch': 'arm64' + }, + 'do_not_install': { + 'not-arch': 'arm64' + } + } + + result = installs_squash.collect_data( + self.final_dict, objs, 'test_element') + + expected = { + 'install.d': { + 'install': [('test_package', 'test_element'), + ('test_arm64_package', 'test_element')] + } + } + + self.assertThat(result, IsMatchingInstallList(expected)) + + @mock.patch.object(os, 'environ', dict(DIB_FEATURE='1', **os.environ)) + def test_skip_when(self): + '''Exercise the when flag''' + objs = { + 'skipped_package': { + 'when': 'DIB_FEATURE=0' + }, + 'not_skipped_package': { + 'when': 'DIB_FEATURE=1' + }, + 'not_equal_package': { + 'when': 'DIB_FEATURE!=0' + }, + 'not_equal_skipped_package': { + 'when': 'DIB_FEATURE!=1' + }, + } + + result = installs_squash.collect_data( + self.final_dict, objs, 'test_element') + + expected = { + 'install.d': { + 'install': [('not_skipped_package', 'test_element'), + ('not_equal_package', 'test_element')] + } + } + + self.assertThat(result, IsMatchingInstallList(expected)) + + def test_skip_no_var(self): + '''Exercise the skip_when missing variable failure case''' + objs = { + 'package': { + 'when': 'MISSING_VAR=1' + }, + } + + self.assertRaises(RuntimeError, installs_squash.collect_data, + self.final_dict, objs, 'test_element') diff --git a/releasenotes/notes/skip-packages-env-c97e7b4820f9bfda.yaml b/releasenotes/notes/skip-packages-env-c97e7b4820f9bfda.yaml new file mode 100644 index 00000000..a5414a8e --- /dev/null +++ b/releasenotes/notes/skip-packages-env-c97e7b4820f9bfda.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + The `package-installs` element now supports skipping installation + of packages based on an environment variable specified in the + config file. See the `package-installs` element documentation for + full details.