Merge "Fix package-installs-v2 output"

diskimage_builder/elements/package-installs/bin/package-installs-v2

@@ -18,14 +18,18 @@ from __future__ import print_function
 
 import argparse
 import json
+import os
 import subprocess
 import sys
 
+from collections import defaultdict
+
 
 # run a command, return output
 #  if follow is set, output will be echoed to stdout
 def process_output(cmdline, follow=False):
     proc = subprocess.Popen(cmdline, stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT,
                             universal_newlines=True)
     if follow:
         print("Running command: %s" % cmdline)
@@ -48,6 +52,8 @@ def process_output(cmdline, follow=False):
 
 
 def main():
+    dbg_level = int(os.getenv('DIB_DEBUG_TRACE', '0'))
+
     parser = argparse.ArgumentParser(
         description="Install or uninstall packages for a specific phase based"
                     " on package-installs files.")
@@ -75,13 +81,31 @@ def main():
         print("Nothing to %s" % install)
         sys.exit(0)
 
+    # sort the list by element, this way we only do one pkg-map call
+    # per element
+    by_element = defaultdict(list)
     for (pkg, element) in install_packages:
-        print("%sing %s from %s" % (install, pkg, element))
+        by_element[element].append(pkg)
-        pkg_map_args = ['pkg-map', '--missing-ok', '--element', element, pkg]
+
+    for element, packages in by_element.items():
+        print("Map %s for %s: %s" % (install, element, ', '.join(packages)))
+
+        # Only trace pkg-map for higher levels of debugging.  Note
+        # that pkg-map debug output comes out on stderr, which is
+        # captured into the output by process_output.  We filter by
+        # "prefix" so we don't think the debug lines are packages!
+        pkg_map_args = ['pkg-map', '--prefix', '-',
+                        '--missing-ok', '--element', element]
+        if dbg_level > 1:
+            pkg_map_args.append('--debug')
+        pkg_map_args.extend(packages)
 
         try:
-            map_output = process_output(pkg_map_args)
+            follow = True if dbg_level > 1 else False
-            pkgs.extend(map_output.strip().split('\n'))
+            map_output = process_output(pkg_map_args, follow=follow)
+            map_output = map_output.strip().split('\n')
+            map_output = [m[1:] for m in map_output if m.startswith('-')]
+            pkgs.extend(map_output)
         except subprocess.CalledProcessError as e:
             if e.returncode == 1:
                 if args.noop:
@@ -105,7 +129,7 @@ def main():
         try:
             process_output(install_args, follow=True)
         except subprocess.CalledProcessError as e:
-            print("install failed with error %s" % e.output)
+            print("install-packages failed with returncode %d" % e.returncode)
             sys.exit(1)
 
 

diskimage_builder/elements/pkg-map/bin/pkg-map

@@ -18,13 +18,10 @@ import argparse
 import json
 import logging
 import os
-import pprint
 import sys
 
 
-def eprint(msg):
+log = logging.getLogger()
-    sys.stderr.write(msg)
-    sys.stderr.write("\n")
 
 
 def os_family(distro):
@@ -60,47 +57,61 @@ def main():
                         help='Do not consider missing mappings an error.'
                              ' Causes packages where no mapping is set to be'
                              ' printed.')
+    # This tool has traditionally output status and debug messages on
+    # stderr.  The problem is if a caller has stderr > stdout then
+    # actual output gets messed in with the logs.  This allows callers
+    # to disambiguate actual output by specifying a unique prefix.
+    parser.add_argument('--prefix', default='',
+                        help='Output mapped packages with this prefix')
     parser.add_argument('--debug', dest='debug', action="store_true",
                         help="Enable debugging output")
 
     args, extra = parser.parse_known_args()
 
-    if args.debug:
+    # Logs have traditionally gone to stderr with this tool.  Maintain
-        logging.basicConfig(level=logging.DEBUG)
+    # compatability
+    level = logging.DEBUG if args.debug else logging.INFO
+    logging.basicConfig(stream=sys.stderr, level=level)
 
     if not args.element and not args.pkg_map:
-        eprint('Please specify an --element argument.')
+        log.error('Please specify an --element argument.')
         sys.exit(1)
 
     if args.element and args.pkg_map:
-        eprint('Specify either --element or --pkg-map')
+        log.error('Specify either --element or --pkg-map')
         sys.exit(1)
 
     if not args.distro:
-        eprint('Please specify a --distro argument or set DISTRO_NAME.')
+        log.error('Please specify a --distro argument or set DISTRO_NAME.')
         sys.exit(1)
 
     if args.pkg_map:
+        # specifying the pkg-map by hand is just for manual testing
+        element = "<%s>" % args.pkg_map
         map_file = args.pkg_map
     else:
-        map_file = '/usr/share/pkg-map/%s' % args.element
+        element = args.element
+        map_file = '/usr/share/pkg-map/%s' % element
+
+    log.info("Mapping for %s : %s" % (element, ' '.join(extra)))
 
-    logging.debug("Map file is %s" % map_file)
     if not os.path.exists(map_file):
-        if os.environ.get('DIB_DEBUG_TRACE', '0') != '0':
-            eprint('Map file for %s element does not exist.' % args.element)
         if args.missing_ok:
+            log.info("No package map for %s, done" % element)
             for name in extra:
-                print(name)
+                print('%s%s' % (args.prefix, name))
             sys.exit(0)
-        sys.exit(2)
+        else:
+            log.error('Required pkg-map for %s element does not exist.'
+                      % args.element)
+            sys.exit(2)
 
     with open(map_file) as fd:
         try:
             package_names = json.loads(fd.read())
-            logging.debug(pprint.pformat(package_names))
+            # log.debug(pprint.pformat(package_names))
         except ValueError:
-            eprint('Unable to parse %s' % map_file)
+            log.error('Unable to parse %s' % map_file)
             raise
 
     # Parse mapping data in this form using release/distro/family/default
@@ -158,21 +169,31 @@ def main():
         except KeyError:
             pass
 
+    # log.debug(pprint.pformat(name_map))
+
     for name in extra:
         pkg_name = name_map.get(name)
         if pkg_name:
-            print(pkg_name)
+            log.debug("map %s -> %s" % (name, pkg_name))
+            print('%s%s' % (args.prefix, pkg_name))
         elif name in name_map:
+            log.debug("map %s -> <skip>" % (name))
             continue
         else:
-            err_msg = 'Missing package name for distro/element: %s/%s'
-            eprint(err_msg % (args.distro, args.element))
             if args.missing_ok:
-                print(name)
+                log.debug("pass -> %s" % (name))
+                print('%s%s' % (args.prefix, name))
             else:
+                log.error("%s has no valid mapping for package %s" %
+                          (element, name))
                 sys.exit(1)
 
     sys.exit(0)
 
 if __name__ == '__main__':
     main()
+
+# Tell emacs to use python-mode
+# Local variables:
+# mode: python
+# End: