From b62ed1823c98a13790cba844a1dad9463e8e7fe6 Mon Sep 17 00:00:00 2001 From: Tobias Henkel Date: Thu, 1 Mar 2018 14:04:10 +0000 Subject: [PATCH] Fix encoding issue during processing output When using the package-installs element there can be some encoding problems if the package installation emits unparsable output [1]. However in this case we just want to forward the output to the console which normally can handle this correctly. In order to fix this switch off universal_newlines processing such that we just operate on bytes. Further we have to decode the lines without setting the locale and ignoring errors. This is required because print encodes without setting the locale and thus we need to filter/modify the stream such that it doesn't crash. [1] Traceback: 2018-03-01 09:58:00.515 | Traceback (most recent call last): 2018-03-01 09:58:00.515 | File "/usr/local/bin/package-installs-v2", line 137, in 2018-03-01 09:58:00.515 | main() 2018-03-01 09:58:00.515 | File "/usr/local/bin/package-installs-v2", line 130, in main 2018-03-01 09:58:00.515 | process_output(install_args, follow=True) 2018-03-01 09:58:00.515 | for line in iter(proc.stdout.readline, ''): 2018-03-01 09:58:00.515 | File "/usr/lib/python3.5/encodings/ascii.py", line 26, in decode 2018-03-01 09:58:00.515 | return codecs.ascii_decode(input, self.errors)[0] 2018-03-01 09:58:00.515 | UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 34: ordinal not in range(128) Change-Id: Ie4af9b4523459a630cfb98d09093bfe9ef7aa61e --- .../elements/package-installs/bin/package-installs-v2 | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/diskimage_builder/elements/package-installs/bin/package-installs-v2 b/diskimage_builder/elements/package-installs/bin/package-installs-v2 index 22544000..6578683c 100755 --- a/diskimage_builder/elements/package-installs/bin/package-installs-v2 +++ b/diskimage_builder/elements/package-installs/bin/package-installs-v2 @@ -18,6 +18,7 @@ from __future__ import print_function import argparse import json +import locale import os import subprocess import sys @@ -29,19 +30,20 @@ from collections import defaultdict # if follow is set, output will be echoed to stdout def process_output(cmdline, follow=False): proc = subprocess.Popen(cmdline, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True) + stderr=subprocess.STDOUT) if follow: print("Running command: %s" % cmdline) out = "" with proc.stdout: - for line in iter(proc.stdout.readline, ''): + for line in iter(proc.stdout.readline, b''): + line = line.decode(encoding=locale.getpreferredencoding(False), + errors='backslashreplace') out += line print("> %s" % line, end="") proc.wait() print("returncode: %d" % proc.returncode) else: - out = proc.communicate()[0] + out = proc.communicate()[0].decode(errors='backslashreplace') if proc.returncode: e = subprocess.CalledProcessError(proc.returncode, cmdline)