classes/lib: Update to use python3 command pipeline decoding

In python3, strings are unicode by default. We need to encode/decode
from command pipelines and other places where we interface with the
real world using the correct locales. This patch updates various
call sites to use the correct encoding/decodings.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

8 files changed, 29 insertions, 31 deletions

diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass
index 581d532693..e3b5c44a09 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -233,7 +233,7 @@ python buildhistory_emit_pkghistory() {
                 key = item[0]
                 if key.endswith('_' + pkg):
                     key = key[:-len(pkg)-1]
-                pkgdata[key] = item[1].decode('utf-8').decode('string_escape')
+                pkgdata[key] = item[1]
 
         pkge = pkgdata.get('PKGE', '0')
         pkgv = pkgdata['PKGV']
@@ -288,14 +288,12 @@ python buildhistory_emit_pkghistory() {
 
 
 def write_recipehistory(rcpinfo, d):
-    import codecs
-
     bb.debug(2, "Writing recipe history")
 
     pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True)
 
     infofile = os.path.join(pkghistdir, "latest")
-    with codecs.open(infofile, "w", encoding='utf8') as f:
+    with open(infofile, "w") as f:
         if rcpinfo.pe != "0":
             f.write(u"PE = %s\n" %  rcpinfo.pe)
         f.write(u"PV = %s\n" %  rcpinfo.pv)
@@ -305,8 +303,6 @@ def write_recipehistory(rcpinfo, d):
 
 
 def write_pkghistory(pkginfo, d):
-    import codecs
-
     bb.debug(2, "Writing package history for package %s" % pkginfo.name)
 
     pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True)
@@ -316,7 +312,7 @@ def write_pkghistory(pkginfo, d):
         bb.utils.mkdirhier(pkgpath)
 
     infofile = os.path.join(pkgpath, "latest")
-    with codecs.open(infofile, "w", encoding='utf8') as f:
+    with open(infofile, "w") as f:
         if pkginfo.pe != "0":
             f.write(u"PE = %s\n" %  pkginfo.pe)
         f.write(u"PV = %s\n" %  pkginfo.pv)
@@ -349,7 +345,7 @@ def write_pkghistory(pkginfo, d):
         filevarpath = os.path.join(pkgpath, "latest.%s" % filevar)
         val = pkginfo.filevars[filevar]
         if val:
-            with codecs.open(filevarpath, "w", encoding='utf8') as f:
+            with open(filevarpath, "w") as f:
                 f.write(val)
         else:
             if os.path.exists(filevarpath):
diff --git a/meta/classes/chrpath.bbclass b/meta/classes/chrpath.bbclass
index 9c68855ab2..cdd7f27600 100644
--- a/meta/classes/chrpath.bbclass
+++ b/meta/classes/chrpath.bbclass
@@ -10,6 +10,8 @@ def process_file_linux(cmd, fpath, rootdir, baseprefix, tmpdir, d):
     if p.returncode != 0:
         return
 
+    err = err.decode('utf-8')
+
     # Handle RUNPATH as well as RPATH
     err = err.replace("RUNPATH=","RPATH=")
     # Throw away everything other than the rpath list
diff --git a/meta/classes/externalsrc.bbclass b/meta/classes/externalsrc.bbclass
index da7eb4781c..b7140a321d 100644
--- a/meta/classes/externalsrc.bbclass
+++ b/meta/classes/externalsrc.bbclass
@@ -145,7 +145,7 @@ def srctree_hash_files(d):
             env = os.environ.copy()
             env['GIT_INDEX_FILE'] = tmp_index.name
             subprocess.check_output(['git', 'add', '.'], cwd=s_dir, env=env)
-            sha1 = subprocess.check_output(['git', 'write-tree'], cwd=s_dir, env=env)
+            sha1 = subprocess.check_output(['git', 'write-tree'], cwd=s_dir, env=env).decode("utf-8")
         with open(oe_hash_file, 'w') as fobj:
             fobj.write(sha1)
         ret = oe_hash_file + ':True'
diff --git a/meta/classes/insane.bbclass b/meta/classes/insane.bbclass
index 71999ada34..9b2337cdbf 100644
--- a/meta/classes/insane.bbclass
+++ b/meta/classes/insane.bbclass
@@ -399,7 +399,7 @@ def package_qa_check_unsafe_references_in_binaries(path, name, d, elf, messages)
         sysroot_path_usr = sysroot_path + exec_prefix
 
         try:
-            ldd_output = bb.process.Popen(["prelink-rtld", "--root", sysroot_path, path], stdout=sub.PIPE).stdout.read()
+            ldd_output = bb.process.Popen(["prelink-rtld", "--root", sysroot_path, path], stdout=sub.PIPE).stdout.read().decode("utf-8")
         except bb.process.CmdError:
             error_msg = pn + ": prelink-rtld aborted when processing %s" % path
             package_qa_handle_error("unsafe-references-in-binaries", error_msg, d)
@@ -986,12 +986,12 @@ def package_qa_check_expanded_d(path,name,d,elf,messages):
     return sane
 
 def package_qa_check_encoding(keys, encode, d):
-    def check_encoding(key,enc):
+    def check_encoding(key, enc):
         sane = True
         value = d.getVar(key, True)
         if value:
             try:
-                s = unicode(value, enc)
+                s = value.encode(enc)
             except UnicodeDecodeError as e:
                 error_msg = "%s has non %s characters" % (key,enc)
                 sane = False
@@ -1217,7 +1217,7 @@ Missing inherit gettext?""" % (gt, config))
         try:
             flag = "WARNING: unrecognized options:"
             log = os.path.join(d.getVar('B', True), 'config.log')
-            output = subprocess.check_output(['grep', '-F', flag, log]).replace(', ', ' ')
+            output = subprocess.check_output(['grep', '-F', flag, log]).decode("utf-8").replace(', ', ' ')
             options = set()
             for line in output.splitlines():
                 options |= set(line.partition(flag)[2].split())
diff --git a/meta/classes/libc-package.bbclass b/meta/classes/libc-package.bbclass
index 467d567923..70f479bb41 100644
--- a/meta/classes/libc-package.bbclass
+++ b/meta/classes/libc-package.bbclass
@@ -150,6 +150,7 @@ python package_do_split_gconvs () {
         c_re = re.compile('^copy "(.*)"')
         i_re = re.compile('^include "(\w+)".*')
         for l in f.readlines():
+            l = l.decode("latin-1")
             m = c_re.match(l) or i_re.match(l)
             if m:
                 dp = legitimize_package_name('%s%s-gconv-%s' % (mlprefix, bpn, m.group(1)))
@@ -171,6 +172,7 @@ python package_do_split_gconvs () {
         c_re = re.compile('^copy "(.*)"')
         i_re = re.compile('^include "(\w+)".*')
         for l in f.readlines():
+            l = l.decode("latin-1")
             m = c_re.match(l) or i_re.match(l)
             if m:
                 dp = legitimize_package_name('%s%s-charmap-%s' % (mlprefix, bpn, m.group(1)))
@@ -191,6 +193,7 @@ python package_do_split_gconvs () {
         c_re = re.compile('^copy "(.*)"')
         i_re = re.compile('^include "(\w+)".*')
         for l in f.readlines():
+            l = l.decode("latin-1")
             m = c_re.match(l) or i_re.match(l)
             if m:
                 dp = legitimize_package_name(mlprefix+bpn+'-localedata-%s' % m.group(1))
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index a4125a0e98..501004ed48 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -63,7 +63,7 @@ def legitimize_package_name(s):
     def fixutf(m):
         cp = m.group(1)
         if cp:
-            return ('\u%s' % cp).decode('unicode_escape').encode('utf-8')
+            return ('\\u%s' % cp).encode('latin-1').decode('unicode_escape')
 
     # Handle unicode codepoints encoded as <U0123>, as in glibc locale files.
     s = re.sub('<U([0-9A-Fa-f]{1,4})>', fixutf, s)
@@ -1259,8 +1259,8 @@ python emit_pkgdata() {
     def write_if_exists(f, pkg, var):
         def encode(str):
             import codecs
-            c = codecs.getencoder("string_escape")
-            return c(str)[0]
+            c = codecs.getencoder("unicode_escape")
+            return c(str)[0].decode("latin1")
 
         val = d.getVar('%s_%s' % (var, pkg), True)
         if val:
diff --git a/meta/classes/package_deb.bbclass b/meta/classes/package_deb.bbclass
index bb5220ed00..e35f427ea2 100644
--- a/meta/classes/package_deb.bbclass
+++ b/meta/classes/package_deb.bbclass
@@ -173,7 +173,7 @@ python do_package_deb () {
                 # Special behavior for description...
                 if 'DESCRIPTION' in fs:
                      summary = localdata.getVar('SUMMARY', True) or localdata.getVar('DESCRIPTION', True) or "."
-                     ctrlfile.write('Description: %s\n' % unicode(summary,'utf-8'))
+                     ctrlfile.write('Description: %s\n' % summary)
                      description = localdata.getVar('DESCRIPTION', True) or "."
                      description = textwrap.dedent(description).strip()
                      if '\\n' in description:
@@ -182,29 +182,25 @@ python do_package_deb () {
                              # We don't limit the width when manually indent, but we do
                              # need the textwrap.fill() to set the initial_indent and
                              # subsequent_indent, so set a large width
-                             ctrlfile.write('%s\n' % unicode(textwrap.fill(t, width=100000, initial_indent=' ', subsequent_indent=' '),'utf-8'))
+                             ctrlfile.write('%s\n' % textwrap.fill(t, width=100000, initial_indent=' ', subsequent_indent=' '))
                      else:
                          # Auto indent
-                         ctrlfile.write('%s\n' % unicode(textwrap.fill(description.strip(), width=74, initial_indent=' ', subsequent_indent=' '),'utf-8'))
+                         ctrlfile.write('%s\n' % textwrap.fill(description.strip(), width=74, initial_indent=' ', subsequent_indent=' '))
 
                 else:
-                     ctrlfile.write(unicode(c % tuple(pullData(fs, localdata)),'utf-8'))
+                     ctrlfile.write(c % tuple(pullData(fs, localdata)))
         except KeyError:
             import sys
             (type, value, traceback) = sys.exc_info()
             bb.utils.unlockfile(lf)
             ctrlfile.close()
             raise bb.build.FuncFailed("Missing field for deb generation: %s" % value)
-        except UnicodeDecodeError:
-            bb.utils.unlockfile(lf)
-            ctrlfile.close()
-            raise bb.build.FuncFailed("Non UTF-8 characters found in one of the fields")
 
         # more fields
 
         custom_fields_chunk = get_package_additional_metadata("deb", localdata)
         if custom_fields_chunk is not None:
-            ctrlfile.write(unicode(custom_fields_chunk))
+            ctrlfile.write(custom_fields_chunk)
             ctrlfile.write("\n")
 
         mapping_rename_hook(localdata)
@@ -255,17 +251,17 @@ python do_package_deb () {
         rconflicts = bb.utils.explode_dep_versions2(localdata.getVar("RCONFLICTS", True) or "")
         debian_cmp_remap(rconflicts)
         if rdepends:
-            ctrlfile.write("Depends: %s\n" % unicode(bb.utils.join_deps(rdepends)))
+            ctrlfile.write("Depends: %s\n" % bb.utils.join_deps(rdepends))
         if rsuggests:
-            ctrlfile.write("Suggests: %s\n" % unicode(bb.utils.join_deps(rsuggests)))
+            ctrlfile.write("Suggests: %s\n" % bb.utils.join_deps(rsuggests))
         if rrecommends:
-            ctrlfile.write("Recommends: %s\n" % unicode(bb.utils.join_deps(rrecommends)))
+            ctrlfile.write("Recommends: %s\n" % bb.utils.join_deps(rrecommends))
         if rprovides:
-            ctrlfile.write("Provides: %s\n" % unicode(bb.utils.join_deps(rprovides)))
+            ctrlfile.write("Provides: %s\n" % bb.utils.join_deps(rprovides))
         if rreplaces:
-            ctrlfile.write("Replaces: %s\n" % unicode(bb.utils.join_deps(rreplaces)))
+            ctrlfile.write("Replaces: %s\n" % bb.utils.join_deps(rreplaces))
         if rconflicts:
-            ctrlfile.write("Conflicts: %s\n" % unicode(bb.utils.join_deps(rconflicts)))
+            ctrlfile.write("Conflicts: %s\n" % bb.utils.join_deps(rconflicts))
         ctrlfile.close()
 
         for script in ["preinst", "postinst", "prerm", "postrm"]:
diff --git a/meta/classes/toaster.bbclass b/meta/classes/toaster.bbclass
index 1a70f14a92..1878fe095d 100644
--- a/meta/classes/toaster.bbclass
+++ b/meta/classes/toaster.bbclass
@@ -33,6 +33,7 @@ python toaster_layerinfo_dumpdata() {
 
     def _get_git_branch(layer_path):
         branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0]
+        branch = branch.decode('utf-8')
         branch = branch.replace('refs/heads/', '').rstrip()
         return branch