From 23bdf6884b886c43aa4997da3a9bccdcb00954a6 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Feb 27 2019 15:06:27 +0000
Subject: [PATCH 1/5] expect string keys in fixEncodingRecurse


---

diff --git a/hub/kojihub.py b/hub/kojihub.py
index edf82e8..4d01ce3 100644
--- a/hub/kojihub.py
+++ b/hub/kojihub.py
@@ -10253,7 +10253,8 @@ class RootExports(object):
             raise koji.GenericError('either rpmID or taskID and filepath must be specified')
 
         headers = koji.get_header_fields(rpm_path, headers)
-        return koji.fixEncodingRecurse(headers, remove_nonprintable=True)
+        return koji.fixEncodingRecurse(headers, remove_nonprintable=True,
+                ignore_keys=True)
 
     queryRPMSigs = staticmethod(query_rpm_sigs)
 
diff --git a/koji/__init__.py b/koji/__init__.py
index f93d9ea..0a84ba0 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -3110,7 +3110,7 @@ def fixEncoding(value, fallback='iso8859-15', remove_nonprintable=False):
         return s
 
 
-def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False):
+def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False, ignore_keys=False):
     """Recursively fix string encoding in an object
 
     Similar behavior to fixEncoding, but recursive
@@ -3123,7 +3123,9 @@ def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False):
         ret = {}
         for k in value:
             v = fixEncodingRecurse(value[k], fallback=fallback, remove_nonprintable=remove_nonprintable)
-            k = fixEncodingRecurse(k, fallback=fallback, remove_nonprintable=remove_nonprintable)
+            # expect, that keys are never really binary
+            if not ignore_keys:
+                k = fixEncodingRecurse(k, fallback=fallback, remove_nonprintable=remove_nonprintable)
             ret[k] = v
         return ret
     elif isinstance(value, six.text_type):

From 6356aa505c1947c725719c4eaea5ccdf00a610f9 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Feb 27 2019 15:06:27 +0000
Subject: [PATCH 2/5] convert xmlrpc.Binary to string before fixEncoding


Fixes: https://pagure.io/koji/issue/1282

---

diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py
index 04248dc..460f7d1 100644
--- a/www/kojiweb/index.py
+++ b/www/kojiweb/index.py
@@ -1382,9 +1382,9 @@ def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-i
             values['enhances'] = server.getRPMDeps(rpm['id'], koji.DEP_ENHANCE)
             values['enhances'].sort(key=_sortbyname)
         headers = server.getRPMHeaders(rpm['id'], headers=['summary', 'description', 'license'])
-        values['summary'] = koji.fixEncoding(headers.get('summary'))
-        values['description'] = koji.fixEncoding(headers.get('description'))
-        values['license'] = koji.fixEncoding(headers.get('license'))
+        values['summary'] = koji.fixEncoding(str(headers.get('summary')))
+        values['description'] = koji.fixEncoding(str(headers.get('description')))
+        values['license'] = koji.fixEncoding(str(headers.get('license')))
     buildroots = kojiweb.util.paginateMethod(server, values, 'listBuildroots', kw={'rpmID': rpm['id']},
                                              start=buildrootStart, dataName='buildroots', prefix='buildroot',
                                              order=buildrootOrder)

From 10ee8905f532a68a96cb889e23af99625266af7a Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Feb 27 2019 16:11:39 +0000
Subject: [PATCH 3/5] fix typo


---

diff --git a/koji/daemon.py b/koji/daemon.py
index 5cb255e..3d9766f 100644
--- a/koji/daemon.py
+++ b/koji/daemon.py
@@ -135,7 +135,7 @@ def log_output(session, path, args, outfile, uploadpath, cwd=None, logerror=0, a
             if fd:
                 try:
                     if six.PY3:
-                        os.write(fs, msg.encode('utf-8'))
+                        os.write(fd, msg.encode('utf-8'))
                     else:
                         os.write(fd, msg)
                     os.close(fd)

From 7842556c43d2bfeb6a86c2e69864ee53d6cb65b2 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Feb 27 2019 16:13:40 +0000
Subject: [PATCH 4/5] skip fixEncoding for py3


---

diff --git a/hub/kojihub.py b/hub/kojihub.py
index 4d01ce3..7472fd5 100644
--- a/hub/kojihub.py
+++ b/hub/kojihub.py
@@ -4355,10 +4355,7 @@ def _get_zipfile_list(archive_id, zippath):
         return result
     with zipfile.ZipFile(zippath, 'r') as archive:
         for entry in archive.infolist():
-            if six.PY2:
-                filename = koji.fixEncoding(entry.filename)
-            else:
-                filename = entry.filename
+            filename = koji.fixEncoding(entry.filename)
             result.append({'archive_id': archive_id,
                            'name': filename,
                            'size': entry.file_size,
@@ -4383,10 +4380,7 @@ def _get_tarball_list(archive_id, tarpath):
         return result
     with tarfile.open(tarpath, 'r') as archive:
         for entry in archive:
-            if six.PY2:
-                filename = koji.fixEncoding(entry.name)
-            else:
-                filename = entry.name
+            filename = koji.fixEncoding(entry.name)
             result.append({'archive_id': archive_id,
                            'name': filename,
                            'size': entry.size,
@@ -6362,10 +6356,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No
     archiveinfo = {'buildroot_id': buildroot_id}
     archiveinfo['build_id'] = buildinfo['id']
     if metadata_only:
-        if six.PY2:
-            filename = koji.fixEncoding(fileinfo['filename'])
-        else:
-            filename = fileinfo['filename']
+        filename = koji.fixEncoding(fileinfo['filename'])
         archiveinfo['filename'] = filename
         archiveinfo['size'] = fileinfo['filesize']
         archiveinfo['checksum'] = fileinfo['checksum']
@@ -6376,10 +6367,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No
         archiveinfo['checksum_type'] = koji.CHECKSUM_TYPES[fileinfo['checksum_type']]
         archiveinfo['metadata_only'] = True
     else:
-        if six.PY2:
-            filename = koji.fixEncoding(os.path.basename(filepath))
-        else:
-            filename = os.path.basename(filepath)
+        filename = koji.fixEncoding(os.path.basename(filepath))
         archiveinfo['filename'] = filename
         archiveinfo['size'] = os.path.getsize(filepath)
         # trust values computed on hub (CG_Importer.prep_outputs)
@@ -6501,8 +6489,7 @@ def _import_archive_file(filepath, destdir):
     be created.
     """
     fname = os.path.basename(filepath)
-    if six.PY2:
-        fname = koji.fixEncoding(fname)
+    fname = koji.fixEncoding(fname)
     final_path = "%s/%s" % (destdir, fname)
     if os.path.exists(final_path):
         raise koji.GenericError("Error importing archive file, %s already exists" % final_path)
@@ -7687,10 +7674,7 @@ def parse_json(value, desc=None, errstr=None):
     if value is None:
         return value
     try:
-        if six.PY2:
-            return koji.fixEncodingRecurse(json.loads(value))
-        else:
-            return json.loads(value)
+        return koji.fixEncodingRecurse(json.loads(value))
     except Exception:
         if errstr is None:
             if desc is None:
@@ -9691,9 +9675,8 @@ class RootExports(object):
         for (cltime, clname, cltext) in zip(fields['changelogtime'], fields['changelogname'],
                                             fields['changelogtext']):
             cldate = datetime.datetime.fromtimestamp(cltime).isoformat(' ')
-            if six.PY2:
-                clname = koji.fixEncoding(clname)
-                cltext = koji.fixEncoding(cltext)
+            clname = koji.fixEncoding(clname)
+            cltext = koji.fixEncoding(cltext)
 
             if author and author != clname:
                 continue
@@ -10253,8 +10236,7 @@ class RootExports(object):
             raise koji.GenericError('either rpmID or taskID and filepath must be specified')
 
         headers = koji.get_header_fields(rpm_path, headers)
-        return koji.fixEncodingRecurse(headers, remove_nonprintable=True,
-                ignore_keys=True)
+        return koji.fixEncodingRecurse(headers, remove_nonprintable=True)
 
     queryRPMSigs = staticmethod(query_rpm_sigs)
 
diff --git a/koji/__init__.py b/koji/__init__.py
index 0a84ba0..a004d7d 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -3063,10 +3063,16 @@ def _taskLabel(taskInfo):
         return '%s (%s)' % (method, arch)
 
 CONTROL_CHARS = [chr(i) for i in range(32)]
-NONPRINTABLE_CHARS = six.b(''.join([c for c in CONTROL_CHARS if c not in '\r\n\t']))
+NONPRINTABLE_CHARS = ''.join([c for c in CONTROL_CHARS if c not in '\r\n\t'])
+if six.PY3:
+    NONPRINTABLE_CHARS_TABLE = dict.fromkeys(map(ord, NONPRINTABLE_CHARS), None)
 def removeNonprintable(value):
     # expects raw-encoded string, not unicode
-    return value.translate(None, NONPRINTABLE_CHARS)
+    if six.PY2:
+        return value.translate(None, NONPRINTABLE_CHARS)
+    else:
+        return value.translate(NONPRINTABLE_CHARS_TABLE)
+
 
 
 def _fix_print(value):
@@ -3089,6 +3095,12 @@ def fixEncoding(value, fallback='iso8859-15', remove_nonprintable=False):
     If value is not valid UTF-8 to begin with, assume it is
     encoded in the 'fallback' charset.
     """
+    if six.PY3:
+        if remove_nonprintable:
+            return removeNonprintable(value)
+        else:
+            return value
+
     if not value:
         return six.b('')
 
@@ -3110,11 +3122,15 @@ def fixEncoding(value, fallback='iso8859-15', remove_nonprintable=False):
         return s
 
 
-def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False, ignore_keys=False):
+def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False):
     """Recursively fix string encoding in an object
 
     Similar behavior to fixEncoding, but recursive
     """
+    if six.PY3:
+        # don't bother with fixing in py3
+        return value
+
     if isinstance(value, tuple):
         return tuple([fixEncodingRecurse(x, fallback=fallback, remove_nonprintable=remove_nonprintable) for x in value])
     elif isinstance(value, list):
@@ -3123,9 +3139,7 @@ def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False, 
         ret = {}
         for k in value:
             v = fixEncodingRecurse(value[k], fallback=fallback, remove_nonprintable=remove_nonprintable)
-            # expect, that keys are never really binary
-            if not ignore_keys:
-                k = fixEncodingRecurse(k, fallback=fallback, remove_nonprintable=remove_nonprintable)
+            k = fixEncodingRecurse(k, fallback=fallback, remove_nonprintable=remove_nonprintable)
             ret[k] = v
         return ret
     elif isinstance(value, six.text_type):
diff --git a/tests/test_builder/test_build_notification.py b/tests/test_builder/test_build_notification.py
index 3abb8a9..0d35f6e 100644
--- a/tests/test_builder/test_build_notification.py
+++ b/tests/test_builder/test_build_notification.py
@@ -100,5 +100,5 @@ class TestBuildNotification(unittest.TestCase):
         self.assertEqual(recipients, ["user@example.com"])
         fn = os.path.join(os.path.dirname(__file__), 'data/calls', 'build_notif_1', 'message.txt')
         with open(fn, 'rb') as fp:
-            msg_expect = fp.read()
+            msg_expect = fp.read().decode()
         self.assertEqual(message, msg_expect)
diff --git a/tests/test_lib/test_fixEncoding.py b/tests/test_lib/test_fixEncoding.py
index 0388a99..e53622e 100644
--- a/tests/test_lib/test_fixEncoding.py
+++ b/tests/test_lib/test_fixEncoding.py
@@ -18,52 +18,48 @@ class FixEncodingTestCase(unittest.TestCase):
 
     simple_values = [
         # [ value, fixed ]
-        ['', six.b('')],
-        [u'', six.b('')],
-        [u'góðan daginn', six.b('g\xc3\xb3\xc3\xb0an daginn')],
-        [u'hej', six.b('hej')],
-        [u'zdravstvuite', six.b('zdravstvuite')],
-        [u'céad míle fáilte', six.b('c\xc3\xa9ad m\xc3\xadle f\xc3\xa1ilte')],
-        [u'dobrý den', six.b('dobr\xc3\xbd den')],
-        [u'hylô', six.b('hyl\xc3\xb4')],
-        [u'jó napot', six.b('j\xc3\xb3 napot')],
-        [u'tervehdys', six.b('tervehdys')],
-        [u'olá', six.b('ol\xc3\xa1')],
-        [u'grüezi', six.b('gr\xc3\xbcezi')],
-        [u'dobre dan', six.b('dobre dan')],
-        [u'hello', six.b('hello')],
-        [u'bună ziua', six.b('bun\xc4\x83 ziua')],
-        [u'こんにちは', six.b('\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf')],
-        [u'你好', six.b('\xe4\xbd\xa0\xe5\xa5\xbd')],
-        [u'नमस्कार',  six.b('\xe0\xa4\xa8\xe0\xa4\xae\xe0\xa4\xb8\xe0\xa5\x8d\xe0\xa4\x95\xe0\xa4\xbe\xe0\xa4\xb0')],
-        [u'안녕하세요', six.b('\xec\x95\x88\xeb\x85\x95\xed\x95\x98\xec\x84\xb8\xec\x9a\x94')],
+        ['', ''],
+        [u'', ''],
+        [u'góðan daginn', 'g\xc3\xb3\xc3\xb0an daginn'],
+        [u'hej', 'hej'],
+        [u'zdravstvuite', 'zdravstvuite'],
+        [u'céad míle fáilte', 'c\xc3\xa9ad m\xc3\xadle f\xc3\xa1ilte'],
+        [u'dobrý den', 'dobr\xc3\xbd den'],
+        [u'hylô', 'hyl\xc3\xb4'],
+        [u'jó napot', 'j\xc3\xb3 napot'],
+        [u'tervehdys', 'tervehdys'],
+        [u'olá', 'ol\xc3\xa1'],
+        [u'grüezi', 'gr\xc3\xbcezi'],
+        [u'dobre dan', 'dobre dan'],
+        [u'hello', 'hello'],
+        [u'bună ziua', 'bun\xc4\x83 ziua'],
+        [u'こんにちは', '\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf'],
+        [u'你好', '\xe4\xbd\xa0\xe5\xa5\xbd'],
+        [u'नमस्कार',  '\xe0\xa4\xa8\xe0\xa4\xae\xe0\xa4\xb8\xe0\xa5\x8d\xe0\xa4\x95\xe0\xa4\xbe\xe0\xa4\xb0'],
+        [u'안녕하세요', '\xec\x95\x88\xeb\x85\x95\xed\x95\x98\xec\x84\xb8\xec\x9a\x94'],
     ]
 
     def test_fixEncoding(self):
         """Test the fixEncoding function"""
         for a, b in self.simple_values:
-            self.assertEqual(koji.fixEncoding(a), b)
             self.assertEqual(koji.fixEncoding(b), b)
-            c = a.encode('utf16')
-            self.assertEqual(koji.fixEncoding(c, fallback='utf16'), b)
-            d = a[:-3] + u'\x00\x01' + a[-3:]
-            self.assertEqual(koji.fixEncoding(d, remove_nonprintable=True), b)
+            if six.PY2:
+                self.assertEqual(koji.fixEncoding(a), b)
+                c = a.encode('utf16')
+                self.assertEqual(koji.fixEncoding(c, fallback='utf16'), b)
+                d = a[:-3] + u'\x00\x01' + a[-3:]
+                self.assertEqual(koji.fixEncoding(d, remove_nonprintable=True), b)
+            else:
+                self.assertEqual(koji.fixEncoding(a), a)
 
-    @mock.patch('sys.stdout', new_callable=six.StringIO)
-    def test_fix_print(self, stdout):
+    def test_fix_print(self):
         """Test the _fix_print function"""
-        expected = ''
+        actual, expected = [], []
         for a, b in self.simple_values:
-            if six.PY3:
-                self.assertEqual(koji._fix_print(b), a)
-            else:
-                self.assertEqual(koji._fix_print(b), b)
-            print(koji._fix_print(b))
-            if six.PY3:
-                expected = expected + a + '\n'
-            else:
-                expected = expected + b + '\n'
-        actual = stdout.getvalue()
+            actual.append(koji._fix_print(b))
+            expected.append(b)
+        expected = '\n'.join(expected)
+        actual = '\n'.join(actual)
         self.assertEqual(actual, expected)
 
     complex_values = [
@@ -73,21 +69,24 @@ class FixEncodingTestCase(unittest.TestCase):
         [None, None],
         [[], []],
         [{u'a': 'a' , 'b' : {'c': u'c\x00'}},
-         {six.b('a'): six.b('a') , six.b('b') : {six.b('c'):  six.b('c\x00')}}],
+         {'a': 'a' , 'b' : {'c':  'c\x00'}}],
         # iso8859-15 fallback
-        ['g\xf3\xf0an daginn', six.b('g\xc3\xb3\xc3\xb0an daginn')],
+        ['g\xf3\xf0an daginn', 'g\xc3\xb3\xc3\xb0an daginn'],
     ]
 
     nonprint = [
-        ['hello\0world\0', six.b('helloworld')],
-        [u'hello\0world\0', six.b('helloworld')],
-        [[u'hello\0world\0'], [six.b('helloworld')]],
-        [{0: u'hello\0world\0'}, {0: six.b('helloworld')}],
-        [[{0: u'hello\0world\0'}], [{0: six.b('helloworld')}]],
+        ['hello\0world\0', 'helloworld'],
+        [u'hello\0world\0', 'helloworld'],
+        [[u'hello\0world\0'], ['helloworld']],
+        [{0: u'hello\0world\0'}, {0: 'helloworld'}],
+        [[{0: u'hello\0world\0'}], [{0: 'helloworld'}]],
     ]
 
     def test_fixEncodingRecurse(self):
         """Test the fixEncodingRecurse function"""
+        if six.PY3:
+            # don't test for py3
+            return
         for a, b in self.simple_values:
             self.assertEqual(koji.fixEncoding(a), b)
         for a, b in self.complex_values:
diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py
index 460f7d1..04248dc 100644
--- a/www/kojiweb/index.py
+++ b/www/kojiweb/index.py
@@ -1382,9 +1382,9 @@ def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-i
             values['enhances'] = server.getRPMDeps(rpm['id'], koji.DEP_ENHANCE)
             values['enhances'].sort(key=_sortbyname)
         headers = server.getRPMHeaders(rpm['id'], headers=['summary', 'description', 'license'])
-        values['summary'] = koji.fixEncoding(str(headers.get('summary')))
-        values['description'] = koji.fixEncoding(str(headers.get('description')))
-        values['license'] = koji.fixEncoding(str(headers.get('license')))
+        values['summary'] = koji.fixEncoding(headers.get('summary'))
+        values['description'] = koji.fixEncoding(headers.get('description'))
+        values['license'] = koji.fixEncoding(headers.get('license'))
     buildroots = kojiweb.util.paginateMethod(server, values, 'listBuildroots', kw={'rpmID': rpm['id']},
                                              start=buildrootStart, dataName='buildroots', prefix='buildroot',
                                              order=buildrootOrder)
diff --git a/www/lib/kojiweb/util.py b/www/lib/kojiweb/util.py
index ed4571f..f96b131 100644
--- a/www/lib/kojiweb/util.py
+++ b/www/lib/kojiweb/util.py
@@ -531,9 +531,6 @@ def escapeHTML(value):
         return value
 
     value = koji.fixEncoding(value)
-    if six.PY3:
-        # it is bytes now, so decode to str
-        value = value.decode()
     return value.replace('&', '&amp;').\
            replace('<', '&lt;').\
            replace('>', '&gt;')

From e0ab35a26693e388ccb4787dbf9ec32cc420e5ce Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: Mar 04 2019 14:47:34 +0000
Subject: [PATCH 5/5] honor remove_nonprintable in fixEncodingRecurse under py3


---

diff --git a/koji/__init__.py b/koji/__init__.py
index a004d7d..d6ae484 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -3127,7 +3127,7 @@ def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False):
 
     Similar behavior to fixEncoding, but recursive
     """
-    if six.PY3:
+    if six.PY3 and not remove_nonprintable:
         # don't bother with fixing in py3
         return value
 
@@ -3142,12 +3142,12 @@ def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False):
             k = fixEncodingRecurse(k, fallback=fallback, remove_nonprintable=remove_nonprintable)
             ret[k] = v
         return ret
-    elif isinstance(value, six.text_type):
+    elif six.PY2 and isinstance(value, six.text_type):
         if remove_nonprintable:
             return removeNonprintable(value.encode('utf8'))
         else:
             return value.encode('utf8')
-    elif isinstance(value, str):
+    elif six.PY2 and isinstance(value, str):
         # value is a str, but may be encoded in utf8 or some
         # other non-ascii charset.  Try to verify it's utf8, and if not,
         # decode it using the fallback encoding.
@@ -3159,6 +3159,8 @@ def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False):
             return removeNonprintable(s)
         else:
             return s
+    elif six.PY3 and isinstance(value, str) and remove_nonprintable:
+        return removeNonprintable(value)
     else:
         return value
 

