From e60d07c3bfd472cf61c6469087ca3c0f7f1b8a76 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Jun 22 2017 08:49:21 +0000
Subject: [PATCH 1/3] Patch xmlrpclib for 64-bit integers


Related: https://pagure.io/koji/issue/464

---

diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py
index 6f848d8..3be9ce7 100644
--- a/hub/kojixmlrpc.py
+++ b/hub/kojixmlrpc.py
@@ -61,6 +61,24 @@ class Marshaller(xmlrpclib.Marshaller):
         self.dump_string(value, write)
     dispatch[datetime.datetime] = dump_datetime
 
+    MAXI8 = 2 ** 64 - 1
+    MINI8 = -2 ** 64
+    def dump_i8(self, value, write):
+        # python2's xmlrpclib doesn't support i8 extension for marshalling,
+        # but can unmarshall it correctly.
+        if value > Marshaller.MAXI8 or value < Marshaller.MINI8:
+            raise OverflowError, "long int exceeds XML-RPC limits"
+        elif value > xmlrpclib.MAXINT or value < xmlrpclib.MININT:
+            write("<value><i8>")
+            write(str(int(value)))
+            write("</i8></value>\n")
+        else:
+            write("<value><int>")
+            write(str(int(value)))
+            write("</int></value>\n")
+    dispatch[types.LongType] = dump_i8
+    dispatch[types.IntType] = dump_i8
+
 xmlrpclib.Marshaller = Marshaller
 
 

From e345ba199bdc8e44c3efca564c6038e09fd74b45 Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Jun 22 2017 08:49:21 +0000
Subject: [PATCH 2/3] remove encode_int in favour of i8 xmlrpc extension


---

diff --git a/devtools/fakehub b/devtools/fakehub
index fed749d..4e4ba07 100755
--- a/devtools/fakehub
+++ b/devtools/fakehub
@@ -7,7 +7,6 @@ import os
 import os.path
 import pprint
 import sys
-import six.moves.xmlrpc_client
 from six.moves import cStringIO
 from six.moves.urllib.parse import quote
 
@@ -58,7 +57,7 @@ def get_request():
         else:
             args.append(nice_literal(s))
     args = koji.encode_args(*args, **kwargs)
-    request = six.moves.xmlrpc_client.dumps(args, method, allow_none=1)
+    request = koji.dumps(args, method, allow_none=1)
     return request
 
 
@@ -68,7 +67,7 @@ def start_response(status, headers):
 
 
 def parse_response(data):
-    p, u = six.moves.xmlrpc_client.getparser()
+    p, u = koji.getparser()
     for chunk in data:
         p.feed(chunk)
     p.close()
diff --git a/hub/kojihub.py b/hub/kojihub.py
index 4fce02b..4a1f041 100644
--- a/hub/kojihub.py
+++ b/hub/kojihub.py
@@ -559,8 +559,7 @@ def make_task(method, arglist, **opts):
             raise koji.GenericError("invalid channel policy")
 
     # encode xmlrpc request
-    opts['request'] = xmlrpclib.dumps(tuple(arglist), methodname=method,
-                                      allow_none=1)
+    opts['request'] = koji.dumps(tuple(arglist), methodname=method, allow_none=1)
     opts['state'] = koji.TASK_STATES['FREE']
     opts['method'] = method
     koji.plugin.run_callbacks('preTaskStateChange', attribute='state', old=None, new='FREE', info=opts)
@@ -3599,7 +3598,7 @@ def get_next_release(build_info):
 
 def _fix_rpm_row(row):
     if 'size' in row:
-        row['size'] = koji.encode_int(row['size'])
+        row['size'] = row['size']
     if 'extra' in row:
         row['extra'] = parse_json(row['extra'], desc='rpm extra')
     return row
@@ -8809,8 +8808,6 @@ class RootExports(object):
         contents = base64.decodestring(data)
         del data
         # we will accept offset and size as strings to work around xmlrpc limits
-        offset = koji.decode_int(offset)
-        size = koji.decode_int(size)
         if isinstance(md5sum, basestring):
             # this case is for backwards compatibility
             verify = "md5"
@@ -8911,7 +8908,7 @@ class RootExports(object):
             st = os.fstat(fd)
             if not stat.S_ISREG(st.st_mode):
                 raise koji.GenericError("Not a regular file: %s" % fn)
-            data['size'] = koji.encode_int(st.st_size)
+            data['size'] = st.st_size
             data['mtime'] = st.st_mtime
             if verify:
                 sum_cls = get_verify_class(verify)
@@ -8927,7 +8924,7 @@ class RootExports(object):
                     length += len(chunk)
                     chksum.update(chunk)
                     chunk = os.read(fd, 8192)
-                data['sumlength'] = koji.encode_int(length)
+                data['sumlength'] = length
                 data['hexdigest'] = chksum.hexdigest()
             return data
         finally:
@@ -12577,9 +12574,9 @@ def handle_upload(environ):
         # this will also remove our lock
         os.close(fd)
     ret = {
-        'size': koji.encode_int(size),
+        'size': size,
         'fileverify': verify,
-        'offset': koji.encode_int(offset),
+        'offset': offset,
     }
     if verify:
         # unsigned 32bit - could be too big for xmlrpc
diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py
index 3be9ce7..d826d5c 100644
--- a/hub/kojixmlrpc.py
+++ b/hub/kojixmlrpc.py
@@ -19,18 +19,14 @@
 #       Mike McLean <mikem@redhat.com>
 
 from ConfigParser import RawConfigParser
-import datetime
 import inspect
 import logging
 import os
 import sys
 import time
 import traceback
-import types
 import pprint
 import resource
-import xmlrpclib
-from xmlrpclib import getparser, dumps, Fault
 from koji.server import WSGIWrapper
 
 import koji
@@ -39,49 +35,11 @@ import koji.db
 import koji.plugin
 import koji.policy
 import koji.util
+# import xmlrpclib functions from koji to use tweaked Marshaller
+from koji import getparser, dumps, Fault
 from koji.context import context
 
 
-# Workaround to allow xmlrpclib deal with iterators
-class Marshaller(xmlrpclib.Marshaller):
-
-    dispatch = xmlrpclib.Marshaller.dispatch.copy()
-
-    def dump_generator(self, value, write):
-        dump = self.__dump
-        write("<value><array><data>\n")
-        for v in value:
-            dump(v, write)
-        write("</data></array></value>\n")
-    dispatch[types.GeneratorType] = dump_generator
-
-    def dump_datetime(self, value, write):
-        # For backwards compatibility, we return datetime objects as strings
-        value = value.isoformat(' ')
-        self.dump_string(value, write)
-    dispatch[datetime.datetime] = dump_datetime
-
-    MAXI8 = 2 ** 64 - 1
-    MINI8 = -2 ** 64
-    def dump_i8(self, value, write):
-        # python2's xmlrpclib doesn't support i8 extension for marshalling,
-        # but can unmarshall it correctly.
-        if value > Marshaller.MAXI8 or value < Marshaller.MINI8:
-            raise OverflowError, "long int exceeds XML-RPC limits"
-        elif value > xmlrpclib.MAXINT or value < xmlrpclib.MININT:
-            write("<value><i8>")
-            write(str(int(value)))
-            write("</i8></value>\n")
-        else:
-            write("<value><int>")
-            write(str(int(value)))
-            write("</int></value>\n")
-    dispatch[types.LongType] = dump_i8
-    dispatch[types.IntType] = dump_i8
-
-xmlrpclib.Marshaller = Marshaller
-
-
 class HandlerRegistry(object):
     """Track handlers for RPC calls"""
 
diff --git a/koji/__init__.py b/koji/__init__.py
index 2b9fe24..51ee1c5 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -73,13 +73,55 @@ import struct
 import tempfile
 import time
 import traceback
+import types
 from . import util
 import warnings
-import six.moves.xmlrpc_client
 import xml.sax
 import xml.sax.handler
-from six.moves.xmlrpc_client import loads, dumps, Fault
 import six.moves.urllib
+from six.moves.xmlrpc_client import getparser, loads, dumps, Fault
+
+# Workaround to allow xmlrpclib deal with iterators and 64-bit ints
+class Marshaller(six.moves.xmlrpc_client.Marshaller):
+
+    dispatch = six.moves.xmlrpc_client.Marshaller.dispatch.copy()
+
+    def dump_generator(self, value, write):
+        dump = self.__dump
+        write("<value><array><data>\n")
+        for v in value:
+            dump(v, write)
+        write("</data></array></value>\n")
+    dispatch[types.GeneratorType] = dump_generator
+
+    def dump_datetime(self, value, write):
+        # For backwards compatibility, we return datetime objects as strings
+        value = value.isoformat(' ')
+        self.dump_string(value, write)
+    dispatch[datetime.datetime] = dump_datetime
+
+    MAXI8 = 2 ** 64 - 1
+    MINI8 = -2 ** 64
+    def dump_i8(self, value, write):
+        # python2's xmlrpclib doesn't support i8 extension for marshalling,
+        # but can unmarshall it correctly.
+        if value > Marshaller.MAXI8 or value < Marshaller.MINI8:
+            raise OverflowError, "long int exceeds XML-RPC limits"
+        elif value > six.moves.xmlrpc_client.MAXINT or \
+             value < six.moves.xmlrpc_client.MININT:
+            write("<value><i8>")
+            write(str(int(value)))
+            write("</i8></value>\n")
+        else:
+            write("<value><int>")
+            write(str(int(value)))
+            write("</int></value>\n")
+    dispatch[types.LongType] = dump_i8
+    dispatch[types.IntType] = dump_i8
+
+six.moves.xmlrpc_client.Marshaller = Marshaller
+six.moves.xmlrpc_client.FastMarshaller = None
+
 
 PROFILE_MODULES = {}  # {module_name: module_instance}
 
@@ -452,23 +494,6 @@ def decode_args2(args, names, strict=True):
     ret.update(opts)
     return ret
 
-## BEGIN kojikamid dup
-
-def encode_int(n):
-    """If n is too large for a 32bit signed, convert it to a string"""
-    if n <= 2147483647:
-        return n
-    #else
-    return str(n)
-## END kojikamid dup
-
-def decode_int(n):
-    """If n is not an integer, attempt to convert it"""
-    if isinstance(n, six.integer_types):
-        return n
-    #else
-    return int(n)
-
 #commonly used functions
 
 def safe_xmlrpc_loads(s):
@@ -2382,7 +2407,7 @@ class ClientSession(object):
         return ret
 
     def _read_xmlrpc_response(self, response):
-        p, u = six.moves.xmlrpc_client.getparser()
+        p, u = getparser()
         for chunk in response.iter_content(8192):
             if self.opts.get('debug_xmlrpc', False):
                 print("body: %r" % chunk)
@@ -2643,7 +2668,7 @@ class ClientSession(object):
             while True:
                 if debug:
                     self.logger.debug("uploadFile(%r,%r,%r,%r,%r,...)" %(path, name, sz, digest, offset))
-                if self.callMethod('uploadFile', path, name, encode_int(sz), digest, encode_int(offset), data, **volopts):
+                if self.callMethod('uploadFile', path, name, sz, digest, offset, data, **volopts):
                     break
                 if tries <= retries:
                     tries += 1
diff --git a/koji/daemon.py b/koji/daemon.py
index 9706437..71f8bf0 100644
--- a/koji/daemon.py
+++ b/koji/daemon.py
@@ -34,7 +34,6 @@ import time
 import sys
 import traceback
 import errno
-import xmlrpclib
 
 
 def incremental_upload(session, fname, fd, path, retries=5, logger=None):
@@ -61,7 +60,7 @@ def incremental_upload(session, fname, fd, path, retries=5, logger=None):
 
         tries = 0
         while True:
-            if session.uploadFile(path, fname, koji.encode_int(size), digest, koji.encode_int(offset), data):
+            if session.uploadFile(path, fname, size, digest, offset, data):
                 break
 
             if tries <= retries:
@@ -1209,12 +1208,12 @@ class TaskManager(object):
         try:
             response = (handler.run(),)
             # note that we wrap response in a singleton tuple
-            response = xmlrpclib.dumps(response, methodresponse=1, allow_none=1)
+            response = koji.dumps(response, methodresponse=1, allow_none=1)
             self.logger.info("RESPONSE: %r" % response)
             self.session.host.closeTask(handler.id, response)
             return
-        except xmlrpclib.Fault, fault:
-            response = xmlrpclib.dumps(fault)
+        except koji.Fault, fault:
+            response = koji.dumps(fault)
             tb = ''.join(traceback.format_exception(*sys.exc_info())).replace(r"\n", "\n")
             self.logger.warn("FAULT:\n%s" % tb)
         except (SystemExit, koji.tasks.ServerExit, KeyboardInterrupt):
@@ -1233,7 +1232,7 @@ class TaskManager(object):
             if issubclass(e_class, koji.GenericError):
                 #just pass it through
                 tb = str(e)
-            response = xmlrpclib.dumps(xmlrpclib.Fault(faultCode, tb))
+            response = koji.dumps(koji.Fault(faultCode, tb))
 
         # if we get here, then we're handling an exception, so fail the task
         self.session.host.failTask(handler.id, response)
diff --git a/vm/kojikamid.py b/vm/kojikamid.py
index 0499c74..386f86e 100755
--- a/vm/kojikamid.py
+++ b/vm/kojikamid.py
@@ -305,7 +305,7 @@ class WindowsBuild(object):
         offset = 0
         checksum = hashlib.md5()
         while True:
-            encoded = self.server.getFile(buildinfo, fileinfo, encode_int(offset), 1048576, type)
+            encoded = self.server.getFile(buildinfo, fileinfo, offset, 1048576, type)
             if not encoded:
                 break
             data = base64.b64decode(encoded)
@@ -568,7 +568,7 @@ def upload_file(server, prefix, path):
         if not data:
             break
         encoded = base64.b64encode(data)
-        server.upload(path, encode_int(offset), encoded)
+        server.upload(path, offset, encoded)
         offset += len(data)
         sum.update(data)
     fobj.close()
diff --git a/vm/kojivmd b/vm/kojivmd
index 3ca7342..b4774bf 100755
--- a/vm/kojivmd
+++ b/vm/kojivmd
@@ -720,8 +720,8 @@ class VMExecTask(BaseTaskHandler):
         """
         remotepath = os.path.dirname(os.path.join(self.getUploadDir(), filepath))
         filename = os.path.basename(filepath)
-        self.session.uploadFile(remotepath, filename, koji.encode_int(size),
-                                md5sum, koji.encode_int(offset), data)
+        self.session.uploadFile(remotepath, filename, size,
+                                md5sum, offset, data)
 
     def verifyChecksum(self, path, checksum, algo='sha1'):
         local_path = os.path.abspath(os.path.join(self.output_dir, path))
diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py
index caa65a7..237e7d0 100644
--- a/www/kojiweb/index.py
+++ b/www/kojiweb/index.py
@@ -774,14 +774,13 @@ def getfile(environ, taskID, name, volume='DEFAULT', offset=None, size=None):
 
 def _chunk_file(server, environ, taskID, name, offset, size, volume):
     remaining = size
-    encode_int = koji.encode_int
     while True:
         if remaining <= 0:
             break
         chunk_size = 1048576
         if remaining < chunk_size:
             chunk_size = remaining
-        content = server.downloadTaskOutput(taskID, name, offset=encode_int(offset), size=chunk_size, volume=volume)
+        content = server.downloadTaskOutput(taskID, name, offset=offset, size=chunk_size, volume=volume)
         if not content:
             break
         yield content

From 6eea21cd36512abc09b9a3f9183fa26f10885e5c Mon Sep 17 00:00:00 2001
From: Tomas Kopecek <tkopecek@redhat.com>
Date: Jun 22 2017 08:50:15 +0000
Subject: [PATCH 3/3] docs update for xmlrpc i8 extension


---

diff --git a/docs/source/using_the_koji_build_system.rst b/docs/source/using_the_koji_build_system.rst
index d913219..b2f1d57 100644
--- a/docs/source/using_the_koji_build_system.rst
+++ b/docs/source/using_the_koji_build_system.rst
@@ -566,3 +566,6 @@ All features supported by command-line client are also accessible by XMLRPC
 API. You can get listing of all available calls, arguments and basic help via
 calling `koji list-api` command. This call will also provide you API
 extensions provided by plugins in that particular koji instance.
+
+In addition to xmlrpc standard we are using `i8` extension for large
+integers. Datetimes are exchanged as text strings in iso format.

