opendev
/
gear
Code Issues Proposed changes

Parse admin requests 2880 times faster 

Both returning the incomplete data from the isComplete method, and
scanning the entire incomplete string for the terminator each time
are O(n^2).

Also, returning the string is *very* slow, accounting for 99.6% of
the speedup; scanning the string each time accounts for 0.6%.

Both changes together find the terminus of a 481MiB response in
about 1.6 seconds compared to 76 minutes prior to this change.

Change-Id: I3614e8f3ff8ad7c3d9ca6da1b520b89cd9d5d603
This commit is contained in:
James E. Blair 2016-07-17 08:05:57 -07:00
parent 550873b76a
commit 2defbeaa1b
1 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
gear/__init__.py
View File

@ -314,7 +314,8 @@ class Connection(object):
admin_request = self._getAdminRequest()
if admin:
complete, remainder = admin_request.isComplete(raw_bytes)
raw_bytes = remainder
if remainder is not None:
raw_bytes = remainder
if complete:
return admin_request
else:
@ -431,6 +432,7 @@ class AdminRequest(object):
command = None
arguments = []
response = None
_complete_position = 0
def __init__(self, *arguments):
self.wait_event = threading.Event()
@ -451,8 +453,10 @@ class AdminRequest(object):
def isComplete(self, data):
x = -1
end_index_newline = data.find(b'\n.\n')
end_index_return = data.find(b'\r\n.\r\n')
start = self._complete_position
start = max(self._complete_position - 4, 0)
end_index_newline = data.find(b'\n.\n', start)
end_index_return = data.find(b'\r\n.\r\n', start)
if end_index_newline != -1:
x = end_index_newline + 3
elif end_index_return != -1:
@ -461,11 +465,12 @@ class AdminRequest(object):
x = 2
elif data.startswith(b'.\r\n'):
x = 3
self._complete_position = len(data)
if x != -1:
self.response = data[:x]
return (True, data[x:])
else:
return (False, data)
return (False, None)
def setComplete(self):
self.wait_event.set()
@ -530,7 +535,7 @@ class CancelJobAdminRequest(AdminRequest):
self.response = data[:x]
return (True, data[x:])
else:
return (False, data)
return (False, None)
class VersionAdminRequest(AdminRequest):
@ -551,7 +556,7 @@ class VersionAdminRequest(AdminRequest):
self.response = data[:x]
return (True, data[x:])
else:
return (False, data)
return (False, None)
class WorkersAdminRequest(AdminRequest):
@ -2279,7 +2284,7 @@ class ServerAdminRequest(AdminRequest):
x = end_index_newline + 1
return (True, data[x:])
else:
return (False, data)
return (False, None)
class NonBlockingConnection(Connection):