Merge "Workaround is required to prevent deadlock in task serializers."
This commit is contained in:
commit
5062cdb7e8
|
@ -52,7 +52,13 @@ def _initialize_worker(serializers_factory, context):
|
||||||
globals()['__factory'] = serializers_factory(context)
|
globals()['__factory'] = serializers_factory(context)
|
||||||
|
|
||||||
|
|
||||||
def _serialize_task_for_node_in_worker(node_and_task):
|
def _serialize_task_for_node_in_worker(node_and_task_and_err):
|
||||||
|
node_and_task, err = node_and_task_and_err
|
||||||
|
|
||||||
|
# NOTE(el): See comment for `wrap_task_iter` method.
|
||||||
|
if err:
|
||||||
|
raise err
|
||||||
|
|
||||||
return _serialize_task_for_node(globals()['__factory'], node_and_task)
|
return _serialize_task_for_node(globals()['__factory'], node_and_task)
|
||||||
|
|
||||||
|
|
||||||
|
@ -72,6 +78,23 @@ class SingleWorkerConcurrencyPolicy(object):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def wrap_task_iter(iterator):
|
||||||
|
"""Workaround is required to prevent deadlock.
|
||||||
|
|
||||||
|
Due to a bug in core python library [1], if iterator passed
|
||||||
|
to imap or imap_unordered raises an exception, it hangs in
|
||||||
|
deadlock forever, the issue is fixed in python 2.7.10, but
|
||||||
|
upstream repositories have much older version.
|
||||||
|
|
||||||
|
[1] http://bugs.python.org/issue23051
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
for i in iterator:
|
||||||
|
yield i, None
|
||||||
|
except Exception as e:
|
||||||
|
yield (None, None), e
|
||||||
|
|
||||||
|
|
||||||
class MultiProcessingConcurrencyPolicy(object):
|
class MultiProcessingConcurrencyPolicy(object):
|
||||||
def __init__(self, workers_num):
|
def __init__(self, workers_num):
|
||||||
self.workers_num = workers_num
|
self.workers_num = workers_num
|
||||||
|
@ -92,7 +115,7 @@ class MultiProcessingConcurrencyPolicy(object):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = pool.imap_unordered(
|
result = pool.imap_unordered(
|
||||||
_serialize_task_for_node_in_worker, tasks
|
_serialize_task_for_node_in_worker, wrap_task_iter(tasks)
|
||||||
)
|
)
|
||||||
for r in result:
|
for r in result:
|
||||||
yield r
|
yield r
|
||||||
|
|
Loading…
Reference in New Issue