Improve get optimal count of max_worker for pool
In the upstream of futures library, the DEFAULT value max_worker for ThreadPoolExecutor and ProcessPoolExecutor has been changed to: ThreadPoolExecutor max_worker = cpu_count() * 5 Reference: https://github.com/agronholm/pythonfutures/blob/master/concurrent/futures/thread.py#L109 ProcessPoolExecutor max_worker = cpu_count() Reference: https://github.com/agronholm/pythonfutures/blob/master/concurrent/futures/process.py#L278 So we should do it as well. Change-Id: Id68e38a639c41426f125180ad335fb1fb97cba59 Closes-bug: #1739034
This commit is contained in:
parent
6bb1d09ba6
commit
8fff2242ff
|
@ -201,7 +201,7 @@ class ProcessPoolExecutor(_process.ProcessPoolExecutor):
|
|||
|
||||
def __init__(self, max_workers=None):
|
||||
if max_workers is None:
|
||||
max_workers = _utils.get_optimal_thread_count()
|
||||
max_workers = _utils.get_optimal_process_count()
|
||||
super(ProcessPoolExecutor, self).__init__(max_workers=max_workers)
|
||||
if self._max_workers <= 0:
|
||||
raise ValueError("Max workers must be greater than zero")
|
||||
|
|
|
@ -128,14 +128,19 @@ def get_callback_name(cb):
|
|||
return ".".join(segments)
|
||||
|
||||
|
||||
def get_optimal_thread_count(default=2):
|
||||
def get_optimal_thread_count(default=5):
|
||||
"""Try to guess optimal thread count for current system."""
|
||||
try:
|
||||
return multiprocessing.cpu_count() + 1
|
||||
return multiprocessing.cpu_count() * 5
|
||||
except NotImplementedError:
|
||||
return default
|
||||
|
||||
|
||||
def get_optimal_process_count(default=1):
|
||||
"""Try to guess optimal process count for current system."""
|
||||
try:
|
||||
return multiprocessing.cpu_count()
|
||||
except NotImplementedError:
|
||||
# NOTE(harlowja): apparently may raise so in this case we will
|
||||
# just setup two threads since it's hard to know what else we
|
||||
# should do in this situation.
|
||||
return default
|
||||
|
||||
|
||||
|
|
|
@ -0,0 +1,8 @@
|
|||
---
|
||||
features:
|
||||
- Improve get_optimal_thread_count() function for
|
||||
choosing a new default value which is 5 * cpu_count()
|
||||
for max_workers used by ThreadPoolExecutor.
|
||||
- Add a new get_optimal_process_count() function
|
||||
for choosing default value which is same as
|
||||
cpu_count() used by ProcessPoolExecutor.
|
Loading…
Reference in New Issue