Improve get optimal count of max_worker for pool

In the upstream of futures library, the DEFAULT value max_worker
for ThreadPoolExecutor and ProcessPoolExecutor has been changed to:

ThreadPoolExecutor

max_worker = cpu_count() * 5

Reference:
https://github.com/agronholm/pythonfutures/blob/master/concurrent/futures/thread.py#L109

ProcessPoolExecutor

max_worker = cpu_count()

Reference:
https://github.com/agronholm/pythonfutures/blob/master/concurrent/futures/process.py#L278

So we should do it as well.

Change-Id: Id68e38a639c41426f125180ad335fb1fb97cba59
Closes-bug: #1739034
This commit is contained in:
Chenjun Shen 2017-12-19 15:19:56 +01:00
parent 6bb1d09ba6
commit 8fff2242ff
3 changed files with 19 additions and 6 deletions

View File

@ -201,7 +201,7 @@ class ProcessPoolExecutor(_process.ProcessPoolExecutor):
def __init__(self, max_workers=None):
if max_workers is None:
max_workers = _utils.get_optimal_thread_count()
max_workers = _utils.get_optimal_process_count()
super(ProcessPoolExecutor, self).__init__(max_workers=max_workers)
if self._max_workers <= 0:
raise ValueError("Max workers must be greater than zero")

View File

@ -128,14 +128,19 @@ def get_callback_name(cb):
return ".".join(segments)
def get_optimal_thread_count(default=2):
def get_optimal_thread_count(default=5):
"""Try to guess optimal thread count for current system."""
try:
return multiprocessing.cpu_count() + 1
return multiprocessing.cpu_count() * 5
except NotImplementedError:
return default
def get_optimal_process_count(default=1):
"""Try to guess optimal process count for current system."""
try:
return multiprocessing.cpu_count()
except NotImplementedError:
# NOTE(harlowja): apparently may raise so in this case we will
# just setup two threads since it's hard to know what else we
# should do in this situation.
return default

View File

@ -0,0 +1,8 @@
---
features:
- Improve get_optimal_thread_count() function for
choosing a new default value which is 5 * cpu_count()
for max_workers used by ThreadPoolExecutor.
- Add a new get_optimal_process_count() function
for choosing default value which is same as
cpu_count() used by ProcessPoolExecutor.