Add scheduler_thread_pool_size configuration

Add scheduler_thread_pool_size configuration and set default value to
1000 to match oslo_service default for service threadgroup [1]

[1] https://github.com/openstack/oslo.service/blob/master/oslo_service/service.py#L718

Change-Id: Ie971f8321a472ef34396faed81aa676eaa006701

releasenotes/notes/scheduler-thread-pool-size-40905866197ef8bd.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Added scheduler thread pool size configuration value and changed default
+    thread pool size for scheduler from 10 to 1000. This fix prevents problems
+    when a large number of cluster operations are executed simultaneously.

senlin/common/config.py

@@ -107,6 +107,9 @@ engine_opts = [
                default=60,
                help=_('Maximum time since last check-in for a service to be '
                       'considered up.')),
+    cfg.IntOpt('scheduler_thread_pool_size',
+               default=1000,
+               help=_('Maximum number of threads to use for scheduler.')),
 ]
 cfg.CONF.register_opts(engine_opts)
 

senlin/engine/scheduler.py

@@ -33,7 +33,8 @@ class ThreadGroupManager(object):
 
     def __init__(self):
         super(ThreadGroupManager, self).__init__()
-        self.group = threadgroup.ThreadGroup()
+        self.group = threadgroup.ThreadGroup(
+            thread_pool_size=cfg.CONF.scheduler_thread_pool_size)
 
         # Create dummy service task, because when there is nothing queued
         # on self.tg the process exits

senlin/tests/unit/engine/test_node.py

@@ -974,7 +974,7 @@ class TestNode(base.SenlinTestCase):
             'FAKE_KEY1': 'FAKE_VALUE1',
             'FAKE_KEY2': 'FAKE_VALUE2',
         }
-        final_inputs = jsonutils.dumps(final_dict)
+        final_inputs = jsonutils.dumps(final_dict, sort_keys=False)
         wfc.execution_create.assert_called_once_with('foo', final_inputs)
 
     def test_run_workflow_failed_creation(self):