Make eventlet.tpool's thread count configurable in object server

If you're running servers_per_port > 0 and threads_per_disk = 0 (as it
should be with servers_per_port on), each object-server process will
have 20 IO threads waiting around to service eventlet.tpool
calls. This is far too many; with servers_per_port, there's no real
benefit to having so many IO threads.

This commit makes it so that, when servers_per_port > 0, each object
server defaults to having one main thread and one IO thread.

Also, eventlet's tpool size is now configurable via the object-server
config file. If a tpool size is set, that's what we'll use regardless
of servers_per_port. This allows operators with an excess of threads
to remove some regardless of servers_per_port.

Change-Id: I8f8914b7e70f2510393eb7c5e6be9708631ac027
Closes-Bug: 1554233

doc/source/deployment_guide.rst

@@ -651,6 +651,19 @@ ionice_priority                None                   I/O scheduling priority of
                                                       priority of the process. Work only with
                                                       ionice_class.
                                                       Ignored if IOPRIO_CLASS_IDLE is set.
+eventlet_tpool_num_threads     auto                   The number of threads in eventlet's thread pool.
+                                                      Most IO will occur in the object server's main
+                                                      thread, but certain "heavy" IO operations will
+                                                      occur in separate IO threads, managed by
+                                                      eventlet.
+                                                      The default value is auto, whose actual value
+                                                      is dependant on the servers_per_port value.
+                                                      If servers_per_port is zero then it uses
+                                                      eventlet's default (currently 20 threads).
+                                                      If the servers_per_port is nonzero then it'll
+                                                      only use 1 thread per process.
+                                                      This value can be overridden with an integer
+                                                      value.
 =============================  ====================== ===============================================
 
 [object-replicator]

etc/object-server.conf-sample

@@ -124,6 +124,29 @@ use = egg:swift#object
 #
 # auto_create_account_prefix = .
 #
+
+# The number of threads in eventlet's thread pool. Most IO will occur
+# in the object server's main thread, but certain "heavy" IO
+# operations will occur in separate IO threads, managed by eventlet.
+#
+# The default value is auto, whose actual value is dependant on the
+# servers_per_port value:
+#
+#  - When servers_per_port is zero, the default value of
+#    eventlet_tpool_num_threads is empty, which uses eventlet's default
+#    (currently 20 threads).
+#
+#  - When servers_per_port is nonzero, the default value of
+#    eventlet_tpool_num_threads is 1.
+#
+# But you may override this value to any integer value.
+#
+# Note that this value is threads per object-server process, so to
+# compute the total number of IO threads on a node, you must multiply
+# this by the number of object-server processes on the node.
+#
+# eventlet_tpool_num_threads = auto
+
 # Configure parameter for creating specific server
 # To handle all verbs, including replication verbs, do not specify
 # "replication_server" (this is the default). To only handle replication,

swift/obj/server.py

@@ -26,14 +26,15 @@ import math
 from swift import gettext_ as _
 from hashlib import md5
 
-from eventlet import sleep, wsgi, Timeout
+from eventlet import sleep, wsgi, Timeout, tpool
 from eventlet.greenthread import spawn
 
 from swift.common.utils import public, get_logger, \
     config_true_value, timing_stats, replication, \
     normalize_delete_at_timestamp, get_log_line, Timestamp, \
     get_expirer_container, parse_mime_headers, \
-    iter_multipart_mime_documents, extract_swift_bytes, safe_json_loads
+    iter_multipart_mime_documents, extract_swift_bytes, safe_json_loads, \
+    config_auto_int_value
 from swift.common.bufferedhttp import http_connect
 from swift.common.constraints import check_object_creation, \
     valid_timestamp, check_utf8
@@ -198,6 +199,34 @@ class ObjectController(BaseStorageServer):
         self.replication_failure_ratio = float(
             conf.get('replication_failure_ratio') or 1.0)
 
+        servers_per_port = int(conf.get('servers_per_port', '0') or 0)
+        if servers_per_port:
+            # The typical servers-per-port deployment also uses one port per
+            # disk, so you really get N servers per disk. In that case,
+            # having a pool of 20 threads per server per disk is far too
+            # much. For example, given a 60-disk chassis and 4 servers per
+            # disk, the default configuration will give us 21 threads per
+            # server (the main thread plus the twenty tpool threads), for a
+            # total of around 60 * 21 * 4 = 5040 threads. This is clearly
+            # too high.
+            #
+            # Instead, we use a tpool size of 1, giving us 2 threads per
+            # process. In the example above, that's 60 * 2 * 4 = 480
+            # threads, which is reasonable since there are 240 processes.
+            default_tpool_size = 1
+        else:
+            # If we're not using servers-per-port, then leave the tpool size
+            # alone. The default (20) is typically good enough for one
+            # object server handling requests for many disks.
+            default_tpool_size = None
+
+        tpool_size = config_auto_int_value(
+            conf.get('eventlet_tpool_num_threads'),
+            default_tpool_size)
+
+        if tpool_size:
+            tpool.set_num_threads(tpool_size)
+
     def get_diskfile(self, device, partition, account, container, obj,
                      policy, **kwargs):
         """

test/unit/obj/test_server.py

@@ -101,6 +101,39 @@ def fake_spawn():
                 gt.wait()
 
 
+class TestTpoolSize(unittest.TestCase):
+    def test_default_config(self):
+        with mock.patch('eventlet.tpool.set_num_threads') as mock_snt:
+            object_server.ObjectController({})
+        self.assertEqual([], mock_snt.mock_calls)
+
+    def test_explicit_setting(self):
+        conf = {'eventlet_tpool_num_threads': '17'}
+        with mock.patch('eventlet.tpool.set_num_threads') as mock_snt:
+            object_server.ObjectController(conf)
+        self.assertEqual([mock.call(17)], mock_snt.mock_calls)
+
+    def test_servers_per_port_no_explicit_setting(self):
+        conf = {'servers_per_port': '3'}
+        with mock.patch('eventlet.tpool.set_num_threads') as mock_snt:
+            object_server.ObjectController(conf)
+        self.assertEqual([mock.call(1)], mock_snt.mock_calls)
+
+    def test_servers_per_port_with_explicit_setting(self):
+        conf = {'eventlet_tpool_num_threads': '17',
+                'servers_per_port': '3'}
+        with mock.patch('eventlet.tpool.set_num_threads') as mock_snt:
+            object_server.ObjectController(conf)
+        self.assertEqual([mock.call(17)], mock_snt.mock_calls)
+
+    def test_servers_per_port_empty(self):
+        # run_wsgi is robust to this, so we should be too
+        conf = {'servers_per_port': ''}
+        with mock.patch('eventlet.tpool.set_num_threads') as mock_snt:
+            object_server.ObjectController(conf)
+        self.assertEqual([], mock_snt.mock_calls)
+
+
 @patch_policies(test_policies)
 class TestObjectController(unittest.TestCase):
     """Test swift.obj.server.ObjectController"""