diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index 8752b947..fa404bb3 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -651,6 +651,19 @@ ionice_priority None I/O scheduling priority of priority of the process. Work only with ionice_class. Ignored if IOPRIO_CLASS_IDLE is set. +eventlet_tpool_num_threads auto The number of threads in eventlet's thread pool. + Most IO will occur in the object server's main + thread, but certain "heavy" IO operations will + occur in separate IO threads, managed by + eventlet. + The default value is auto, whose actual value + is dependant on the servers_per_port value. + If servers_per_port is zero then it uses + eventlet's default (currently 20 threads). + If the servers_per_port is nonzero then it'll + only use 1 thread per process. + This value can be overridden with an integer + value. ============================= ====================== =============================================== [object-replicator] diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index 224906e1..69d63ef6 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -124,6 +124,29 @@ use = egg:swift#object # # auto_create_account_prefix = . # + +# The number of threads in eventlet's thread pool. Most IO will occur +# in the object server's main thread, but certain "heavy" IO +# operations will occur in separate IO threads, managed by eventlet. +# +# The default value is auto, whose actual value is dependant on the +# servers_per_port value: +# +# - When servers_per_port is zero, the default value of +# eventlet_tpool_num_threads is empty, which uses eventlet's default +# (currently 20 threads). +# +# - When servers_per_port is nonzero, the default value of +# eventlet_tpool_num_threads is 1. +# +# But you may override this value to any integer value. +# +# Note that this value is threads per object-server process, so to +# compute the total number of IO threads on a node, you must multiply +# this by the number of object-server processes on the node. +# +# eventlet_tpool_num_threads = auto + # Configure parameter for creating specific server # To handle all verbs, including replication verbs, do not specify # "replication_server" (this is the default). To only handle replication, diff --git a/swift/obj/server.py b/swift/obj/server.py index 1efa3997..e33adb81 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -26,14 +26,15 @@ import math from swift import gettext_ as _ from hashlib import md5 -from eventlet import sleep, wsgi, Timeout +from eventlet import sleep, wsgi, Timeout, tpool from eventlet.greenthread import spawn from swift.common.utils import public, get_logger, \ config_true_value, timing_stats, replication, \ normalize_delete_at_timestamp, get_log_line, Timestamp, \ get_expirer_container, parse_mime_headers, \ - iter_multipart_mime_documents, extract_swift_bytes, safe_json_loads + iter_multipart_mime_documents, extract_swift_bytes, safe_json_loads, \ + config_auto_int_value from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, \ valid_timestamp, check_utf8 @@ -198,6 +199,34 @@ class ObjectController(BaseStorageServer): self.replication_failure_ratio = float( conf.get('replication_failure_ratio') or 1.0) + servers_per_port = int(conf.get('servers_per_port', '0') or 0) + if servers_per_port: + # The typical servers-per-port deployment also uses one port per + # disk, so you really get N servers per disk. In that case, + # having a pool of 20 threads per server per disk is far too + # much. For example, given a 60-disk chassis and 4 servers per + # disk, the default configuration will give us 21 threads per + # server (the main thread plus the twenty tpool threads), for a + # total of around 60 * 21 * 4 = 5040 threads. This is clearly + # too high. + # + # Instead, we use a tpool size of 1, giving us 2 threads per + # process. In the example above, that's 60 * 2 * 4 = 480 + # threads, which is reasonable since there are 240 processes. + default_tpool_size = 1 + else: + # If we're not using servers-per-port, then leave the tpool size + # alone. The default (20) is typically good enough for one + # object server handling requests for many disks. + default_tpool_size = None + + tpool_size = config_auto_int_value( + conf.get('eventlet_tpool_num_threads'), + default_tpool_size) + + if tpool_size: + tpool.set_num_threads(tpool_size) + def get_diskfile(self, device, partition, account, container, obj, policy, **kwargs): """ diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 853ff122..e33c0b31 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -101,6 +101,39 @@ def fake_spawn(): gt.wait() +class TestTpoolSize(unittest.TestCase): + def test_default_config(self): + with mock.patch('eventlet.tpool.set_num_threads') as mock_snt: + object_server.ObjectController({}) + self.assertEqual([], mock_snt.mock_calls) + + def test_explicit_setting(self): + conf = {'eventlet_tpool_num_threads': '17'} + with mock.patch('eventlet.tpool.set_num_threads') as mock_snt: + object_server.ObjectController(conf) + self.assertEqual([mock.call(17)], mock_snt.mock_calls) + + def test_servers_per_port_no_explicit_setting(self): + conf = {'servers_per_port': '3'} + with mock.patch('eventlet.tpool.set_num_threads') as mock_snt: + object_server.ObjectController(conf) + self.assertEqual([mock.call(1)], mock_snt.mock_calls) + + def test_servers_per_port_with_explicit_setting(self): + conf = {'eventlet_tpool_num_threads': '17', + 'servers_per_port': '3'} + with mock.patch('eventlet.tpool.set_num_threads') as mock_snt: + object_server.ObjectController(conf) + self.assertEqual([mock.call(17)], mock_snt.mock_calls) + + def test_servers_per_port_empty(self): + # run_wsgi is robust to this, so we should be too + conf = {'servers_per_port': ''} + with mock.patch('eventlet.tpool.set_num_threads') as mock_snt: + object_server.ObjectController(conf) + self.assertEqual([], mock_snt.mock_calls) + + @patch_policies(test_policies) class TestObjectController(unittest.TestCase): """Test swift.obj.server.ObjectController"""