From 66c14b2aff94baf5a5e8479c43c975c41e1098ca Mon Sep 17 00:00:00 2001 From: Andrea Ieri Date: Thu, 23 Jul 2020 15:57:46 -0400 Subject: [PATCH] Ensure lockup_timeout is never < 2*rsync_timeout Change-Id: Id0afc210b7be8ac7c740753ef06b51db0bae410c Closes-Bug: 1888725 --- config.yaml | 6 ++++++ lib/swift_storage_context.py | 7 +++++++ templates/object-server-replicator.conf | 1 + templates/object-server.conf | 1 + unit_tests/test_swift_storage_context.py | 17 +++++++++++++++-- 5 files changed, 30 insertions(+), 2 deletions(-) diff --git a/config.yaml b/config.yaml index dd21c9d..ee580e7 100644 --- a/config.yaml +++ b/config.yaml @@ -114,6 +114,12 @@ options: type: int description: | Max duration of a partition rsync (in seconds). + object-lockup-timeout: + default: 1800 + type: int + description: | + Attempt to kill all workers if nothing replicates for this amount of seconds + Note: this will always be set to no less than 2*object-rsync-timeout. object-handoffs-first: type: boolean default: False diff --git a/lib/swift_storage_context.py b/lib/swift_storage_context.py index cac9349..ae07a6b 100644 --- a/lib/swift_storage_context.py +++ b/lib/swift_storage_context.py @@ -107,6 +107,13 @@ class SwiftStorageServerContext(OSContextGenerator): 'statsd_port': config('statsd-port'), 'statsd_sample_rate': config('statsd-sample-rate'), } + + # ensure lockup_timeout > rsync_timeout. See bug 1575277 + ctxt['object_lockup_timeout'] = max( + config('object-lockup-timeout'), + 2*ctxt['object_rsync_timeout'] + ) + if config('node-timeout'): node_timeout = config('node-timeout') ctxt['node_timeout'] = node_timeout diff --git a/templates/object-server-replicator.conf b/templates/object-server-replicator.conf index 962ad37..70ae5fb 100644 --- a/templates/object-server-replicator.conf +++ b/templates/object-server-replicator.conf @@ -24,3 +24,4 @@ replication_server = true [object-replicator] concurrency = {{ object_replicator_concurrency }} rsync_timeout = {{ object_rsync_timeout }} +lockup_timeout = {{ object_lockup_timeout }} diff --git a/templates/object-server.conf b/templates/object-server.conf index 2717e51..0b408c2 100644 --- a/templates/object-server.conf +++ b/templates/object-server.conf @@ -35,6 +35,7 @@ http_timeout = {{ http_timeout }} [object-replicator] concurrency = {{ object_replicator_concurrency }} rsync_timeout = {{ object_rsync_timeout }} +lockup_timeout = {{ object_lockup_timeout }} {% if object_handoffs_first %} handoffs_first = True {% endif %} diff --git a/unit_tests/test_swift_storage_context.py b/unit_tests/test_swift_storage_context.py index a9a24a9..32e6b40 100644 --- a/unit_tests/test_swift_storage_context.py +++ b/unit_tests/test_swift_storage_context.py @@ -81,6 +81,17 @@ class SwiftStorageContextTests(CharmTestCase): ctxt.enable_rsyncd() _file.write.assert_called_with('RSYNC_ENABLE=true\n') + def test_swift_storage_lockup_timeout_too_low(self): + self.test_config.set('object-rsync-timeout', 1000) + self.test_config.set('object-lockup-timeout', 1050) + ctxt = swift_context.SwiftStorageServerContext() + result = ctxt() + ex = { + 'object_rsync_timeout': 1000, + 'object_lockup_timeout': 2000, + } + self.assertDictContainsSubset(ex, result) + def test_swift_storage_server_context(self): self.unit_private_ip.return_value = '10.0.0.5' self.test_config.set('account-server-port', '500') @@ -94,7 +105,8 @@ class SwiftStorageContextTests(CharmTestCase): self.test_config.set('account-max-connections', '10') self.test_config.set('container-max-connections', '10') self.test_config.set('object-max-connections', '10') - self.test_config.set('object-rsync-timeout', '950') + self.test_config.set('object-rsync-timeout', 950) + self.test_config.set('object-lockup-timeout', 3000) self.test_config.set('object-handoffs-first', True) self.test_config.set('file-allocation-reserve', '10737418240') ctxt = swift_context.SwiftStorageServerContext() @@ -112,7 +124,8 @@ class SwiftStorageContextTests(CharmTestCase): 'account_max_connections': '10', 'container_max_connections': '10', 'object_max_connections': '10', - 'object_rsync_timeout': '950', + 'object_rsync_timeout': 950, + 'object_lockup_timeout': 3000, 'fallocate_reserve': '10737418240', 'standalone_replicator': True, 'object_handoffs_first': True,