Ensure lockup_timeout is never < 2*rsync_timeout

Change-Id: Id0afc210b7be8ac7c740753ef06b51db0bae410c
Closes-Bug: 1888725

config.yaml

@@ -114,6 +114,12 @@ options:
     type: int
     description: |
       Max duration of a partition rsync (in seconds).
+  object-lockup-timeout:
+    default: 1800
+    type: int
+    description: |
+      Attempt to kill all workers if nothing replicates for this amount of seconds
+      Note: this will always be set to no less than 2*object-rsync-timeout.
   object-handoffs-first:
     type: boolean
     default: False

lib/swift_storage_context.py

@@ -107,6 +107,13 @@ class SwiftStorageServerContext(OSContextGenerator):
             'statsd_port': config('statsd-port'),
             'statsd_sample_rate': config('statsd-sample-rate'),
         }
+
+        # ensure lockup_timeout > rsync_timeout. See bug 1575277
+        ctxt['object_lockup_timeout'] = max(
+            config('object-lockup-timeout'),
+            2*ctxt['object_rsync_timeout']
+        )
+
         if config('node-timeout'):
             node_timeout = config('node-timeout')
             ctxt['node_timeout'] = node_timeout

templates/object-server-replicator.conf

@@ -24,3 +24,4 @@ replication_server = true
 [object-replicator]
 concurrency = {{ object_replicator_concurrency }}
 rsync_timeout = {{ object_rsync_timeout }}
+lockup_timeout = {{ object_lockup_timeout }}

templates/object-server.conf

@@ -35,6 +35,7 @@ http_timeout = {{ http_timeout }}
 [object-replicator]
 concurrency = {{ object_replicator_concurrency }}
 rsync_timeout = {{ object_rsync_timeout }}
+lockup_timeout = {{ object_lockup_timeout }}
 {% if object_handoffs_first %}
 handoffs_first = True
 {% endif %}

unit_tests/test_swift_storage_context.py

@@ -81,6 +81,17 @@ class SwiftStorageContextTests(CharmTestCase):
             ctxt.enable_rsyncd()
             _file.write.assert_called_with('RSYNC_ENABLE=true\n')
 
+    def test_swift_storage_lockup_timeout_too_low(self):
+        self.test_config.set('object-rsync-timeout', 1000)
+        self.test_config.set('object-lockup-timeout', 1050)
+        ctxt = swift_context.SwiftStorageServerContext()
+        result = ctxt()
+        ex = {
+            'object_rsync_timeout': 1000,
+            'object_lockup_timeout': 2000,
+        }
+        self.assertDictContainsSubset(ex, result)
+
     def test_swift_storage_server_context(self):
         self.unit_private_ip.return_value = '10.0.0.5'
         self.test_config.set('account-server-port', '500')
@@ -94,7 +105,8 @@ class SwiftStorageContextTests(CharmTestCase):
         self.test_config.set('account-max-connections', '10')
         self.test_config.set('container-max-connections', '10')
         self.test_config.set('object-max-connections', '10')
-        self.test_config.set('object-rsync-timeout', '950')
+        self.test_config.set('object-rsync-timeout', 950)
+        self.test_config.set('object-lockup-timeout', 3000)
         self.test_config.set('object-handoffs-first', True)
         self.test_config.set('file-allocation-reserve', '10737418240')
         ctxt = swift_context.SwiftStorageServerContext()
@@ -112,7 +124,8 @@ class SwiftStorageContextTests(CharmTestCase):
             'account_max_connections': '10',
             'container_max_connections': '10',
             'object_max_connections': '10',
-            'object_rsync_timeout': '950',
+            'object_rsync_timeout': 950,
+            'object_lockup_timeout': 3000,
             'fallocate_reserve': '10737418240',
             'standalone_replicator': True,
             'object_handoffs_first': True,