Add support for object replication handoffs_first

In extreme cases of object rebalance or node failure, swift environments
with millions of objects may have a hard time with objects landing in
handoff or misplaced partitions and the operator of the cloud may need
to prioritize migrating data in handoff partitions back to primary
partitions over the 3 primary partitions replicating amongst each other.

To allow for this, the object-server.conf [object-replicator]
configuration can have handoffs_first set to True which will force
the object-replicators to focus on handoff partitions before syncing
primary partitions for any given object partition.

Change-Id: I8b44c287567a0e6d634def0b13baf0fe4ad4aa7b
Closes-Bug: 1878087

config.yaml

@@ -114,6 +114,13 @@ options:
     type: int
     description: |
       Max duration of a partition rsync (in seconds).
+  object-handoffs-first:
+    type: boolean
+    default: False
+    description: |
+      If set to True, partitions that are not supposed to be on the node will
+      be replicated first. The default setting should not be changed, except
+      for extreme situations.
   nagios-check-params:
     default: "-m -r 60 180 10 20"
     type: string

lib/swift_storage_context.py

@@ -101,6 +101,7 @@ class SwiftStorageServerContext(OSContextGenerator):
             'object_replicator_concurrency': config(
                 'object-replicator-concurrency'),
             'object_rsync_timeout': config('object-rsync-timeout'),
+            'object_handoffs_first': config('object-handoffs-first'),
             'statsd_host': config('statsd-host'),
             'statsd_port': config('statsd-port'),
             'statsd_sample_rate': config('statsd-sample-rate'),

templates/object-server.conf

@@ -26,6 +26,9 @@ threads_per_disk = {{ object_server_threads_per_disk }}
 [object-replicator]
 concurrency = {{ object_replicator_concurrency }}
 rsync_timeout = {{ object_rsync_timeout }}
+{% if object_handoffs_first %}
+handoffs_first = True
+{% endif %}
 
 [object-sync]
 {% endif %}

unit_tests/test_swift_storage_context.py

@@ -95,6 +95,7 @@ class SwiftStorageContextTests(CharmTestCase):
         self.test_config.set('container-max-connections', '10')
         self.test_config.set('object-max-connections', '10')
         self.test_config.set('object-rsync-timeout', '950')
+        self.test_config.set('object-handoffs-first', True)
         ctxt = swift_context.SwiftStorageServerContext()
         result = ctxt()
         ex = {
@@ -112,6 +113,7 @@ class SwiftStorageContextTests(CharmTestCase):
             'object_max_connections': '10',
             'object_rsync_timeout': '950',
             'standalone_replicator': True,
+            'object_handoffs_first': True,
             'statsd_host': '',
             'statsd_port': 3125,
             'statsd_sample_rate': 1.0