Do not sync suffixes when remote rejects reconstructor sync

The commit a0fcca1e makes reconstructor not sync suffixes when remote
reject reconstructor revert. However, the exact same logic should
be applied to SYNC job as well. REPLICATE requests aren't generally
needed when using SSYC (which the reconstructor always does).

If a ssync_sender fails to finish a sync the reconstructor should skip
the REPLICATE call entirely and move on to the next partition without
causing any useless remote IO.

Change-Id: Ida50539e645ea7e2950ba668c7f031a8d10da787
Closes-Bug: #1665141

swift/obj/reconstructor.py

@@ -850,7 +850,8 @@ class ObjectReconstructor(Daemon):
             success, _ = ssync_sender(
                 self, node, job, suffixes)()
             # let remote end know to rehash it's suffixes
-            self.rehash_remote(node, job, suffixes)
+            if success:
+                self.rehash_remote(node, job, suffixes)
             # update stats for this attempt
             self.suffix_sync += len(suffixes)
             self.logger.update_stats('suffix.syncs', len(suffixes))

test/unit/obj/test_reconstructor.py

@@ -4113,6 +4113,11 @@ class TestObjectReconstructor(BaseTestObjectReconstructor):
                 (node['replication_ip'], '/%s/0' % node['device']),
                 (node['replication_ip'], '/%s/0/123-abc' % node['device']),
             ])
+        # the first (primary sync_to) node's rehash_remote will be skipped
+        first_node = part_nodes[0]
+        expected_suffix_calls.remove(
+            (first_node['replication_ip'], '/%s/0/123-abc'
+             % first_node['device']))
 
         ssync_calls = []
         with mock_ssync_sender(ssync_calls,