Add slo_manifest_hook callback

... to allow other middlewares to impose additional constraints on
or make edits to SLO manifests before being written.

The callback takes a single argument: the python list that represents
the manifest to be written. All the normal list operations listed at
https://docs.python.org/2/library/stdtypes.html#mutable-sequence-types
are available to make changes to that before SLO serializes it as JSON.

The callback may return a list of problematic segments; each item in the
list should be a tuple of

    (quoted object name, description of problem)

This will be useful both for s3api minimum segment size validation and
creating tar large objects.

Change-Id: I198c5196e0221a72b14597a06e5ce3c4b2bbf436
Related-Bug: #1636663

swift/common/middleware/slo.py

@@ -1227,6 +1227,15 @@ class StaticLargeObject(object):
                         data_for_storage[i] = seg_data
                         total_size += segment_length
 
+            # Middleware left of SLO can add a callback to the WSGI
+            # environment to perform additional validation and/or
+            # manipulation on the manifest that will be written.
+            hook = req.environ.get('swift.callback.slo_manifest_hook')
+            if hook:
+                more_problems = hook(data_for_storage)
+                if more_problems:
+                    problem_segments.extend(more_problems)
+
             if problem_segments:
                 err = HTTPBadRequest(content_type=out_content_type)
                 resp_dict = {}

test/unit/common/middleware/test_slo.py

@@ -964,6 +964,59 @@ class TestSloPutManifest(SloTestCase):
         self.assertEqual('a', manifest_data[0]['hash'])
         self.assertEqual('b', manifest_data[1]['hash'])
 
+    def test_handle_multipart_put_with_manipulator_callback(self):
+        def data_inserter(manifest):
+            for i in range(len(manifest), -1, -1):
+                manifest.insert(i, {'data': 'WA=='})
+
+        good_data = json.dumps([
+            {'path': '/checktest/a_1'},
+            {'path': '/checktest/b_2'}])
+        req = Request.blank(
+            '/v1/AUTH_test/checktest/man_3?multipart-manifest=put',
+            environ={'REQUEST_METHOD': 'PUT',
+                     'swift.callback.slo_manifest_hook': data_inserter},
+            body=good_data)
+        status, headers, body = self.call_slo(req)
+        self.assertEqual(self.app.call_count, 3)
+
+        # Check that we still populated the manifest properly from our HEADs
+        req = Request.blank(
+            '/v1/AUTH_test/checktest/man_3?multipart-manifest=put',
+            environ={'REQUEST_METHOD': 'GET'})
+        status, headers, body = self.call_app(req)
+        manifest_data = json.loads(body)
+        self.assertEqual([
+            {k: v for k, v in item.items()
+             if k in ('name', 'bytes', 'hash', 'data')}
+            for item in manifest_data
+        ], [
+            {'data': 'WA=='},
+            {'name': '/checktest/a_1', 'bytes': 1, 'hash': 'a'},
+            {'data': 'WA=='},
+            {'name': '/checktest/b_2', 'bytes': 2, 'hash': 'b'},
+            {'data': 'WA=='},
+        ])
+
+    def test_handle_multipart_put_with_validator_callback(self):
+        def complainer(manifest):
+            return [(item['name'], "Don't wanna") for item in manifest]
+
+        good_data = json.dumps([
+            {'path': '/checktest/a_1'},
+            {'path': '/checktest/b_2'}])
+        req = Request.blank(
+            '/v1/AUTH_test/checktest/man_3?multipart-manifest=put',
+            environ={'REQUEST_METHOD': 'PUT',
+                     'swift.callback.slo_manifest_hook': complainer},
+            body=good_data)
+        status, headers, body = self.call_slo(req)
+        self.assertEqual(self.app.call_count, 2)
+        self.assertEqual(status, '400 Bad Request')
+        body = body.split('\n')
+        self.assertIn("/checktest/a_1, Don't wanna", body)
+        self.assertIn("/checktest/b_2, Don't wanna", body)
+
     def test_handle_unsatisfiable_ranges(self):
         bad_data = json.dumps(
             [{'path': '/checktest/a_1', 'etag': None,