Increase httplib._MAXHEADERS

Python 2.7.9+ and 3.2.6+ limits the number of maximum headers in httplib to 100
[1,2,3]. This setting is too low for Swift.

By default the maximum number of allowed headers depends on the number of max
allowed metadata settings plus a default value of 32 for regular http headers.
If for some reason this is not enough (custom middleware for example) it can be
increased with the extra_header_count constraint.

[1] https://bugs.python.org/issue16037
[2] https://hg.python.org/cpython/raw-file/15c95b7d81dc/Misc/NEWS
[3] https://hg.python.org/cpython/raw-file/v3.2.6/Misc/NEWS

Co-Authored-By: Clay Gerrard <clay.gerrard@gmail.com>
Co-Authored-By: Matthew Oliver <matt@oliver.net.au>
Co-Authored-By: Thomas Herve <therve@redhat.com>

Cherry picked from commit edfca861b6

Conflicts:
	swift/common/constraints.py
	test/functional/__init__.py
	test/functional/swift_test_client.py
	test/unit/proxy/test_server.py

Change-Id: I388fd697ec88476024b0e9f1ae75ba35ff765282

etc/swift.conf-sample

@@ -89,6 +89,14 @@ default = yes
 #max_header_size = 8192
 
 
+# By default the maximum number of allowed headers depends on the number of max
+# allowed metadata settings plus a default value of 32 for regular http
+# headers.  If for some reason this is not enough (custom middleware for
+# example) it can be increased with the extra_header_count constraint.
+
+#extra_header_count = 32
+
+
 # max_object_name_length is the max number of bytes in the utf8 encoding
 # of an object name
 

swift/common/bufferedhttp.py

@@ -27,13 +27,18 @@ BufferedHTTPResponse.
 """
 
 from swift import gettext_ as _
+from swift.common import constraints
 from urllib import quote
 import logging
 import time
 
+import eventlet
 from eventlet.green.httplib import CONTINUE, HTTPConnection, HTTPMessage, \
     HTTPResponse, HTTPSConnection, _UNKNOWN
 
+httplib = eventlet.import_patched('httplib')
+httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT
+
 
 class BufferedHTTPResponse(HTTPResponse):
     """HTTPResponse class that buffers reading of headers"""

swift/common/constraints.py

@@ -35,6 +35,7 @@ CONTAINER_LISTING_LIMIT = 10000
 ACCOUNT_LISTING_LIMIT = 10000
 MAX_ACCOUNT_NAME_LENGTH = 256
 MAX_CONTAINER_NAME_LENGTH = 256
+EXTRA_HEADER_COUNT = 0
 
 # If adding an entry to DEFAULT_CONSTRAINTS, note that
 # these constraints are automatically published by the
@@ -52,6 +53,7 @@ DEFAULT_CONSTRAINTS = {
     'account_listing_limit': ACCOUNT_LISTING_LIMIT,
     'max_account_name_length': MAX_ACCOUNT_NAME_LENGTH,
     'max_container_name_length': MAX_CONTAINER_NAME_LENGTH,
+    'extra_header_count': EXTRA_HEADER_COUNT,
 }
 
 SWIFT_CONSTRAINTS_LOADED = False
@@ -99,6 +101,13 @@ FORMAT2CONTENT_TYPE = {'plain': 'text/plain', 'json': 'application/json',
                        'xml': 'application/xml'}
 
 
+# By default the maximum number of allowed headers depends on the number of max
+# allowed metadata settings plus a default value of 32 for regular http
+# headers.  If for some reason this is not enough (custom middleware for
+# example) it can be increased with the extra_header_count constraint.
+MAX_HEADER_COUNT = MAX_META_COUNT + 32 + max(EXTRA_HEADER_COUNT, 0)
+
+
 def check_metadata(req, target_type):
     """
     Check metadata sent in the request headers.  This should only check

test/functional/__init__.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import httplib
 import os
 import sys
 import pickle
@@ -51,6 +52,8 @@ from swift.container import server as container_server
 from swift.obj import server as object_server, mem_server as mem_object_server
 import swift.proxy.controllers.obj
 
+httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT
+
 # In order to get the proper blocking behavior of sockets without using
 # threads, where we can set an arbitrary timeout for some piece of code under
 # test, we use eventlet with the standard socket library patched. We have to

test/functional/swift_test_client.py

@@ -28,8 +28,12 @@ from nose import SkipTest
 from xml.dom import minidom
 from swiftclient import get_auth
 
+from swift.common import constraints
+
 from test import safe_repr
 
+httplib._MAXHEADERS = constraints.MAX_HEADER_COUNT
+
 
 class AuthenticationFailed(Exception):
     pass

test/unit/proxy/test_server.py

@@ -7245,7 +7245,7 @@ class TestSwiftInfo(unittest.TestCase):
         self.assertEqual(si['account_autocreate'], False)
         # this next test is deliberately brittle in order to alert if
         # other items are added to swift info
-        self.assertEqual(len(si), 16)
+        self.assertEqual(len(si), 17)
 
         self.assertTrue('policies' in si)
         sorted_pols = sorted(si['policies'], key=operator.itemgetter('name'))