Eventlet green threads not released back to pool

Presently, the wsgi server allows persist connections. Hence even after
the response is sent to the client, it doesn't close the client socket
connection. Because of this problem, the green thread is not released
back to the pool.

In order to close the client socket connection explicitly after the
response is sent and read successfully by the client, you simply have to
set keepalive to False when you create a wsgi server.

DocImpact:
Added http_keepalive option (default=True).

Conflicts:
        doc/source/configuring.rst
        etc/glance-api.conf
        glance/common/wsgi.py
        glance/tests/unit/test_opts.py

SecurityImpact

Closes-Bug: #1361360
Change-Id: I93aaca24935a4f3096210233097dd6b8c5440176
(cherry picked from commit 16a821e00d)

doc/source/configuring.rst

@@ -1331,3 +1331,14 @@ glance-registry service separately, by default they place at
 to make profiling work as designed operator needs to make those values of HMAC
 key be consistent for all services in your deployment. Without HMAC key the
 profiling will not be triggered even profiling feature is enabled.
+
+Configuring http_keepalive option
+----------------------------------
+
+* ``http_keepalive=<True|False>``
+
+If False, server will return the header "Connection: close", If True, server
+will return "Connection: Keep-Alive" in its responses. In order to close the
+client socket connection explicitly after the response is sent and read
+successfully by the client, you simply have to set this option to False when
+you create a wsgi server.

etc/glance-api.conf

@@ -109,6 +109,13 @@ backlog = 4096
 # and 'store_type'.
 #location_strategy = location_order
 
+# http_keepalive option. If False, server will return the header
+# "Connection: close", If True, server will return "Connection: Keep-Alive"
+# in its responses. In order to close the client socket connection
+# explicitly after the response is sent and read successfully by the client,
+# you simply have to set this option to False when you create a wsgi server.
+#http_keepalive = True
+
 # ================= Syslog Options ============================
 
 # Send logs to syslog (/dev/log) instead of to file specified

etc/glance-registry.conf

@@ -58,6 +58,13 @@ limit_param_default = 25
 # Default: False
 #sqlalchemy_debug = True
 
+# http_keepalive option. If False, server will return the header
+# "Connection: close", If True, server will return "Connection: Keep-Alive"
+# in its responses. In order to close the client socket connection
+# explicitly after the response is sent and read successfully by the client,
+# you simply have to set this option to False when you create a wsgi server.
+#http_keepalive = True
+
 # ================= Syslog Options ============================
 
 # Send logs to syslog (/dev/log) instead of to file specified

glance/common/wsgi.py

@@ -89,6 +89,9 @@ eventlet_opts = [
                       'max_header_line may need to be increased when using '
                       'large tokens (typically those generated by the '
                       'Keystone v3 API with big service catalogs')),
+    cfg.BoolOpt('http_keepalive', default=True,
+                help=_('If False, closes the client socket connection '
+                       'explicitly.')),
 ]
 
 profiler_opts = [
@@ -340,7 +343,8 @@ class Server(object):
                                  self.application,
                                  log=logging.WritableLogger(self.logger),
                                  custom_pool=self.pool,
-                                 debug=False)
+                                 debug=False,
+                                 keepalive=CONF.http_keepalive)
         except socket.error as err:
             if err[0] != errno.EINVAL:
                 raise
@@ -351,7 +355,8 @@ class Server(object):
         self.logger.info(_("Starting single process server"))
         eventlet.wsgi.server(sock, application, custom_pool=self.pool,
                              log=logging.WritableLogger(self.logger),
-                             debug=False)
+                             debug=False,
+                             keepalive=CONF.http_keepalive)
 
 
 class Middleware(object):

glance/tests/unit/common/test_wsgi.py

@@ -466,6 +466,28 @@ class ServerTest(test_utils.BaseTestCase):
         actual = wsgi.Server(threads=1).create_pool()
         self.assertIsInstance(actual, eventlet.greenpool.GreenPool)
 
+    @mock.patch.object(wsgi, 'get_socket')
+    def test_http_keepalive(self, mock_get_socket):
+        fake_socket = 'fake_socket'
+        mock_get_socket.return_value = 'fake_socket'
+        self.config(http_keepalive=False)
+        self.config(workers=0)
+
+        server = wsgi.Server(threads=1)
+        # mocking eventlet.wsgi server method to check it is called with
+        # configured 'http_keepalive' value.
+        with mock.patch.object(eventlet.wsgi,
+                               'server') as mock_server:
+            fake_application = "fake-application"
+            server.start(fake_application, 0)
+            server.wait()
+            mock_server.assert_called_once_with(fake_socket,
+                                                fake_application,
+                                                log=mock.ANY,
+                                                debug=False,
+                                                custom_pool=server.pool,
+                                                keepalive=False)
+
 
 class TestHelpers(test_utils.BaseTestCase):
 

glance/tests/unit/test_opts.py

@@ -145,7 +145,8 @@ class OptsTestCase(utils.BaseTestCase):
             'eventlet_executor_pool_size',
             'store_type_preference',
             'flavor',
-            'config_file'
+            'config_file',
+            'http_keepalive',
         ]
 
         self._check_opt_groups(opt_list, expected_opt_groups)
@@ -205,7 +206,8 @@ class OptsTestCase(utils.BaseTestCase):
             'eventlet_hub',
             'max_header_line',
             'flavor',
-            'config_file'
+            'config_file',
+            'http_keepalive',
         ]
 
         self._check_opt_groups(opt_list, expected_opt_groups)