Eventlet green threads not released back to pool

Presently, the wsgi server allows persist connections hence even after
the response is sent to the client, it doesn't close the client socket
connection.
Because of this problem, the green thread is not released back to the pool.

In order to close the client socket connection explicitly after the
response is sent and read successfully by the client, you simply have to
set keepalive to False when you create a wsgi server.

Add a parameter to take advantage of the new(ish) eventlet socket timeout
behaviour. Allows closing idle client connections after a period of
time, eg:

$ time nc localhost 8776
real    1m0.063s

Setting 'client_socket_timeout = 0' means do not timeout.

DocImpact:
Added wsgi_keep_alive option (default=True).
Added client_socket_timeout option (default=900).

SecurityImpact

Closes-Bug: #1361360
Change-Id: I03b9c5c64f4bd8bca78dfc83199ef17d9a7ea5b7

keystone/common/config.py

@@ -1007,6 +1007,15 @@ FILE_OPTIONS = {
                    deprecated_group='DEFAULT',
                    help='The port number which the admin service listens '
                         'on.'),
+        cfg.BoolOpt('wsgi_keep_alive', default=True,
+                    help="If set to false, disables keepalives on the server; "
+                         "all connections will be closed after serving one "
+                         "request."),
+        cfg.IntOpt('client_socket_timeout', default=900,
+                   help="Timeout for socket operations on a client "
+                        "connection. If an incoming connection is idle for "
+                        "this number of seconds it will be closed. A value "
+                        "of '0' means wait forever."),
         cfg.BoolOpt('tcp_keepalive', default=False,
                     deprecated_name='tcp_keepalive',
                     deprecated_group='DEFAULT',

keystone/common/environment/eventlet_server.py

@@ -25,12 +25,16 @@ import sys
 import eventlet
 import eventlet.wsgi
 import greenlet
+from oslo_config import cfg
 from oslo_log import log
 from oslo_log import loggers
 
 from keystone.i18n import _LE, _LI
 
 
+CONF = cfg.CONF
+
+
 LOG = log.getLogger(__name__)
 
 # The size of a pool that is used to spawn a single green thread in which
@@ -182,10 +186,12 @@ class Server(object):
     def _run(self, application, socket):
         """Start a WSGI server with a new green thread pool."""
         logger = log.getLogger('eventlet.wsgi.server')
+        socket_timeout = CONF.eventlet_server.client_socket_timeout or None
         try:
-            eventlet.wsgi.server(socket, application,
-                                 log=EventletFilteringLogger(logger),
-                                 debug=False)
+            eventlet.wsgi.server(
+                socket, application, log=EventletFilteringLogger(logger),
+                debug=False, keepalive=CONF.eventlet_server.wsgi_keep_alive,
+                socket_timeout=socket_timeout)
         except greenlet.GreenletExit:
             # Wait until all servers have completed running
             pass

keystone/tests/unit/test_wsgi.py

@@ -13,9 +13,11 @@
 # under the License.
 
 import gettext
+import random
 import socket
 import uuid
 
+import eventlet
 import mock
 import oslo_i18n
 from oslo_serialization import jsonutils
@@ -425,3 +427,45 @@ class ServerTest(tests.TestCase):
                                                     1)
 
         self.assertTrue(mock_listen.called)
+
+    def test_client_socket_timeout(self):
+        # mocking server method of eventlet.wsgi to check it is called with
+        # configured 'client_socket_timeout' value.
+
+        # setting random socket_timeout between 1 to 10, so that after given
+        # seconds client connection will be closed if it is idle.
+        socket_timeout = random.randint(1, 10)
+        self.config_fixture.config(group='eventlet_server',
+                                   client_socket_timeout=socket_timeout)
+        server = environment.Server(mock.MagicMock(), host=self.host,
+                                    port=self.port)
+        with mock.patch.object(eventlet.wsgi, 'server') as mock_server:
+            fake_application = uuid.uuid4().hex
+            fake_socket = uuid.uuid4().hex
+            server._run(fake_application, fake_socket)
+            mock_server.assert_called_once_with(fake_socket,
+                                                fake_application,
+                                                debug=mock.ANY,
+                                                socket_timeout=socket_timeout,
+                                                log=mock.ANY,
+                                                keepalive=mock.ANY)
+
+    def test_wsgi_keep_alive(self):
+        # mocking server method of eventlet.wsgi to check it is called with
+        # configured 'wsgi_keep_alive' value.
+        wsgi_keepalive = False
+        self.config_fixture.config(group='eventlet_server',
+                                   wsgi_keep_alive=wsgi_keepalive)
+
+        server = environment.Server(mock.MagicMock(), host=self.host,
+                                    port=self.port)
+        with mock.patch.object(eventlet.wsgi, 'server') as mock_server:
+            fake_application = uuid.uuid4().hex
+            fake_socket = uuid.uuid4().hex
+            server._run(fake_application, fake_socket)
+            mock_server.assert_called_once_with(fake_socket,
+                                                fake_application,
+                                                debug=mock.ANY,
+                                                socket_timeout=mock.ANY,
+                                                log=mock.ANY,
+                                                keepalive=wsgi_keepalive)