Adds tcp_keepalive and tcp_keepidle config options

Currently the wsgi server will not close connections once requests
complete and will not enable keepalive on it's wsgi sockets. This can
be a problem for those who need to align the server keepalive with
load balancer timeouts without modifying system keepalive settings.

To remedy this we add new config options tcp_keepalive and
tcp_keepidle which are disabled by default to remain backwards
compatible.

DocImpact: "new config options for wsgi tcp_keepalive & tcp_keepidle"
Closes-bug: bug 1260406
Co-authored-by: Hirofumi Ichihara <ichihara.hirofumi@lab.ntt.co.jp>
Change-Id: Ic53402c57e1ebe44cde4c18e5e15200dcbbcb04b

bin/keystone-all

@@ -56,7 +56,9 @@ CONF = config.CONF
 
 def create_server(conf, name, host, port):
     app = deploy.loadapp('config:%s' % conf, name=name)
-    server = environment.Server(app, host=host, port=port)
+    server = environment.Server(app, host=host, port=port,
+                                keepalive=CONF.tcp_keepalive,
+                                keepidle=CONF.tcp_keepidle)
     if CONF.ssl.enable:
         server.set_ssl(CONF.ssl.certfile, CONF.ssl.keyfile,
                        CONF.ssl.ca_certs, CONF.ssl.cert_required)

etc/keystone.conf.sample

@@ -15,6 +15,14 @@
 # The port number which the public admin listens on
 # admin_port = 35357
 
+# Set this to True if you want to enable TCP_KEEPALIVE on server sockets i.e.
+# sockets used by the keystone wsgi server for client connections.
+# tcp_keepalive = False
+
+# Sets the value of TCP_KEEPIDLE in seconds for each server socket. Only
+# applies if tcp_keepalive is True. Not supported on OS X.
+# tcp_keepidle = 600
+
 # The base endpoint URLs for keystone that are advertised to clients
 # (NOTE: this does NOT affect how keystone listens for connections)
 # public_endpoint = http://localhost:%(public_port)s/

keystone/common/config.py

@@ -47,7 +47,17 @@ FILE_OPTIONS = {
         cfg.StrOpt('member_role_id',
                    default='9fe2ff9ee4384b1894a90878d3e92bab'),
         cfg.StrOpt('member_role_name', default='_member_'),
-        cfg.IntOpt('crypt_strength', default=40000)],
+        cfg.IntOpt('crypt_strength', default=40000),
+        cfg.BoolOpt('tcp_keepalive', default=False,
+                    help=("Set this to True if you want to enable "
+                          "TCP_KEEPALIVE on server sockets i.e. sockets used "
+                          "by the keystone wsgi server for client "
+                          "connections")),
+        cfg.IntOpt('tcp_keepidle',
+                   default=600,
+                   help=("Sets the value of TCP_KEEPIDLE in seconds for each "
+                         "server socket. Only applies if tcp_keepalive is "
+                         "True. Not supported on OS X."))],
     'identity': [
         cfg.StrOpt('default_domain_id', default='default'),
         cfg.BoolOpt('domain_specific_drivers_enabled',

keystone/common/environment/eventlet_server.py

@@ -35,7 +35,8 @@ LOG = log.getLogger(__name__)
 class Server(object):
     """Server class to manage multiple WSGI sockets and applications."""
 
-    def __init__(self, application, host=None, port=None, threads=1000):
+    def __init__(self, application, host=None, port=None, threads=1000,
+                 keepalive=False, keepidle=None):
         self.application = application
         self.host = host or '0.0.0.0'
         self.port = port or 0
@@ -44,6 +45,8 @@ class Server(object):
         self.greenthread = None
         self.do_ssl = False
         self.cert_required = False
+        self.keepalive = keepalive
+        self.keepidle = keepidle
 
     def start(self, key=None, backlog=128):
         """Run a WSGI server with the given application."""
@@ -77,6 +80,15 @@ class Server(object):
                                           ca_certs=self.ca_certs)
             _socket = sslsocket
 
+        # Optionally enable keepalive on the wsgi socket.
+        if self.keepalive:
+            _socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+
+            # This option isn't available in the OS X version of eventlet
+            if hasattr(socket, 'TCP_KEEPIDLE') and self.keepidle is not None:
+                _socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE,
+                                   self.keepidle)
+
         self.greenthread = self.pool.spawn(self._run,
                                            self.application,
                                            _socket)

keystone/tests/test_wsgi.py

@@ -16,7 +16,10 @@
 
 from babel import localedata
 import gettext
+import mock
+import socket
 
+from keystone.common import environment
 from keystone.common import wsgi
 from keystone import exception
 from keystone.openstack.common.fixture import moxstubout
@@ -249,3 +252,62 @@ class LocalizedResponseTest(tests.TestCase):
         # are lazy-translated.
         self.assertIsInstance(_('The resource could not be found.'),
                               gettextutils.Message)
+
+
+class ServerTest(tests.TestCase):
+
+    def setUp(self):
+        super(ServerTest, self).setUp()
+        environment.use_eventlet()
+        self.host = '127.0.0.1'
+        self.port = '1234'
+
+    @mock.patch('eventlet.listen')
+    @mock.patch('socket.getaddrinfo')
+    def test_keepalive_unset(self, mock_getaddrinfo, mock_listen):
+        mock_getaddrinfo.return_value = [(1, 2, 3, 4, 5)]
+        mock_sock = mock.Mock()
+        mock_sock.setsockopt = mock.Mock()
+
+        mock_listen.return_value = mock_sock
+        server = environment.Server(mock.MagicMock(), host=self.host,
+                                    port=self.port)
+        server.start()
+        self.assertTrue(mock_listen.called)
+        self.assertFalse(mock_sock.setsockopt.called)
+
+    @mock.patch('eventlet.listen')
+    @mock.patch('socket.getaddrinfo')
+    def test_keepalive_set(self, mock_getaddrinfo, mock_listen):
+        mock_getaddrinfo.return_value = [(1, 2, 3, 4, 5)]
+        mock_sock = mock.Mock()
+        mock_sock.setsockopt = mock.Mock()
+
+        mock_listen.return_value = mock_sock
+        server = environment.Server(mock.MagicMock(), host=self.host,
+                                    port=self.port, keepalive=True)
+        server.start()
+        mock_sock.setsockopt.assert_called_once_with(socket.SOL_SOCKET,
+                                                     socket.SO_KEEPALIVE,
+                                                     1)
+        self.assertTrue(mock_listen.called)
+
+    @mock.patch('eventlet.listen')
+    @mock.patch('socket.getaddrinfo')
+    def test_keepalive_and_keepidle_set(self, mock_getaddrinfo, mock_listen):
+        mock_getaddrinfo.return_value = [(1, 2, 3, 4, 5)]
+        mock_sock = mock.Mock()
+        mock_sock.setsockopt = mock.Mock()
+
+        mock_listen.return_value = mock_sock
+        server = environment.Server(mock.MagicMock(), host=self.host,
+                                    port=self.port, keepalive=True,
+                                    keepidle=1)
+        server.start()
+        self.assertEqual(mock_sock.setsockopt.call_count, 2)
+        # Test the last set of call args i.e. for the keepidle
+        mock_sock.setsockopt.assert_called_with(socket.IPPROTO_TCP,
+                                                socket.TCP_KEEPIDLE,
+                                                1)
+
+        self.assertTrue(mock_listen.called)