Nova-api service throws error when SIGHUP is sent

Added reset method in WSGIService class.

After adding reset method when SIGHUP signal is sent to
wsgi service parent process,then it sends SIGHUP signal
to all of its child processes. Each child process handles
SIGHUP signal by first stopping the service and then calls
service start method again. When it stops the service, it
kills the eventlet thread, which internally closes the wsgi
server socket object. This server socket object is now not
usable again and it throws following error, while restarting
the service:

error: [Errno 9] Bad file descriptor

To resolve 'Bad file descriptor' error, creating duplicate
socket object, every time service starts.

When the wsgi service is stopped, it sets the green pool
size to 0, so resizing the green pool to default value,
on service restart.

Closes-Bug: #1334651

Change-Id: If1035deaf8b31f2712d88f0112fdd2e9e9dc7cb0

nova/service.py

@@ -353,6 +353,14 @@ class WSGIService(object):
         self.port = self.server.port
         self.backdoor_port = None
 
+    def reset(self):
+        """Reset server greenpool size to default.
+
+        :returns: None
+
+        """
+        self.server.reset()
+
     def _get_manager(self):
         """Initialize a Manager object appropriate for this service.
 

nova/tests/integrated/test_multiprocess_api.py

@@ -207,6 +207,20 @@ class MultiprocessWSGITest(integrated_helpers._IntegratedTestBase):
         self.assertTrue(os.WIFEXITED(status))
         self.assertEqual(os.WEXITSTATUS(status), 0)
 
+    def test_restart_sighup(self):
+        start_workers = self._spawn()
+
+        os.kill(self.pid, signal.SIGHUP)
+
+        # Wait at most 5 seconds to restart a worker
+        cond = lambda: start_workers == self._get_workers()
+        timeout = 5
+        self._wait(cond, timeout)
+
+        # Make sure worker pids match
+        end_workers = self._get_workers()
+        self.assertEqual(start_workers, end_workers)
+
 
 class MultiprocessWSGITestV3(client.TestOpenStackClientV3Mixin,
                              MultiprocessWSGITest):

nova/tests/test_service.py

@@ -327,6 +327,20 @@ class TestWSGIService(test.TestCase):
         self.assertNotEqual(0, test_service.port)
         test_service.stop()
 
+    def test_reset_pool_size_to_default(self):
+        test_service = service.WSGIService("test_service")
+        test_service.start()
+
+        # Stopping the service, which in turn sets pool size to 0
+        test_service.stop()
+        self.assertEqual(test_service.server._pool.size, 0)
+
+        # Resetting pool size to default
+        test_service.reset()
+        test_service.start()
+        self.assertEqual(test_service.server._pool.size,
+                         CONF.wsgi_default_pool_size)
+
 
 class TestLauncher(test.TestCase):
 

nova/tests/test_wsgi.py

@@ -157,6 +157,20 @@ class TestWSGIServer(test.NoDBTestCase):
         server.stop()
         server.wait()
 
+    def test_reset_pool_size_to_default(self):
+        server = nova.wsgi.Server("test_resize", None,
+            host="127.0.0.1", max_url_len=16384)
+        server.start()
+
+        # Stopping the server, which in turn sets pool size to 0
+        server.stop()
+        self.assertEqual(server._pool.size, 0)
+
+        # Resetting pool size to default
+        server.reset()
+        server.start()
+        self.assertEqual(server._pool.size, CONF.wsgi_default_pool_size)
+
 
 class TestWSGIServerWithSSL(test.NoDBTestCase):
     """WSGI server with SSL tests."""

nova/wsgi.py

@@ -102,7 +102,8 @@ class Server(object):
         self.app = app
         self._server = None
         self._protocol = protocol
-        self._pool = eventlet.GreenPool(pool_size or self.default_pool_size)
+        self.pool_size = pool_size or self.default_pool_size
+        self._pool = eventlet.GreenPool(self.pool_size)
         self._logger = logging.getLogger("nova.%s.wsgi.server" % self.name)
         self._wsgi_logger = logging.WritableLogger(self._logger)
         self._use_ssl = use_ssl
@@ -141,6 +142,12 @@ class Server(object):
 
         :returns: None
         """
+        # The server socket object will be closed after server exits,
+        # but the underlying file descriptor will remain open, and will
+        # give bad file descriptor error. So duplicating the socket object,
+        # to keep file descriptor usable.
+
+        dup_socket = self._socket.dup()
         if self._use_ssl:
             try:
                 ca_file = CONF.ssl_ca_file
@@ -175,18 +182,18 @@ class Server(object):
                     ssl_kwargs['ca_certs'] = ca_file
                     ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED
 
-                self._socket = eventlet.wrap_ssl(self._socket,
-                                                 **ssl_kwargs)
+                dup_socket = eventlet.wrap_ssl(dup_socket,
+                                               **ssl_kwargs)
 
-                self._socket.setsockopt(socket.SOL_SOCKET,
-                                        socket.SO_REUSEADDR, 1)
+                dup_socket.setsockopt(socket.SOL_SOCKET,
+                                      socket.SO_REUSEADDR, 1)
                 # sockets can hang around forever without keepalive
-                self._socket.setsockopt(socket.SOL_SOCKET,
-                                        socket.SO_KEEPALIVE, 1)
+                dup_socket.setsockopt(socket.SOL_SOCKET,
+                                      socket.SO_KEEPALIVE, 1)
 
                 # This option isn't available in the OS X version of eventlet
                 if hasattr(socket, 'TCP_KEEPIDLE'):
-                    self._socket.setsockopt(socket.IPPROTO_TCP,
+                    dup_socket.setsockopt(socket.IPPROTO_TCP,
                                     socket.TCP_KEEPIDLE,
                                     CONF.tcp_keepidle)
 
@@ -197,7 +204,7 @@ class Server(object):
 
         wsgi_kwargs = {
             'func': eventlet.wsgi.server,
-            'sock': self._socket,
+            'sock': dup_socket,
             'site': self.app,
             'protocol': self._protocol,
             'custom_pool': self._pool,
@@ -211,6 +218,14 @@ class Server(object):
 
         self._server = eventlet.spawn(**wsgi_kwargs)
 
+    def reset(self):
+        """Reset server greenpool size to default.
+
+        :returns: None
+
+        """
+        self._pool.resize(self.pool_size)
+
     def stop(self):
         """Stop this server.