Merge "Consume sslutils and wsgi modules from oslo.service"

doc/source/devref/neutron_api.rst

@@ -50,6 +50,13 @@ Neutron API is not very stable, and there are cases when a desired change in
 neutron tree is expected to trigger breakage for one or more external
 repositories under the neutron tent. Below you can find a list of known
 incompatible changes that could or are known to trigger those breakages.
+The changes are listed in reverse chronological order (newer at the top).
+
+* change: Consume sslutils and wsgi modules from oslo.service.
+
+  - commit: Ibfdf07e665fcfcd093a0e31274e1a6116706aec2
+  - solution: switch using oslo_service.wsgi.Router; stop using neutron.wsgi.Router.
+  - severity: Low (some out-of-tree plugins might be affected).
 
 * change: oslo.service adopted.
 

etc/neutron.conf

@@ -344,15 +344,21 @@
 # use_ssl = False
 
 # Certificate file to use when starting API server securely
+# This option is deprecated for removal in the N release, please
+# use cert_file option from [ssl] section instead.
 # ssl_cert_file = /path/to/certfile
 
 # Private key file to use when starting API server securely
+# This option is deprecated for removal in the N release, please
+# use key_file option from [ssl] section instead.
 # ssl_key_file = /path/to/keyfile
 
 # CA certificate file to use when starting API server securely to
 # verify connecting clients. This is an optional parameter only required if
 # API clients need to authenticate to the API server using SSL certificates
 # signed by a trusted CA
+# This option is deprecated for removal in the N release, please
+# use ca_file option from [ssl] section instead.
 # ssl_ca_file = /path/to/cafile
 # ======== end of WSGI parameters related to the API server ==========
 
@@ -1038,3 +1044,21 @@ lock_path = $state_path/lock
 [qos]
 # Drivers list to use to send the update notification
 # notification_drivers = message_queue
+
+[ssl]
+
+#
+# From oslo.service.sslutils
+#
+
+# CA certificate file to use to verify connecting clients. (string
+# value)
+#ca_file = <None>
+
+# Certificate file to use when starting the server securely. (string
+# value)
+#cert_file = <None>
+
+# Private key file to use when starting the server securely. (string
+# value)
+#key_file = <None>

neutron/agent/metadata/namespace_proxy.py

@@ -15,6 +15,7 @@
 import httplib2
 from oslo_config import cfg
 from oslo_log import log as logging
+from oslo_service import wsgi as base_wsgi
 from oslo_utils import encodeutils
 import six
 import six.moves.urllib.parse as urlparse
@@ -45,7 +46,7 @@ class NetworkMetadataProxyHandler(object):
         if network_id is None and router_id is None:
             raise exceptions.NetworkIdOrRouterIdRequiredError()
 
-    @webob.dec.wsgify(RequestClass=webob.Request)
+    @webob.dec.wsgify(RequestClass=base_wsgi.Request)
     def __call__(self, req):
         LOG.debug("Request: %s", req)
         try:

neutron/api/v2/router.py

@@ -15,6 +15,7 @@
 
 from oslo_config import cfg
 from oslo_log import log as logging
+from oslo_service import wsgi as base_wsgi
 import routes as routes_mapper
 import six
 import six.moves.urllib.parse as urlparse
@@ -66,7 +67,7 @@ class Index(wsgi.Application):
         return webob.Response(body=body, content_type=content_type)
 
 
-class APIRouter(wsgi.Router):
+class APIRouter(base_wsgi.Router):
 
     @classmethod
     def factory(cls, global_config, **local_config):

neutron/common/config.py

@@ -17,7 +17,6 @@
 Routines for configuring Neutron
 """
 
-import os
 import sys
 
 from keystoneclient import auth
@@ -26,7 +25,8 @@ from oslo_config import cfg
 from oslo_db import options as db_options
 from oslo_log import log as logging
 import oslo_messaging
-from paste import deploy
+from oslo_service import _options
+from oslo_service import wsgi
 
 from neutron.api.v2 import attributes
 from neutron.common import utils
@@ -42,8 +42,6 @@ core_opts = [
                help=_("The host IP to bind to")),
     cfg.IntOpt('bind_port', default=9696,
                help=_("The port to bind to")),
-    cfg.StrOpt('api_paste_config', default="api-paste.ini",
-               help=_("The API paste config file to use")),
     cfg.StrOpt('api_extensions_path', default="",
                help=_("The path for API extensions")),
     cfg.StrOpt('auth_strategy', default='keystone',
@@ -153,6 +151,9 @@ core_cli_opts = [
 # Register the configuration options
 cfg.CONF.register_opts(core_opts)
 cfg.CONF.register_cli_opts(core_cli_opts)
+# TODO(eezhova): Replace it with wsgi.register_opts(CONF) when oslo.service
+# 0.10.0 releases.
+cfg.CONF.register_opts(_options.wsgi_opts)
 
 # Ensure that the control exchange is set correctly
 oslo_messaging.set_transport_defaults(control_exchange='neutron')
@@ -232,24 +233,7 @@ def load_paste_app(app_name):
     """Builds and returns a WSGI app from a paste config file.
 
     :param app_name: Name of the application to load
-    :raises ConfigFilesNotFoundError when config file cannot be located
-    :raises RuntimeError when application cannot be loaded from config file
     """
-
+    loader = wsgi.Loader(cfg.CONF)
-    config_path = cfg.CONF.find_file(cfg.CONF.api_paste_config)
+    app = loader.load_app(app_name)
-    if not config_path:
-        raise cfg.ConfigFilesNotFoundError(
-            config_files=[cfg.CONF.api_paste_config])
-    config_path = os.path.abspath(config_path)
-    LOG.info(_LI("Config paste file: %s"), config_path)
-
-    try:
-        app = deploy.loadapp("config:%s" % config_path, name=app_name)
-    except (LookupError, ImportError):
-        msg = (_("Unable to load %(app_name)s from "
-                 "configuration file %(config_path)s.") %
-               {'app_name': app_name,
-                'config_path': config_path})
-        LOG.exception(msg)
-        raise RuntimeError(msg)
     return app

neutron/tests/unit/api/test_extensions.py

@@ -19,6 +19,7 @@ import mock
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_serialization import jsonutils
+from oslo_service import wsgi as base_wsgi
 import routes
 import six
 import webob
@@ -48,7 +49,7 @@ _get_path = test_base._get_path
 extensions_path = ':'.join(neutron.tests.unit.extensions.__path__)
 
 
-class ExtensionsTestApp(wsgi.Router):
+class ExtensionsTestApp(base_wsgi.Router):
 
     def __init__(self, options={}):
         mapper = routes.Mapper()

neutron/tests/unit/common/test_config.py

@@ -1,31 +0,0 @@
-# Copyright (c) 2012 OpenStack Foundation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import mock
-from oslo_config import cfg
-
-from neutron.common import config
-from neutron.tests import base
-
-
-class ConfigurationTest(base.BaseTestCase):
-
-    def test_load_paste_app_not_found(self):
-        self.config(api_paste_config='no_such_file.conf')
-        with mock.patch.object(cfg.CONF, 'find_file', return_value=None) as ff:
-            e = self.assertRaises(cfg.ConfigFilesNotFoundError,
-                                  config.load_paste_app, 'app')
-            ff.assert_called_once_with('no_such_file.conf')
-            self.assertEqual(['no_such_file.conf'], e.config_files)

neutron/tests/unit/test_wsgi.py

@@ -713,139 +713,3 @@ class FaultTest(base.BaseTestCase):
             "/", method='POST', headers={'Content-Type': "unknow"})
         response = my_fault(request)
         self.assertEqual(415, response.status_int)
-
-
-class TestWSGIServerWithSSL(base.BaseTestCase):
-    """WSGI server tests."""
-
-    def setUp(self):
-        super(TestWSGIServerWithSSL, self).setUp()
-        if six.PY3:
-            self.skip("bug/1482633")
-
-    @mock.patch("exceptions.RuntimeError")
-    @mock.patch("os.path.exists")
-    def test__check_ssl_settings(self, exists_mock, runtime_error_mock):
-        exists_mock.return_value = True
-        CONF.set_default('use_ssl', True)
-        CONF.set_default("ssl_cert_file", 'certificate.crt')
-        CONF.set_default("ssl_key_file", 'privatekey.key')
-        CONF.set_default("ssl_ca_file", 'cacert.pem')
-        wsgi.Server("test_app")
-        self.assertFalse(runtime_error_mock.called)
-
-    @mock.patch("os.path.exists")
-    def test__check_ssl_settings_no_ssl_cert_file_fails(self, exists_mock):
-        exists_mock.side_effect = [False]
-        CONF.set_default('use_ssl', True)
-        CONF.set_default("ssl_cert_file", "/no/such/file")
-        self.assertRaises(RuntimeError, wsgi.Server, "test_app")
-
-    @mock.patch("os.path.exists")
-    def test__check_ssl_settings_no_ssl_key_file_fails(self, exists_mock):
-        exists_mock.side_effect = [True, False]
-        CONF.set_default('use_ssl', True)
-        CONF.set_default("ssl_cert_file", 'certificate.crt')
-        CONF.set_default("ssl_key_file", "/no/such/file")
-        self.assertRaises(RuntimeError, wsgi.Server, "test_app")
-
-    @mock.patch("os.path.exists")
-    def test__check_ssl_settings_no_ssl_ca_file_fails(self, exists_mock):
-        exists_mock.side_effect = [True, True, False]
-        CONF.set_default('use_ssl', True)
-        CONF.set_default("ssl_cert_file", 'certificate.crt')
-        CONF.set_default("ssl_key_file", 'privatekey.key')
-        CONF.set_default("ssl_ca_file", "/no/such/file")
-        self.assertRaises(RuntimeError, wsgi.Server, "test_app")
-
-    @mock.patch("ssl.wrap_socket")
-    @mock.patch("os.path.exists")
-    def _test_wrap_ssl(self, exists_mock, wrap_socket_mock, **kwargs):
-        exists_mock.return_value = True
-        sock = mock.Mock()
-        CONF.set_default("ssl_cert_file", 'certificate.crt')
-        CONF.set_default("ssl_key_file", 'privatekey.key')
-        ssl_kwargs = {'server_side': True,
-                      'certfile': CONF.ssl_cert_file,
-                      'keyfile': CONF.ssl_key_file,
-                      'cert_reqs': ssl.CERT_NONE,
-                      }
-        if kwargs:
-            ssl_kwargs.update(**kwargs)
-        server = wsgi.Server("test_app")
-        server.wrap_ssl(sock)
-        wrap_socket_mock.assert_called_once_with(sock, **ssl_kwargs)
-
-    def test_wrap_ssl(self):
-        self._test_wrap_ssl()
-
-    def test_wrap_ssl_ca_file(self):
-        CONF.set_default("ssl_ca_file", 'cacert.pem')
-        ssl_kwargs = {'ca_certs': CONF.ssl_ca_file,
-                      'cert_reqs': ssl.CERT_REQUIRED
-                      }
-        self._test_wrap_ssl(**ssl_kwargs)
-
-    def test_app_using_ssl(self):
-        CONF.set_default('use_ssl', True)
-        CONF.set_default("ssl_cert_file",
-                         os.path.join(TEST_VAR_DIR, 'certificate.crt'))
-        CONF.set_default("ssl_key_file",
-                         os.path.join(TEST_VAR_DIR, 'privatekey.key'))
-
-        greetings = 'Hello, World!!!'
-
-        @webob.dec.wsgify
-        def hello_world(req):
-            return greetings
-
-        server = wsgi.Server("test_app")
-        server.start(hello_world, 0, host="127.0.0.1")
-
-        response = open_no_proxy('https://127.0.0.1:%d/' % server.port)
-
-        self.assertEqual(greetings, response.read())
-
-        server.stop()
-
-    def test_app_using_ssl_combined_cert_and_key(self):
-        CONF.set_default('use_ssl', True)
-        CONF.set_default("ssl_cert_file",
-                         os.path.join(TEST_VAR_DIR, 'certandkey.pem'))
-
-        greetings = 'Hello, World!!!'
-
-        @webob.dec.wsgify
-        def hello_world(req):
-            return greetings
-
-        server = wsgi.Server("test_app")
-        server.start(hello_world, 0, host="127.0.0.1")
-
-        response = open_no_proxy('https://127.0.0.1:%d/' % server.port)
-
-        self.assertEqual(greetings, response.read())
-
-        server.stop()
-
-    def test_app_using_ipv6_and_ssl(self):
-        CONF.set_default('use_ssl', True)
-        CONF.set_default("ssl_cert_file",
-                         os.path.join(TEST_VAR_DIR, 'certificate.crt'))
-        CONF.set_default("ssl_key_file",
-                         os.path.join(TEST_VAR_DIR, 'privatekey.key'))
-
-        greetings = 'Hello, World!!!'
-
-        @webob.dec.wsgify
-        def hello_world(req):
-            return greetings
-
-        server = wsgi.Server("test_app")
-        server.start(hello_world, 0, host="::1")
-
-        response = open_no_proxy('https://[::1]:%d/' % server.port)
-
-        self.assertEqual(greetings, response.read())
-
-        server.stop()

neutron/wsgi.py

@@ -19,9 +19,7 @@ Utility methods for working with WSGI servers
 from __future__ import print_function
 
 import errno
-import os
 import socket
-import ssl
 import sys
 import time
 
@@ -30,10 +28,12 @@ from oslo_config import cfg
 import oslo_i18n
 from oslo_log import log as logging
 from oslo_serialization import jsonutils
+from oslo_service import _options
 from oslo_service import service as common_service
+from oslo_service import sslutils
 from oslo_service import systemd
+from oslo_service import wsgi
 from oslo_utils import excutils
-import routes.middleware
 import six
 import webob.dec
 import webob.exc
@@ -50,44 +50,19 @@ socket_opts = [
                default=4096,
                help=_("Number of backlog requests to configure "
                       "the socket with")),
-    cfg.IntOpt('tcp_keepidle',
-               default=600,
-               help=_("Sets the value of TCP_KEEPIDLE in seconds for each "
-                      "server socket. Not supported on OS X.")),
     cfg.IntOpt('retry_until_window',
                default=30,
                help=_("Number of seconds to keep retrying to listen")),
-    cfg.IntOpt('max_header_line',
-               default=16384,
-               help=_("Max header line to accommodate large tokens")),
     cfg.BoolOpt('use_ssl',
                 default=False,
                 help=_('Enable SSL on the API server')),
-    cfg.StrOpt('ssl_ca_file',
-               help=_("CA certificate file to use to verify "
-                      "connecting clients")),
-    cfg.StrOpt('ssl_cert_file',
-               help=_("Certificate file to use when starting "
-                      "the server securely")),
-    cfg.StrOpt('ssl_key_file',
-               help=_("Private key file to use when starting "
-                      "the server securely")),
-    cfg.BoolOpt('wsgi_keep_alive',
-                default=True,
-                help=_("Determines if connections are allowed to be held "
-                     "open by clients after a request is fulfilled. A value "
-                     "of False will ensure that the socket connection will "
-                     "be explicitly closed once a response has been sent to "
-                     "the client.")),
-    cfg.IntOpt('client_socket_timeout', default=900,
-               help=_("Timeout for client connections socket operations. "
-                    "If an incoming connection is idle for this number of "
-                    "seconds it will be closed. A value of '0' means "
-                    "wait forever.")),
 ]
 
 CONF = cfg.CONF
 CONF.register_opts(socket_opts)
+# TODO(eezhova): Replace it with wsgi.register_opts(CONF) when oslo.service
+# 0.10.0 releases.
+CONF.register_opts(_options.wsgi_opts)
 
 LOG = logging.getLogger(__name__)
 
@@ -118,7 +93,7 @@ class WorkerService(worker.NeutronWorker):
         # Duplicate a socket object to keep a file descriptor usable.
         dup_sock = self._service._socket.dup()
         if CONF.use_ssl:
-            dup_sock = self._service.wrap_ssl(dup_sock)
+            dup_sock = sslutils.wrap(CONF, dup_sock)
         self._server = self._service.pool.spawn(self._service._run,
                                                 self._application,
                                                 dup_sock)
@@ -152,7 +127,7 @@ class Server(object):
         # wsgi server to wait forever.
         self.client_socket_timeout = CONF.client_socket_timeout or None
         if CONF.use_ssl:
-            self._check_ssl_settings()
+            sslutils.is_enabled(CONF)
 
     def _get_socket(self, host, port, backlog):
         bind_addr = (host, port)
@@ -201,37 +176,6 @@ class Server(object):
 
         return sock
 
-    @staticmethod
-    def _check_ssl_settings():
-        if not os.path.exists(CONF.ssl_cert_file):
-            raise RuntimeError(_("Unable to find ssl_cert_file "
-                                 ": %s") % CONF.ssl_cert_file)
-
-        # ssl_key_file is optional because the key may be embedded in the
-        # certificate file
-        if CONF.ssl_key_file and not os.path.exists(CONF.ssl_key_file):
-            raise RuntimeError(_("Unable to find "
-                                 "ssl_key_file : %s") % CONF.ssl_key_file)
-
-        # ssl_ca_file is optional
-        if CONF.ssl_ca_file and not os.path.exists(CONF.ssl_ca_file):
-            raise RuntimeError(_("Unable to find ssl_ca_file "
-                                 ": %s") % CONF.ssl_ca_file)
-
-    @staticmethod
-    def wrap_ssl(sock):
-        ssl_kwargs = {'server_side': True,
-                      'certfile': CONF.ssl_cert_file,
-                      'keyfile': CONF.ssl_key_file,
-                      'cert_reqs': ssl.CERT_NONE,
-                      }
-
-        if CONF.ssl_ca_file:
-            ssl_kwargs['ca_certs'] = CONF.ssl_ca_file
-            ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED
-
-        return ssl.wrap_socket(sock, **ssl_kwargs)
-
     def start(self, application, port, host='0.0.0.0', workers=0):
         """Run a WSGI server with the given application."""
         self._host = host
@@ -353,7 +297,7 @@ class Middleware(object):
         return self.process_response(response)
 
 
-class Request(webob.Request):
+class Request(wsgi.Request):
 
     def best_match_content_type(self):
         """Determine the most acceptable content-type.
@@ -710,63 +654,6 @@ class Debug(Middleware):
         print()
 
 
-class Router(object):
-    """WSGI middleware that maps incoming requests to WSGI apps."""
-
-    def __init__(self, mapper):
-        """Create a router for the given routes.Mapper.
-
-        Each route in `mapper` must specify a 'controller', which is a
-        WSGI app to call.  You'll probably want to specify an 'action' as
-        well and have your controller be a wsgi.Controller, who will route
-        the request to the action method.
-
-        Examples:
-          mapper = routes.Mapper()
-          sc = ServerController()
-
-          # Explicit mapping of one route to a controller+action
-          mapper.connect(None, "/svrlist", controller=sc, action="list")
-
-          # Actions are all implicitly defined
-          mapper.resource("network", "networks", controller=nc)
-
-          # Pointing to an arbitrary WSGI app.  You can specify the
-          # {path_info:.*} parameter so the target app can be handed just that
-          # section of the URL.
-          mapper.connect(None, "/v1.0/{path_info:.*}", controller=BlogApp())
-        """
-        self.map = mapper
-        self._router = routes.middleware.RoutesMiddleware(self._dispatch,
-                                                          self.map)
-
-    @webob.dec.wsgify
-    def __call__(self, req):
-        """Route the incoming request to a controller based on self.map.
-
-        If no match, return a 404.
-        """
-        return self._router
-
-    @staticmethod
-    @webob.dec.wsgify(RequestClass=Request)
-    def _dispatch(req):
-        """Dispatch a Request.
-
-        Called by self._router after matching the incoming request to a route
-        and putting the information into req.environ. Either returns 404
-        or the routed WSGI app's response.
-        """
-        match = req.environ['wsgiorg.routing_args'][1]
-        if not match:
-            language = req.best_match_language()
-            msg = _('The resource could not be found.')
-            msg = oslo_i18n.translate(msg, language)
-            return webob.exc.HTTPNotFound(explanation=msg)
-        app = match['controller']
-        return app
-
-
 class Resource(Application):
     """WSGI app that handles (de)serialization and controller dispatch.