diff --git a/contrib/ci/pre_test_hook.sh b/contrib/ci/pre_test_hook.sh index fd6f799b0c..e5f4ce9f5c 100755 --- a/contrib/ci/pre_test_hook.sh +++ b/contrib/ci/pre_test_hook.sh @@ -85,6 +85,12 @@ elif [[ "$DRIVER" == "dummy" ]]; then driver_path="manila.tests.share.drivers.dummy.DummyDriver" DEFAULT_EXTRA_SPECS="'snapshot_support=True create_share_from_snapshot_support=True revert_to_snapshot_support=True mount_snapshot_support=True'" echo "MANILA_SERVICE_IMAGE_ENABLED=False" >> $localconf + + # Run dummy driver CI job using standalone approach for running + # manila API service just because we need to test this approach too, + # that is very useful for development needs. + echo "MANILA_USE_MOD_WSGI=False" >> $localconf + echo "SHARE_DRIVER=$driver_path" >> $localconf echo "SUPPRESS_ERRORS_IN_CLEANUP=False" >> $localconf echo "MANILA_REPLICA_STATE_UPDATE_INTERVAL=10" >> $localconf diff --git a/devstack/apache-manila.template b/devstack/apache-manila.template new file mode 100644 index 0000000000..5049954d07 --- /dev/null +++ b/devstack/apache-manila.template @@ -0,0 +1,25 @@ +Listen %PORT% +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)" manila_combined + + + WSGIDaemonProcess manila-api processes=%APIWORKERS% threads=2 user=%USER% display-name=%{GROUP} + WSGIProcessGroup manila-api + WSGIScriptAlias / %MANILA_BIN_DIR%/manila-wsgi + WSGIApplicationGroup %{GLOBAL} + WSGIPassAuthorization On + = 2.4> + ErrorLogFormat "%{cu}t %M" + + ErrorLog /var/log/%APACHE_NAME%/manila_api.log + CustomLog /var/log/%APACHE_NAME%/manila_api_access.log manila_combined + + + = 2.4> + Require all granted + + + Order allow,deny + Allow from all + + + diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 54cc3402bc..5cdd0b5823 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -59,6 +59,22 @@ function cleanup_manila { _clean_zfsonlinux_data } +# _config_manila_apache_wsgi() - Configure manila-api wsgi application. +function _config_manila_apache_wsgi { + local manila_api_apache_conf + local venv_path="" + manila_api_apache_conf=$(apache_site_config_for manila-api) + + sudo cp $MANILA_DIR/devstack/apache-manila.template $manila_api_apache_conf + sudo sed -e " + s|%APACHE_NAME%|$APACHE_NAME|g; + s|%MANILA_BIN_DIR%|$MANILA_BIN_DIR|g; + s|%PORT%|$MANILA_SERVICE_PORT|g; + s|%APIWORKERS%|$API_WORKERS|g; + s|%USER%|$STACK_USER|g; + " -i $manila_api_apache_conf +} + # configure_default_backends - configures default Manila backends with generic driver. function configure_default_backends { # Configure two default backends with generic drivers onboard @@ -257,6 +273,10 @@ function configure_manila { MANILA_CONFIGURE_GROUPS=${MANILA_CONFIGURE_GROUPS:-"$MANILA_ENABLED_BACKENDS"} set_config_opts $MANILA_CONFIGURE_GROUPS set_config_opts DEFAULT + + if [ $(trueorfalse False MANILA_USE_MOD_WSGI) == True ]; then + _config_manila_apache_wsgi + fi } @@ -759,7 +779,14 @@ function configure_samba { # start_manila_api - starts manila API services and checks its availability function start_manila_api { - run_process m-api "$MANILA_BIN_DIR/manila-api --config-file $MANILA_CONF" + if [ $(trueorfalse False MANILA_USE_MOD_WSGI) == True ]; then + install_apache_wsgi + enable_apache_site manila-api + restart_apache_server + tail_log m-api /var/log/$APACHE_NAME/manila_api.log + else + run_process m-api "$MANILA_BIN_DIR/manila-api --config-file $MANILA_CONF" + fi echo "Waiting for Manila API to start..." if ! wait_for_service $SERVICE_TIMEOUT $MANILA_SERVICE_PROTOCOL://$MANILA_SERVICE_HOST:$MANILA_SERVICE_PORT; then @@ -788,8 +815,16 @@ function start_manila { # stop_manila - Stop running processes function stop_manila { - # Kill the manila processes - for serv in m-api m-sch m-shr m-dat; do + # Disable manila api service + if [ $(trueorfalse False MANILA_USE_MOD_WSGI) == True ]; then + disable_apache_site manila-api + restart_apache_server + else + stop_process m-api + fi + + # Kill all other manila processes + for serv in m-sch m-shr m-dat; do stop_process $serv done } diff --git a/devstack/settings b/devstack/settings index cc3c68c369..42f1a3f253 100644 --- a/devstack/settings +++ b/devstack/settings @@ -73,10 +73,9 @@ MANILA_DEFAULT_SHARE_GROUP_TYPE_SPECS=${MANILA_DEFAULT_SHARE_GROUP_TYPE_SPECS:-' # Public facing bits MANILA_SERVICE_HOST=${MANILA_SERVICE_HOST:-$SERVICE_HOST} MANILA_SERVICE_PORT=${MANILA_SERVICE_PORT:-8786} -MANILA_SERVICE_PORT_INT=${MANILA_SERVICE_PORT_INT:-18776} +MANILA_SERVICE_PORT_INT=${MANILA_SERVICE_PORT_INT:-18786} MANILA_SERVICE_PROTOCOL=${MANILA_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL} - # Support entry points installation of console scripts if [[ -d $MANILA_DIR/bin ]]; then MANILA_BIN_DIR=$MANILA_DIR/bin @@ -84,7 +83,6 @@ else MANILA_BIN_DIR=$(get_python_exec_prefix) fi - # Common opts SHARE_NAME_PREFIX=${SHARE_NAME_PREFIX:-share-} MANILA_ENABLED_SHARE_PROTOCOLS=${ENABLED_SHARE_PROTOCOLS:-"NFS,CIFS"} @@ -97,6 +95,14 @@ MANILA_SERVICE_SECGROUP="manila-service" # migrations again. MANILA_USE_DOWNGRADE_MIGRATIONS=${MANILA_USE_DOWNGRADE_MIGRATIONS:-"False"} +# Toggle for deploying manila-api service under Apache web server with enabled 'mod_wsgi' plugin. +# Disabled by default, which means running manila-api service as standalone +# eventlet-based WSGI application. +# Set it as True, because starting with Pike it is requirement from +# 'governance' project. See: +# https://governance.openstack.org/tc/goals/pike/deploy-api-in-wsgi.html#completion-criteria +MANILA_USE_MOD_WSGI=${MANILA_USE_MOD_WSGI:-True} + # Common info for Generic driver(s) SHARE_DRIVER=${SHARE_DRIVER:-manila.share.drivers.generic.GenericShareDriver} diff --git a/manila/api/middleware/auth.py b/manila/api/middleware/auth.py index 1e798c9305..cd6b5490f4 100644 --- a/manila/api/middleware/auth.py +++ b/manila/api/middleware/auth.py @@ -27,7 +27,7 @@ import webob.exc from manila.api.openstack import wsgi from manila import context from manila.i18n import _ -from manila import wsgi as base_wsgi +from manila.wsgi import common as base_wsgi use_forwarded_for_opt = cfg.BoolOpt( 'use_forwarded_for', diff --git a/manila/api/middleware/fault.py b/manila/api/middleware/fault.py index 5c1434b64f..e24e95a446 100644 --- a/manila/api/middleware/fault.py +++ b/manila/api/middleware/fault.py @@ -21,7 +21,7 @@ import webob.exc from manila.api.openstack import wsgi from manila import utils -from manila import wsgi as base_wsgi +from manila.wsgi import common as base_wsgi LOG = log.getLogger(__name__) diff --git a/manila/api/openstack/__init__.py b/manila/api/openstack/__init__.py index 3897d4c5c2..040abc28fb 100644 --- a/manila/api/openstack/__init__.py +++ b/manila/api/openstack/__init__.py @@ -19,11 +19,11 @@ WSGI middleware for OpenStack API controllers. """ from oslo_log import log +from oslo_service import wsgi as base_wsgi import routes from manila.api.openstack import wsgi from manila.i18n import _ -from manila import wsgi as base_wsgi LOG = log.getLogger(__name__) @@ -117,13 +117,3 @@ class APIRouter(base_wsgi.Router): def _setup_routes(self, mapper, ext_mgr): raise NotImplementedError - - -class FaultWrapper(base_wsgi.Middleware): - def __init__(self, application): - LOG.warning('manila.api.openstack:FaultWrapper is deprecated. ' - 'Please use ' - 'manila.api.middleware.fault:FaultWrapper instead.') - # Avoid circular imports from here. - from manila.api.middleware import fault - super(FaultWrapper, self).__init__(fault.FaultWrapper(application)) diff --git a/manila/api/openstack/wsgi.py b/manila/api/openstack/wsgi.py index e1b1bc32d7..2f984f0535 100644 --- a/manila/api/openstack/wsgi.py +++ b/manila/api/openstack/wsgi.py @@ -31,7 +31,8 @@ from manila.common import constants from manila import exception from manila.i18n import _ from manila import policy -from manila import wsgi +from manila import utils +from manila.wsgi import common as wsgi LOG = log.getLogger(__name__) @@ -860,15 +861,20 @@ class Resource(wsgi.Application): if hasattr(response, 'headers'): for hdr, val in response.headers.items(): - # Headers must be utf-8 strings - response.headers[hdr] = six.text_type(val) + val = utils.convert_str(val) + response.headers[hdr] = val if not request.api_version_request.is_null(): response.headers[API_VERSION_REQUEST_HEADER] = ( request.api_version_request.get_string()) if request.api_version_request.experimental: + # NOTE(vponomaryov): Translate our boolean header + # to string explicitly to avoid 'TypeError' failure + # running manila API under Apache + mod-wsgi. + # It is safe to do so, because all headers are returned as + # strings anyway. response.headers[EXPERIMENTAL_API_REQUEST_HEADER] = ( - request.api_version_request.experimental) + '%s' % request.api_version_request.experimental) response.headers['Vary'] = API_VERSION_REQUEST_HEADER return response @@ -1280,14 +1286,19 @@ class Fault(webob.exc.HTTPException): 'message': self.wrapped_exc.explanation}} if code == 413: retry = self.wrapped_exc.headers['Retry-After'] - fault_data[fault_name]['retryAfter'] = retry + fault_data[fault_name]['retryAfter'] = '%s' % retry if not req.api_version_request.is_null(): self.wrapped_exc.headers[API_VERSION_REQUEST_HEADER] = ( req.api_version_request.get_string()) if req.api_version_request.experimental: + # NOTE(vponomaryov): Translate our boolean header + # to string explicitly to avoid 'TypeError' failure + # running manila API under Apache + mod-wsgi. + # It is safe to do so, because all headers are returned as + # strings anyway. self.wrapped_exc.headers[EXPERIMENTAL_API_REQUEST_HEADER] = ( - req.api_version_request.experimental) + '%s' % req.api_version_request.experimental) self.wrapped_exc.headers['Vary'] = API_VERSION_REQUEST_HEADER content_type = req.best_match_content_type() @@ -1330,7 +1341,7 @@ class OverLimitFault(webob.exc.HTTPException): def _retry_after(retry_time): delay = int(math.ceil(retry_time - time.time())) retry_after = delay if delay > 0 else 0 - headers = {'Retry-After': '%d' % retry_after} + headers = {'Retry-After': '%s' % retry_after} return headers @webob.dec.wsgify(RequestClass=Request) diff --git a/manila/api/v1/limits.py b/manila/api/v1/limits.py index 66c04ec6f9..85209d6d6d 100644 --- a/manila/api/v1/limits.py +++ b/manila/api/v1/limits.py @@ -33,7 +33,7 @@ from manila.api.openstack import wsgi from manila.api.views import limits as limits_views from manila.i18n import _ from manila import quota -from manila import wsgi as base_wsgi +from manila.wsgi import common as base_wsgi QUOTAS = quota.QUOTAS diff --git a/manila/common/config.py b/manila/common/config.py index bc4feb73e6..a69cdcddbe 100644 --- a/manila/common/config.py +++ b/manila/common/config.py @@ -41,9 +41,6 @@ log.register_options(CONF) core_opts = [ - cfg.StrOpt('api_paste_config', - default="api-paste.ini", - help='File name for the paste.deploy config for manila-api.'), cfg.StrOpt('state_path', default='/var/lib/manila', help="Top-level directory for maintaining manila's state."), diff --git a/manila/exception.py b/manila/exception.py index d23d63a649..65abbe5ede 100644 --- a/manila/exception.py +++ b/manila/exception.py @@ -386,7 +386,7 @@ class WillNotSchedule(ManilaException): class QuotaError(ManilaException): message = _("Quota exceeded: code=%(code)s.") code = 413 - headers = {'Retry-After': 0} + headers = {'Retry-After': '0'} safe = True diff --git a/manila/opts.py b/manila/opts.py index 930f202628..65b9afba46 100644 --- a/manila/opts.py +++ b/manila/opts.py @@ -85,7 +85,7 @@ import manila.share.hook import manila.share.manager import manila.volume import manila.volume.cinder -import manila.wsgi +import manila.wsgi.eventlet_server # List of *all* options in [DEFAULT] namespace of manila. @@ -165,8 +165,7 @@ _global_opt_lists = [ manila.share.hook.hook_options, manila.share.manager.share_manager_opts, manila.volume._volume_opts, - manila.wsgi.eventlet_opts, - manila.wsgi.socket_opts, + manila.wsgi.eventlet_server.socket_opts, ] _opts = [ diff --git a/manila/service.py b/manila/service.py index 7687f9d556..03786f7821 100644 --- a/manila/service.py +++ b/manila/service.py @@ -26,6 +26,7 @@ from oslo_log import log import oslo_messaging as messaging from oslo_service import loopingcall from oslo_service import service +from oslo_service import wsgi from oslo_utils import importutils from manila import context @@ -34,7 +35,6 @@ from manila import db from manila import exception from manila import rpc from manila import version -from manila import wsgi LOG = log.getLogger(__name__) @@ -283,7 +283,7 @@ class WSGIService(service.ServiceBase): """ self.name = name self.manager = self._get_manager() - self.loader = loader or wsgi.Loader() + self.loader = loader or wsgi.Loader(CONF) if not rpc.initialized(): rpc.init(CONF) self.app = self.loader.load_app(name) @@ -296,10 +296,13 @@ class WSGIService(service.ServiceBase): "greater than 1. Input value ignored." % {'name': name}) # Reset workers to default self.workers = None - self.server = wsgi.Server(name, - self.app, - host=self.host, - port=self.port) + self.server = wsgi.Server( + CONF, + name, + self.app, + host=self.host, + port=self.port, + ) def _get_manager(self): """Initialize a Manager object appropriate for this service. diff --git a/manila/tests/api/fakes.py b/manila/tests/api/fakes.py index c104b8835b..f7c93c0908 100644 --- a/manila/tests/api/fakes.py +++ b/manila/tests/api/fakes.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo_service import wsgi from oslo_utils import timeutils from oslo_utils import uuidutils import routes @@ -33,7 +34,6 @@ from manila.api import versions from manila.common import constants from manila import context from manila import exception -from manila import wsgi FAKE_UUID = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' diff --git a/manila/tests/api/middleware/test_faults.py b/manila/tests/api/middleware/test_faults.py index a540187e72..a15e8fda8c 100644 --- a/manila/tests/api/middleware/test_faults.py +++ b/manila/tests/api/middleware/test_faults.py @@ -19,7 +19,9 @@ import webob import webob.dec import webob.exc +from manila.api.middleware import fault from manila.api.openstack import wsgi +from manila import exception from manila import test @@ -72,7 +74,7 @@ class TestFaults(test.TestCase): "overLimit": { "message": "sorry", "code": 413, - "retryAfter": 4, + "retryAfter": '4', }, } actual = jsonutils.loads(response.body) @@ -109,3 +111,76 @@ class TestFaults(test.TestCase): """Ensure the status_int is set correctly on faults.""" fault = wsgi.Fault(webob.exc.HTTPBadRequest(explanation='what?')) self.assertEqual(400, fault.status_int) + + +class ExceptionTest(test.TestCase): + + def _wsgi_app(self, inner_app): + return fault.FaultWrapper(inner_app) + + def _do_test_exception_safety_reflected_in_faults(self, expose): + class ExceptionWithSafety(exception.ManilaException): + safe = expose + + @webob.dec.wsgify + def fail(req): + raise ExceptionWithSafety('some explanation') + + api = self._wsgi_app(fail) + resp = webob.Request.blank('/').get_response(api) + self.assertIn('{"computeFault', six.text_type(resp.body), resp.body) + expected = ('ExceptionWithSafety: some explanation' if expose else + 'The server has either erred or is incapable ' + 'of performing the requested operation.') + self.assertIn(expected, six.text_type(resp.body), resp.body) + self.assertEqual(500, resp.status_int, resp.body) + + def test_safe_exceptions_are_described_in_faults(self): + self._do_test_exception_safety_reflected_in_faults(True) + + def test_unsafe_exceptions_are_not_described_in_faults(self): + self._do_test_exception_safety_reflected_in_faults(False) + + def _do_test_exception_mapping(self, exception_type, msg): + @webob.dec.wsgify + def fail(req): + raise exception_type(msg) + + api = self._wsgi_app(fail) + resp = webob.Request.blank('/').get_response(api) + self.assertIn(msg, six.text_type(resp.body), resp.body) + self.assertEqual(exception_type.code, resp.status_int, resp.body) + + if hasattr(exception_type, 'headers'): + for (key, value) in exception_type.headers.items(): + self.assertIn(key, resp.headers) + self.assertEqual(value, resp.headers[key]) + + def test_quota_error_mapping(self): + self._do_test_exception_mapping(exception.QuotaError, 'too many used') + + def test_non_manila_notfound_exception_mapping(self): + class ExceptionWithCode(Exception): + code = 404 + + self._do_test_exception_mapping(ExceptionWithCode, + 'NotFound') + + def test_non_manila_exception_mapping(self): + class ExceptionWithCode(Exception): + code = 417 + + self._do_test_exception_mapping(ExceptionWithCode, + 'Expectation failed') + + def test_exception_with_none_code_throws_500(self): + class ExceptionWithNoneCode(Exception): + code = None + + @webob.dec.wsgify + def fail(req): + raise ExceptionWithNoneCode() + + api = self._wsgi_app(fail) + resp = webob.Request.blank('/').get_response(api) + self.assertEqual(500, resp.status_int) diff --git a/manila/tests/api/test_versions.py b/manila/tests/api/test_versions.py index c36623f435..83f0abba10 100644 --- a/manila/tests/api/test_versions.py +++ b/manila/tests/api/test_versions.py @@ -185,7 +185,7 @@ class ExperimentalAPITestCase(test.TestCase): self.assertEqual('2.0', response.headers[version_header_name]) if experimental: - self.assertEqual(experimental, + self.assertEqual('%s' % experimental, response.headers.get(experimental_header_name)) else: self.assertNotIn(experimental_header_name, response.headers) diff --git a/manila/tests/api/test_wsgi.py b/manila/tests/api/test_wsgi.py index bb53f76a97..7310213e8a 100644 --- a/manila/tests/api/test_wsgi.py +++ b/manila/tests/api/test_wsgi.py @@ -19,33 +19,19 @@ Test WSGI basics and provide some helper functions for other WSGI tests. """ -from manila import test - +from oslo_service import wsgi import routes -import six import webob -from manila import wsgi +from manila import test +from manila.wsgi import common as common_wsgi class Test(test.TestCase): - def test_debug(self): - - class Application(wsgi.Application): - """Dummy application to test debug.""" - - def __call__(self, environ, start_response): - start_response("200", [("X-Test", "checking")]) - return [six.b('Test result')] - - application = wsgi.Debug(Application()) - result = webob.Request.blank('/').get_response(application) - self.assertEqual(six.b("Test result"), result.body) - def test_router(self): - class Application(wsgi.Application): + class Application(common_wsgi.Application): """Test application to call from router.""" def __call__(self, environ, start_response): diff --git a/manila/tests/conf_fixture.py b/manila/tests/conf_fixture.py index 3bea20bda9..a468f7b7c8 100644 --- a/manila/tests/conf_fixture.py +++ b/manila/tests/conf_fixture.py @@ -17,6 +17,7 @@ import os from oslo_policy import opts +from oslo_service import wsgi from manila.common import config @@ -38,6 +39,7 @@ def set_defaults(conf): _safe_set_of_opts(conf, 'service_instance_user', 'fake_user') _API_PASTE_PATH = os.path.abspath(os.path.join(CONF.state_path, 'etc/manila/api-paste.ini')) + wsgi.register_opts(conf) _safe_set_of_opts(conf, 'api_paste_config', _API_PASTE_PATH) _safe_set_of_opts(conf, 'share_driver', 'manila.tests.fake_driver.FakeShareDriver') diff --git a/manila/tests/test_service.py b/manila/tests/test_service.py index 22874a4dbd..c0d78d5ae2 100644 --- a/manila/tests/test_service.py +++ b/manila/tests/test_service.py @@ -24,6 +24,7 @@ Unit Tests for remote procedure calls using queue import ddt import mock from oslo_config import cfg +from oslo_service import wsgi from manila import context from manila import db @@ -32,7 +33,6 @@ from manila import manager from manila import service from manila import test from manila import utils -from manila import wsgi test_service_opts = [ cfg.StrOpt("fake_manager", @@ -224,5 +224,5 @@ class TestWSGIService(test.TestCase): # Resetting pool size to default self.test_service.reset() self.test_service.start() - self.assertEqual(1000, self.test_service.server._pool.size) + self.assertGreater(self.test_service.server._pool.size, 0) wsgi.Loader.load_app.assert_called_once_with("test_service") diff --git a/manila/tests/test_utils.py b/manila/tests/test_utils.py index 9fdf051084..a979ed6eab 100644 --- a/manila/tests/test_utils.py +++ b/manila/tests/test_utils.py @@ -25,6 +25,7 @@ from oslo_config import cfg from oslo_utils import timeutils from oslo_utils import uuidutils import paramiko +import six from webob import exc import manila @@ -744,3 +745,39 @@ class ShareMigrationHelperTestCase(test.TestCase): self.assertRaises(expected_exc, utils.wait_for_access_update, self.context, db, fake_instance, 1) + + +@ddt.ddt +class ConvertStrTestCase(test.TestCase): + + def test_convert_str_str_input(self): + self.mock_object(utils.encodeutils, 'safe_encode') + input_value = six.text_type("string_input") + + output_value = utils.convert_str(input_value) + + if six.PY2: + utils.encodeutils.safe_encode.assert_called_once_with(input_value) + self.assertEqual( + utils.encodeutils.safe_encode.return_value, output_value) + else: + self.assertEqual(0, utils.encodeutils.safe_encode.call_count) + self.assertEqual(input_value, output_value) + + def test_convert_str_bytes_input(self): + self.mock_object(utils.encodeutils, 'safe_encode') + if six.PY2: + input_value = six.binary_type("binary_input") + else: + input_value = six.binary_type("binary_input", "utf-8") + + output_value = utils.convert_str(input_value) + + if six.PY2: + utils.encodeutils.safe_encode.assert_called_once_with(input_value) + self.assertEqual( + utils.encodeutils.safe_encode.return_value, output_value) + else: + self.assertEqual(0, utils.encodeutils.safe_encode.call_count) + self.assertIsInstance(output_value, six.string_types) + self.assertEqual(six.text_type("binary_input"), output_value) diff --git a/manila/tests/test_wsgi.py b/manila/tests/test_wsgi.py deleted file mode 100644 index 05789a2d8f..0000000000 --- a/manila/tests/test_wsgi.py +++ /dev/null @@ -1,334 +0,0 @@ -# Copyright 2011 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Unit tests for `manila.wsgi`.""" - -import os.path -import ssl -import tempfile - -import ddt -import eventlet -import mock -from oslo_config import cfg -from oslo_utils import netutils -import six -from six.moves import urllib -import testtools -import webob -import webob.dec - -from manila.api.middleware import fault -from manila import exception -from manila import test -from manila import utils -import manila.wsgi - -CONF = cfg.CONF - -TEST_VAR_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), - 'var')) - - -class TestLoaderNothingExists(test.TestCase): - """Loader tests where os.path.exists always returns False.""" - - def test_config_not_found(self): - self.assertRaises( - manila.exception.ConfigNotFound, - manila.wsgi.Loader, - 'nonexistent_file.ini', - ) - - -class TestLoaderNormalFilesystem(test.TestCase): - """Loader tests with normal filesystem (unmodified os.path module).""" - - _paste_config = """ -[app:test_app] -use = egg:Paste#static -document_root = /tmp - """ - - def setUp(self): - super(TestLoaderNormalFilesystem, self).setUp() - self.config = tempfile.NamedTemporaryFile(mode="w+t") - self.config.write(self._paste_config.lstrip()) - self.config.seek(0) - self.config.flush() - self.loader = manila.wsgi.Loader(self.config.name) - self.addCleanup(self.config.close) - - def test_config_found(self): - self.assertEqual(self.config.name, self.loader.config_path) - - def test_app_not_found(self): - self.assertRaises( - manila.exception.PasteAppNotFound, - self.loader.load_app, - "non-existent app", - ) - - def test_app_found(self): - url_parser = self.loader.load_app("test_app") - self.assertEqual("/tmp", url_parser.directory) - - -@ddt.ddt -class TestWSGIServer(test.TestCase): - """WSGI server tests.""" - - def test_no_app(self): - server = manila.wsgi.Server("test_app", None, host="127.0.0.1", port=0) - self.assertEqual("test_app", server.name) - - def test_start_random_port(self): - server = manila.wsgi.Server("test_random_port", None, host="127.0.0.1") - server.start() - self.assertNotEqual(0, server.port) - server.stop() - server.wait() - - @testtools.skipIf(not netutils.is_ipv6_enabled(), - "Test requires an IPV6 configured interface") - @testtools.skipIf(utils.is_eventlet_bug105(), - 'Eventlet bug #105 affect test results.') - def test_start_random_port_with_ipv6(self): - server = manila.wsgi.Server("test_random_port", - None, - host="::1") - server.start() - self.assertEqual("::1", server.host) - self.assertNotEqual(0, server.port) - server.stop() - server.wait() - - def test_start_with_default_tcp_options(self): - server = manila.wsgi.Server("test_tcp_options", - None, - host="127.0.0.1") - self.mock_object( - netutils, 'set_tcp_keepalive') - server.start() - netutils.set_tcp_keepalive.assert_called_once_with( - mock.ANY, tcp_keepalive=True, tcp_keepalive_count=None, - tcp_keepalive_interval=None, tcp_keepidle=600) - - def test_start_with_custom_tcp_options(self): - CONF.set_default("tcp_keepalive", False) - CONF.set_default("tcp_keepalive_count", 33) - CONF.set_default("tcp_keepalive_interval", 22) - CONF.set_default("tcp_keepidle", 11) - server = manila.wsgi.Server("test_tcp_options", - None, - host="127.0.0.1") - self.mock_object( - netutils, 'set_tcp_keepalive') - server.start() - netutils.set_tcp_keepalive.assert_called_once_with( - mock.ANY, tcp_keepalive=False, tcp_keepalive_count=33, - tcp_keepalive_interval=22, tcp_keepidle=11) - - def test_app(self): - self.mock_object( - eventlet, 'spawn', mock.Mock(side_effect=eventlet.spawn)) - greetings = 'Hello, World!!!' - - def hello_world(env, start_response): - if env['PATH_INFO'] != '/': - start_response('404 Not Found', - [('Content-Type', 'text/plain')]) - return ['Not Found\r\n'] - start_response('200 OK', [('Content-Type', 'text/plain')]) - return [greetings] - - server = manila.wsgi.Server( - "test_app", hello_world, host="127.0.0.1", port=0) - server.start() - - response = urllib.request.urlopen('http://127.0.0.1:%d/' % server.port) - self.assertEqual(six.b(greetings), response.read()) - - # Verify provided parameters to eventlet.spawn func - eventlet.spawn.assert_called_once_with( - func=eventlet.wsgi.server, - sock=mock.ANY, - site=server.app, - protocol=server._protocol, - custom_pool=server._pool, - log=server._logger, - socket_timeout=server.client_socket_timeout, - keepalive=manila.wsgi.CONF.wsgi_keep_alive, - ) - - server.stop() - - @ddt.data(0, 0.1, 1, None) - def test_init_server_with_socket_timeout(self, client_socket_timeout): - CONF.set_default("client_socket_timeout", client_socket_timeout) - server = manila.wsgi.Server( - "test_app", lambda *args, **kwargs: None, host="127.0.0.1", port=0) - self.assertEqual(client_socket_timeout, server.client_socket_timeout) - - @testtools.skipIf(six.PY3, "bug/1482633") - def test_app_using_ssl(self): - CONF.set_default("ssl_cert_file", - os.path.join(TEST_VAR_DIR, 'certificate.crt')) - CONF.set_default("ssl_key_file", - os.path.join(TEST_VAR_DIR, 'privatekey.key')) - - greetings = 'Hello, World!!!' - - @webob.dec.wsgify - def hello_world(req): - return greetings - - server = manila.wsgi.Server( - "test_app", hello_world, host="127.0.0.1", port=0) - server.start() - - if hasattr(ssl, '_create_unverified_context'): - response = urllib.request.urlopen( - 'https://127.0.0.1:%d/' % server.port, - context=ssl._create_unverified_context()) - else: - response = urllib.request.urlopen( - 'https://127.0.0.1:%d/' % server.port) - - self.assertEqual(greetings, response.read()) - - server.stop() - - @testtools.skipIf(not netutils.is_ipv6_enabled(), - "Test requires an IPV6 configured interface") - @testtools.skipIf(utils.is_eventlet_bug105(), - 'Eventlet bug #105 affect test results.') - @testtools.skipIf(six.PY3, "bug/1482633") - def test_app_using_ipv6_and_ssl(self): - CONF.set_default("ssl_cert_file", - os.path.join(TEST_VAR_DIR, 'certificate.crt')) - CONF.set_default("ssl_key_file", - os.path.join(TEST_VAR_DIR, 'privatekey.key')) - - greetings = 'Hello, World!!!' - - @webob.dec.wsgify - def hello_world(req): - return greetings - - server = manila.wsgi.Server("test_app", - hello_world, - host="::1", - port=0) - server.start() - - if hasattr(ssl, '_create_unverified_context'): - response = urllib.request.urlopen( - 'https://[::1]:%d/' % server.port, - context=ssl._create_unverified_context()) - else: - response = urllib.request.urlopen( - 'https://[::1]:%d/' % server.port) - - self.assertEqual(greetings, response.read()) - - server.stop() - - def test_reset_pool_size_to_default(self): - server = manila.wsgi.Server("test_resize", None, host="127.0.0.1") - server.start() - - # Stopping the server, which in turn sets pool size to 0 - server.stop() - self.assertEqual(0, server._pool.size) - - # Resetting pool size to default - server.reset() - server.start() - self.assertEqual(1000, server._pool.size) - - -class ExceptionTest(test.TestCase): - - def _wsgi_app(self, inner_app): - return fault.FaultWrapper(inner_app) - - def _do_test_exception_safety_reflected_in_faults(self, expose): - class ExceptionWithSafety(exception.ManilaException): - safe = expose - - @webob.dec.wsgify - def fail(req): - raise ExceptionWithSafety('some explanation') - - api = self._wsgi_app(fail) - resp = webob.Request.blank('/').get_response(api) - self.assertIn('{"computeFault', six.text_type(resp.body), resp.body) - expected = ('ExceptionWithSafety: some explanation' if expose else - 'The server has either erred or is incapable ' - 'of performing the requested operation.') - self.assertIn(expected, six.text_type(resp.body), resp.body) - self.assertEqual(500, resp.status_int, resp.body) - - def test_safe_exceptions_are_described_in_faults(self): - self._do_test_exception_safety_reflected_in_faults(True) - - def test_unsafe_exceptions_are_not_described_in_faults(self): - self._do_test_exception_safety_reflected_in_faults(False) - - def _do_test_exception_mapping(self, exception_type, msg): - @webob.dec.wsgify - def fail(req): - raise exception_type(msg) - - api = self._wsgi_app(fail) - resp = webob.Request.blank('/').get_response(api) - self.assertIn(msg, six.text_type(resp.body), resp.body) - self.assertEqual(exception_type.code, resp.status_int, resp.body) - - if hasattr(exception_type, 'headers'): - for (key, value) in exception_type.headers.items(): - self.assertIn(key, resp.headers) - self.assertEqual(value, resp.headers[key]) - - def test_quota_error_mapping(self): - self._do_test_exception_mapping(exception.QuotaError, 'too many used') - - def test_non_manila_notfound_exception_mapping(self): - class ExceptionWithCode(Exception): - code = 404 - - self._do_test_exception_mapping(ExceptionWithCode, - 'NotFound') - - def test_non_manila_exception_mapping(self): - class ExceptionWithCode(Exception): - code = 417 - - self._do_test_exception_mapping(ExceptionWithCode, - 'Expectation failed') - - def test_exception_with_none_code_throws_500(self): - class ExceptionWithNoneCode(Exception): - code = None - - @webob.dec.wsgify - def fail(req): - raise ExceptionWithNoneCode() - - api = self._wsgi_app(fail) - resp = webob.Request.blank('/').get_response(api) - self.assertEqual(500, resp.status_int) diff --git a/manila/tests/wsgi/__init__.py b/manila/tests/wsgi/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/manila/tests/wsgi/test_common.py b/manila/tests/wsgi/test_common.py new file mode 100644 index 0000000000..0b9f18be85 --- /dev/null +++ b/manila/tests/wsgi/test_common.py @@ -0,0 +1,45 @@ +# Copyright 2017 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from manila import test +from manila.wsgi import common + + +class FakeApp(common.Application): + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + + +class WSGICommonTestCase(test.TestCase): + + def test_application_factory(self): + fake_global_config = mock.Mock() + kwargs = {"k1": "v1", "k2": "v2"} + + result = FakeApp.factory(fake_global_config, **kwargs) + + fake_global_config.assert_not_called() + self.assertIsInstance(result, FakeApp) + for k, v in kwargs.items(): + self.assertTrue(hasattr(result, k)) + self.assertEqual(getattr(result, k), v) + + def test_application___call__(self): + self.assertRaises( + NotImplementedError, + common.Application(), 'fake_environ', 'fake_start_response') diff --git a/manila/tests/wsgi/test_wsgi.py b/manila/tests/wsgi/test_wsgi.py new file mode 100644 index 0000000000..335f6b3562 --- /dev/null +++ b/manila/tests/wsgi/test_wsgi.py @@ -0,0 +1,45 @@ +# Copyright 2017 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from manila import test +from manila.wsgi import wsgi + + +class WSGITestCase(test.TestCase): + + def test_initialize_application(self): + self.mock_object(wsgi.log, 'register_options') + self.mock_object(wsgi.cfg.ConfigOpts, '__call__') + self.mock_object(wsgi.config, 'verify_share_protocols') + self.mock_object(wsgi.log, 'setup') + self.mock_object(wsgi.rpc, 'init') + self.mock_object(wsgi.wsgi, 'Loader') + wsgi.sys.argv = ['--verbose', '--debug'] + + result = wsgi.initialize_application() + + self.assertEqual( + wsgi.wsgi.Loader.return_value.load_app.return_value, result) + wsgi.log.register_options.assert_called_once_with(mock.ANY) + wsgi.cfg.ConfigOpts.__call__.assert_called_once_with( + mock.ANY, project="manila", version=wsgi.version.version_string()) + wsgi.config.verify_share_protocols.assert_called_once_with() + wsgi.log.setup.assert_called_once_with(mock.ANY, "manila") + wsgi.rpc.init.assert_called_once_with(mock.ANY) + wsgi.wsgi.Loader.assert_called_once_with(mock.ANY) + wsgi.wsgi.Loader.return_value.load_app.assert_called_once_with( + name='osapi_share') diff --git a/manila/utils.py b/manila/utils.py index 3d312a23e5..ec9bc02b6f 100644 --- a/manila/utils.py +++ b/manila/utils.py @@ -36,6 +36,7 @@ from oslo_concurrency import lockutils from oslo_concurrency import processutils from oslo_config import cfg from oslo_log import log +from oslo_utils import encodeutils from oslo_utils import importutils from oslo_utils import netutils from oslo_utils import strutils @@ -520,6 +521,24 @@ def require_driver_initialized(func): return wrapper +def convert_str(text): + """Convert to native string. + + Convert bytes and Unicode strings to native strings: + + * convert to bytes on Python 2: + encode Unicode using encodeutils.safe_encode() + * convert to Unicode on Python 3: decode bytes from UTF-8 + """ + if six.PY2: + return encodeutils.safe_encode(text) + else: + if isinstance(text, bytes): + return text.decode('utf-8') + else: + return text + + def translate_string_size_to_float(string, multiplier='G'): """Translates human-readable storage size to float value. diff --git a/manila/wsgi.py b/manila/wsgi.py deleted file mode 100644 index 69c8463a76..0000000000 --- a/manila/wsgi.py +++ /dev/null @@ -1,551 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Utility methods for working with WSGI servers.""" - -from __future__ import print_function - -import errno -import os -import socket -import ssl -import sys -import time - -import eventlet -import eventlet.wsgi -import greenlet -from oslo_config import cfg -from oslo_log import log -from oslo_service import service -from oslo_utils import excutils -from oslo_utils import netutils -from paste import deploy -import routes.middleware -import webob.dec -import webob.exc - -from manila.common import config -from manila import exception -from manila.i18n import _ - -socket_opts = [ - cfg.IntOpt('backlog', - default=4096, - help="Number of backlog requests to configure the socket " - "with."), - cfg.BoolOpt('tcp_keepalive', - default=True, - help="Sets the value of TCP_KEEPALIVE (True/False) for each " - "server socket."), - cfg.IntOpt('tcp_keepidle', - default=600, - help="Sets the value of TCP_KEEPIDLE in seconds for each " - "server socket. Not supported on OS X."), - cfg.IntOpt('tcp_keepalive_interval', - help="Sets the value of TCP_KEEPINTVL in seconds for each " - "server socket. Not supported on OS X."), - cfg.IntOpt('tcp_keepalive_count', - help="Sets the value of TCP_KEEPCNT for each " - "server socket. Not supported on OS X."), - cfg.StrOpt('ssl_ca_file', - help="CA certificate file to use to verify " - "connecting clients."), - cfg.StrOpt('ssl_cert_file', - help="Certificate file to use when starting " - "the server securely."), - cfg.StrOpt('ssl_key_file', - help="Private key file to use when starting " - "the server securely."), -] - -eventlet_opts = [ - cfg.IntOpt('max_header_line', - default=16384, - help="Maximum line size of message headers to be accepted. " - "Option max_header_line may need to be increased when " - "using large tokens (typically those generated by the " - "Keystone v3 API with big service catalogs)."), - cfg.IntOpt('client_socket_timeout', - default=900, - help="Timeout for client connections socket operations. " - "If an incoming connection is idle for this number of " - "seconds it will be closed. A value of '0' means " - "wait forever."), - cfg.BoolOpt('wsgi_keep_alive', - default=True, - help='If False, closes the client socket connection ' - 'explicitly. Setting it to True to maintain backward ' - 'compatibility. Recommended setting is set it to False.'), -] - -CONF = cfg.CONF -CONF.register_opts(socket_opts) -CONF.register_opts(eventlet_opts) - -LOG = log.getLogger(__name__) - - -class Server(service.ServiceBase): - """Server class to manage a WSGI server, serving a WSGI application.""" - - default_pool_size = 1000 - - def __init__(self, name, app, host=None, port=None, pool_size=None, - protocol=eventlet.wsgi.HttpProtocol, backlog=128): - """Initialize, but do not start, a WSGI server. - - :param name: Pretty name for logging. - :param app: The WSGI application to serve. - :param host: IP address to serve the application. - :param port: Port number to server the application. - :param pool_size: Maximum number of eventlets to spawn concurrently. - :returns: None - - """ - eventlet.wsgi.MAX_HEADER_LINE = CONF.max_header_line - self.client_socket_timeout = CONF.client_socket_timeout - self.name = name - self.app = app - self._host = host or "0.0.0.0" - self._port = port or 0 - self._server = None - self._socket = None - self._protocol = protocol - self.pool_size = pool_size or self.default_pool_size - self._pool = eventlet.GreenPool(self.pool_size) - self._logger = log.getLogger("eventlet.wsgi.server") - - if backlog < 1: - raise exception.InvalidInput( - reason='The backlog must be more than 1') - - bind_addr = (host, port) - # TODO(dims): eventlet's green dns/socket module does not actually - # support IPv6 in getaddrinfo(). We need to get around this in the - # future or monitor upstream for a fix - try: - info = socket.getaddrinfo(bind_addr[0], - bind_addr[1], - socket.AF_UNSPEC, - socket.SOCK_STREAM)[0] - family = info[0] - bind_addr = info[-1] - except Exception: - family = socket.AF_INET - - cert_file = CONF.ssl_cert_file - key_file = CONF.ssl_key_file - ca_file = CONF.ssl_ca_file - self._use_ssl = cert_file or key_file - - if cert_file and not os.path.exists(cert_file): - raise RuntimeError(_("Unable to find cert_file : %s") % cert_file) - - if ca_file and not os.path.exists(ca_file): - raise RuntimeError(_("Unable to find ca_file : %s") % ca_file) - - if key_file and not os.path.exists(key_file): - raise RuntimeError(_("Unable to find key_file : %s") % key_file) - - if self._use_ssl and (not cert_file or not key_file): - raise RuntimeError(_("When running server in SSL mode, you must " - "specify both a cert_file and key_file " - "option value in your configuration file")) - - retry_until = time.time() + 30 - while not self._socket and time.time() < retry_until: - try: - self._socket = eventlet.listen( - bind_addr, backlog=backlog, family=family) - except socket.error as err: - if err.args[0] != errno.EADDRINUSE: - raise - eventlet.sleep(0.1) - - if not self._socket: - raise RuntimeError(_("Could not bind to %(host)s:%(port)s " - "after trying for 30 seconds") % - {'host': host, 'port': port}) - - (self._host, self._port) = self._socket.getsockname()[0:2] - LOG.info("%(name)s listening on %(_host)s:%(_port)s", - {'name': self.name, '_host': self._host, '_port': self._port}) - - def start(self): - """Start serving a WSGI application. - - :returns: None - :raises: manila.exception.InvalidInput - - """ - # The server socket object will be closed after server exits, - # but the underlying file descriptor will remain open, and will - # give bad file descriptor error. So duplicating the socket object, - # to keep file descriptor usable. - - config.set_middleware_defaults() - dup_socket = self._socket.dup() - - netutils.set_tcp_keepalive( - dup_socket, - tcp_keepalive=CONF.tcp_keepalive, - tcp_keepidle=CONF.tcp_keepidle, - tcp_keepalive_interval=CONF.tcp_keepalive_interval, - tcp_keepalive_count=CONF.tcp_keepalive_count - ) - - if self._use_ssl: - try: - ssl_kwargs = { - 'server_side': True, - 'certfile': CONF.ssl_cert_file, - 'keyfile': CONF.ssl_key_file, - 'cert_reqs': ssl.CERT_NONE, - } - - if CONF.ssl_ca_file: - ssl_kwargs['ca_certs'] = CONF.ssl_ca_file - ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED - - dup_socket = ssl.wrap_socket(dup_socket, - **ssl_kwargs) - - dup_socket.setsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR, 1) - - except Exception: - with excutils.save_and_reraise_exception(): - LOG.error( - ("Failed to start %(name)s on %(_host)s:%(_port)s " - "with SSL support."), - {"name": self.name, "_host": self._host, - "_port": self._port} - ) - - wsgi_kwargs = { - 'func': eventlet.wsgi.server, - 'sock': dup_socket, - 'site': self.app, - 'protocol': self._protocol, - 'custom_pool': self._pool, - 'log': self._logger, - 'socket_timeout': self.client_socket_timeout, - 'keepalive': CONF.wsgi_keep_alive, - } - - self._server = eventlet.spawn(**wsgi_kwargs) - - @property - def host(self): - return self._host - - @property - def port(self): - return self._port - - def stop(self): - """Stop this server. - - This is not a very nice action, as currently the method by which a - server is stopped is by killing its eventlet. - - :returns: None - - """ - LOG.info("Stopping WSGI server.") - if self._server is not None: - # Resize pool to stop new requests from being processed - self._pool.resize(0) - self._server.kill() - - def wait(self): - """Block, until the server has stopped. - - Waits on the server's eventlet to finish, then returns. - - :returns: None - - """ - try: - if self._server is not None: - self._pool.waitall() - self._server.wait() - except greenlet.GreenletExit: - LOG.info("WSGI server has stopped.") - - def reset(self): - """Reset server greenpool size to default. - - :returns: None - """ - self._pool.resize(self.pool_size) - - -class Request(webob.Request): - pass - - -class Application(object): - """Base WSGI application wrapper. Subclasses need to implement __call__.""" - - @classmethod - def factory(cls, global_config, **local_config): - """Used for paste app factories in paste.deploy config files. - - Any local configuration (that is, values under the [app:APPNAME] - section of the paste config) will be passed into the `__init__` method - as kwargs. - - A hypothetical configuration would look like: - - [app:wadl] - latest_version = 1.3 - paste.app_factory = manila.api.fancy_api:Wadl.factory - - which would result in a call to the `Wadl` class as - - import manila.api.fancy_api - fancy_api.Wadl(latest_version='1.3') - - You could of course re-implement the `factory` method in subclasses, - but using the kwarg passing it shouldn't be necessary. - - """ - return cls(**local_config) - - def __call__(self, environ, start_response): - r"""Subclasses will probably want to implement __call__ like this: - - @webob.dec.wsgify(RequestClass=Request) - def __call__(self, req): - # Any of the following objects work as responses: - - # Option 1: simple string - res = 'message\n' - - # Option 2: a nicely formatted HTTP exception page - res = exc.HTTPForbidden(detail='Nice try') - - # Option 3: a webob Response object (in case you need to play with - # headers, or you want to be treated like an iterable, or or or) - res = Response(); - res.app_iter = open('somefile') - - # Option 4: any wsgi app to be run next - res = self.application - - # Option 5: you can get a Response object for a wsgi app, too, to - # play with headers etc - res = req.get_response(self.application) - - # You can then just return your response... - return res - # ... or set req.response and return None. - req.response = res - - See the end of http://pythonpaste.org/webob/modules/dec.html - for more info. - - """ - raise NotImplementedError(_('You must implement __call__')) - - -class Middleware(Application): - """Base WSGI middleware. - - These classes require an application to be - initialized that will be called next. By default the middleware will - simply call its wrapped app, or you can override __call__ to customize its - behavior. - - """ - - @classmethod - def factory(cls, global_config, **local_config): - """Used for paste app factories in paste.deploy config files. - - Any local configuration (that is, values under the [filter:APPNAME] - section of the paste config) will be passed into the `__init__` method - as kwargs. - - A hypothetical configuration would look like: - - [filter:analytics] - redis_host = 127.0.0.1 - paste.filter_factory = manila.api.analytics:Analytics.factory - - which would result in a call to the `Analytics` class as - - import manila.api.analytics - analytics.Analytics(app_from_paste, redis_host='127.0.0.1') - - You could of course re-implement the `factory` method in subclasses, - but using the kwarg passing it shouldn't be necessary. - - """ - def _factory(app): - return cls(app, **local_config) - return _factory - - def __init__(self, application): - self.application = application - - def process_request(self, req): - """Called on each request. - - If this returns None, the next application down the stack will be - executed. If it returns a response then that response will be returned - and execution will stop here. - - """ - return None - - def process_response(self, response): - """Do whatever you'd like to the response.""" - return response - - @webob.dec.wsgify(RequestClass=Request) - def __call__(self, req): - response = self.process_request(req) - if response: - return response - response = req.get_response(self.application) - return self.process_response(response) - - -class Debug(Middleware): - """Helper class for debugging a WSGI application. - - Can be inserted into any WSGI application chain to get information - about the request and response. - - """ - - @webob.dec.wsgify(RequestClass=Request) - def __call__(self, req): - print(('*' * 40) + ' REQUEST ENVIRON') - for key, value in req.environ.items(): - print(key, '=', value) - print() - resp = req.get_response(self.application) - - print(('*' * 40) + ' RESPONSE HEADERS') - for (key, value) in resp.headers.items(): - print(key, '=', value) - print() - - resp.app_iter = self.print_generator(resp.app_iter) - - return resp - - @staticmethod - def print_generator(app_iter): - """Iterator that prints the contents of a wrapper string.""" - print(('*' * 40) + ' BODY') - for part in app_iter: - sys.stdout.write(part.decode()) - sys.stdout.flush() - yield part - print() - - -class Router(object): - """WSGI middleware that maps incoming requests to WSGI apps.""" - - def __init__(self, mapper): - """Create a router for the given routes.Mapper. - - Each route in `mapper` must specify a 'controller', which is a - WSGI app to call. You'll probably want to specify an 'action' as - well and have your controller be an object that can route - the request to the action-specific method. - - Examples: - mapper = routes.Mapper() - sc = ServerController() - - # Explicit mapping of one route to a controller+action - mapper.connect(None, '/svrlist', controller=sc, action='list') - - # Actions are all implicitly defined - mapper.resource('server', 'servers', controller=sc) - - # Pointing to an arbitrary WSGI app. You can specify the - # {path_info:.*} parameter so the target app can be handed just that - # section of the URL. - mapper.connect(None, '/v1.0/{path_info:.*}', controller=BlogApp()) - - """ - self.map = mapper - self._router = routes.middleware.RoutesMiddleware(self._dispatch, - self.map) - - @webob.dec.wsgify(RequestClass=Request) - def __call__(self, req): - """Route the incoming request to a controller based on self.map. - - If no match, return a 404. - - """ - return self._router - - @staticmethod - @webob.dec.wsgify(RequestClass=Request) - def _dispatch(req): - """Dispatch the request to the appropriate controller. - - Called by self._router after matching the incoming request to a route - and putting the information into req.environ. Either returns 404 - or the routed WSGI app's response. - - """ - match = req.environ['wsgiorg.routing_args'][1] - if not match: - return webob.exc.HTTPNotFound() - app = match['controller'] - return app - - -class Loader(object): - """Used to load WSGI applications from paste configurations.""" - - def __init__(self, config_path=None): - """Initialize the loader, and attempt to find the config. - - :param config_path: Full or relative path to the paste config. - :returns: None - - """ - config_path = config_path or CONF.api_paste_config - self.config_path = CONF.find_file(config_path) - if not self.config_path: - raise exception.ConfigNotFound(path=config_path) - - def load_app(self, name): - """Return the paste URLMap wrapped WSGI application. - - :param name: Name of the application to load. - :returns: Paste URLMap object wrapping the requested application. - :raises: `manila.exception.PasteAppNotFound` - - """ - try: - return deploy.loadapp("config:%s" % self.config_path, name=name) - except LookupError as err: - LOG.error(err) - raise exception.PasteAppNotFound(name=name, path=self.config_path) diff --git a/manila/wsgi/__init__.py b/manila/wsgi/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/manila/wsgi/common.py b/manila/wsgi/common.py new file mode 100644 index 0000000000..86f92c2318 --- /dev/null +++ b/manila/wsgi/common.py @@ -0,0 +1,155 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Utility methods for working with WSGI servers.""" + +import webob.dec +import webob.exc + +from manila.i18n import _ + + +class Request(webob.Request): + pass + + +class Application(object): + """Base WSGI application wrapper. Subclasses need to implement __call__.""" + + @classmethod + def factory(cls, global_config, **local_config): + """Used for paste app factories in paste.deploy config files. + + Any local configuration (that is, values under the [app:APPNAME] + section of the paste config) will be passed into the `__init__` method + as kwargs. + + A hypothetical configuration would look like: + + [app:wadl] + latest_version = 1.3 + paste.app_factory = manila.api.fancy_api:Wadl.factory + + which would result in a call to the `Wadl` class as + + import manila.api.fancy_api + fancy_api.Wadl(latest_version='1.3') + + You could of course re-implement the `factory` method in subclasses, + but using the kwarg passing it shouldn't be necessary. + + """ + return cls(**local_config) + + def __call__(self, environ, start_response): + r"""Subclasses will probably want to implement __call__ like this: + + @webob.dec.wsgify(RequestClass=Request) + def __call__(self, req): + # Any of the following objects work as responses: + + # Option 1: simple string + res = 'message\n' + + # Option 2: a nicely formatted HTTP exception page + res = exc.HTTPForbidden(detail='Nice try') + + # Option 3: a webob Response object (in case you need to play with + # headers, or you want to be treated like an iterable, or or or) + res = Response(); + res.app_iter = open('somefile') + + # Option 4: any wsgi app to be run next + res = self.application + + # Option 5: you can get a Response object for a wsgi app, too, to + # play with headers etc + res = req.get_response(self.application) + + # You can then just return your response... + return res + # ... or set req.response and return None. + req.response = res + + See the end of http://pythonpaste.org/webob/modules/dec.html + for more info. + + """ + raise NotImplementedError(_('You must implement __call__')) + + +class Middleware(Application): + """Base WSGI middleware. + + These classes require an application to be + initialized that will be called next. By default the middleware will + simply call its wrapped app, or you can override __call__ to customize its + behavior. + + """ + + @classmethod + def factory(cls, global_config, **local_config): + """Used for paste app factories in paste.deploy config files. + + Any local configuration (that is, values under the [filter:APPNAME] + section of the paste config) will be passed into the `__init__` method + as kwargs. + + A hypothetical configuration would look like: + + [filter:analytics] + redis_host = 127.0.0.1 + paste.filter_factory = manila.api.analytics:Analytics.factory + + which would result in a call to the `Analytics` class as + + import manila.api.analytics + analytics.Analytics(app_from_paste, redis_host='127.0.0.1') + + You could of course re-implement the `factory` method in subclasses, + but using the kwarg passing it shouldn't be necessary. + + """ + def _factory(app): + return cls(app, **local_config) + return _factory + + def __init__(self, application): + self.application = application + + def process_request(self, req): + """Called on each request. + + If this returns None, the next application down the stack will be + executed. If it returns a response then that response will be returned + and execution will stop here. + + """ + return None + + def process_response(self, response): + """Do whatever you'd like to the response.""" + return response + + @webob.dec.wsgify(RequestClass=Request) + def __call__(self, req): + response = self.process_request(req) + if response: + return response + response = req.get_response(self.application) + return self.process_response(response) diff --git a/manila/wsgi/eventlet_server.py b/manila/wsgi/eventlet_server.py new file mode 100644 index 0000000000..bba7077c83 --- /dev/null +++ b/manila/wsgi/eventlet_server.py @@ -0,0 +1,59 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Utility methods for working with WSGI servers.""" + +import socket + +from oslo_config import cfg +from oslo_service import wsgi +from oslo_utils import netutils + +socket_opts = [ + cfg.BoolOpt('tcp_keepalive', + default=True, + help="Sets the value of TCP_KEEPALIVE (True/False) for each " + "server socket."), + cfg.IntOpt('tcp_keepalive_interval', + help="Sets the value of TCP_KEEPINTVL in seconds for each " + "server socket. Not supported on OS X."), + cfg.IntOpt('tcp_keepalive_count', + help="Sets the value of TCP_KEEPCNT for each " + "server socket. Not supported on OS X."), +] + +CONF = cfg.CONF +CONF.register_opts(socket_opts) + + +class Server(wsgi.Server): + """Server class to manage a WSGI server, serving a WSGI application.""" + + def _set_socket_opts(self, _socket): + _socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + # NOTE(praneshp): Call set_tcp_keepalive in oslo to set + # tcp keepalive parameters. Sockets can hang around forever + # without keepalive + netutils.set_tcp_keepalive( + _socket, + self.conf.tcp_keepalive, + self.conf.tcp_keepidle, + self.conf.tcp_keepalive_count, + self.conf.tcp_keepalive_interval, + ) + return _socket diff --git a/manila/wsgi/wsgi.py b/manila/wsgi/wsgi.py new file mode 100644 index 0000000000..31ac2621e7 --- /dev/null +++ b/manila/wsgi/wsgi.py @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Manila OS API WSGI application.""" + + +import sys + +from oslo_config import cfg +from oslo_log import log +from oslo_service import wsgi + +from manila import i18n +i18n.enable_lazy() + +# Need to register global_opts +from manila.common import config +from manila import rpc +from manila import version + +CONF = cfg.CONF + + +def initialize_application(): + log.register_options(CONF) + CONF(sys.argv[1:], project="manila", version=version.version_string()) + config.verify_share_protocols() + log.setup(CONF, "manila") + rpc.init(CONF) + return wsgi.Loader(CONF).load_app(name='osapi_share') diff --git a/releasenotes/notes/added-possibility-to-run-manila-api-with-web-servers-that-support-wsgi-apps-cfffe0b789f8670a.yaml b/releasenotes/notes/added-possibility-to-run-manila-api-with-web-servers-that-support-wsgi-apps-cfffe0b789f8670a.yaml new file mode 100644 index 0000000000..bf5ea27ac1 --- /dev/null +++ b/releasenotes/notes/added-possibility-to-run-manila-api-with-web-servers-that-support-wsgi-apps-cfffe0b789f8670a.yaml @@ -0,0 +1,8 @@ +--- +features: + - Manila API service now can be run using web servers that support + WSGI applications. +upgrade: + - Deprecated path 'manila.api.openstack:FaultWrapper' to 'FaultWrapper' + was removed and now only current path is available, which + is 'manila.api.middleware.fault:FaultWrapper'. diff --git a/setup.cfg b/setup.cfg index e7030516be..606ad9ce4b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,6 +34,8 @@ console_scripts = manila-rootwrap = oslo_rootwrap.cmd:main manila-scheduler = manila.cmd.scheduler:main manila-share = manila.cmd.share:main +wsgi_scripts = + manila-wsgi = manila.wsgi.wsgi:initialize_application manila.scheduler.filters = AvailabilityZoneFilter = manila.scheduler.filters.availability_zone:AvailabilityZoneFilter CapabilitiesFilter = manila.scheduler.filters.capabilities:CapabilitiesFilter