From 16bfc82962785409d1373571608508de75122645 Mon Sep 17 00:00:00 2001 From: Valeriy Ponomaryov Date: Tue, 21 Mar 2017 17:25:26 +0300 Subject: [PATCH] Add possibility to run 'manila-api' with wsgi web servers One of the goals for Pike [1] is to make each API service be able to run under web servers that support WSGI applications, such as Apache (+mod-wsgi) and Nginx (+uWSGI). Do following to address governance requirements: - Split existing manila/wsgi.py module into 3 modules: First (manila/wsgi/eventlet_server.py) is used by eventlet-based WSGI application approach. Second (manila/wsgi/wsgi.py) is used for WSGI web servers. And third (manila/wsgi/common.py) is common code for both. All three are made in cinder-like way to have alike-approach. - Reuse common code from "oslo_service/wsgi.py" module that allows us to remove code duplication. - Delete config opts that are defined by newly reused common code. - Register new entry point that will be manila wsgi app: "manila-wsgi". - Fix "manila/api/openstack/wsgi.py" module to be compatible with str/bytes handling approach used by Apache mod-wsgi plugin using different python versions (2/3). - Add web server config template "devstack/apache-manila.template" - Add devstack support where usage of this feature can be enabled or disabled using "MANILA_USE_MOD_WSGI" env var. It is set to "True" by default, because it is requirement for Pike release - to have it running in all CI jobs. Disable it only for one CI job that uses dummy driver and tests various manila core features that are not covered by other CI jobs. [1] https://governance.openstack.org/tc/goals/pike/deploy-api-in-wsgi.html Partially-Implements BluePrint wsgi-web-servers-support DocImpact Change-Id: Ibdef3c6810b65a5d6f3611e2d0079c635ee523ab --- contrib/ci/pre_test_hook.sh | 6 + devstack/apache-manila.template | 25 + devstack/plugin.sh | 41 +- devstack/settings | 12 +- manila/api/middleware/auth.py | 2 +- manila/api/middleware/fault.py | 2 +- manila/api/openstack/__init__.py | 12 +- manila/api/openstack/wsgi.py | 25 +- manila/api/v1/limits.py | 2 +- manila/common/config.py | 3 - manila/exception.py | 2 +- manila/opts.py | 5 +- manila/service.py | 15 +- manila/tests/api/fakes.py | 2 +- manila/tests/api/middleware/test_faults.py | 77 ++- manila/tests/api/test_versions.py | 2 +- manila/tests/api/test_wsgi.py | 22 +- manila/tests/conf_fixture.py | 2 + manila/tests/test_service.py | 4 +- manila/tests/test_utils.py | 37 ++ manila/tests/test_wsgi.py | 334 ----------- manila/tests/wsgi/__init__.py | 0 manila/tests/wsgi/test_common.py | 45 ++ manila/tests/wsgi/test_wsgi.py | 45 ++ manila/utils.py | 19 + manila/wsgi.py | 551 ------------------ manila/wsgi/__init__.py | 0 manila/wsgi/common.py | 155 +++++ manila/wsgi/eventlet_server.py | 59 ++ manila/wsgi/wsgi.py | 39 ++ ...at-support-wsgi-apps-cfffe0b789f8670a.yaml | 8 + setup.cfg | 2 + 32 files changed, 607 insertions(+), 948 deletions(-) create mode 100644 devstack/apache-manila.template delete mode 100644 manila/tests/test_wsgi.py create mode 100644 manila/tests/wsgi/__init__.py create mode 100644 manila/tests/wsgi/test_common.py create mode 100644 manila/tests/wsgi/test_wsgi.py delete mode 100644 manila/wsgi.py create mode 100644 manila/wsgi/__init__.py create mode 100644 manila/wsgi/common.py create mode 100644 manila/wsgi/eventlet_server.py create mode 100644 manila/wsgi/wsgi.py create mode 100644 releasenotes/notes/added-possibility-to-run-manila-api-with-web-servers-that-support-wsgi-apps-cfffe0b789f8670a.yaml diff --git a/contrib/ci/pre_test_hook.sh b/contrib/ci/pre_test_hook.sh index fd6f799b0c..e5f4ce9f5c 100755 --- a/contrib/ci/pre_test_hook.sh +++ b/contrib/ci/pre_test_hook.sh @@ -85,6 +85,12 @@ elif [[ "$DRIVER" == "dummy" ]]; then driver_path="manila.tests.share.drivers.dummy.DummyDriver" DEFAULT_EXTRA_SPECS="'snapshot_support=True create_share_from_snapshot_support=True revert_to_snapshot_support=True mount_snapshot_support=True'" echo "MANILA_SERVICE_IMAGE_ENABLED=False" >> $localconf + + # Run dummy driver CI job using standalone approach for running + # manila API service just because we need to test this approach too, + # that is very useful for development needs. + echo "MANILA_USE_MOD_WSGI=False" >> $localconf + echo "SHARE_DRIVER=$driver_path" >> $localconf echo "SUPPRESS_ERRORS_IN_CLEANUP=False" >> $localconf echo "MANILA_REPLICA_STATE_UPDATE_INTERVAL=10" >> $localconf diff --git a/devstack/apache-manila.template b/devstack/apache-manila.template new file mode 100644 index 0000000000..5049954d07 --- /dev/null +++ b/devstack/apache-manila.template @@ -0,0 +1,25 @@ +Listen %PORT% +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)" manila_combined + + + WSGIDaemonProcess manila-api processes=%APIWORKERS% threads=2 user=%USER% display-name=%{GROUP} + WSGIProcessGroup manila-api + WSGIScriptAlias / %MANILA_BIN_DIR%/manila-wsgi + WSGIApplicationGroup %{GLOBAL} + WSGIPassAuthorization On + = 2.4> + ErrorLogFormat "%{cu}t %M" + + ErrorLog /var/log/%APACHE_NAME%/manila_api.log + CustomLog /var/log/%APACHE_NAME%/manila_api_access.log manila_combined + + + = 2.4> + Require all granted + + + Order allow,deny + Allow from all + + + diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 54cc3402bc..5cdd0b5823 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -59,6 +59,22 @@ function cleanup_manila { _clean_zfsonlinux_data } +# _config_manila_apache_wsgi() - Configure manila-api wsgi application. +function _config_manila_apache_wsgi { + local manila_api_apache_conf + local venv_path="" + manila_api_apache_conf=$(apache_site_config_for manila-api) + + sudo cp $MANILA_DIR/devstack/apache-manila.template $manila_api_apache_conf + sudo sed -e " + s|%APACHE_NAME%|$APACHE_NAME|g; + s|%MANILA_BIN_DIR%|$MANILA_BIN_DIR|g; + s|%PORT%|$MANILA_SERVICE_PORT|g; + s|%APIWORKERS%|$API_WORKERS|g; + s|%USER%|$STACK_USER|g; + " -i $manila_api_apache_conf +} + # configure_default_backends - configures default Manila backends with generic driver. function configure_default_backends { # Configure two default backends with generic drivers onboard @@ -257,6 +273,10 @@ function configure_manila { MANILA_CONFIGURE_GROUPS=${MANILA_CONFIGURE_GROUPS:-"$MANILA_ENABLED_BACKENDS"} set_config_opts $MANILA_CONFIGURE_GROUPS set_config_opts DEFAULT + + if [ $(trueorfalse False MANILA_USE_MOD_WSGI) == True ]; then + _config_manila_apache_wsgi + fi } @@ -759,7 +779,14 @@ function configure_samba { # start_manila_api - starts manila API services and checks its availability function start_manila_api { - run_process m-api "$MANILA_BIN_DIR/manila-api --config-file $MANILA_CONF" + if [ $(trueorfalse False MANILA_USE_MOD_WSGI) == True ]; then + install_apache_wsgi + enable_apache_site manila-api + restart_apache_server + tail_log m-api /var/log/$APACHE_NAME/manila_api.log + else + run_process m-api "$MANILA_BIN_DIR/manila-api --config-file $MANILA_CONF" + fi echo "Waiting for Manila API to start..." if ! wait_for_service $SERVICE_TIMEOUT $MANILA_SERVICE_PROTOCOL://$MANILA_SERVICE_HOST:$MANILA_SERVICE_PORT; then @@ -788,8 +815,16 @@ function start_manila { # stop_manila - Stop running processes function stop_manila { - # Kill the manila processes - for serv in m-api m-sch m-shr m-dat; do + # Disable manila api service + if [ $(trueorfalse False MANILA_USE_MOD_WSGI) == True ]; then + disable_apache_site manila-api + restart_apache_server + else + stop_process m-api + fi + + # Kill all other manila processes + for serv in m-sch m-shr m-dat; do stop_process $serv done } diff --git a/devstack/settings b/devstack/settings index cc3c68c369..42f1a3f253 100644 --- a/devstack/settings +++ b/devstack/settings @@ -73,10 +73,9 @@ MANILA_DEFAULT_SHARE_GROUP_TYPE_SPECS=${MANILA_DEFAULT_SHARE_GROUP_TYPE_SPECS:-' # Public facing bits MANILA_SERVICE_HOST=${MANILA_SERVICE_HOST:-$SERVICE_HOST} MANILA_SERVICE_PORT=${MANILA_SERVICE_PORT:-8786} -MANILA_SERVICE_PORT_INT=${MANILA_SERVICE_PORT_INT:-18776} +MANILA_SERVICE_PORT_INT=${MANILA_SERVICE_PORT_INT:-18786} MANILA_SERVICE_PROTOCOL=${MANILA_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL} - # Support entry points installation of console scripts if [[ -d $MANILA_DIR/bin ]]; then MANILA_BIN_DIR=$MANILA_DIR/bin @@ -84,7 +83,6 @@ else MANILA_BIN_DIR=$(get_python_exec_prefix) fi - # Common opts SHARE_NAME_PREFIX=${SHARE_NAME_PREFIX:-share-} MANILA_ENABLED_SHARE_PROTOCOLS=${ENABLED_SHARE_PROTOCOLS:-"NFS,CIFS"} @@ -97,6 +95,14 @@ MANILA_SERVICE_SECGROUP="manila-service" # migrations again. MANILA_USE_DOWNGRADE_MIGRATIONS=${MANILA_USE_DOWNGRADE_MIGRATIONS:-"False"} +# Toggle for deploying manila-api service under Apache web server with enabled 'mod_wsgi' plugin. +# Disabled by default, which means running manila-api service as standalone +# eventlet-based WSGI application. +# Set it as True, because starting with Pike it is requirement from +# 'governance' project. See: +# https://governance.openstack.org/tc/goals/pike/deploy-api-in-wsgi.html#completion-criteria +MANILA_USE_MOD_WSGI=${MANILA_USE_MOD_WSGI:-True} + # Common info for Generic driver(s) SHARE_DRIVER=${SHARE_DRIVER:-manila.share.drivers.generic.GenericShareDriver} diff --git a/manila/api/middleware/auth.py b/manila/api/middleware/auth.py index 1e798c9305..cd6b5490f4 100644 --- a/manila/api/middleware/auth.py +++ b/manila/api/middleware/auth.py @@ -27,7 +27,7 @@ import webob.exc from manila.api.openstack import wsgi from manila import context from manila.i18n import _ -from manila import wsgi as base_wsgi +from manila.wsgi import common as base_wsgi use_forwarded_for_opt = cfg.BoolOpt( 'use_forwarded_for', diff --git a/manila/api/middleware/fault.py b/manila/api/middleware/fault.py index 5c1434b64f..e24e95a446 100644 --- a/manila/api/middleware/fault.py +++ b/manila/api/middleware/fault.py @@ -21,7 +21,7 @@ import webob.exc from manila.api.openstack import wsgi from manila import utils -from manila import wsgi as base_wsgi +from manila.wsgi import common as base_wsgi LOG = log.getLogger(__name__) diff --git a/manila/api/openstack/__init__.py b/manila/api/openstack/__init__.py index 3897d4c5c2..040abc28fb 100644 --- a/manila/api/openstack/__init__.py +++ b/manila/api/openstack/__init__.py @@ -19,11 +19,11 @@ WSGI middleware for OpenStack API controllers. """ from oslo_log import log +from oslo_service import wsgi as base_wsgi import routes from manila.api.openstack import wsgi from manila.i18n import _ -from manila import wsgi as base_wsgi LOG = log.getLogger(__name__) @@ -117,13 +117,3 @@ class APIRouter(base_wsgi.Router): def _setup_routes(self, mapper, ext_mgr): raise NotImplementedError - - -class FaultWrapper(base_wsgi.Middleware): - def __init__(self, application): - LOG.warning('manila.api.openstack:FaultWrapper is deprecated. ' - 'Please use ' - 'manila.api.middleware.fault:FaultWrapper instead.') - # Avoid circular imports from here. - from manila.api.middleware import fault - super(FaultWrapper, self).__init__(fault.FaultWrapper(application)) diff --git a/manila/api/openstack/wsgi.py b/manila/api/openstack/wsgi.py index e1b1bc32d7..2f984f0535 100644 --- a/manila/api/openstack/wsgi.py +++ b/manila/api/openstack/wsgi.py @@ -31,7 +31,8 @@ from manila.common import constants from manila import exception from manila.i18n import _ from manila import policy -from manila import wsgi +from manila import utils +from manila.wsgi import common as wsgi LOG = log.getLogger(__name__) @@ -860,15 +861,20 @@ class Resource(wsgi.Application): if hasattr(response, 'headers'): for hdr, val in response.headers.items(): - # Headers must be utf-8 strings - response.headers[hdr] = six.text_type(val) + val = utils.convert_str(val) + response.headers[hdr] = val if not request.api_version_request.is_null(): response.headers[API_VERSION_REQUEST_HEADER] = ( request.api_version_request.get_string()) if request.api_version_request.experimental: + # NOTE(vponomaryov): Translate our boolean header + # to string explicitly to avoid 'TypeError' failure + # running manila API under Apache + mod-wsgi. + # It is safe to do so, because all headers are returned as + # strings anyway. response.headers[EXPERIMENTAL_API_REQUEST_HEADER] = ( - request.api_version_request.experimental) + '%s' % request.api_version_request.experimental) response.headers['Vary'] = API_VERSION_REQUEST_HEADER return response @@ -1280,14 +1286,19 @@ class Fault(webob.exc.HTTPException): 'message': self.wrapped_exc.explanation}} if code == 413: retry = self.wrapped_exc.headers['Retry-After'] - fault_data[fault_name]['retryAfter'] = retry + fault_data[fault_name]['retryAfter'] = '%s' % retry if not req.api_version_request.is_null(): self.wrapped_exc.headers[API_VERSION_REQUEST_HEADER] = ( req.api_version_request.get_string()) if req.api_version_request.experimental: + # NOTE(vponomaryov): Translate our boolean header + # to string explicitly to avoid 'TypeError' failure + # running manila API under Apache + mod-wsgi. + # It is safe to do so, because all headers are returned as + # strings anyway. self.wrapped_exc.headers[EXPERIMENTAL_API_REQUEST_HEADER] = ( - req.api_version_request.experimental) + '%s' % req.api_version_request.experimental) self.wrapped_exc.headers['Vary'] = API_VERSION_REQUEST_HEADER content_type = req.best_match_content_type() @@ -1330,7 +1341,7 @@ class OverLimitFault(webob.exc.HTTPException): def _retry_after(retry_time): delay = int(math.ceil(retry_time - time.time())) retry_after = delay if delay > 0 else 0 - headers = {'Retry-After': '%d' % retry_after} + headers = {'Retry-After': '%s' % retry_after} return headers @webob.dec.wsgify(RequestClass=Request) diff --git a/manila/api/v1/limits.py b/manila/api/v1/limits.py index 66c04ec6f9..85209d6d6d 100644 --- a/manila/api/v1/limits.py +++ b/manila/api/v1/limits.py @@ -33,7 +33,7 @@ from manila.api.openstack import wsgi from manila.api.views import limits as limits_views from manila.i18n import _ from manila import quota -from manila import wsgi as base_wsgi +from manila.wsgi import common as base_wsgi QUOTAS = quota.QUOTAS diff --git a/manila/common/config.py b/manila/common/config.py index bc4feb73e6..a69cdcddbe 100644 --- a/manila/common/config.py +++ b/manila/common/config.py @@ -41,9 +41,6 @@ log.register_options(CONF) core_opts = [ - cfg.StrOpt('api_paste_config', - default="api-paste.ini", - help='File name for the paste.deploy config for manila-api.'), cfg.StrOpt('state_path', default='/var/lib/manila', help="Top-level directory for maintaining manila's state."), diff --git a/manila/exception.py b/manila/exception.py index d23d63a649..65abbe5ede 100644 --- a/manila/exception.py +++ b/manila/exception.py @@ -386,7 +386,7 @@ class WillNotSchedule(ManilaException): class QuotaError(ManilaException): message = _("Quota exceeded: code=%(code)s.") code = 413 - headers = {'Retry-After': 0} + headers = {'Retry-After': '0'} safe = True diff --git a/manila/opts.py b/manila/opts.py index 930f202628..65b9afba46 100644 --- a/manila/opts.py +++ b/manila/opts.py @@ -85,7 +85,7 @@ import manila.share.hook import manila.share.manager import manila.volume import manila.volume.cinder -import manila.wsgi +import manila.wsgi.eventlet_server # List of *all* options in [DEFAULT] namespace of manila. @@ -165,8 +165,7 @@ _global_opt_lists = [ manila.share.hook.hook_options, manila.share.manager.share_manager_opts, manila.volume._volume_opts, - manila.wsgi.eventlet_opts, - manila.wsgi.socket_opts, + manila.wsgi.eventlet_server.socket_opts, ] _opts = [ diff --git a/manila/service.py b/manila/service.py index 7687f9d556..03786f7821 100644 --- a/manila/service.py +++ b/manila/service.py @@ -26,6 +26,7 @@ from oslo_log import log import oslo_messaging as messaging from oslo_service import loopingcall from oslo_service import service +from oslo_service import wsgi from oslo_utils import importutils from manila import context @@ -34,7 +35,6 @@ from manila import db from manila import exception from manila import rpc from manila import version -from manila import wsgi LOG = log.getLogger(__name__) @@ -283,7 +283,7 @@ class WSGIService(service.ServiceBase): """ self.name = name self.manager = self._get_manager() - self.loader = loader or wsgi.Loader() + self.loader = loader or wsgi.Loader(CONF) if not rpc.initialized(): rpc.init(CONF) self.app = self.loader.load_app(name) @@ -296,10 +296,13 @@ class WSGIService(service.ServiceBase): "greater than 1. Input value ignored." % {'name': name}) # Reset workers to default self.workers = None - self.server = wsgi.Server(name, - self.app, - host=self.host, - port=self.port) + self.server = wsgi.Server( + CONF, + name, + self.app, + host=self.host, + port=self.port, + ) def _get_manager(self): """Initialize a Manager object appropriate for this service. diff --git a/manila/tests/api/fakes.py b/manila/tests/api/fakes.py index c104b8835b..f7c93c0908 100644 --- a/manila/tests/api/fakes.py +++ b/manila/tests/api/fakes.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo_service import wsgi from oslo_utils import timeutils from oslo_utils import uuidutils import routes @@ -33,7 +34,6 @@ from manila.api import versions from manila.common import constants from manila import context from manila import exception -from manila import wsgi FAKE_UUID = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' diff --git a/manila/tests/api/middleware/test_faults.py b/manila/tests/api/middleware/test_faults.py index a540187e72..a15e8fda8c 100644 --- a/manila/tests/api/middleware/test_faults.py +++ b/manila/tests/api/middleware/test_faults.py @@ -19,7 +19,9 @@ import webob import webob.dec import webob.exc +from manila.api.middleware import fault from manila.api.openstack import wsgi +from manila import exception from manila import test @@ -72,7 +74,7 @@ class TestFaults(test.TestCase): "overLimit": { "message": "sorry", "code": 413, - "retryAfter": 4, + "retryAfter": '4', }, } actual = jsonutils.loads(response.body) @@ -109,3 +111,76 @@ class TestFaults(test.TestCase): """Ensure the status_int is set correctly on faults.""" fault = wsgi.Fault(webob.exc.HTTPBadRequest(explanation='what?')) self.assertEqual(400, fault.status_int) + + +class ExceptionTest(test.TestCase): + + def _wsgi_app(self, inner_app): + return fault.FaultWrapper(inner_app) + + def _do_test_exception_safety_reflected_in_faults(self, expose): + class ExceptionWithSafety(exception.ManilaException): + safe = expose + + @webob.dec.wsgify + def fail(req): + raise ExceptionWithSafety('some explanation') + + api = self._wsgi_app(fail) + resp = webob.Request.blank('/').get_response(api) + self.assertIn('{"computeFault', six.text_type(resp.body), resp.body) + expected = ('ExceptionWithSafety: some explanation' if expose else + 'The server has either erred or is incapable ' + 'of performing the requested operation.') + self.assertIn(expected, six.text_type(resp.body), resp.body) + self.assertEqual(500, resp.status_int, resp.body) + + def test_safe_exceptions_are_described_in_faults(self): + self._do_test_exception_safety_reflected_in_faults(True) + + def test_unsafe_exceptions_are_not_described_in_faults(self): + self._do_test_exception_safety_reflected_in_faults(False) + + def _do_test_exception_mapping(self, exception_type, msg): + @webob.dec.wsgify + def fail(req): + raise exception_type(msg) + + api = self._wsgi_app(fail) + resp = webob.Request.blank('/').get_response(api) + self.assertIn(msg, six.text_type(resp.body), resp.body) + self.assertEqual(exception_type.code, resp.status_int, resp.body) + + if hasattr(exception_type, 'headers'): + for (key, value) in exception_type.headers.items(): + self.assertIn(key, resp.headers) + self.assertEqual(value, resp.headers[key]) + + def test_quota_error_mapping(self): + self._do_test_exception_mapping(exception.QuotaError, 'too many used') + + def test_non_manila_notfound_exception_mapping(self): + class ExceptionWithCode(Exception): + code = 404 + + self._do_test_exception_mapping(ExceptionWithCode, + 'NotFound') + + def test_non_manila_exception_mapping(self): + class ExceptionWithCode(Exception): + code = 417 + + self._do_test_exception_mapping(ExceptionWithCode, + 'Expectation failed') + + def test_exception_with_none_code_throws_500(self): + class ExceptionWithNoneCode(Exception): + code = None + + @webob.dec.wsgify + def fail(req): + raise ExceptionWithNoneCode() + + api = self._wsgi_app(fail) + resp = webob.Request.blank('/').get_response(api) + self.assertEqual(500, resp.status_int) diff --git a/manila/tests/api/test_versions.py b/manila/tests/api/test_versions.py index c36623f435..83f0abba10 100644 --- a/manila/tests/api/test_versions.py +++ b/manila/tests/api/test_versions.py @@ -185,7 +185,7 @@ class ExperimentalAPITestCase(test.TestCase): self.assertEqual('2.0', response.headers[version_header_name]) if experimental: - self.assertEqual(experimental, + self.assertEqual('%s' % experimental, response.headers.get(experimental_header_name)) else: self.assertNotIn(experimental_header_name, response.headers) diff --git a/manila/tests/api/test_wsgi.py b/manila/tests/api/test_wsgi.py index bb53f76a97..7310213e8a 100644 --- a/manila/tests/api/test_wsgi.py +++ b/manila/tests/api/test_wsgi.py @@ -19,33 +19,19 @@ Test WSGI basics and provide some helper functions for other WSGI tests. """ -from manila import test - +from oslo_service import wsgi import routes -import six import webob -from manila import wsgi +from manila import test +from manila.wsgi import common as common_wsgi class Test(test.TestCase): - def test_debug(self): - - class Application(wsgi.Application): - """Dummy application to test debug.""" - - def __call__(self, environ, start_response): - start_response("200", [("X-Test", "checking")]) - return [six.b('Test result')] - - application = wsgi.Debug(Application()) - result = webob.Request.blank('/').get_response(application) - self.assertEqual(six.b("Test result"), result.body) - def test_router(self): - class Application(wsgi.Application): + class Application(common_wsgi.Application): """Test application to call from router.""" def __call__(self, environ, start_response): diff --git a/manila/tests/conf_fixture.py b/manila/tests/conf_fixture.py index 3bea20bda9..a468f7b7c8 100644 --- a/manila/tests/conf_fixture.py +++ b/manila/tests/conf_fixture.py @@ -17,6 +17,7 @@ import os from oslo_policy import opts +from oslo_service import wsgi from manila.common import config @@ -38,6 +39,7 @@ def set_defaults(conf): _safe_set_of_opts(conf, 'service_instance_user', 'fake_user') _API_PASTE_PATH = os.path.abspath(os.path.join(CONF.state_path, 'etc/manila/api-paste.ini')) + wsgi.register_opts(conf) _safe_set_of_opts(conf, 'api_paste_config', _API_PASTE_PATH) _safe_set_of_opts(conf, 'share_driver', 'manila.tests.fake_driver.FakeShareDriver') diff --git a/manila/tests/test_service.py b/manila/tests/test_service.py index 22874a4dbd..c0d78d5ae2 100644 --- a/manila/tests/test_service.py +++ b/manila/tests/test_service.py @@ -24,6 +24,7 @@ Unit Tests for remote procedure calls using queue import ddt import mock from oslo_config import cfg +from oslo_service import wsgi from manila import context from manila import db @@ -32,7 +33,6 @@ from manila import manager from manila import service from manila import test from manila import utils -from manila import wsgi test_service_opts = [ cfg.StrOpt("fake_manager", @@ -224,5 +224,5 @@ class TestWSGIService(test.TestCase): # Resetting pool size to default self.test_service.reset() self.test_service.start() - self.assertEqual(1000, self.test_service.server._pool.size) + self.assertGreater(self.test_service.server._pool.size, 0) wsgi.Loader.load_app.assert_called_once_with("test_service") diff --git a/manila/tests/test_utils.py b/manila/tests/test_utils.py index 9fdf051084..a979ed6eab 100644 --- a/manila/tests/test_utils.py +++ b/manila/tests/test_utils.py @@ -25,6 +25,7 @@ from oslo_config import cfg from oslo_utils import timeutils from oslo_utils import uuidutils import paramiko +import six from webob import exc import manila @@ -744,3 +745,39 @@ class ShareMigrationHelperTestCase(test.TestCase): self.assertRaises(expected_exc, utils.wait_for_access_update, self.context, db, fake_instance, 1) + + +@ddt.ddt +class ConvertStrTestCase(test.TestCase): + + def test_convert_str_str_input(self): + self.mock_object(utils.encodeutils, 'safe_encode') + input_value = six.text_type("string_input") + + output_value = utils.convert_str(input_value) + + if six.PY2: + utils.encodeutils.safe_encode.assert_called_once_with(input_value) + self.assertEqual( + utils.encodeutils.safe_encode.return_value, output_value) + else: + self.assertEqual(0, utils.encodeutils.safe_encode.call_count) + self.assertEqual(input_value, output_value) + + def test_convert_str_bytes_input(self): + self.mock_object(utils.encodeutils, 'safe_encode') + if six.PY2: + input_value = six.binary_type("binary_input") + else: + input_value = six.binary_type("binary_input", "utf-8") + + output_value = utils.convert_str(input_value) + + if six.PY2: + utils.encodeutils.safe_encode.assert_called_once_with(input_value) + self.assertEqual( + utils.encodeutils.safe_encode.return_value, output_value) + else: + self.assertEqual(0, utils.encodeutils.safe_encode.call_count) + self.assertIsInstance(output_value, six.string_types) + self.assertEqual(six.text_type("binary_input"), output_value) diff --git a/manila/tests/test_wsgi.py b/manila/tests/test_wsgi.py deleted file mode 100644 index 05789a2d8f..0000000000 --- a/manila/tests/test_wsgi.py +++ /dev/null @@ -1,334 +0,0 @@ -# Copyright 2011 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Unit tests for `manila.wsgi`.""" - -import os.path -import ssl -import tempfile - -import ddt -import eventlet -import mock -from oslo_config import cfg -from oslo_utils import netutils -import six -from six.moves import urllib -import testtools -import webob -import webob.dec - -from manila.api.middleware import fault -from manila import exception -from manila import test -from manila import utils -import manila.wsgi - -CONF = cfg.CONF - -TEST_VAR_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), - 'var')) - - -class TestLoaderNothingExists(test.TestCase): - """Loader tests where os.path.exists always returns False.""" - - def test_config_not_found(self): - self.assertRaises( - manila.exception.ConfigNotFound, - manila.wsgi.Loader, - 'nonexistent_file.ini', - ) - - -class TestLoaderNormalFilesystem(test.TestCase): - """Loader tests with normal filesystem (unmodified os.path module).""" - - _paste_config = """ -[app:test_app] -use = egg:Paste#static -document_root = /tmp - """ - - def setUp(self): - super(TestLoaderNormalFilesystem, self).setUp() - self.config = tempfile.NamedTemporaryFile(mode="w+t") - self.config.write(self._paste_config.lstrip()) - self.config.seek(0) - self.config.flush() - self.loader = manila.wsgi.Loader(self.config.name) - self.addCleanup(self.config.close) - - def test_config_found(self): - self.assertEqual(self.config.name, self.loader.config_path) - - def test_app_not_found(self): - self.assertRaises( - manila.exception.PasteAppNotFound, - self.loader.load_app, - "non-existent app", - ) - - def test_app_found(self): - url_parser = self.loader.load_app("test_app") - self.assertEqual("/tmp", url_parser.directory) - - -@ddt.ddt -class TestWSGIServer(test.TestCase): - """WSGI server tests.""" - - def test_no_app(self): - server = manila.wsgi.Server("test_app", None, host="127.0.0.1", port=0) - self.assertEqual("test_app", server.name) - - def test_start_random_port(self): - server = manila.wsgi.Server("test_random_port", None, host="127.0.0.1") - server.start() - self.assertNotEqual(0, server.port) - server.stop() - server.wait() - - @testtools.skipIf(not netutils.is_ipv6_enabled(), - "Test requires an IPV6 configured interface") - @testtools.skipIf(utils.is_eventlet_bug105(), - 'Eventlet bug #105 affect test results.') - def test_start_random_port_with_ipv6(self): - server = manila.wsgi.Server("test_random_port", - None, - host="::1") - server.start() - self.assertEqual("::1", server.host) - self.assertNotEqual(0, server.port) - server.stop() - server.wait() - - def test_start_with_default_tcp_options(self): - server = manila.wsgi.Server("test_tcp_options", - None, - host="127.0.0.1") - self.mock_object( - netutils, 'set_tcp_keepalive') - server.start() - netutils.set_tcp_keepalive.assert_called_once_with( - mock.ANY, tcp_keepalive=True, tcp_keepalive_count=None, - tcp_keepalive_interval=None, tcp_keepidle=600) - - def test_start_with_custom_tcp_options(self): - CONF.set_default("tcp_keepalive", False) - CONF.set_default("tcp_keepalive_count", 33) - CONF.set_default("tcp_keepalive_interval", 22) - CONF.set_default("tcp_keepidle", 11) - server = manila.wsgi.Server("test_tcp_options", - None, - host="127.0.0.1") - self.mock_object( - netutils, 'set_tcp_keepalive') - server.start() - netutils.set_tcp_keepalive.assert_called_once_with( - mock.ANY, tcp_keepalive=False, tcp_keepalive_count=33, - tcp_keepalive_interval=22, tcp_keepidle=11) - - def test_app(self): - self.mock_object( - eventlet, 'spawn', mock.Mock(side_effect=eventlet.spawn)) - greetings = 'Hello, World!!!' - - def hello_world(env, start_response): - if env['PATH_INFO'] != '/': - start_response('404 Not Found', - [('Content-Type', 'text/plain')]) - return ['Not Found\r\n'] - start_response('200 OK', [('Content-Type', 'text/plain')]) - return [greetings] - - server = manila.wsgi.Server( - "test_app", hello_world, host="127.0.0.1", port=0) - server.start() - - response = urllib.request.urlopen('http://127.0.0.1:%d/' % server.port) - self.assertEqual(six.b(greetings), response.read()) - - # Verify provided parameters to eventlet.spawn func - eventlet.spawn.assert_called_once_with( - func=eventlet.wsgi.server, - sock=mock.ANY, - site=server.app, - protocol=server._protocol, - custom_pool=server._pool, - log=server._logger, - socket_timeout=server.client_socket_timeout, - keepalive=manila.wsgi.CONF.wsgi_keep_alive, - ) - - server.stop() - - @ddt.data(0, 0.1, 1, None) - def test_init_server_with_socket_timeout(self, client_socket_timeout): - CONF.set_default("client_socket_timeout", client_socket_timeout) - server = manila.wsgi.Server( - "test_app", lambda *args, **kwargs: None, host="127.0.0.1", port=0) - self.assertEqual(client_socket_timeout, server.client_socket_timeout) - - @testtools.skipIf(six.PY3, "bug/1482633") - def test_app_using_ssl(self): - CONF.set_default("ssl_cert_file", - os.path.join(TEST_VAR_DIR, 'certificate.crt')) - CONF.set_default("ssl_key_file", - os.path.join(TEST_VAR_DIR, 'privatekey.key')) - - greetings = 'Hello, World!!!' - - @webob.dec.wsgify - def hello_world(req): - return greetings - - server = manila.wsgi.Server( - "test_app", hello_world, host="127.0.0.1", port=0) - server.start() - - if hasattr(ssl, '_create_unverified_context'): - response = urllib.request.urlopen( - 'https://127.0.0.1:%d/' % server.port, - context=ssl._create_unverified_context()) - else: - response = urllib.request.urlopen( - 'https://127.0.0.1:%d/' % server.port) - - self.assertEqual(greetings, response.read()) - - server.stop() - - @testtools.skipIf(not netutils.is_ipv6_enabled(), - "Test requires an IPV6 configured interface") - @testtools.skipIf(utils.is_eventlet_bug105(), - 'Eventlet bug #105 affect test results.') - @testtools.skipIf(six.PY3, "bug/1482633") - def test_app_using_ipv6_and_ssl(self): - CONF.set_default("ssl_cert_file", - os.path.join(TEST_VAR_DIR, 'certificate.crt')) - CONF.set_default("ssl_key_file", - os.path.join(TEST_VAR_DIR, 'privatekey.key')) - - greetings = 'Hello, World!!!' - - @webob.dec.wsgify - def hello_world(req): - return greetings - - server = manila.wsgi.Server("test_app", - hello_world, - host="::1", - port=0) - server.start() - - if hasattr(ssl, '_create_unverified_context'): - response = urllib.request.urlopen( - 'https://[::1]:%d/' % server.port, - context=ssl._create_unverified_context()) - else: - response = urllib.request.urlopen( - 'https://[::1]:%d/' % server.port) - - self.assertEqual(greetings, response.read()) - - server.stop() - - def test_reset_pool_size_to_default(self): - server = manila.wsgi.Server("test_resize", None, host="127.0.0.1") - server.start() - - # Stopping the server, which in turn sets pool size to 0 - server.stop() - self.assertEqual(0, server._pool.size) - - # Resetting pool size to default - server.reset() - server.start() - self.assertEqual(1000, server._pool.size) - - -class ExceptionTest(test.TestCase): - - def _wsgi_app(self, inner_app): - return fault.FaultWrapper(inner_app) - - def _do_test_exception_safety_reflected_in_faults(self, expose): - class ExceptionWithSafety(exception.ManilaException): - safe = expose - - @webob.dec.wsgify - def fail(req): - raise ExceptionWithSafety('some explanation') - - api = self._wsgi_app(fail) - resp = webob.Request.blank('/').get_response(api) - self.assertIn('{"computeFault', six.text_type(resp.body), resp.body) - expected = ('ExceptionWithSafety: some explanation' if expose else - 'The server has either erred or is incapable ' - 'of performing the requested operation.') - self.assertIn(expected, six.text_type(resp.body), resp.body) - self.assertEqual(500, resp.status_int, resp.body) - - def test_safe_exceptions_are_described_in_faults(self): - self._do_test_exception_safety_reflected_in_faults(True) - - def test_unsafe_exceptions_are_not_described_in_faults(self): - self._do_test_exception_safety_reflected_in_faults(False) - - def _do_test_exception_mapping(self, exception_type, msg): - @webob.dec.wsgify - def fail(req): - raise exception_type(msg) - - api = self._wsgi_app(fail) - resp = webob.Request.blank('/').get_response(api) - self.assertIn(msg, six.text_type(resp.body), resp.body) - self.assertEqual(exception_type.code, resp.status_int, resp.body) - - if hasattr(exception_type, 'headers'): - for (key, value) in exception_type.headers.items(): - self.assertIn(key, resp.headers) - self.assertEqual(value, resp.headers[key]) - - def test_quota_error_mapping(self): - self._do_test_exception_mapping(exception.QuotaError, 'too many used') - - def test_non_manila_notfound_exception_mapping(self): - class ExceptionWithCode(Exception): - code = 404 - - self._do_test_exception_mapping(ExceptionWithCode, - 'NotFound') - - def test_non_manila_exception_mapping(self): - class ExceptionWithCode(Exception): - code = 417 - - self._do_test_exception_mapping(ExceptionWithCode, - 'Expectation failed') - - def test_exception_with_none_code_throws_500(self): - class ExceptionWithNoneCode(Exception): - code = None - - @webob.dec.wsgify - def fail(req): - raise ExceptionWithNoneCode() - - api = self._wsgi_app(fail) - resp = webob.Request.blank('/').get_response(api) - self.assertEqual(500, resp.status_int) diff --git a/manila/tests/wsgi/__init__.py b/manila/tests/wsgi/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/manila/tests/wsgi/test_common.py b/manila/tests/wsgi/test_common.py new file mode 100644 index 0000000000..0b9f18be85 --- /dev/null +++ b/manila/tests/wsgi/test_common.py @@ -0,0 +1,45 @@ +# Copyright 2017 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from manila import test +from manila.wsgi import common + + +class FakeApp(common.Application): + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + + +class WSGICommonTestCase(test.TestCase): + + def test_application_factory(self): + fake_global_config = mock.Mock() + kwargs = {"k1": "v1", "k2": "v2"} + + result = FakeApp.factory(fake_global_config, **kwargs) + + fake_global_config.assert_not_called() + self.assertIsInstance(result, FakeApp) + for k, v in kwargs.items(): + self.assertTrue(hasattr(result, k)) + self.assertEqual(getattr(result, k), v) + + def test_application___call__(self): + self.assertRaises( + NotImplementedError, + common.Application(), 'fake_environ', 'fake_start_response') diff --git a/manila/tests/wsgi/test_wsgi.py b/manila/tests/wsgi/test_wsgi.py new file mode 100644 index 0000000000..335f6b3562 --- /dev/null +++ b/manila/tests/wsgi/test_wsgi.py @@ -0,0 +1,45 @@ +# Copyright 2017 Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from manila import test +from manila.wsgi import wsgi + + +class WSGITestCase(test.TestCase): + + def test_initialize_application(self): + self.mock_object(wsgi.log, 'register_options') + self.mock_object(wsgi.cfg.ConfigOpts, '__call__') + self.mock_object(wsgi.config, 'verify_share_protocols') + self.mock_object(wsgi.log, 'setup') + self.mock_object(wsgi.rpc, 'init') + self.mock_object(wsgi.wsgi, 'Loader') + wsgi.sys.argv = ['--verbose', '--debug'] + + result = wsgi.initialize_application() + + self.assertEqual( + wsgi.wsgi.Loader.return_value.load_app.return_value, result) + wsgi.log.register_options.assert_called_once_with(mock.ANY) + wsgi.cfg.ConfigOpts.__call__.assert_called_once_with( + mock.ANY, project="manila", version=wsgi.version.version_string()) + wsgi.config.verify_share_protocols.assert_called_once_with() + wsgi.log.setup.assert_called_once_with(mock.ANY, "manila") + wsgi.rpc.init.assert_called_once_with(mock.ANY) + wsgi.wsgi.Loader.assert_called_once_with(mock.ANY) + wsgi.wsgi.Loader.return_value.load_app.assert_called_once_with( + name='osapi_share') diff --git a/manila/utils.py b/manila/utils.py index 3d312a23e5..ec9bc02b6f 100644 --- a/manila/utils.py +++ b/manila/utils.py @@ -36,6 +36,7 @@ from oslo_concurrency import lockutils from oslo_concurrency import processutils from oslo_config import cfg from oslo_log import log +from oslo_utils import encodeutils from oslo_utils import importutils from oslo_utils import netutils from oslo_utils import strutils @@ -520,6 +521,24 @@ def require_driver_initialized(func): return wrapper +def convert_str(text): + """Convert to native string. + + Convert bytes and Unicode strings to native strings: + + * convert to bytes on Python 2: + encode Unicode using encodeutils.safe_encode() + * convert to Unicode on Python 3: decode bytes from UTF-8 + """ + if six.PY2: + return encodeutils.safe_encode(text) + else: + if isinstance(text, bytes): + return text.decode('utf-8') + else: + return text + + def translate_string_size_to_float(string, multiplier='G'): """Translates human-readable storage size to float value. diff --git a/manila/wsgi.py b/manila/wsgi.py deleted file mode 100644 index 69c8463a76..0000000000 --- a/manila/wsgi.py +++ /dev/null @@ -1,551 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Utility methods for working with WSGI servers.""" - -from __future__ import print_function - -import errno -import os -import socket -import ssl -import sys -import time - -import eventlet -import eventlet.wsgi -import greenlet -from oslo_config import cfg -from oslo_log import log -from oslo_service import service -from oslo_utils import excutils -from oslo_utils import netutils -from paste import deploy -import routes.middleware -import webob.dec -import webob.exc - -from manila.common import config -from manila import exception -from manila.i18n import _ - -socket_opts = [ - cfg.IntOpt('backlog', - default=4096, - help="Number of backlog requests to configure the socket " - "with."), - cfg.BoolOpt('tcp_keepalive', - default=True, - help="Sets the value of TCP_KEEPALIVE (True/False) for each " - "server socket."), - cfg.IntOpt('tcp_keepidle', - default=600, - help="Sets the value of TCP_KEEPIDLE in seconds for each " - "server socket. Not supported on OS X."), - cfg.IntOpt('tcp_keepalive_interval', - help="Sets the value of TCP_KEEPINTVL in seconds for each " - "server socket. Not supported on OS X."), - cfg.IntOpt('tcp_keepalive_count', - help="Sets the value of TCP_KEEPCNT for each " - "server socket. Not supported on OS X."), - cfg.StrOpt('ssl_ca_file', - help="CA certificate file to use to verify " - "connecting clients."), - cfg.StrOpt('ssl_cert_file', - help="Certificate file to use when starting " - "the server securely."), - cfg.StrOpt('ssl_key_file', - help="Private key file to use when starting " - "the server securely."), -] - -eventlet_opts = [ - cfg.IntOpt('max_header_line', - default=16384, - help="Maximum line size of message headers to be accepted. " - "Option max_header_line may need to be increased when " - "using large tokens (typically those generated by the " - "Keystone v3 API with big service catalogs)."), - cfg.IntOpt('client_socket_timeout', - default=900, - help="Timeout for client connections socket operations. " - "If an incoming connection is idle for this number of " - "seconds it will be closed. A value of '0' means " - "wait forever."), - cfg.BoolOpt('wsgi_keep_alive', - default=True, - help='If False, closes the client socket connection ' - 'explicitly. Setting it to True to maintain backward ' - 'compatibility. Recommended setting is set it to False.'), -] - -CONF = cfg.CONF -CONF.register_opts(socket_opts) -CONF.register_opts(eventlet_opts) - -LOG = log.getLogger(__name__) - - -class Server(service.ServiceBase): - """Server class to manage a WSGI server, serving a WSGI application.""" - - default_pool_size = 1000 - - def __init__(self, name, app, host=None, port=None, pool_size=None, - protocol=eventlet.wsgi.HttpProtocol, backlog=128): - """Initialize, but do not start, a WSGI server. - - :param name: Pretty name for logging. - :param app: The WSGI application to serve. - :param host: IP address to serve the application. - :param port: Port number to server the application. - :param pool_size: Maximum number of eventlets to spawn concurrently. - :returns: None - - """ - eventlet.wsgi.MAX_HEADER_LINE = CONF.max_header_line - self.client_socket_timeout = CONF.client_socket_timeout - self.name = name - self.app = app - self._host = host or "0.0.0.0" - self._port = port or 0 - self._server = None - self._socket = None - self._protocol = protocol - self.pool_size = pool_size or self.default_pool_size - self._pool = eventlet.GreenPool(self.pool_size) - self._logger = log.getLogger("eventlet.wsgi.server") - - if backlog < 1: - raise exception.InvalidInput( - reason='The backlog must be more than 1') - - bind_addr = (host, port) - # TODO(dims): eventlet's green dns/socket module does not actually - # support IPv6 in getaddrinfo(). We need to get around this in the - # future or monitor upstream for a fix - try: - info = socket.getaddrinfo(bind_addr[0], - bind_addr[1], - socket.AF_UNSPEC, - socket.SOCK_STREAM)[0] - family = info[0] - bind_addr = info[-1] - except Exception: - family = socket.AF_INET - - cert_file = CONF.ssl_cert_file - key_file = CONF.ssl_key_file - ca_file = CONF.ssl_ca_file - self._use_ssl = cert_file or key_file - - if cert_file and not os.path.exists(cert_file): - raise RuntimeError(_("Unable to find cert_file : %s") % cert_file) - - if ca_file and not os.path.exists(ca_file): - raise RuntimeError(_("Unable to find ca_file : %s") % ca_file) - - if key_file and not os.path.exists(key_file): - raise RuntimeError(_("Unable to find key_file : %s") % key_file) - - if self._use_ssl and (not cert_file or not key_file): - raise RuntimeError(_("When running server in SSL mode, you must " - "specify both a cert_file and key_file " - "option value in your configuration file")) - - retry_until = time.time() + 30 - while not self._socket and time.time() < retry_until: - try: - self._socket = eventlet.listen( - bind_addr, backlog=backlog, family=family) - except socket.error as err: - if err.args[0] != errno.EADDRINUSE: - raise - eventlet.sleep(0.1) - - if not self._socket: - raise RuntimeError(_("Could not bind to %(host)s:%(port)s " - "after trying for 30 seconds") % - {'host': host, 'port': port}) - - (self._host, self._port) = self._socket.getsockname()[0:2] - LOG.info("%(name)s listening on %(_host)s:%(_port)s", - {'name': self.name, '_host': self._host, '_port': self._port}) - - def start(self): - """Start serving a WSGI application. - - :returns: None - :raises: manila.exception.InvalidInput - - """ - # The server socket object will be closed after server exits, - # but the underlying file descriptor will remain open, and will - # give bad file descriptor error. So duplicating the socket object, - # to keep file descriptor usable. - - config.set_middleware_defaults() - dup_socket = self._socket.dup() - - netutils.set_tcp_keepalive( - dup_socket, - tcp_keepalive=CONF.tcp_keepalive, - tcp_keepidle=CONF.tcp_keepidle, - tcp_keepalive_interval=CONF.tcp_keepalive_interval, - tcp_keepalive_count=CONF.tcp_keepalive_count - ) - - if self._use_ssl: - try: - ssl_kwargs = { - 'server_side': True, - 'certfile': CONF.ssl_cert_file, - 'keyfile': CONF.ssl_key_file, - 'cert_reqs': ssl.CERT_NONE, - } - - if CONF.ssl_ca_file: - ssl_kwargs['ca_certs'] = CONF.ssl_ca_file - ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED - - dup_socket = ssl.wrap_socket(dup_socket, - **ssl_kwargs) - - dup_socket.setsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR, 1) - - except Exception: - with excutils.save_and_reraise_exception(): - LOG.error( - ("Failed to start %(name)s on %(_host)s:%(_port)s " - "with SSL support."), - {"name": self.name, "_host": self._host, - "_port": self._port} - ) - - wsgi_kwargs = { - 'func': eventlet.wsgi.server, - 'sock': dup_socket, - 'site': self.app, - 'protocol': self._protocol, - 'custom_pool': self._pool, - 'log': self._logger, - 'socket_timeout': self.client_socket_timeout, - 'keepalive': CONF.wsgi_keep_alive, - } - - self._server = eventlet.spawn(**wsgi_kwargs) - - @property - def host(self): - return self._host - - @property - def port(self): - return self._port - - def stop(self): - """Stop this server. - - This is not a very nice action, as currently the method by which a - server is stopped is by killing its eventlet. - - :returns: None - - """ - LOG.info("Stopping WSGI server.") - if self._server is not None: - # Resize pool to stop new requests from being processed - self._pool.resize(0) - self._server.kill() - - def wait(self): - """Block, until the server has stopped. - - Waits on the server's eventlet to finish, then returns. - - :returns: None - - """ - try: - if self._server is not None: - self._pool.waitall() - self._server.wait() - except greenlet.GreenletExit: - LOG.info("WSGI server has stopped.") - - def reset(self): - """Reset server greenpool size to default. - - :returns: None - """ - self._pool.resize(self.pool_size) - - -class Request(webob.Request): - pass - - -class Application(object): - """Base WSGI application wrapper. Subclasses need to implement __call__.""" - - @classmethod - def factory(cls, global_config, **local_config): - """Used for paste app factories in paste.deploy config files. - - Any local configuration (that is, values under the [app:APPNAME] - section of the paste config) will be passed into the `__init__` method - as kwargs. - - A hypothetical configuration would look like: - - [app:wadl] - latest_version = 1.3 - paste.app_factory = manila.api.fancy_api:Wadl.factory - - which would result in a call to the `Wadl` class as - - import manila.api.fancy_api - fancy_api.Wadl(latest_version='1.3') - - You could of course re-implement the `factory` method in subclasses, - but using the kwarg passing it shouldn't be necessary. - - """ - return cls(**local_config) - - def __call__(self, environ, start_response): - r"""Subclasses will probably want to implement __call__ like this: - - @webob.dec.wsgify(RequestClass=Request) - def __call__(self, req): - # Any of the following objects work as responses: - - # Option 1: simple string - res = 'message\n' - - # Option 2: a nicely formatted HTTP exception page - res = exc.HTTPForbidden(detail='Nice try') - - # Option 3: a webob Response object (in case you need to play with - # headers, or you want to be treated like an iterable, or or or) - res = Response(); - res.app_iter = open('somefile') - - # Option 4: any wsgi app to be run next - res = self.application - - # Option 5: you can get a Response object for a wsgi app, too, to - # play with headers etc - res = req.get_response(self.application) - - # You can then just return your response... - return res - # ... or set req.response and return None. - req.response = res - - See the end of http://pythonpaste.org/webob/modules/dec.html - for more info. - - """ - raise NotImplementedError(_('You must implement __call__')) - - -class Middleware(Application): - """Base WSGI middleware. - - These classes require an application to be - initialized that will be called next. By default the middleware will - simply call its wrapped app, or you can override __call__ to customize its - behavior. - - """ - - @classmethod - def factory(cls, global_config, **local_config): - """Used for paste app factories in paste.deploy config files. - - Any local configuration (that is, values under the [filter:APPNAME] - section of the paste config) will be passed into the `__init__` method - as kwargs. - - A hypothetical configuration would look like: - - [filter:analytics] - redis_host = 127.0.0.1 - paste.filter_factory = manila.api.analytics:Analytics.factory - - which would result in a call to the `Analytics` class as - - import manila.api.analytics - analytics.Analytics(app_from_paste, redis_host='127.0.0.1') - - You could of course re-implement the `factory` method in subclasses, - but using the kwarg passing it shouldn't be necessary. - - """ - def _factory(app): - return cls(app, **local_config) - return _factory - - def __init__(self, application): - self.application = application - - def process_request(self, req): - """Called on each request. - - If this returns None, the next application down the stack will be - executed. If it returns a response then that response will be returned - and execution will stop here. - - """ - return None - - def process_response(self, response): - """Do whatever you'd like to the response.""" - return response - - @webob.dec.wsgify(RequestClass=Request) - def __call__(self, req): - response = self.process_request(req) - if response: - return response - response = req.get_response(self.application) - return self.process_response(response) - - -class Debug(Middleware): - """Helper class for debugging a WSGI application. - - Can be inserted into any WSGI application chain to get information - about the request and response. - - """ - - @webob.dec.wsgify(RequestClass=Request) - def __call__(self, req): - print(('*' * 40) + ' REQUEST ENVIRON') - for key, value in req.environ.items(): - print(key, '=', value) - print() - resp = req.get_response(self.application) - - print(('*' * 40) + ' RESPONSE HEADERS') - for (key, value) in resp.headers.items(): - print(key, '=', value) - print() - - resp.app_iter = self.print_generator(resp.app_iter) - - return resp - - @staticmethod - def print_generator(app_iter): - """Iterator that prints the contents of a wrapper string.""" - print(('*' * 40) + ' BODY') - for part in app_iter: - sys.stdout.write(part.decode()) - sys.stdout.flush() - yield part - print() - - -class Router(object): - """WSGI middleware that maps incoming requests to WSGI apps.""" - - def __init__(self, mapper): - """Create a router for the given routes.Mapper. - - Each route in `mapper` must specify a 'controller', which is a - WSGI app to call. You'll probably want to specify an 'action' as - well and have your controller be an object that can route - the request to the action-specific method. - - Examples: - mapper = routes.Mapper() - sc = ServerController() - - # Explicit mapping of one route to a controller+action - mapper.connect(None, '/svrlist', controller=sc, action='list') - - # Actions are all implicitly defined - mapper.resource('server', 'servers', controller=sc) - - # Pointing to an arbitrary WSGI app. You can specify the - # {path_info:.*} parameter so the target app can be handed just that - # section of the URL. - mapper.connect(None, '/v1.0/{path_info:.*}', controller=BlogApp()) - - """ - self.map = mapper - self._router = routes.middleware.RoutesMiddleware(self._dispatch, - self.map) - - @webob.dec.wsgify(RequestClass=Request) - def __call__(self, req): - """Route the incoming request to a controller based on self.map. - - If no match, return a 404. - - """ - return self._router - - @staticmethod - @webob.dec.wsgify(RequestClass=Request) - def _dispatch(req): - """Dispatch the request to the appropriate controller. - - Called by self._router after matching the incoming request to a route - and putting the information into req.environ. Either returns 404 - or the routed WSGI app's response. - - """ - match = req.environ['wsgiorg.routing_args'][1] - if not match: - return webob.exc.HTTPNotFound() - app = match['controller'] - return app - - -class Loader(object): - """Used to load WSGI applications from paste configurations.""" - - def __init__(self, config_path=None): - """Initialize the loader, and attempt to find the config. - - :param config_path: Full or relative path to the paste config. - :returns: None - - """ - config_path = config_path or CONF.api_paste_config - self.config_path = CONF.find_file(config_path) - if not self.config_path: - raise exception.ConfigNotFound(path=config_path) - - def load_app(self, name): - """Return the paste URLMap wrapped WSGI application. - - :param name: Name of the application to load. - :returns: Paste URLMap object wrapping the requested application. - :raises: `manila.exception.PasteAppNotFound` - - """ - try: - return deploy.loadapp("config:%s" % self.config_path, name=name) - except LookupError as err: - LOG.error(err) - raise exception.PasteAppNotFound(name=name, path=self.config_path) diff --git a/manila/wsgi/__init__.py b/manila/wsgi/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/manila/wsgi/common.py b/manila/wsgi/common.py new file mode 100644 index 0000000000..86f92c2318 --- /dev/null +++ b/manila/wsgi/common.py @@ -0,0 +1,155 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Utility methods for working with WSGI servers.""" + +import webob.dec +import webob.exc + +from manila.i18n import _ + + +class Request(webob.Request): + pass + + +class Application(object): + """Base WSGI application wrapper. Subclasses need to implement __call__.""" + + @classmethod + def factory(cls, global_config, **local_config): + """Used for paste app factories in paste.deploy config files. + + Any local configuration (that is, values under the [app:APPNAME] + section of the paste config) will be passed into the `__init__` method + as kwargs. + + A hypothetical configuration would look like: + + [app:wadl] + latest_version = 1.3 + paste.app_factory = manila.api.fancy_api:Wadl.factory + + which would result in a call to the `Wadl` class as + + import manila.api.fancy_api + fancy_api.Wadl(latest_version='1.3') + + You could of course re-implement the `factory` method in subclasses, + but using the kwarg passing it shouldn't be necessary. + + """ + return cls(**local_config) + + def __call__(self, environ, start_response): + r"""Subclasses will probably want to implement __call__ like this: + + @webob.dec.wsgify(RequestClass=Request) + def __call__(self, req): + # Any of the following objects work as responses: + + # Option 1: simple string + res = 'message\n' + + # Option 2: a nicely formatted HTTP exception page + res = exc.HTTPForbidden(detail='Nice try') + + # Option 3: a webob Response object (in case you need to play with + # headers, or you want to be treated like an iterable, or or or) + res = Response(); + res.app_iter = open('somefile') + + # Option 4: any wsgi app to be run next + res = self.application + + # Option 5: you can get a Response object for a wsgi app, too, to + # play with headers etc + res = req.get_response(self.application) + + # You can then just return your response... + return res + # ... or set req.response and return None. + req.response = res + + See the end of http://pythonpaste.org/webob/modules/dec.html + for more info. + + """ + raise NotImplementedError(_('You must implement __call__')) + + +class Middleware(Application): + """Base WSGI middleware. + + These classes require an application to be + initialized that will be called next. By default the middleware will + simply call its wrapped app, or you can override __call__ to customize its + behavior. + + """ + + @classmethod + def factory(cls, global_config, **local_config): + """Used for paste app factories in paste.deploy config files. + + Any local configuration (that is, values under the [filter:APPNAME] + section of the paste config) will be passed into the `__init__` method + as kwargs. + + A hypothetical configuration would look like: + + [filter:analytics] + redis_host = 127.0.0.1 + paste.filter_factory = manila.api.analytics:Analytics.factory + + which would result in a call to the `Analytics` class as + + import manila.api.analytics + analytics.Analytics(app_from_paste, redis_host='127.0.0.1') + + You could of course re-implement the `factory` method in subclasses, + but using the kwarg passing it shouldn't be necessary. + + """ + def _factory(app): + return cls(app, **local_config) + return _factory + + def __init__(self, application): + self.application = application + + def process_request(self, req): + """Called on each request. + + If this returns None, the next application down the stack will be + executed. If it returns a response then that response will be returned + and execution will stop here. + + """ + return None + + def process_response(self, response): + """Do whatever you'd like to the response.""" + return response + + @webob.dec.wsgify(RequestClass=Request) + def __call__(self, req): + response = self.process_request(req) + if response: + return response + response = req.get_response(self.application) + return self.process_response(response) diff --git a/manila/wsgi/eventlet_server.py b/manila/wsgi/eventlet_server.py new file mode 100644 index 0000000000..bba7077c83 --- /dev/null +++ b/manila/wsgi/eventlet_server.py @@ -0,0 +1,59 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Utility methods for working with WSGI servers.""" + +import socket + +from oslo_config import cfg +from oslo_service import wsgi +from oslo_utils import netutils + +socket_opts = [ + cfg.BoolOpt('tcp_keepalive', + default=True, + help="Sets the value of TCP_KEEPALIVE (True/False) for each " + "server socket."), + cfg.IntOpt('tcp_keepalive_interval', + help="Sets the value of TCP_KEEPINTVL in seconds for each " + "server socket. Not supported on OS X."), + cfg.IntOpt('tcp_keepalive_count', + help="Sets the value of TCP_KEEPCNT for each " + "server socket. Not supported on OS X."), +] + +CONF = cfg.CONF +CONF.register_opts(socket_opts) + + +class Server(wsgi.Server): + """Server class to manage a WSGI server, serving a WSGI application.""" + + def _set_socket_opts(self, _socket): + _socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + # NOTE(praneshp): Call set_tcp_keepalive in oslo to set + # tcp keepalive parameters. Sockets can hang around forever + # without keepalive + netutils.set_tcp_keepalive( + _socket, + self.conf.tcp_keepalive, + self.conf.tcp_keepidle, + self.conf.tcp_keepalive_count, + self.conf.tcp_keepalive_interval, + ) + return _socket diff --git a/manila/wsgi/wsgi.py b/manila/wsgi/wsgi.py new file mode 100644 index 0000000000..31ac2621e7 --- /dev/null +++ b/manila/wsgi/wsgi.py @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Manila OS API WSGI application.""" + + +import sys + +from oslo_config import cfg +from oslo_log import log +from oslo_service import wsgi + +from manila import i18n +i18n.enable_lazy() + +# Need to register global_opts +from manila.common import config +from manila import rpc +from manila import version + +CONF = cfg.CONF + + +def initialize_application(): + log.register_options(CONF) + CONF(sys.argv[1:], project="manila", version=version.version_string()) + config.verify_share_protocols() + log.setup(CONF, "manila") + rpc.init(CONF) + return wsgi.Loader(CONF).load_app(name='osapi_share') diff --git a/releasenotes/notes/added-possibility-to-run-manila-api-with-web-servers-that-support-wsgi-apps-cfffe0b789f8670a.yaml b/releasenotes/notes/added-possibility-to-run-manila-api-with-web-servers-that-support-wsgi-apps-cfffe0b789f8670a.yaml new file mode 100644 index 0000000000..bf5ea27ac1 --- /dev/null +++ b/releasenotes/notes/added-possibility-to-run-manila-api-with-web-servers-that-support-wsgi-apps-cfffe0b789f8670a.yaml @@ -0,0 +1,8 @@ +--- +features: + - Manila API service now can be run using web servers that support + WSGI applications. +upgrade: + - Deprecated path 'manila.api.openstack:FaultWrapper' to 'FaultWrapper' + was removed and now only current path is available, which + is 'manila.api.middleware.fault:FaultWrapper'. diff --git a/setup.cfg b/setup.cfg index e7030516be..606ad9ce4b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,6 +34,8 @@ console_scripts = manila-rootwrap = oslo_rootwrap.cmd:main manila-scheduler = manila.cmd.scheduler:main manila-share = manila.cmd.share:main +wsgi_scripts = + manila-wsgi = manila.wsgi.wsgi:initialize_application manila.scheduler.filters = AvailabilityZoneFilter = manila.scheduler.filters.availability_zone:AvailabilityZoneFilter CapabilitiesFilter = manila.scheduler.filters.capabilities:CapabilitiesFilter