From 7fc492ea79ab0dbbc3d50bf477ae2a1586281dda Mon Sep 17 00:00:00 2001 From: Marc Koderer Date: Wed, 9 Mar 2016 14:51:58 +0100 Subject: [PATCH] Fix keystone v3 issues for all clients This fix basically ports nova fix https://review.openstack.org/#/c/136931/ plus additional bug fixing to Manila. It creates a common class to get used for all clients (neutron, nova, cinder). Idea is to create an auth session and pass this to the client object instead of let the client do that. The auth session will be created by a config group which means the configuration for the clients needed to be heavily reworked. Patch is also backward compatible with old options but flag them as deprecated. DocImpact Change-Id: Ic211a11308a3295409467efd88bff413482ee58d Closes-bug: #1555093 --- contrib/ci/pre_test_hook.sh | 3 + devstack/plugin.sh | 14 +- manila/common/client_auth.py | 113 +++++++++++++ manila/compute/nova.py | 159 ++++++++++-------- manila/network/neutron/api.py | 118 ++++++++----- manila/opts.py | 3 + manila/test.py | 2 + manila/tests/common/test_client_auth.py | 114 +++++++++++++ manila/tests/fake_client_exception_class.py | 22 +++ .../tests/network/neutron/test_neutron_api.py | 10 +- manila/tests/volume/test_cinder.py | 4 +- manila/volume/cinder.py | 149 ++++++++-------- requirements.txt | 1 + 13 files changed, 516 insertions(+), 196 deletions(-) create mode 100644 manila/common/client_auth.py create mode 100644 manila/tests/common/test_client_auth.py create mode 100644 manila/tests/fake_client_exception_class.py diff --git a/contrib/ci/pre_test_hook.sh b/contrib/ci/pre_test_hook.sh index 58aa52b72b..62434779df 100755 --- a/contrib/ci/pre_test_hook.sh +++ b/contrib/ci/pre_test_hook.sh @@ -27,6 +27,9 @@ echo "TEMPEST_SERVICES+=,manila" >> $localrc_path echo "VOLUME_BACKING_FILE_SIZE=22G" >> $localrc_path echo "CINDER_LVM_TYPE=thin" >> $localrc_path +# NOTE(mkoderer): switch to keystone v3 by default +echo "IDENTITY_API_VERSION=3" >> $localrc_path + # NOTE(vponomaryov): Set oversubscription ratio for Cinder LVM driver # bigger than 1.0, because in CI we do not need such small value. # It will allow us to avoid exceeding real capacity in CI test runs. diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 5cfe41af4f..ea5936aa78 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -179,10 +179,6 @@ function configure_manila { iniset $MANILA_CONF DEFAULT state_path $MANILA_STATE_PATH iniset $MANILA_CONF DEFAULT default_share_type $MANILA_DEFAULT_SHARE_TYPE - iniset $MANILA_CONF DEFAULT nova_admin_password $SERVICE_PASSWORD - iniset $MANILA_CONF DEFAULT cinder_admin_password $SERVICE_PASSWORD - iniset $MANILA_CONF DEFAULT neutron_admin_password $SERVICE_PASSWORD - iniset $MANILA_CONF DEFAULT enabled_share_protocols $MANILA_ENABLED_SHARE_PROTOCOLS iniset $MANILA_CONF oslo_concurrency lock_path $MANILA_LOCK_PATH @@ -191,6 +187,16 @@ function configure_manila { iniset $MANILA_CONF DEFAULT lvm_share_volume_group $SHARE_GROUP + if is_service_enabled neutron; then + configure_auth_token_middleware $MANILA_CONF neutron $MANILA_AUTH_CACHE_DIR neutron + fi + if is_service_enabled nova; then + configure_auth_token_middleware $MANILA_CONF nova $MANILA_AUTH_CACHE_DIR nova + fi + if is_service_enabled cinder; then + configure_auth_token_middleware $MANILA_CONF cinder $MANILA_AUTH_CACHE_DIR cinder + fi + # Note: set up config group does not mean that this backend will be enabled. # To enable it, specify its name explicitly using "enabled_share_backends" opt. configure_default_backends diff --git a/manila/common/client_auth.py b/manila/common/client_auth.py new file mode 100644 index 0000000000..2dc94b5789 --- /dev/null +++ b/manila/common/client_auth.py @@ -0,0 +1,113 @@ +# Copyright 2016 SAP SE +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy + +from keystoneauth1 import loading as ks_loading +from keystoneauth1.loading._plugins.identity import v2 +from oslo_config import cfg +from oslo_log import log + +from manila import exception +from manila.i18n import _ +from manila.i18n import _LW + +CONF = cfg.CONF +LOG = log.getLogger(__name__) + +"""Helper class to support keystone v2 and v3 for clients + +Builds auth and session context before instantiation of the actual +client. In order to build this context a dedicated config group is +needed to load all needed parameters dynamically. + + +""" + + +class AuthClientLoader(object): + def __init__(self, client_class, exception_module, cfg_group, + deprecated_opts_for_v2=None): + self.client_class = client_class + self.exception_module = exception_module + self.group = cfg_group + self.admin_auth = None + self.conf = CONF + self.session = None + self.auth_plugin = None + self.deprecated_opts_for_v2 = deprecated_opts_for_v2 + + @staticmethod + def list_opts(group): + """Generates a list of config option for a given group + + :param group: group name + :return: list of auth default configuration + """ + opts = copy.deepcopy(ks_loading.register_session_conf_options( + CONF, group)) + opts.insert(0, ks_loading.get_auth_common_conf_options()[0]) + + for plugin_option in ks_loading.get_auth_plugin_conf_options( + 'password'): + found = False + for option in opts: + if option.name == plugin_option.name: + found = True + break + if not found: + opts.append(plugin_option) + opts.sort(key=lambda x: x.name) + return [(group, opts)] + + def _load_auth_plugin(self): + if self.admin_auth: + return self.admin_auth + self.auth_plugin = ks_loading.load_auth_from_conf_options( + CONF, self.group) + + if self.deprecated_opts_for_v2 and not self.auth_plugin: + LOG.warn(_LW("Not specifying auth options is deprecated")) + self.auth_plugin = v2.Password().load_from_options( + **self.deprecated_opts_for_v2) + + if self.auth_plugin: + return self.auth_plugin + + msg = _('Cannot load auth plugin for %s') % self.group + raise self.exception_module.Unauthorized(message=msg) + + def get_client(self, context, admin=False, **kwargs): + """Get's the client with the correct auth/session context + + """ + auth_plugin = None + + if not self.session: + self.session = ks_loading.load_session_from_conf_options( + self.conf, self.group) + + if admin or (context.is_admin and not context.auth_token): + if not self.admin_auth: + self.admin_auth = self._load_auth_plugin() + auth_plugin = self.admin_auth + else: + # NOTE(mkoderer): Manila basically needs admin clients for + # it's actions. If needed this must be enhanced later + raise exception.ManilaException( + _("Client (%s) is not flagged as admin") % self.group) + + return self.client_class(session=self.session, auth=auth_plugin, + **kwargs) diff --git a/manila/compute/nova.py b/manila/compute/nova.py index 5b784bc6dd..e3766d6377 100644 --- a/manila/compute/nova.py +++ b/manila/compute/nova.py @@ -16,107 +16,118 @@ Handles all requests to Nova. """ +from keystoneauth1 import loading as ks_loading from novaclient import client as nova_client from novaclient import exceptions as nova_exception -from novaclient import service_catalog from novaclient import utils from oslo_config import cfg from oslo_log import log import six +from manila.common import client_auth from manila.common.config import core_opts from manila.db import base from manila import exception from manila.i18n import _ -nova_opts = [ +NOVA_GROUP = 'nova' + +nova_deprecated_opts = [ + cfg.StrOpt('nova_admin_username', + default='nova', + help='Nova admin username.', + deprecated_group='DEFAULT', + deprecated_for_removal=True, + deprecated_reason="This option isn't used any longer. Please " + "use [nova] username instead."), + cfg.StrOpt('nova_admin_password', + help='Nova admin password.', + deprecated_group='DEFAULT', + deprecated_for_removal=True, + deprecated_reason="This option isn't used any longer. Please " + "use [nova] password instead."), + cfg.StrOpt('nova_admin_tenant_name', + default='service', + help='Nova admin tenant name.', + deprecated_group='DEFAULT', + deprecated_for_removal=True, + deprecated_reason="This option isn't used any longer. Please " + "use [nova] tenant instead."), + cfg.StrOpt('nova_admin_auth_url', + default='http://localhost:5000/v2.0', + help='Identity service URL.', + deprecated_group='DEFAULT', + deprecated_for_removal=True, + deprecated_reason="This option isn't used any longer. Please " + "use [nova] url instead."), cfg.StrOpt('nova_catalog_info', default='compute:nova:publicURL', help='Info to match when looking for nova in the service ' 'catalog. Format is separated values of the form: ' - '::'), + '::', + deprecated_group='DEFAULT', + deprecated_for_removal=True, + deprecated_reason="This option isn't used any longer."), cfg.StrOpt('nova_catalog_admin_info', default='compute:nova:adminURL', - help='Same as nova_catalog_info, but for admin endpoint.'), - cfg.StrOpt('nova_ca_certificates_file', - help='Location of CA certificates file to use for nova client ' - 'requests.'), - cfg.BoolOpt('nova_api_insecure', - default=False, - help='Allow to perform insecure SSL requests to nova.'), - cfg.StrOpt('nova_admin_username', - default='nova', - help='Nova admin username.'), - cfg.StrOpt('nova_admin_password', - help='Nova admin password.'), - cfg.StrOpt('nova_admin_tenant_name', - default='service', - help='Nova admin tenant name.'), - cfg.StrOpt('nova_admin_auth_url', - default='http://localhost:5000/v2.0', - help='Identity service URL.'), - cfg.StrOpt('nova_api_microversion', - default='2.10', - help='Version of Nova API to be used.'), + help='Same as nova_catalog_info, but for admin endpoint.', + deprecated_group='DEFAULT', + deprecated_for_removal=True, + deprecated_reason="This option isn't used any longer."), ] +nova_opts = [ + cfg.StrOpt('api_microversion', + default='2.10', + deprecated_group="DEFAULT", + deprecated_name="nova_api_microversion", + help='Version of Nova API to be used.'), + cfg.StrOpt('ca_certificates_file', + deprecated_group="DEFAULT", + deprecated_name="nova_ca_certificates_file", + help='Location of CA certificates file to use for nova client ' + 'requests.'), + cfg.BoolOpt('api_insecure', + default=False, + deprecated_group="DEFAULT", + deprecated_name="nova_api_insecure", + help='Allow to perform insecure SSL requests to nova.'), + ] + CONF = cfg.CONF -CONF.register_opts(nova_opts) +CONF.register_opts(nova_deprecated_opts) CONF.register_opts(core_opts) +CONF.register_opts(nova_opts, NOVA_GROUP) +ks_loading.register_session_conf_options(CONF, NOVA_GROUP) +ks_loading.register_auth_conf_options(CONF, NOVA_GROUP) LOG = log.getLogger(__name__) +def list_opts(): + return client_auth.AuthClientLoader.list_opts(NOVA_GROUP) + +auth_obj = None + + def novaclient(context): - if context.is_admin and context.project_id is None: - c = nova_client.Client( - CONF.nova_api_microversion, - CONF.nova_admin_username, - CONF.nova_admin_password, - CONF.nova_admin_tenant_name, - CONF.nova_admin_auth_url, - insecure=CONF.nova_api_insecure, - cacert=CONF.nova_ca_certificates_file, - ) - c.authenticate() - return c - - compat_catalog = { - 'access': {'serviceCatalog': context.service_catalog or []} - } - sc = service_catalog.ServiceCatalog(compat_catalog) - - nova_catalog_info = CONF.nova_catalog_info - - info = nova_catalog_info - service_type, service_name, endpoint_type = info.split(':') - # extract the region if set in configuration - if CONF.os_region_name: - attr = 'region' - filter_value = CONF.os_region_name - else: - attr = None - filter_value = None - url = sc.url_for(attr=attr, - filter_value=filter_value, - service_type=service_type, - service_name=service_name, - endpoint_type=endpoint_type) - - LOG.debug('Novaclient connection created using URL: %s', url) - - c = nova_client.Client(context.user_id, - context.auth_token, - context.project_id, - auth_url=url, - insecure=CONF.nova_api_insecure, - cacert=CONF.nova_ca_certificates_file, - extensions=[]) - # noauth extracts user_id:project_id from auth_token - c.client.auth_token = context.auth_token or '%s:%s' % (context.user_id, - context.project_id) - c.client.management_url = url - return c + global auth_obj + if not auth_obj: + deprecated_opts_for_v2 = { + 'username': CONF.nova_admin_username, + 'password': CONF.nova_admin_password, + 'tenant_name': CONF.nova_admin_tenant_name, + 'auth_url': CONF.nova_admin_auth_url, + } + auth_obj = client_auth.AuthClientLoader( + client_class=nova_client.Client, + exception_module=nova_exception, + cfg_group=NOVA_GROUP, + deprecated_opts_for_v2=deprecated_opts_for_v2) + return auth_obj.get_client(context, + version=CONF[NOVA_GROUP].api_microversion, + insecure=CONF[NOVA_GROUP].api_insecure, + cacert=CONF[NOVA_GROUP].ca_certificates_file) def _untranslate_server_summary_view(server): diff --git a/manila/network/neutron/api.py b/manila/network/neutron/api.py index 332d4e8695..24a8e8758c 100644 --- a/manila/network/neutron/api.py +++ b/manila/network/neutron/api.py @@ -14,69 +14,98 @@ # License for the specific language governing permissions and limitations # under the License. +from keystoneauth1 import loading as ks_loading from neutronclient.common import exceptions as neutron_client_exc from neutronclient.v2_0 import client as clientv20 from oslo_config import cfg from oslo_log import log +from manila.common import client_auth from manila import context from manila import exception from manila.i18n import _LE from manila.network.neutron import constants as neutron_constants -neutron_opts = [ - cfg.StrOpt( - 'neutron_url', - default='http://127.0.0.1:9696', - deprecated_group='DEFAULT', - help='URL for connecting to neutron.'), - cfg.IntOpt( - 'neutron_url_timeout', - default=30, - deprecated_group='DEFAULT', - help='Timeout value for connecting to neutron in seconds.'), +NEUTRON_GROUP = 'neutron' + +neutron_deprecated_opts = [ cfg.StrOpt( 'neutron_admin_username', default='neutron', deprecated_group='DEFAULT', + deprecated_for_removal=True, + deprecated_reason="This option isn't used any longer. Please use " + "[neutron] username instead.", help='Username for connecting to neutron in admin context.'), cfg.StrOpt( 'neutron_admin_password', help='Password for connecting to neutron in admin context.', deprecated_group='DEFAULT', + deprecated_for_removal=True, + deprecated_reason="This option isn't used any longer. Please use " + "[neutron] password instead.", secret=True), cfg.StrOpt( 'neutron_admin_project_name', default='service', deprecated_group='DEFAULT', deprecated_name='neutron_admin_tenant_name', + deprecated_for_removal=True, + deprecated_reason="This option isn't used any longer. Please use " + "[neutron] project instead.", help='Project name for connecting to Neutron in admin context.'), cfg.StrOpt( 'neutron_admin_auth_url', default='http://localhost:5000/v2.0', deprecated_group='DEFAULT', + deprecated_for_removal=True, + deprecated_reason="This option isn't used any longer. Please use " + "[neutron] auth_url instead.", help='Auth URL for connecting to neutron in admin context.'), +] + +neutron_opts = [ + cfg.StrOpt( + 'url', + default='http://127.0.0.1:9696', + deprecated_group="DEFAULT", + deprecated_name="neutron_url", + help='URL for connecting to neutron.'), + cfg.IntOpt( + 'url_timeout', + default=30, + deprecated_group="DEFAULT", + deprecated_name="neutron_url_timeout", + help='Timeout value for connecting to neutron in seconds.'), cfg.BoolOpt( - 'neutron_api_insecure', + 'api_insecure', default=False, - deprecated_group='DEFAULT', + deprecated_group="DEFAULT", help='If set, ignore any SSL validation issues.'), cfg.StrOpt( - 'neutron_auth_strategy', + 'auth_strategy', default='keystone', - deprecated_group='DEFAULT', + deprecated_group="DEFAULT", help='Auth strategy for connecting to neutron in admin context.'), cfg.StrOpt( - 'neutron_ca_certificates_file', - deprecated_group='DEFAULT', + 'ca_certificates_file', + deprecated_for_removal=True, + deprecated_group="DEFAULT", help='Location of CA certificates file to use for ' 'neutron client requests.'), + cfg.StrOpt( + 'region_name', + help='Region name for connecting to neutron in admin context') ] CONF = cfg.CONF LOG = log.getLogger(__name__) +def list_opts(): + return client_auth.AuthClientLoader.list_opts(NEUTRON_GROUP) + + class API(object): """API for interacting with the neutron 2.x API. @@ -85,39 +114,38 @@ class API(object): def __init__(self, config_group_name=None): self.config_group_name = config_group_name or 'DEFAULT' - CONF.register_opts(neutron_opts, group=self.config_group_name) + + ks_loading.register_session_conf_options(CONF, NEUTRON_GROUP) + ks_loading.register_auth_conf_options(CONF, NEUTRON_GROUP) + CONF.register_opts(neutron_opts, NEUTRON_GROUP) + CONF.register_opts(neutron_deprecated_opts, + group=self.config_group_name) + self.configuration = getattr(CONF, self.config_group_name, CONF) self.last_neutron_extension_sync = None self.extensions = {} - self.client = self.get_client(context.get_admin_context()) + self.auth_obj = None - def _get_client(self, token=None): - params = { - 'endpoint_url': self.configuration.neutron_url, - 'timeout': self.configuration.neutron_url_timeout, - 'insecure': self.configuration.neutron_api_insecure, - 'ca_cert': self.configuration.neutron_ca_certificates_file, - } - if token: - params['token'] = token - params['auth_strategy'] = None - else: - params['username'] = self.configuration.neutron_admin_username - params['tenant_name'] = ( - self.configuration.neutron_admin_project_name) - params['password'] = self.configuration.neutron_admin_password - params['auth_url'] = self.configuration.neutron_admin_auth_url - params['auth_strategy'] = self.configuration.neutron_auth_strategy - return clientv20.Client(**params) + @property + def client(self): + return self.get_client(context.get_admin_context()) def get_client(self, context): - if context.is_admin: - token = None - elif not context.auth_token: - raise neutron_client_exc.Unauthorized() - else: - token = context.auth_token - return self._get_client(token=token) + if not self.auth_obj: + config = CONF[self.config_group_name] + v2_deprecated_opts = { + 'username': config.neutron_admin_username, + 'password': config.neutron_admin_password, + 'tenant_name': config.neutron_admin_project_name, + 'auth_url': config.neutron_admin_auth_url, + } + self.auth_obj = client_auth.AuthClientLoader( + client_class=clientv20.Client, + exception_module=neutron_client_exc, + cfg_group=NEUTRON_GROUP, + deprecated_opts_for_v2=v2_deprecated_opts) + + return self.auth_obj.get_client(self, context) @property def admin_project_id(self): @@ -127,7 +155,7 @@ class API(object): except neutron_client_exc.NeutronClientException as e: raise exception.NetworkException(code=e.status_code, message=e.message) - return self.client.httpclient.auth_tenant_id + return self.client.httpclient.get_project_id() def get_all_admin_project_networks(self): search_opts = {'tenant_id': self.admin_project_id, 'shared': False} diff --git a/manila/opts.py b/manila/opts.py index 5d63c53eb9..ada42e6052 100644 --- a/manila/opts.py +++ b/manila/opts.py @@ -160,6 +160,9 @@ _opts.extend(oslo_concurrency.opts.list_opts()) _opts.extend(oslo_log._options.list_opts()) _opts.extend(oslo_middleware.opts.list_opts()) _opts.extend(oslo_policy.opts.list_opts()) +_opts.extend(manila.network.neutron.api.list_opts()) +_opts.extend(manila.compute.nova.list_opts()) +_opts.extend(manila.volume.cinder.list_opts()) def list_opts(): diff --git a/manila/test.py b/manila/test.py index 8cf0782fc1..352e8782ed 100644 --- a/manila/test.py +++ b/manila/test.py @@ -140,6 +140,8 @@ class TestCase(base_test.BaseTestCase): self.useFixture(self.messaging_conf) rpc.init(CONF) + mock.patch('keystoneauth1.loading.load_auth_from_conf_options').start() + fake_notifier.stub_notifier(self) def tearDown(self): diff --git a/manila/tests/common/test_client_auth.py b/manila/tests/common/test_client_auth.py new file mode 100644 index 0000000000..4d2a0969a5 --- /dev/null +++ b/manila/tests/common/test_client_auth.py @@ -0,0 +1,114 @@ +# Copyright 2016 SAP SE +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from keystoneauth1 import loading as auth +from keystoneauth1.loading._plugins.identity import v2 +from oslo_config import cfg + +import mock + +from manila.common import client_auth +from manila import exception +from manila import test +from manila.tests import fake_client_exception_class + + +class ClientAuthTestCase(test.TestCase): + def setUp(self): + super(ClientAuthTestCase, self).setUp() + self.context = mock.Mock() + self.fake_client = mock.Mock() + self.execption_mod = fake_client_exception_class + self.auth = client_auth.AuthClientLoader( + self.fake_client, self.execption_mod, 'foo_group') + + def test_get_client_admin_true(self): + mock_load_session = self.mock_object(auth, + 'load_session_from_conf_options') + + self.auth.get_client(self.context, admin=True) + + mock_load_session.assert_called_once_with(client_auth.CONF, + 'foo_group') + self.fake_client.assert_called_once_with( + session=mock_load_session(), + auth=auth.load_auth_from_conf_options()) + + def test_get_client_admin_false(self): + self.mock_object(auth, 'load_session_from_conf_options') + + self.assertRaises(exception.ManilaException, self.auth.get_client, + self.context, admin=False) + + def test_load_auth_plugin_caching(self): + self.auth.admin_auth = 'admin obj' + result = self.auth._load_auth_plugin() + + self.assertEqual(self.auth.admin_auth, result) + + def test_load_auth_plugin_no_auth(self): + auth.load_auth_from_conf_options.return_value = None + + self.assertRaises(fake_client_exception_class.Unauthorized, + self.auth._load_auth_plugin) + + def test_load_auth_plugin_no_auth_deprecated_opts(self): + auth.load_auth_from_conf_options.return_value = None + self.auth.deprecated_opts_for_v2 = {"username": "foo"} + pwd_mock = self.mock_object(v2, 'Password') + auth_result = mock.Mock() + auth_result.load_from_options = mock.Mock(return_value='foo_auth') + pwd_mock.return_value = auth_result + + result = self.auth._load_auth_plugin() + + pwd_mock.assert_called_once_with() + auth_result.load_from_options.assert_called_once_with(username='foo') + self.assertEqual(result, 'foo_auth') + + @mock.patch.object(auth, 'register_session_conf_options') + @mock.patch.object(auth, 'get_auth_common_conf_options') + @mock.patch.object(auth, 'get_auth_plugin_conf_options') + def test_list_opts(self, auth_conf, common_conf, register): + register.return_value = [cfg.StrOpt('username'), + cfg.StrOpt('password')] + common_conf.return_value = ([cfg.StrOpt('auth_url')]) + auth_conf.return_value = [cfg.StrOpt('password')] + + result = client_auth.AuthClientLoader.list_opts("foo_group") + + self.assertEqual('foo_group', result[0][0]) + for entry in result[0][1]: + self.assertIn(entry.name, ['username', 'auth_url', 'password']) + common_conf.assert_called_once_with() + auth_conf.assert_called_once_with('password') + + @mock.patch.object(auth, 'register_session_conf_options') + @mock.patch.object(auth, 'get_auth_common_conf_options') + @mock.patch.object(auth, 'get_auth_plugin_conf_options') + def test_list_opts_not_found(self, auth_conf, common_conf, register,): + register.return_value = [cfg.StrOpt('username'), + cfg.StrOpt('password')] + common_conf.return_value = ([cfg.StrOpt('auth_url')]) + auth_conf.return_value = [cfg.StrOpt('tenant')] + + result = client_auth.AuthClientLoader.list_opts("foo_group") + + self.assertEqual('foo_group', result[0][0]) + for entry in result[0][1]: + self.assertIn(entry.name, ['username', 'auth_url', 'password', + 'tenant']) + common_conf.assert_called_once_with() + auth_conf.assert_called_once_with('password') diff --git a/manila/tests/fake_client_exception_class.py b/manila/tests/fake_client_exception_class.py new file mode 100644 index 0000000000..c42f77cb06 --- /dev/null +++ b/manila/tests/fake_client_exception_class.py @@ -0,0 +1,22 @@ +# Copyright 2016 SAP SE +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class Unauthorized(Exception): + status_code = 401 + message = "Unauthorized: bad credentials." + + def __init__(self, message=None): + pass diff --git a/manila/tests/network/neutron/test_neutron_api.py b/manila/tests/network/neutron/test_neutron_api.py index 7af2876bfa..6d14768b7c 100644 --- a/manila/tests/network/neutron/test_neutron_api.py +++ b/manila/tests/network/neutron/test_neutron_api.py @@ -96,7 +96,6 @@ class NeutronApiTest(test.TestCase): neutron_api_instance = neutron_api.API() # Verify results - self.assertTrue(clientv20.Client.called) self.assertTrue(hasattr(neutron_api_instance, 'client')) self.assertTrue(hasattr(neutron_api_instance, 'configuration')) self.assertEqual('DEFAULT', neutron_api_instance.config_group_name) @@ -107,6 +106,7 @@ class NeutronApiTest(test.TestCase): # instantiate Neutron API object obj = neutron_api.API(fake_config_group_name) + obj.get_client(mock.Mock()) # Verify results self.assertTrue(clientv20.Client.called) @@ -572,8 +572,8 @@ class NeutronApiTest(test.TestCase): fake_admin_project_id = 'fake_admin_project_id_value' self.neutron_api.client.httpclient = mock.Mock() self.neutron_api.client.httpclient.auth_token = mock.Mock() - self.neutron_api.client.httpclient.auth_tenant_id = ( - fake_admin_project_id) + self.neutron_api.client.httpclient.get_project_id = mock.Mock( + return_value=fake_admin_project_id) admin_project_id = self.neutron_api.admin_project_id @@ -586,8 +586,8 @@ class NeutronApiTest(test.TestCase): self.neutron_api.client.httpclient.auth_token = mock.Mock( return_value=None) self.neutron_api.client.httpclient.authenticate = mock.Mock() - self.neutron_api.client.httpclient.auth_tenant_id = ( - fake_admin_project_id) + self.neutron_api.client.httpclient.get_project_id = mock.Mock( + return_value=fake_admin_project_id) admin_project_id = self.neutron_api.admin_project_id diff --git a/manila/tests/volume/test_cinder.py b/manila/tests/volume/test_cinder.py index cde177bbef..4565c07afe 100644 --- a/manila/tests/volume/test_cinder.py +++ b/manila/tests/volume/test_cinder.py @@ -113,7 +113,7 @@ class CinderApiTestCase(test.TestCase): volume['attach_status'] = "detached" instance = {'availability_zone': 'zone1'} volume['availability_zone'] = 'zone2' - cinder.CONF.set_override('cinder_cross_az_attach', False) + cinder.CONF.set_override('cross_az_attach', False, 'cinder') self.assertRaises(exception.InvalidVolume, self.api.check_attach, self.ctx, volume, instance) volume['availability_zone'] = 'zone1' @@ -125,7 +125,7 @@ class CinderApiTestCase(test.TestCase): volume['attach_status'] = "detached" volume['availability_zone'] = 'zone1' instance = {'availability_zone': 'zone1'} - cinder.CONF.set_override('cinder_cross_az_attach', False) + cinder.CONF.set_override('cross_az_attach', False, 'cinder') self.assertIsNone(self.api.check_attach(self.ctx, volume, instance)) cinder.CONF.reset() diff --git a/manila/volume/cinder.py b/manila/volume/cinder.py index 7f33ba2536..301cb01891 100644 --- a/manila/volume/cinder.py +++ b/manila/volume/cinder.py @@ -20,103 +20,120 @@ Handles all requests relating to volumes + cinder. import copy from cinderclient import exceptions as cinder_exception -from cinderclient import service_catalog from cinderclient.v2 import client as cinder_client +from keystoneauth1 import loading as ks_loading from oslo_config import cfg from oslo_log import log import six +from manila.common import client_auth from manila.common.config import core_opts import manila.context as ctxt from manila.db import base from manila import exception from manila.i18n import _ +CINDER_GROUP = 'cinder' -cinder_opts = [ +cinder_deprecated_opts = [ cfg.StrOpt('cinder_catalog_info', default='volume:cinder:publicURL', help='Info to match when looking for cinder in the service ' 'catalog. Format is separated values of the form: ' - '::'), - cfg.StrOpt('cinder_ca_certificates_file', - help='Location of CA certificates file to use for cinder ' - 'client requests.'), - cfg.IntOpt('cinder_http_retries', - default=3, - help='Number of cinderclient retries on failed HTTP calls.'), - cfg.BoolOpt('cinder_api_insecure', - default=False, - help='Allow to perform insecure SSL requests to cinder.'), - cfg.BoolOpt('cinder_cross_az_attach', - default=True, - help='Allow attaching between instances and volumes in ' - 'different availability zones.'), + '::', + deprecated_group='DEFAULT', + deprecated_for_removal=True, + deprecated_reason="This option isn't used any longer."), cfg.StrOpt('cinder_admin_username', default='cinder', - help='Cinder admin username.'), + help='Cinder admin username.', + deprecated_group='DEFAULT', + deprecated_for_removal=True, + deprecated_reason="This option isn't used any longer. Please " + "use [cinder] username instead."), + cfg.StrOpt('cinder_admin_password', - help='Cinder admin password.'), + help='Cinder admin password.', + deprecated_group='DEFAULT', + deprecated_for_removal=True, + deprecated_reason="This option isn't used any longer. Please " + "use [cinder] password instead."), cfg.StrOpt('cinder_admin_tenant_name', default='service', - help='Cinder admin tenant name.'), + help='Cinder admin tenant name.', + deprecated_group='DEFAULT', + deprecated_for_removal=True, + deprecated_reason="This option isn't used any longer. Please " + "use [cinder] tenant_name instead."), cfg.StrOpt('cinder_admin_auth_url', default='http://localhost:5000/v2.0', - help='Identity service URL.') + help='Identity service URL.', + deprecated_group='DEFAULT', + deprecated_for_removal=True, + deprecated_reason="This option isn't used any longer. Please " + "use [cinder] auth_url instead.") ] +cinder_opts = [ + cfg.BoolOpt('cross_az_attach', + default=True, + deprecated_group="DEFAULT", + deprecated_name="cinder_cross_az_attach", + help='Allow attaching between instances and volumes in ' + 'different availability zones.'), + cfg.StrOpt('ca_certificates_file', + help='Location of CA certificates file to use for cinder ' + 'client requests.', + deprecated_group='DEFAULT', + deprecated_name="cinder_ca_certificates_file"), + cfg.IntOpt('http_retries', + default=3, + help='Number of cinderclient retries on failed HTTP calls.', + deprecated_group='DEFAULT', + deprecated_name="cinder_http_retries"), + cfg.BoolOpt('api_insecure', + default=False, + help='Allow to perform insecure SSL requests to cinder.', + deprecated_group='DEFAULT', + deprecated_name="cinder_api_insecure"), + ] + CONF = cfg.CONF -CONF.register_opts(cinder_opts) +CONF.register_opts(cinder_deprecated_opts) CONF.register_opts(core_opts) +CONF.register_opts(cinder_opts, CINDER_GROUP) +ks_loading.register_session_conf_options(CONF, CINDER_GROUP) +ks_loading.register_auth_conf_options(CONF, CINDER_GROUP) + LOG = log.getLogger(__name__) +def list_opts(): + return client_auth.AuthClientLoader.list_opts(CINDER_GROUP) + + +auth_obj = None + + def cinderclient(context): - if context.is_admin and context.project_id is None: - c = cinder_client.Client(CONF.cinder_admin_username, - CONF.cinder_admin_password, - CONF.cinder_admin_tenant_name, - CONF.cinder_admin_auth_url, - insecure=CONF.cinder_api_insecure, - retries=CONF.cinder_http_retries, - cacert=CONF.cinder_ca_certificates_file) - c.authenticate() - return c - - compat_catalog = { - 'access': {'serviceCatalog': context.service_catalog or []} - } - sc = service_catalog.ServiceCatalog(compat_catalog) - info = CONF.cinder_catalog_info - service_type, service_name, endpoint_type = info.split(':') - # extract the region if set in configuration - if CONF.os_region_name: - attr = 'region' - filter_value = CONF.os_region_name - else: - attr = None - filter_value = None - url = sc.url_for(attr=attr, - filter_value=filter_value, - service_type=service_type, - service_name=service_name, - endpoint_type=endpoint_type) - - LOG.debug('Cinderclient connection created using URL: %s', url) - - c = cinder_client.Client(context.user_id, - context.auth_token, - project_id=context.project_id, - auth_url=url, - insecure=CONF.cinder_api_insecure, - retries=CONF.cinder_http_retries, - cacert=CONF.cinder_ca_certificates_file) - # noauth extracts user_id:project_id from auth_token - c.client.auth_token = context.auth_token or '%s:%s' % (context.user_id, - context.project_id) - c.client.management_url = url - return c + global auth_obj + if not auth_obj: + deprecated_opts_for_v2 = { + 'username': CONF.nova_admin_username, + 'password': CONF.nova_admin_password, + 'tenant_name': CONF.nova_admin_tenant_name, + 'auth_url': CONF.nova_admin_auth_url, + } + auth_obj = client_auth.AuthClientLoader( + client_class=cinder_client.Client, + exception_module=cinder_exception, + cfg_group=CINDER_GROUP, + deprecated_opts_for_v2=deprecated_opts_for_v2) + return auth_obj.get_client(context, + insecure=CONF[CINDER_GROUP].api_insecure, + cacert=CONF[CINDER_GROUP].ca_certificates_file, + retries=CONF[CINDER_GROUP].http_retries) def _untranslate_volume_summary_view(context, vol): @@ -232,7 +249,7 @@ class API(base.Base): if volume['attach_status'] == "attached": msg = _("already attached") raise exception.InvalidVolume(reason=msg) - if instance and not CONF.cinder_cross_az_attach: + if instance and not CONF[CINDER_GROUP].cross_az_attach: if instance['availability_zone'] != volume['availability_zone']: msg = _("Instance and volume not in same availability_zone") raise exception.InvalidVolume(reason=msg) diff --git a/requirements.txt b/requirements.txt index 2fb4d12e9e..0a98c3e712 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,6 +28,7 @@ paramiko>=1.16.0 # LGPL Paste # MIT PasteDeploy>=1.5.0 # MIT python-neutronclient!=4.1.0,>=2.6.0 # Apache-2.0 +keystoneauth1>=2.1.0 # Apache-2.0 keystonemiddleware!=4.1.0,>=4.0.0 # Apache-2.0 requests!=2.9.0,>=2.8.1 # Apache-2.0 retrying!=1.3.0,>=1.2.3 # Apache-2.0