From 51a6096a85bf66e918f5981546841e73e038bceb Mon Sep 17 00:00:00 2001 From: Mehdi Abaakouk Date: Mon, 28 Sep 2015 09:02:25 +0200 Subject: [PATCH] Use keystoneauth1 instead of manual setup This change moves our code to the new keystoneauth1 library. This allows to wipe out all authentification code from Ceilometer. Ceilometer become compatible with all keystone API version and all keystone auth plugin for authentification. This also moves the keystone project discovery to v3 API, to fully removes the keystone v2 client from Ceilometer. Implements blueprint support-keystone-v3 Depends-On: Ia5e924eb58aac7fd53e9fe4a3dbdee102ece3ad7 Depends-On: Ic7bbf9c98eb3f5a5d31da6f313458c4c4d62f59f Change-Id: Id2938c2b323e935b3da35768f1f75ea3ae65bad5 --- ceilometer/agent/discovery/endpoint.py | 12 +- ceilometer/agent/discovery/tenant.py | 8 +- ceilometer/agent/manager.py | 12 +- ceilometer/api/controllers/v2/root.py | 12 +- ceilometer/dispatcher/gnocchi.py | 2 +- ceilometer/dispatcher/gnocchi_client.py | 2 +- ceilometer/energy/kwapi.py | 3 +- ceilometer/image/glance.py | 11 +- ceilometer/keystone_client.py | 182 +++++++++++++----- ceilometer/network/floatingip.py | 3 +- ceilometer/neutron_client.py | 21 +- ceilometer/nova_client.py | 41 ++-- ceilometer/objectstore/rgw.py | 8 +- ceilometer/objectstore/swift.py | 12 +- ceilometer/opts.py | 13 +- ceilometer/service.py | 50 +---- .../functional/api/v2/test_api_upgrade.py | 8 +- ceilometer/tests/unit/agent/test_discovery.py | 18 +- ceilometer/tests/unit/agent/test_manager.py | 2 +- .../tests/unit/dispatcher/test_gnocchi.py | 16 +- ceilometer/tests/unit/image/test_glance.py | 3 +- .../tests/unit/network/services/test_fwaas.py | 6 +- .../tests/unit/network/services/test_lbaas.py | 6 +- .../unit/network/services/test_vpnaas.py | 6 +- .../tests/unit/network/test_floatingip.py | 5 +- ceilometer/tests/unit/objectstore/test_rgw.py | 12 +- .../tests/unit/objectstore/test_swift.py | 12 +- ceilometer/tests/unit/test_novaclient.py | 2 +- setup.cfg | 3 + 29 files changed, 287 insertions(+), 204 deletions(-) diff --git a/ceilometer/agent/discovery/endpoint.py b/ceilometer/agent/discovery/endpoint.py index bf7f5ad2..cb014db0 100644 --- a/ceilometer/agent/discovery/endpoint.py +++ b/ceilometer/agent/discovery/endpoint.py @@ -17,10 +17,11 @@ from oslo_log import log from ceilometer.agent import plugin_base as plugin from ceilometer.i18n import _LW +from ceilometer import keystone_client LOG = log.getLogger(__name__) -cfg.CONF.import_group('service_credentials', 'ceilometer.service') +cfg.CONF.import_group('service_credentials', 'ceilometer.keystone_client') class EndpointDiscovery(plugin.DiscoveryBase): @@ -34,10 +35,11 @@ class EndpointDiscovery(plugin.DiscoveryBase): @staticmethod def discover(manager, param=None): - endpoints = manager.keystone.service_catalog.get_urls( - service_type=param, - endpoint_type=cfg.CONF.service_credentials.os_endpoint_type, - region_name=cfg.CONF.service_credentials.os_region_name) + endpoints = keystone_client.get_service_catalog( + manager.keystone).get_urls( + service_type=param, + interface=cfg.CONF.service_credentials.interface, + region_name=cfg.CONF.service_credentials.region_name) if not endpoints: LOG.warning(_LW('No endpoints found for service %s'), "" if param is None else param) diff --git a/ceilometer/agent/discovery/tenant.py b/ceilometer/agent/discovery/tenant.py index 4351dda9..f69af3f9 100644 --- a/ceilometer/agent/discovery/tenant.py +++ b/ceilometer/agent/discovery/tenant.py @@ -19,17 +19,17 @@ from ceilometer.agent import plugin_base as plugin LOG = log.getLogger(__name__) -cfg.CONF.import_group('service_credentials', 'ceilometer.service') +cfg.CONF.import_group('service_credentials', 'ceilometer.keystone_client') class TenantDiscovery(plugin.DiscoveryBase): """Discovery that supplies keystone tenants. This discovery should be used when the pollster's work can't be divided - into smaller pieces than per-tenant. Example of this is the Swift - pollster, which polls account details and does so per-tenant. + into smaller pieces than per-tenants. Example of this is the Swift + pollster, which polls account details and does so per-project. """ def discover(self, manager, param=None): - tenants = manager.keystone.tenants.list() + tenants = manager.keystone.projects.list() return tenants or [] diff --git a/ceilometer/agent/manager.py b/ceilometer/agent/manager.py index 69364137..67e5481e 100644 --- a/ceilometer/agent/manager.py +++ b/ceilometer/agent/manager.py @@ -22,6 +22,7 @@ import collections import itertools import random +from keystoneauth1 import exceptions as ka_exceptions from keystoneclient import exceptions as ks_exceptions from oslo_config import cfg from oslo_context import context @@ -411,7 +412,8 @@ class AgentManager(service_base.BaseService): try: self._keystone = keystone_client.get_client() self._keystone_last_exception = None - except ks_exceptions.ClientException as e: + except (ka_exceptions.ClientException, + ks_exceptions.ClientException) as e: self._keystone = None self._keystone_last_exception = e if self._keystone is not None: @@ -445,8 +447,9 @@ class AgentManager(service_base.BaseService): service_type = getattr( cfg.CONF.service_types, discoverer.KEYSTONE_REQUIRED_FOR_SERVICE) - if not self.keystone.service_catalog.get_endpoints( - service_type=service_type): + if not keystone_client.get_service_catalog( + self.keystone).get_endpoints( + service_type=service_type): LOG.warning(_LW( 'Skipping %(name)s, %(service_type)s service ' 'is not registered in keystone'), @@ -460,7 +463,8 @@ class AgentManager(service_base.BaseService): resources.extend(partitioned) if discovery_cache is not None: discovery_cache[url] = partitioned - except ks_exceptions.ClientException as e: + except (ka_exceptions.ClientException, + ks_exceptions.ClientException) as e: LOG.error(_LE('Skipping %(name)s, keystone issue: ' '%(exc)s'), {'name': name, 'exc': e}) except Exception as err: diff --git a/ceilometer/api/controllers/v2/root.py b/ceilometer/api/controllers/v2/root.py index 561f992c..29d22579 100644 --- a/ceilometer/api/controllers/v2/root.py +++ b/ceilometer/api/controllers/v2/root.py @@ -18,7 +18,7 @@ # License for the specific language governing permissions and limitations # under the License. -from keystoneclient import exceptions +from keystoneauth1 import exceptions from oslo_config import cfg from oslo_log import log from oslo_utils import strutils @@ -114,8 +114,9 @@ class V2Controller(object): self._gnocchi_is_enabled = False else: try: - ks = keystone_client.get_client() - ks.service_catalog.url_for(service_type='metric') + catalog = keystone_client.get_service_catalog( + keystone_client.get_client()) + catalog.url_for(service_type='metric') except exceptions.EndpointNotFound: self._gnocchi_is_enabled = False except exceptions.ClientException: @@ -138,9 +139,10 @@ class V2Controller(object): cfg.CONF.api.aodh_url) else: try: - ks = keystone_client.get_client() + catalog = keystone_client.get_service_catalog( + keystone_client.get_client()) self._aodh_url = self._normalize_aodh_url( - ks.service_catalog.url_for(service_type='alarming')) + catalog.url_for(service_type='alarming')) except exceptions.EndpointNotFound: self._aodh_url = "" except exceptions.ClientException: diff --git a/ceilometer/dispatcher/gnocchi.py b/ceilometer/dispatcher/gnocchi.py index 59b88b40..13f3a829 100644 --- a/ceilometer/dispatcher/gnocchi.py +++ b/ceilometer/dispatcher/gnocchi.py @@ -206,7 +206,7 @@ class GnocchiDispatcher(dispatcher.MeterDispatcherBase): with self._gnocchi_project_id_lock: if self._gnocchi_project_id is None: try: - project = self._ks_client.tenants.find( + project = self._ks_client.projects.find( name=self.conf.dispatcher_gnocchi.filter_project) except Exception: LOG.exception('fail to retrieve user of Gnocchi service') diff --git a/ceilometer/dispatcher/gnocchi_client.py b/ceilometer/dispatcher/gnocchi_client.py index db6b2526..bb7c0cc4 100644 --- a/ceilometer/dispatcher/gnocchi_client.py +++ b/ceilometer/dispatcher/gnocchi_client.py @@ -97,7 +97,7 @@ class Client(object): def _get_headers(self, content_type="application/json"): return { 'Content-Type': content_type, - 'X-Auth-Token': self._ks_client.auth_token, + 'X-Auth-Token': keystone_client.get_auth_token(self._ks_client), } @maybe_retry_if_authentication_error() diff --git a/ceilometer/energy/kwapi.py b/ceilometer/energy/kwapi.py index 7f905729..55cbda48 100644 --- a/ceilometer/energy/kwapi.py +++ b/ceilometer/energy/kwapi.py @@ -21,6 +21,7 @@ import requests import six from ceilometer.agent import plugin_base +from ceilometer import keystone_client from ceilometer import sample @@ -69,7 +70,7 @@ class _Base(plugin_base.PollsterBase): @staticmethod def get_kwapi_client(ksclient, endpoint): """Returns a KwapiClient configured with the proper url and token.""" - return KwapiClient(endpoint, ksclient.auth_token) + return KwapiClient(endpoint, keystone_client.get_auth_token(ksclient)) CACHE_KEY_PROBE = 'kwapi.probes' diff --git a/ceilometer/image/glance.py b/ceilometer/image/glance.py index e47deec5..04080c4d 100644 --- a/ceilometer/image/glance.py +++ b/ceilometer/image/glance.py @@ -22,6 +22,7 @@ from oslo_config import cfg from oslo_utils import timeutils from ceilometer.agent import plugin_base +from ceilometer import keystone_client from ceilometer import sample @@ -55,12 +56,10 @@ class _Base(plugin_base.PollsterBase): @staticmethod def get_glance_client(ksclient, endpoint): # hard-code v1 glance API version selection while v2 API matures - service_credentials = cfg.CONF.service_credentials - return glanceclient.Client('1', endpoint, - token=ksclient.auth_token, - cacert=service_credentials.os_cacert, - insecure=service_credentials.insecure, - timeout=cfg.CONF.http_timeout) + return glanceclient.Client('1', + session=keystone_client.get_session(), + endpoint=endpoint, + auth=ksclient.session.auth) def _get_images(self, ksclient, endpoint): client = self.get_glance_client(ksclient, endpoint) diff --git a/ceilometer/keystone_client.py b/ceilometer/keystone_client.py index 04186d39..f6fd60da 100644 --- a/ceilometer/keystone_client.py +++ b/ceilometer/keystone_client.py @@ -13,67 +13,59 @@ # License for the specific language governing permissions and limitations # under the License. +import os -from keystoneclient import discover as ks_discover -from keystoneclient import exceptions as ks_exception -from keystoneclient import session as ks_session -from keystoneclient.v2_0 import client as ks_client + +from keystoneauth1 import exceptions as ka_exception +from keystoneauth1 import identity as ka_identity +from keystoneauth1 import loading as ka_loading from keystoneclient.v3 import client as ks_client_v3 from oslo_config import cfg +from oslo_log import log -cfg.CONF.import_group('service_credentials', 'ceilometer.service') -cfg.CONF.import_opt('http_timeout', 'ceilometer.service') +LOG = log.getLogger(__name__) + +CFG_GROUP = "service_credentials" -def get_client(): - return ks_client.Client( - username=cfg.CONF.service_credentials.os_username, - password=cfg.CONF.service_credentials.os_password, - tenant_id=cfg.CONF.service_credentials.os_tenant_id, - tenant_name=cfg.CONF.service_credentials.os_tenant_name, - cacert=cfg.CONF.service_credentials.os_cacert, - auth_url=cfg.CONF.service_credentials.os_auth_url, - region_name=cfg.CONF.service_credentials.os_region_name, - insecure=cfg.CONF.service_credentials.insecure, - timeout=cfg.CONF.http_timeout,) +def get_session(requests_session=None): + """Get a ceilometer service credentials auth session.""" + auth_plugin = ka_loading.load_auth_from_conf_options(cfg.CONF, CFG_GROUP) + session = ka_loading.load_session_from_conf_options( + cfg.CONF, CFG_GROUP, auth=auth_plugin, session=requests_session + ) + return session -def get_v3_client(trust_id=None): +def get_client(trust_id=None, requests_session=None): """Return a client for keystone v3 endpoint, optionally using a trust.""" - auth_url = cfg.CONF.service_credentials.os_auth_url - try: - auth_url_noneversion = auth_url.replace('/v2.0', '/') - discover = ks_discover.Discover(auth_url=auth_url_noneversion) - v3_auth_url = discover.url_for('3.0') - if v3_auth_url: - auth_url = v3_auth_url - else: - auth_url = auth_url - except Exception: - auth_url = auth_url.replace('/v2.0', '/v3') - return ks_client_v3.Client( - username=cfg.CONF.service_credentials.os_username, - password=cfg.CONF.service_credentials.os_password, - cacert=cfg.CONF.service_credentials.os_cacert, - auth_url=auth_url, - region_name=cfg.CONF.service_credentials.os_region_name, - insecure=cfg.CONF.service_credentials.insecure, - timeout=cfg.CONF.http_timeout, - trust_id=trust_id) + session = get_session(requests_session=requests_session) + return ks_client_v3.Client(session=session, trust_id=trust_id) + + +def get_service_catalog(client): + return client.session.auth.get_access(client.session).service_catalog + + +def get_auth_token(client): + return client.session.auth.get_access(client.session).auth_token + + +def get_client_on_behalf_user(auth_plugin, trust_id=None, + requests_session=None): + """Return a client for keystone v3 endpoint, optionally using a trust.""" + session = ka_loading.load_session_from_conf_options( + cfg.CONF, CFG_GROUP, auth=auth_plugin, session=requests_session + ) + return ks_client_v3.Client(session=session, trust_id=trust_id) def create_trust_id(trustor_user_id, trustor_project_id, roles, auth_plugin): """Create a new trust using the ceilometer service user.""" - admin_client = get_v3_client() - + admin_client = get_client() trustee_user_id = admin_client.auth_ref.user_id - session = ks_session.Session.construct({ - 'cacert': cfg.CONF.service_credentials.os_cacert, - 'insecure': cfg.CONF.service_credentials.insecure}) - - client = ks_client_v3.Client(session=session, auth=auth_plugin) - + client = get_client_on_behalf_user(auth_plugin=auth_plugin) trust = client.trusts.create(trustor_user=trustor_user_id, trustee_user=trustee_user_id, project=trustor_project_id, @@ -84,12 +76,98 @@ def create_trust_id(trustor_user_id, trustor_project_id, roles, auth_plugin): def delete_trust_id(trust_id, auth_plugin): """Delete a trust previously setup for the ceilometer user.""" - session = ks_session.Session.construct({ - 'cacert': cfg.CONF.service_credentials.os_cacert, - 'insecure': cfg.CONF.service_credentials.insecure}) - - client = ks_client_v3.Client(session=session, auth=auth_plugin) + client = get_client_on_behalf_user(auth_plugin=auth_plugin) try: client.trusts.delete(trust_id) - except ks_exception.NotFound: + except ka_exception.NotFound: pass + + +CLI_OPTS = [ + cfg.StrOpt('region-name', + deprecated_group="DEFAULT", + deprecated_name="os-region-name", + default=os.environ.get('OS_REGION_NAME'), + help='Region name to use for OpenStack service endpoints.'), + cfg.StrOpt('interface', + default=os.environ.get( + 'OS_INTERFACE', os.environ.get('OS_ENDPOINT_TYPE', + 'public')), + deprecated_name="os-endpoint-type", + choices=('public', 'internal', 'admin', 'auth', 'publicURL', + 'internalURL', 'adminURL'), + help='Type of endpoint in Identity service catalog to use for ' + 'communication with OpenStack services.'), +] + +cfg.CONF.register_cli_opts(CLI_OPTS, group=CFG_GROUP) + + +def register_keystoneauth_opts(conf): + ka_loading.register_auth_conf_options(conf, CFG_GROUP) + ka_loading.register_session_conf_options( + conf, CFG_GROUP, + deprecated_opts={'cacert': [ + cfg.DeprecatedOpt('os-cacert', group=CFG_GROUP), + cfg.DeprecatedOpt('os-cacert', group="DEFAULT")] + }) + conf.set_default("auth_type", default="password-ceilometer-legacy", + group=CFG_GROUP) + + +def setup_keystoneauth(conf): + if conf[CFG_GROUP].auth_type == "password-ceilometer-legacy": + LOG.warn("Value 'password-ceilometer-legacy' for '[%s]/auth_type' " + "is deprecated. And will be removed in Ceilometer 7.0. " + "Use 'password' instead.", + CFG_GROUP) + + ka_loading.load_auth_from_conf_options(conf, CFG_GROUP) + + +class LegacyCeilometerKeystoneLoader(ka_loading.BaseLoader): + @property + def plugin_class(self): + return ka_identity.V2Password + + def get_options(self): + options = super(LegacyCeilometerKeystoneLoader, self).get_options() + options.extend([ + ka_loading.Opt( + 'os-username', + default=os.environ.get('OS_USERNAME', 'ceilometer'), + help='User name to use for OpenStack service access.'), + ka_loading.Opt( + 'os-password', + secret=True, + default=os.environ.get('OS_PASSWORD', 'admin'), + help='Password to use for OpenStack service access.'), + ka_loading.Opt( + 'os-tenant-id', + default=os.environ.get('OS_TENANT_ID', ''), + help='Tenant ID to use for OpenStack service access.'), + ka_loading.Opt( + 'os-tenant-name', + default=os.environ.get('OS_TENANT_NAME', 'admin'), + help='Tenant name to use for OpenStack service access.'), + ka_loading.Opt( + 'os-auth-url', + default=os.environ.get('OS_AUTH_URL', + 'http://localhost:5000/v2.0'), + help='Auth URL to use for OpenStack service access.'), + ]) + return options + + def load_from_options(self, **kwargs): + options_map = { + 'os_auth_url': 'auth_url', + 'os_username': 'username', + 'os_password': 'password', + 'os_tenant_name': 'tenant_name', + 'os_tenant_id': 'tenant_id', + } + identity_kwargs = dict((options_map[o.dest], + kwargs.get(o.dest) or o.default) + for o in self.get_options() + if o.dest in options_map) + return self.plugin_class(**identity_kwargs) diff --git a/ceilometer/network/floatingip.py b/ceilometer/network/floatingip.py index 37df3031..7258cf59 100644 --- a/ceilometer/network/floatingip.py +++ b/ceilometer/network/floatingip.py @@ -34,7 +34,8 @@ class FloatingIPPollster(plugin_base.PollsterBase): @staticmethod def _get_floating_ips(ksclient, endpoint): nv = nova_client.Client( - auth_token=ksclient.auth_token, bypass_url=endpoint) + auth=ksclient.session.auth, + endpoint_override=endpoint) return nv.floating_ip_get_all() def _iter_floating_ips(self, ksclient, cache, endpoint): diff --git a/ceilometer/neutron_client.py b/ceilometer/neutron_client.py index 5e8f5d7b..4dfb92e6 100644 --- a/ceilometer/neutron_client.py +++ b/ceilometer/neutron_client.py @@ -19,6 +19,7 @@ from neutronclient.v2_0 import client as clientv20 from oslo_config import cfg from oslo_log import log +from ceilometer import keystone_client SERVICE_OPTS = [ cfg.StrOpt('neutron', @@ -27,8 +28,7 @@ SERVICE_OPTS = [ ] cfg.CONF.register_opts(SERVICE_OPTS, group='service_types') -cfg.CONF.import_opt('http_timeout', 'ceilometer.service') -cfg.CONF.import_group('service_credentials', 'ceilometer.service') +cfg.CONF.import_group('service_credentials', 'ceilometer.keystone_client') LOG = log.getLogger(__name__) @@ -58,22 +58,11 @@ class Client(object): def __init__(self): conf = cfg.CONF.service_credentials params = { - 'insecure': conf.insecure, - 'ca_cert': conf.os_cacert, - 'username': conf.os_username, - 'password': conf.os_password, - 'auth_url': conf.os_auth_url, - 'region_name': conf.os_region_name, - 'endpoint_type': conf.os_endpoint_type, - 'timeout': cfg.CONF.http_timeout, + 'session': keystone_client.get_session(), + 'endpoint_type': conf.interface, + 'region_name': conf.region_name, 'service_type': cfg.CONF.service_types.neutron, } - - if conf.os_tenant_id: - params['tenant_id'] = conf.os_tenant_id - else: - params['tenant_name'] = conf.os_tenant_name - self.client = clientv20.Client(**params) @logged diff --git a/ceilometer/nova_client.py b/ceilometer/nova_client.py index 084c4ea5..0afb92b1 100644 --- a/ceilometer/nova_client.py +++ b/ceilometer/nova_client.py @@ -12,17 +12,22 @@ # under the License. import functools +import logging import novaclient from novaclient import client as nova_client from oslo_config import cfg from oslo_log import log +from ceilometer import keystone_client OPTS = [ cfg.BoolOpt('nova_http_log_debug', default=False, - help='Allow novaclient\'s debug log output.'), + # Added in Mikita + deprecated_for_removal=True, + help=('Allow novaclient\'s debug log output. ' + '(Use default_log_levels instead)')), ] SERVICE_OPTS = [ @@ -34,7 +39,7 @@ SERVICE_OPTS = [ cfg.CONF.register_opts(OPTS) cfg.CONF.register_opts(SERVICE_OPTS, group='service_types') cfg.CONF.import_opt('http_timeout', 'ceilometer.service') -cfg.CONF.import_group('service_credentials', 'ceilometer.service') +cfg.CONF.import_group('service_credentials', 'ceilometer.keystone_client') LOG = log.getLogger(__name__) @@ -55,26 +60,28 @@ def logged(func): class Client(object): """A client which gets information via python-novaclient.""" - def __init__(self, bypass_url=None, auth_token=None): + def __init__(self, endpoint_override=None, auth=None): """Initialize a nova client object.""" conf = cfg.CONF.service_credentials - tenant = conf.os_tenant_id or conf.os_tenant_name + + logger = None + if cfg.CONF.nova_http_log_debug: + logger = logging.getLogger("novaclient-debug") + logger.setLevel(log.DEBUG) + self.nova_client = nova_client.Client( version=2, - username=conf.os_username, - api_key=conf.os_password, - project_id=tenant, - auth_url=conf.os_auth_url, - auth_token=auth_token, - region_name=conf.os_region_name, - endpoint_type=conf.os_endpoint_type, + session=keystone_client.get_session(), + + # nova adapter options + region_name=conf.region_name, + interface=conf.interface, service_type=cfg.CONF.service_types.nova, - bypass_url=bypass_url, - cacert=conf.os_cacert, - insecure=conf.insecure, - timeout=cfg.CONF.http_timeout, - http_log_debug=cfg.CONF.nova_http_log_debug, - no_cache=True) + + # keystone adapter options + endpoint_override=endpoint_override, + auth=auth, + logger=logger) def _with_flavor_and_image(self, instances): flavor_cache = {} diff --git a/ceilometer/objectstore/rgw.py b/ceilometer/objectstore/rgw.py index 6fcf417e..dd205394 100644 --- a/ceilometer/objectstore/rgw.py +++ b/ceilometer/objectstore/rgw.py @@ -22,6 +22,7 @@ from oslo_utils import timeutils import six.moves.urllib.parse as urlparse from ceilometer.agent import plugin_base +from ceilometer import keystone_client from ceilometer import sample LOG = log.getLogger(__name__) @@ -70,9 +71,10 @@ class _Base(plugin_base.PollsterBase): if _Base._ENDPOINT is None: try: conf = cfg.CONF.service_credentials - rgw_url = ksclient.service_catalog.url_for( - service_type=cfg.CONF.service_types.radosgw, - endpoint_type=conf.os_endpoint_type) + rgw_url = keystone_client.get_service_catalog( + ksclient).url_for( + service_type=cfg.CONF.service_types.radosgw, + interface=conf.interface) _Base._ENDPOINT = urlparse.urljoin(rgw_url, '/admin') except exceptions.EndpointNotFound: LOG.debug("Radosgw endpoint not found") diff --git a/ceilometer/objectstore/swift.py b/ceilometer/objectstore/swift.py index c924d867..69661436 100644 --- a/ceilometer/objectstore/swift.py +++ b/ceilometer/objectstore/swift.py @@ -25,6 +25,7 @@ import six.moves.urllib.parse as urlparse from swiftclient import client as swift from ceilometer.agent import plugin_base +from ceilometer import keystone_client from ceilometer import sample @@ -45,7 +46,7 @@ SERVICE_OPTS = [ cfg.CONF.register_opts(OPTS) cfg.CONF.register_opts(SERVICE_OPTS, group='service_types') -cfg.CONF.import_group('service_credentials', 'ceilometer.service') +cfg.CONF.import_group('service_credentials', 'ceilometer.keystone_client') class _Base(plugin_base.PollsterBase): @@ -68,9 +69,10 @@ class _Base(plugin_base.PollsterBase): if _Base._ENDPOINT is None: try: conf = cfg.CONF.service_credentials - _Base._ENDPOINT = ksclient.service_catalog.url_for( - service_type=cfg.CONF.service_types.swift, - endpoint_type=conf.os_endpoint_type) + _Base._ENDPOINT = keystone_client.get_service_catalog( + ksclient).url_for( + service_type=cfg.CONF.service_types.swift, + interface=conf.interface) except exceptions.EndpointNotFound: LOG.debug("Swift endpoint not found") return _Base._ENDPOINT @@ -90,7 +92,7 @@ class _Base(plugin_base.PollsterBase): api_method = '%s_account' % self.METHOD yield (t.id, getattr(swift, api_method) (self._neaten_url(endpoint, t.id), - ksclient.auth_token)) + keystone_client.get_auth_token(ksclient))) @staticmethod def _neaten_url(endpoint, tenant_id): diff --git a/ceilometer/opts.py b/ceilometer/opts.py index 7eb94da3..1902f7ae 100644 --- a/ceilometer/opts.py +++ b/ceilometer/opts.py @@ -13,6 +13,8 @@ # under the License. import itertools +from keystoneauth1 import loading + import ceilometer.agent.manager import ceilometer.api import ceilometer.api.app @@ -35,6 +37,7 @@ import ceilometer.image.glance import ceilometer.ipmi.notifications.ironic import ceilometer.ipmi.platform.intel_node_manager import ceilometer.ipmi.pollsters +import ceilometer.keystone_client import ceilometer.meter.notifications import ceilometer.middleware import ceilometer.network.notifications @@ -103,7 +106,15 @@ def list_opts(): ('publisher_notifier', ceilometer.publisher.messaging.NOTIFIER_OPTS), ('publisher_rpc', ceilometer.publisher.messaging.RPC_OPTS), ('rgw_admin_credentials', ceilometer.objectstore.rgw.CREDENTIAL_OPTS), - ('service_credentials', ceilometer.service.CLI_OPTS), + # NOTE(sileht): the configuration file contains only the options + # for the password plugin that handles keystone v2 and v3 API + # with discovery. But other options are possible. + # Also, the default loaded plugin is password-ceilometer-legacy for + # backward compatibily + ('service_credentials', ( + ceilometer.keystone_client.CLI_OPTS + + loading.get_auth_common_conf_options() + + loading.get_auth_plugin_conf_options('password'))), ('service_types', itertools.chain(ceilometer.energy.kwapi.SERVICE_OPTS, ceilometer.image.glance.SERVICE_OPTS, diff --git a/ceilometer/service.py b/ceilometer/service.py index 670e7a4f..39db8a6a 100644 --- a/ceilometer/service.py +++ b/ceilometer/service.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -import os import socket import sys @@ -23,10 +22,10 @@ import oslo_i18n from oslo_log import log from oslo_reports import guru_meditation_report as gmr +from ceilometer import keystone_client from ceilometer import messaging from ceilometer import version - OPTS = [ cfg.StrOpt('host', default=socket.gethostname(), @@ -40,48 +39,6 @@ OPTS = [ ] cfg.CONF.register_opts(OPTS) -CLI_OPTS = [ - cfg.StrOpt('os-username', - deprecated_group="DEFAULT", - default=os.environ.get('OS_USERNAME', 'ceilometer'), - help='User name to use for OpenStack service access.'), - cfg.StrOpt('os-password', - deprecated_group="DEFAULT", - secret=True, - default=os.environ.get('OS_PASSWORD', 'admin'), - help='Password to use for OpenStack service access.'), - cfg.StrOpt('os-tenant-id', - deprecated_group="DEFAULT", - default=os.environ.get('OS_TENANT_ID', ''), - help='Tenant ID to use for OpenStack service access.'), - cfg.StrOpt('os-tenant-name', - deprecated_group="DEFAULT", - default=os.environ.get('OS_TENANT_NAME', 'admin'), - help='Tenant name to use for OpenStack service access.'), - cfg.StrOpt('os-cacert', - default=os.environ.get('OS_CACERT'), - help='Certificate chain for SSL validation.'), - cfg.StrOpt('os-auth-url', - deprecated_group="DEFAULT", - default=os.environ.get('OS_AUTH_URL', - 'http://localhost:5000/v2.0'), - help='Auth URL to use for OpenStack service access.'), - cfg.StrOpt('os-region-name', - deprecated_group="DEFAULT", - default=os.environ.get('OS_REGION_NAME'), - help='Region name to use for OpenStack service endpoints.'), - cfg.StrOpt('os-endpoint-type', - default=os.environ.get('OS_ENDPOINT_TYPE', 'publicURL'), - help='Type of endpoint in Identity service catalog to use for ' - 'communication with OpenStack services.'), - cfg.BoolOpt('insecure', - default=False, - help='Disables X.509 certificate validation when an ' - 'SSL connection to Identity Service is established.'), -] - -cfg.CONF.register_cli_opts(CLI_OPTS, group="service_credentials") - API_OPT = cfg.IntOpt('workers', default=1, min=1, @@ -108,6 +65,7 @@ COLL_OPT = cfg.IntOpt('workers', 'default value is 1.') cfg.CONF.register_opt(COLL_OPT, 'collector') +keystone_client.register_keystoneauth_opts(cfg.CONF) LOG = log.getLogger(__name__) @@ -119,11 +77,15 @@ def prepare_service(argv=None, config_files=None): ['stevedore=INFO', 'keystoneclient=INFO', 'neutronclient=INFO']) log.set_defaults(default_log_levels=log_levels) + if argv is None: argv = sys.argv cfg.CONF(argv[1:], project='ceilometer', validate_default_values=True, version=version.version_info.version_string(), default_config_files=config_files) + + keystone_client.setup_keystoneauth(cfg.CONF) + log.setup(cfg.CONF, 'ceilometer') # NOTE(liusheng): guru cannot run with service under apache daemon, so when # ceilometer-api running with mod_wsgi, the argv is [], we don't start diff --git a/ceilometer/tests/functional/api/v2/test_api_upgrade.py b/ceilometer/tests/functional/api/v2/test_api_upgrade.py index e2185e17..5d0a1399 100644 --- a/ceilometer/tests/functional/api/v2/test_api_upgrade.py +++ b/ceilometer/tests/functional/api/v2/test_api_upgrade.py @@ -31,7 +31,9 @@ class TestAPIUpgradePath(v2.FunctionalTest): self.CONF.set_override('aodh_url', None, group='api') self.CONF.set_override('meter_dispatchers', ['database']) self.ks = mock.Mock() - self.ks.service_catalog.url_for.side_effect = self._url_for + self.catalog = (self.ks.session.auth.get_access. + return_value.service_catalog) + self.catalog.url_for.side_effect = self._url_for self.useFixture(mockpatch.Patch( 'ceilometer.keystone_client.get_client', return_value=self.ks)) @@ -115,7 +117,7 @@ class TestAPIUpgradePath(v2.FunctionalTest): def test_gnocchi_enabled_without_database_backend_keystone(self): self._setup_keystone_mock() self._do_test_gnocchi_enabled_without_database_backend() - self.ks.service_catalog.url_for.assert_has_calls([ + self.catalog.url_for.assert_has_calls([ mock.call(service_type="alarming"), mock.call(service_type="metric")], any_order=True) @@ -128,7 +130,7 @@ class TestAPIUpgradePath(v2.FunctionalTest): self._setup_keystone_mock() self._do_test_alarm_redirect() self.assertEqual([mock.call(service_type="alarming")], - self.ks.service_catalog.url_for.mock_calls) + self.catalog.url_for.mock_calls) def test_alarm_redirect_configoptions(self): self._setup_osloconfig_options() diff --git a/ceilometer/tests/unit/agent/test_discovery.py b/ceilometer/tests/unit/agent/test_discovery.py index a2827c8d..bf68c26b 100644 --- a/ceilometer/tests/unit/agent/test_discovery.py +++ b/ceilometer/tests/unit/agent/test_discovery.py @@ -31,31 +31,31 @@ class TestEndpointDiscovery(base.BaseTestCase): self.discovery = endpoint.EndpointDiscovery() self.manager = mock.MagicMock() self.CONF = self.useFixture(fixture_config.Config()).conf - self.CONF.set_override('os_endpoint_type', 'test-endpoint-type', + self.CONF.set_override('interface', 'test-endpoint-type', group='service_credentials') - self.CONF.set_override('os_region_name', 'test-region-name', + self.CONF.set_override('region_name', 'test-region-name', group='service_credentials') + self.catalog = (self.manager.keystone.session.auth.get_access. + return_value.service_catalog) def test_keystone_called(self): self.discovery.discover(self.manager, param='test-service-type') expected = [mock.call(service_type='test-service-type', - endpoint_type='test-endpoint-type', + interface='test-endpoint-type', region_name='test-region-name')] - self.assertEqual(expected, - self.manager.keystone.service_catalog.get_urls - .call_args_list) + self.assertEqual(expected, self.catalog.get_urls.call_args_list) def test_keystone_called_no_service_type(self): self.discovery.discover(self.manager) expected = [mock.call(service_type=None, - endpoint_type='test-endpoint-type', + interface='test-endpoint-type', region_name='test-region-name')] self.assertEqual(expected, - self.manager.keystone.service_catalog.get_urls + self.catalog.get_urls .call_args_list) def test_keystone_called_no_endpoints(self): - self.manager.keystone.service_catalog.get_urls.return_value = [] + self.catalog.get_urls.return_value = [] self.assertEqual([], self.discovery.discover(self.manager)) diff --git a/ceilometer/tests/unit/agent/test_manager.py b/ceilometer/tests/unit/agent/test_manager.py index 505def76..c244fc3e 100644 --- a/ceilometer/tests/unit/agent/test_manager.py +++ b/ceilometer/tests/unit/agent/test_manager.py @@ -166,7 +166,7 @@ class TestManager(base.BaseTestCase): class TestPollsterKeystone(agentbase.TestPollster): def get_samples(self, manager, cache, resources): # Just try to use keystone, that will raise an exception - manager.keystone.tenants.list() + manager.keystone.projects.list() class TestPollsterPollingException(agentbase.TestPollster): diff --git a/ceilometer/tests/unit/dispatcher/test_gnocchi.py b/ceilometer/tests/unit/dispatcher/test_gnocchi.py index 532b4eb6..eb2c0609 100644 --- a/ceilometer/tests/unit/dispatcher/test_gnocchi.py +++ b/ceilometer/tests/unit/dispatcher/test_gnocchi.py @@ -92,7 +92,7 @@ class DispatcherTest(base.BaseTestCase): }] ks_client = mock.Mock(auth_token='fake_token') - ks_client.tenants.find.return_value = mock.Mock( + ks_client.projects.find.return_value = mock.Mock( name='gnocchi', id='a2d42c23-d518-46b6-96ab-3fba2e146859') self.useFixture(mockpatch.Patch( 'ceilometer.keystone_client.get_client', @@ -295,8 +295,10 @@ class DispatcherWorkflowTest(base.BaseTestCase, # configuration. self.conf.config(url='http://localhost:8041', group='dispatcher_gnocchi') - ks_client = mock.Mock(auth_token='fake_token') - ks_client.tenants.find.return_value = mock.Mock( + ks_client = mock.Mock() + ks_client.session.auth.get_access.return_value.auth_token = ( + "fake_token") + ks_client.projects.find.return_value = mock.Mock( name='gnocchi', id='a2d42c23-d518-46b6-96ab-3fba2e146859') self.useFixture(mockpatch.Patch( 'ceilometer.keystone_client.get_client', @@ -351,9 +353,11 @@ class DispatcherWorkflowTest(base.BaseTestCase, post_responses.append(MockResponse(self.measure)) if self.measure == 401: - type(self.ks_client).auth_token = mock.PropertyMock( - side_effect=['fake_token', 'new_token', 'new_token', - 'new_token', 'new_token']) + + type(self.ks_client.session.auth.get_access.return_value + ).auth_token = (mock.PropertyMock( + side_effect=['fake_token', 'new_token', 'new_token', + 'new_token', 'new_token'])) headers = {'Content-Type': 'application/json', 'X-Auth-Token': 'new_token'} diff --git a/ceilometer/tests/unit/image/test_glance.py b/ceilometer/tests/unit/image/test_glance.py index c192f7b5..ae338238 100644 --- a/ceilometer/tests/unit/image/test_glance.py +++ b/ceilometer/tests/unit/image/test_glance.py @@ -118,7 +118,8 @@ class TestManager(manager.AgentManager): def __init__(self): super(TestManager, self).__init__() self._keystone = mock.Mock() - self._keystone.service_catalog.get_endpoints = mock.Mock( + access = self._keystone.session.auth.get_access.return_value + access.service_catalog.get_endpoints = mock.Mock( return_value={'image': mock.ANY}) diff --git a/ceilometer/tests/unit/network/services/test_fwaas.py b/ceilometer/tests/unit/network/services/test_fwaas.py index e50f8bf8..8aa55f14 100644 --- a/ceilometer/tests/unit/network/services/test_fwaas.py +++ b/ceilometer/tests/unit/network/services/test_fwaas.py @@ -33,8 +33,10 @@ class _BaseTestFWPollster(base.BaseTestCase): self.context = context.get_admin_context() self.manager = manager.AgentManager() plugin_base._get_keystone = mock.Mock() - plugin_base._get_keystone.service_catalog.get_endpoints = ( - mock.MagicMock(return_value={'network': mock.ANY})) + catalog = (plugin_base._get_keystone.session.auth.get_access. + return_value.service_catalog) + catalog.get_endpoints = mock.MagicMock( + return_value={'network': mock.ANY}) class TestFirewallPollster(_BaseTestFWPollster): diff --git a/ceilometer/tests/unit/network/services/test_lbaas.py b/ceilometer/tests/unit/network/services/test_lbaas.py index 5c721f83..329591a1 100644 --- a/ceilometer/tests/unit/network/services/test_lbaas.py +++ b/ceilometer/tests/unit/network/services/test_lbaas.py @@ -33,8 +33,10 @@ class _BaseTestLBPollster(base.BaseTestCase): self.context = context.get_admin_context() self.manager = manager.AgentManager() plugin_base._get_keystone = mock.Mock() - plugin_base._get_keystone.service_catalog.get_endpoints = ( - mock.MagicMock(return_value={'network': mock.ANY})) + catalog = (plugin_base._get_keystone.session.auth.get_access. + return_value.service_catalog) + catalog.get_endpoints = mock.MagicMock( + return_value={'network': mock.ANY}) class TestLBPoolPollster(_BaseTestLBPollster): diff --git a/ceilometer/tests/unit/network/services/test_vpnaas.py b/ceilometer/tests/unit/network/services/test_vpnaas.py index 395a1b49..880f82fa 100644 --- a/ceilometer/tests/unit/network/services/test_vpnaas.py +++ b/ceilometer/tests/unit/network/services/test_vpnaas.py @@ -33,8 +33,10 @@ class _BaseTestVPNPollster(base.BaseTestCase): self.context = context.get_admin_context() self.manager = manager.AgentManager() plugin_base._get_keystone = mock.Mock() - plugin_base._get_keystone.service_catalog.get_endpoints = ( - mock.MagicMock(return_value={'network': mock.ANY})) + catalog = (plugin_base._get_keystone.session.auth.get_access. + return_value.service_catalog) + catalog.get_endpoints = mock.MagicMock( + return_value={'network': mock.ANY}) class TestVPNServicesPollster(_BaseTestVPNPollster): diff --git a/ceilometer/tests/unit/network/test_floatingip.py b/ceilometer/tests/unit/network/test_floatingip.py index d5a563ad..9395d84f 100644 --- a/ceilometer/tests/unit/network/test_floatingip.py +++ b/ceilometer/tests/unit/network/test_floatingip.py @@ -34,8 +34,9 @@ class TestFloatingIPPollster(base.BaseTestCase): self.context = context.get_admin_context() self.manager = manager.AgentManager() self.manager._keystone = mock.Mock() - self.manager._keystone.service_catalog.get_endpoints = mock.Mock( - return_value={'network': mock.ANY}) + catalog = (self.manager._keystone.session.auth. + get_access.return_value.service_catalog) + catalog.get_endpoints = mock.Mock(return_value={'network': mock.ANY}) self.pollster = floatingip.FloatingIPPollster() fake_ips = self.fake_get_ips() patch_virt = mock.patch('ceilometer.nova_client.Client.' diff --git a/ceilometer/tests/unit/objectstore/test_rgw.py b/ceilometer/tests/unit/objectstore/test_rgw.py index e9c14ec0..424b0d83 100644 --- a/ceilometer/tests/unit/objectstore/test_rgw.py +++ b/ceilometer/tests/unit/objectstore/test_rgw.py @@ -50,8 +50,10 @@ class TestManager(manager.AgentManager): def __init__(self): super(TestManager, self).__init__() - self._keystone = mock.MagicMock() - self._keystone.service_catalog.url_for.return_value = '/endpoint' + self._keystone = mock.Mock() + self._catalog = (self._keystone.session.auth.get_access. + return_value.service_catalog) + self._catalog.url_for.return_value = 'http://foobar/endpoint' class TestRgwPollster(testscenarios.testcase.WithScenarios, @@ -148,7 +150,7 @@ class TestRgwPollster(testscenarios.testcase.WithScenarios, api_method = 'get_%s' % self.pollster.METHOD with mockpatch.PatchObject(rgw_client, api_method, new=mock_method): with mockpatch.PatchObject( - self.manager.keystone.service_catalog, 'url_for', + self.manager._catalog, 'url_for', return_value=endpoint): list(self.pollster.get_samples(self.manager, {}, ASSIGNED_TENANTS)) @@ -163,7 +165,7 @@ class TestRgwPollster(testscenarios.testcase.WithScenarios, with mockpatch.PatchObject(rgw_client, api_method, new=mock.MagicMock()): with mockpatch.PatchObject( - self.manager.keystone.service_catalog, 'url_for', + self.manager._catalog, 'url_for', new=mock_url_for): list(self.pollster.get_samples(self.manager, {}, ASSIGNED_TENANTS)) @@ -173,7 +175,7 @@ class TestRgwPollster(testscenarios.testcase.WithScenarios, def test_endpoint_notfound(self): with mockpatch.PatchObject( - self.manager.keystone.service_catalog, 'url_for', + self.manager._catalog, 'url_for', side_effect=self.fake_ks_service_catalog_url_for): samples = list(self.pollster.get_samples(self.manager, {}, ASSIGNED_TENANTS)) diff --git a/ceilometer/tests/unit/objectstore/test_swift.py b/ceilometer/tests/unit/objectstore/test_swift.py index 9748bf95..47a044db 100644 --- a/ceilometer/tests/unit/objectstore/test_swift.py +++ b/ceilometer/tests/unit/objectstore/test_swift.py @@ -69,6 +69,10 @@ class TestManager(manager.AgentManager): super(TestManager, self).__init__() self._keystone = mock.MagicMock() self._keystone_last_exception = None + self._service_catalog = (self._keystone.session.auth. + get_access.return_value.service_catalog) + self._auth_token = (self._keystone.session.auth. + get_access.return_value.auth_token) class TestSwiftPollster(testscenarios.testcase.WithScenarios, @@ -180,12 +184,12 @@ class TestSwiftPollster(testscenarios.testcase.WithScenarios, api_method = '%s_account' % self.pollster.METHOD with mockpatch.PatchObject(swift_client, api_method, new=mock_method): with mockpatch.PatchObject( - self.manager.keystone.service_catalog, 'url_for', + self.manager._service_catalog, 'url_for', return_value=endpoint): list(self.pollster.get_samples(self.manager, {}, ASSIGNED_TENANTS)) expected = [mock.call(self.pollster._neaten_url(endpoint, t.id), - self.manager.keystone.auth_token) + self.manager._auth_token) for t in ASSIGNED_TENANTS] self.assertEqual(expected, mock_method.call_args_list) @@ -196,7 +200,7 @@ class TestSwiftPollster(testscenarios.testcase.WithScenarios, with mockpatch.PatchObject(swift_client, api_method, new=mock.MagicMock()): with mockpatch.PatchObject( - self.manager.keystone.service_catalog, 'url_for', + self.manager._service_catalog, 'url_for', new=mock_url_for): list(self.pollster.get_samples(self.manager, {}, ASSIGNED_TENANTS)) @@ -206,7 +210,7 @@ class TestSwiftPollster(testscenarios.testcase.WithScenarios, def test_endpoint_notfound(self): with mockpatch.PatchObject( - self.manager.keystone.service_catalog, 'url_for', + self.manager._service_catalog, 'url_for', side_effect=self.fake_ks_service_catalog_url_for): samples = list(self.pollster.get_samples(self.manager, {}, ASSIGNED_TENANTS)) diff --git a/ceilometer/tests/unit/test_novaclient.py b/ceilometer/tests/unit/test_novaclient.py index c592a2ec..d267de5b 100644 --- a/ceilometer/tests/unit/test_novaclient.py +++ b/ceilometer/tests/unit/test_novaclient.py @@ -247,4 +247,4 @@ class TestNovaClient(base.BaseTestCase): def test_with_nova_http_log_debug(self): self.CONF.set_override("nova_http_log_debug", True) self.nv = nova_client.Client() - self.assertTrue(self.nv.nova_client.client.http_log_debug) + self.assertIsNotNone(self.nv.nova_client.client.logger) diff --git a/setup.cfg b/setup.cfg index c39ab704..6903d427 100644 --- a/setup.cfg +++ b/setup.cfg @@ -265,6 +265,9 @@ network.statistics.drivers = oslo.config.opts = ceilometer = ceilometer.opts:list_opts +keystoneauth1.plugin = + password-ceilometer-legacy = ceilometer.keystone_client:LegacyCeilometerKeystoneLoader + [build_sphinx] all_files = 1 build-dir = doc/build