From b71fd4bef46f6a343c9f9f711e959f5f62a98a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tr=C4=99bski?= Date: Sun, 7 May 2017 01:53:12 +0200 Subject: [PATCH] Integrate with keystoneauth Following commit makes enhancements to the keystone handling inside monasca-agent: * using generic password approach that abstracts from underlying keystone version thus allows agent to be used seamlessly with either v2.0 or v3. The only relevant part is the set of parameters that one needs to supply to either monasca-reconfigure or agent.yaml configuration file * using keystone discovery - it simply means that agent will no longer enforce particular keystone version but will allow keystoneauth to pick the best match for given environment Extra: * extracted methods get_session and get_client utilize an aproach presented above and can be used outside of monasca_agent.common.keystone inside checks or detection plugins * make imports to import only modules instead specific objects * removed some redundant methods Story: 2000995 Task: 4191 Needed-By: I579f6bcd5975a32af2a255be41c9b6c4043fa1dc Needed-By: Ifee5b88ccb632222310aafb1081ecb9c9d085150 Change-Id: Iec97e50089ed31ae7ad8244b37cec128817871a5 --- docs/Agent.md | 34 +- monasca_agent/collector/checks/utils.py | 16 +- monasca_agent/collector/checks_d/libvirt.py | 18 +- monasca_agent/collector/checks_d/ovs.py | 25 +- monasca_agent/common/config.py | 10 +- monasca_agent/common/keystone.py | 390 ++++++++++++++------ monasca_agent/forwarder/api/monasca_api.py | 5 +- monasca_setup/main.py | 8 + requirements.txt | 2 + tests/test_keystone.py | 259 ++++++++----- tox.ini | 11 +- 11 files changed, 535 insertions(+), 243 deletions(-) diff --git a/docs/Agent.md b/docs/Agent.md index ba0a1dbe..e1e53719 100644 --- a/docs/Agent.md +++ b/docs/Agent.md @@ -36,10 +36,10 @@ The Agent is composed of the following components: | Component Name | Process Name | Description | | -------------- | ------------ | ----------- | | Supervisor | supervisord | Runs as root, launches all other processes as the user configured to run monasca-agent. This process manages the lifecycle of the Collector, Forwarder and Statsd Daemon. It allows Start, Stop and Restart of all the agent processes together. | -| Collector | monasca-collector | Gathers system & application metrics on a configurable interval and sends them to the Forwarder process. The collector runs various plugins for collection of different plugins.| -| Forwarder | monasca-forwarder | Gathers data from the collector and statsd and submits it to Monasca API over SSL (tcp/17123) | -| Statsd Daemon | monasca-statsd | Statsd engine capable of handling dimensions associated with metrics submitted by a client that supports them. Also supports metrics from the standard statsd client. (udp/8125) | -| Monasca Setup | monasca-setup | The monasca-setup script configures the agent. The Monasca Setup program can also auto-detect and configure certain agent plugins | +| Collector | monasca-collector | Gathers system & application metrics on a configurable interval and sends them to the Forwarder process. The collector runs various plugins for collection of different plugins.| +| Forwarder | monasca-forwarder | Gathers data from the collector and statsd and submits it to Monasca API over SSL (tcp/17123) | +| Statsd Daemon | monasca-statsd | Statsd engine capable of handling dimensions associated with metrics submitted by a client that supports them. Also supports metrics from the standard statsd client. (udp/8125) | +| Monasca Setup | monasca-setup | The monasca-setup script configures the agent. The Monasca Setup program can also auto-detect and configure certain agent plugins | # Installing The Agent (monasca-agent) is available for installation from the Python Package Index (PyPI). To install it, you first need `pip` installed on the node to be monitored. Instructions on installing pip may be found at https://pip.pypa.io/en/latest/installing.html. The Agent will NOT run under any flavor of Windows or Mac OS at this time but has been tested thoroughly on Ubuntu and should work under most flavors of Linux. Support may be added for Mac OS and Windows in the future. Example of an Ubuntu or Debian based install: @@ -112,6 +112,32 @@ All parameters require a '--' before the parameter such as '--verbose'. Run `mon | backlog_send_rate | Integer value of how many batches of buffered measurements to send each time the forwarder flushes data | 1000 | | monasca_statsd_port | Integer value for statsd daemon port number | 8125 | +#### A note around using monasca-agent with different versions of Keystone + +Keystone comes in two version: **v2.0** and **v3**. These versions differ between each +other when it comes to the set of acceptable parameters that client library can send to Keystone API. + +monasca-agent can work with either of versions mentioned above. +However there are certain limitations. Examine a list below to see what +parameters should be provided via monasca-setup (or manually in agent.yaml) to +successfully configure connectivity with Keystone. + +For **v2_0** arguments are: +* ```username``` +* ```password``` +* ```project_id``` (internally mapped to **tenant_id**) +* ```project_name``` (internally mapped to **tenant_name**) + +For **v3** arguments are: +* ```username``` +* ```password``` +* ```project_id``` +* ```project_name``` +* ```project_domain_id``` +* ```project_domain_name``` +* ```user_domain_id``` +* ```user_domain_name``` + ### Providing Arguments to Detection plugins When running individual detection plugins you can specify arguments that augment the configuration created. In some instances the arguments just provide additional information for the detection plugin, for example `monasca-setup -d nova -a disable_http_check=true.` In others detection is skipped entirely and the arguments provide diff --git a/monasca_agent/collector/checks/utils.py b/monasca_agent/collector/checks/utils.py index 87d67406..a4070b93 100644 --- a/monasca_agent/collector/checks/utils.py +++ b/monasca_agent/collector/checks/utils.py @@ -11,14 +11,12 @@ import re import requests from monasca_agent.common import exceptions +from monasca_agent.common import keystone log = logging.getLogger(__name__) DEFAULT_TIMEOUT = 20 -from keystoneclient.v2_0 import client as kc -from monasca_agent.common import keystone - def add_basic_auth(request, username, password): """A helper to add basic authentication to a urllib2 request. @@ -30,14 +28,6 @@ def add_basic_auth(request, username, password): return request -def get_keystone_client(config): - session = keystone.get_session(config) - - return kc.Client(session=session, - endpoint_type=config.get('endpoint_type', 'publicURL'), - region_name=config.get('region_name')) - - def get_tenant_name(tenants, tenant_id): tenant_name = None for tenant in tenants: @@ -51,8 +41,8 @@ def get_tenant_list(config, log): tenants = [] try: log.debug("Retrieving Keystone tenant list") - keystone = get_keystone_client(config) - tenants = keystone.tenants.list() + client = keystone.get_client(**config) + tenants = client.tenants.list() except Exception as e: msg = "Unable to get tenant list from keystone: {0}" log.error(msg.format(e)) diff --git a/monasca_agent/collector/checks_d/libvirt.py b/monasca_agent/collector/checks_d/libvirt.py index 5821f535..53bf7bc8 100644 --- a/monasca_agent/collector/checks_d/libvirt.py +++ b/monasca_agent/collector/checks_d/libvirt.py @@ -1,6 +1,7 @@ #!/bin/env python # (c) Copyright 2014-2016 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -29,15 +30,16 @@ from calendar import timegm from copy import deepcopy from datetime import datetime from datetime import timedelta -from monasca_agent.collector.checks import AgentCheck -from monasca_agent.collector.virt import inspector -from monasca_agent.common import keystone from multiprocessing.dummy import Pool from netaddr import all_matching_cidrs from neutronclient.v2_0 import client as neutron_client from novaclient import client as n_client from novaclient.exceptions import NotFound +from monasca_agent.collector.checks import AgentCheck +from monasca_agent.collector.virt import inspector +from monasca_agent.common import keystone +from monasca_agent import version as ma_version DOM_STATES = {libvirt.VIR_DOMAIN_BLOCKED: 'VM is blocked', libvirt.VIR_DOMAIN_CRASHED: 'VM has crashed', @@ -133,12 +135,14 @@ class LibvirtCheck(AgentCheck): port_cache = None netns = None # Get a list of all instances from the Nova API - session = keystone.get_session(self.init_config) + session = keystone.get_session(**self.init_config) nova_client = n_client.Client( "2.1", session=session, endpoint_type=self.init_config.get("endpoint_type", "publicURL"), service_type="compute", - region_name=self.init_config.get('region_name')) + region_name=self.init_config.get('region_name'), + client_name='monasca-agent[libvirt]', + client_version=ma_version.version_string) self._get_this_host_aggregate(nova_client) instances = nova_client.servers.list( search_opts={'all_tenants': 1, 'host': self.hostname}) @@ -147,7 +151,9 @@ class LibvirtCheck(AgentCheck): nu = neutron_client.Client( session=session, endpoint_type=self.init_config.get("endpoint_type", "publicURL"), - region_name=self.init_config.get('region_name')) + region_name=self.init_config.get('region_name'), + client_name='monasca-agent[libvirt]', + client_version=ma_version.version_string) port_cache = nu.list_ports()['ports'] # Finding existing network namespaces is an indication that either # DVR agent_mode is enabled, or this is all-in-one (like devstack) diff --git a/monasca_agent/collector/checks_d/ovs.py b/monasca_agent/collector/checks_d/ovs.py index 51e96450..576d6ef9 100644 --- a/monasca_agent/collector/checks_d/ovs.py +++ b/monasca_agent/collector/checks_d/ovs.py @@ -1,12 +1,11 @@ #!/bin/env python # (C) Copyright 2016 Hewlett Packard Enterprise Development Company LP +# Copyright 2017 Fujitsu LIMITED -import datetime +from copy import deepcopy import json -import logging import math -import monasca_agent.collector.checks.utils as utils import os import re import socket @@ -14,12 +13,14 @@ import stat import subprocess import time -from copy import deepcopy -from monasca_agent.collector.checks import AgentCheck -from monasca_agent.common import keystone from neutronclient.v2_0 import client as neutron_client from novaclient import client as nova_client +from monasca_agent.collector.checks import AgentCheck +import monasca_agent.collector.checks.utils as utils +from monasca_agent.common import keystone +from monasca_agent import version as ma_version + OVS_CMD = """\ %s --columns=name,external_ids,statistics,options \ --format=json --data=json list Interface\ @@ -56,7 +57,7 @@ class OvsCheck(AgentCheck): else: include_re = include_re + '|' + 'qg.*' self.include_iface_re = re.compile(include_re) - self.session = keystone.get_session(self.init_config) + self.session = keystone.get_session(**self.init_config) def check(self, instance): time_start = time.time() @@ -299,7 +300,9 @@ class OvsCheck(AgentCheck): nc = nova_client.Client(2, session=self.session, endpoint_type=endpoint_type, service_type="compute", - region_name=region_name) + region_name=region_name, + client_name='monasca-agent[ovs]', + client_version=ma_version.version_string) return nc @@ -308,7 +311,9 @@ class OvsCheck(AgentCheck): endpoint_type = self.init_config.get("endpoint_type", "publicURL") return neutron_client.Client(session=self.session, region_name=region_name, - endpoint_type=endpoint_type) + endpoint_type=endpoint_type, + client_name='monasca-agent[ovs]', + client_version=ma_version.version_string) def _run_command(self, command, input=None): self.log.debug("Executing command - {0}".format(command)) @@ -413,7 +418,7 @@ class OvsCheck(AgentCheck): self.log.debug("Retrieving Neutron router data") all_routers_data = self.neutron_client.list_routers() except Exception as e: - self.log.error("Unable to get neutron data: {0}".format(e)) + self.log.exception("Unable to get neutron data: %s", str(e)) return port_cache all_ports_data = all_ports_data['ports'] diff --git a/monasca_agent/common/config.py b/monasca_agent/common/config.py index 8dd358b1..216cedad 100644 --- a/monasca_agent/common/config.py +++ b/monasca_agent/common/config.py @@ -1,13 +1,13 @@ # (C) Copyright 2015-2017 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED import logging import os -import pkg_resources import six import yaml -from monasca_agent.common.exceptions import PathNotFound -import monasca_agent.common.singleton as singleton +from monasca_agent.common import exceptions +from monasca_agent.common import singleton from monasca_agent import version DEFAULT_CONFIG_FILE = '/etc/monasca/agent/agent.yaml' @@ -62,7 +62,7 @@ class Config(object): 'project_domain_name': '', 'project_domain_id': '', 'ca_file': '', - 'insecure': '', + 'insecure': False, 'username': '', 'password': '', 'use_keystone': True, @@ -126,7 +126,7 @@ class Config(object): path = os.path.join(os.path.dirname(self._configFile), 'conf.d') if os.path.exists(path): return path - raise PathNotFound(path) + raise exceptions.PathNotFound(path) def check_yaml(self, conf_path): f = open(conf_path) diff --git a/monasca_agent/common/keystone.py b/monasca_agent/common/keystone.py index 34fbaa11..0276c05b 100644 --- a/monasca_agent/common/keystone.py +++ b/monasca_agent/common/keystone.py @@ -1,110 +1,315 @@ # (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP -# (C) Copyright 2017 KylinCloud +# Copyright 2017 Fujitsu LIMITED import logging -import six from keystoneauth1 import identity from keystoneauth1 import session -from monascaclient import ksclient +from keystoneclient import discover +import six -import monasca_agent.common.singleton as singleton +from monasca_agent.common import singleton +from monasca_agent import version as ma_version -log = logging.getLogger(__name__) +LOG = logging.getLogger(__name__) -def get_session(config): - auth = identity.Password(auth_url=config.get('auth_url'), - username=config.get('username'), - password=config.get('password'), - project_name=config.get('project_name'), - user_domain_name=config.get( - 'user_domain_name', 'default'), - project_domain_name=config.get( - 'project_domain_name', 'default')) - sess = session.Session(auth=auth) +_DEFAULT_SERVICE_TYPE = 'monitoring' +_DEFAULT_ENDPOINT_TYPE = 'public' + + +def _sanitize_args(data): + """Removes keys for which value is None. + + :param data: dictionary with data + :type data: dict + :return: cleaned data + :rtype: dict + + """ + return {k: v for k, v in data.items() if v is not None} + + +def get_session(**kwargs): + """Creates new keystone session. + + Method uses :py:class:`keystoneauth1.identity.Password` + abstracting from underlying Keystone version + + This method is capable of creating a session regardless of + Keystone version (either v2 or v3). However if: + + - using **Keystone v2** following arguments [domain_id, domain_name, + project_domain_id and project_domain_name] should not be set. Keystone V2 + does not support authentication with domain scope. + - using **Keystone v2** following arguments are prohibited: + [user_domain_id, user_domain_name] + - using **Keystone v3** be careful with the scope of authentication. + For more details about scopes refer to identity_tokens_ and v3_identity_ + + .. _v3_api: https://developer.openstack.org/api-ref/identity/v3/index.html?expanded=token-authentication-with-scoped-authorization-detail + .. _identity_tokens: https://docs.openstack.org/admin-guide/identity-tokens.html + + In overall: + + - for **Keystone V2** following arguments are allowed: + [auth_url, user_id, username, password, trust_id, tenant_name, + tenant_id, project_name, project_id]. + * for **Keystone V3** following argumenta are allowed: + [auth_url, user_id, username, password, user_domain_id, user_domain_name, + trust_id, project_id, project_name, project_domain_id, + project_domain_name, domain_id, domain_name, tenant_id, tenant_name] + + However, note that project_id and project_name will override tenant_id + and tenant_name, as in:: + + >>> project_id = project_id or tenant_id + >>> project_name = project_name or tenant_name + + Arguments tenant_id and tenant_name are kept for sake of + backward compatibility between two versions of Keystone. + + Note: + Keystone version is resolved on the runtime + by keystoneauth1 library + + :param string auth_url: URL of keystone service. + :param string username: Username for authentication. + :param string password: Password for authentication. + :param string user_id: User ID for authentication. + :param string user_domain_id: User's domain ID for authentication + (replaced by default_domain_if if set) + :param string user_domain_name: User's domain name for authentication + (replaced by default_domain_name if set) + :param string project_id: Project ID for authentication + :param string project_name: Project Name for authentication + :param string project_domain_id: Project Domain ID for authentication + :param string project_domain_name: Project Domain Name for authentication + :param string tenant_id: Tenant ID for authentication + (replaced by project_id if set) + :param string tenant_name: Tenant Name for authentication + (replaced by project_name if set) + :param string domain_id: Domain ID for authentication. + :param string domain_name: Domain name for authentication + :param string trust_id: Trust ID for authentication. + :param string default_domain_id: Default domain ID for authentication. + :param string default_domain_name: Default domain name for authentication + :param float keystone_timeout: A timeout to pass to requests. This should be a + numerical value indicating some amount (or fraction) + of seconds or 0 for no timeout. (optional, defaults + to 0) + :param bool insecure: Should request be verified or not + (optional, defaults to False) + :param union(string,tuple) ca_file: A client certificate to pass to + requests. These are of the same form as requests expects. + Either a single filename containing both the certificate + and key or a tuple containing the path to the certificate + then a path to the key. (optional) + :param bool reauthenticate: Should reauthenticate if token expires + (optional, defaults to True) + :return: session instance + :rtype: keystoneauth1.session.Session + + """ + + LOG.debug('Initializing keystone session using generic password') + + auth = identity.Password( + auth_url=kwargs.get('auth_url', None), + username=kwargs.get('username', None), + password=kwargs.get('password', None), + user_id=kwargs.get('user_id', None), + user_domain_id=kwargs.get('user_domain_id', None), + user_domain_name=kwargs.get('user_domain_name', None), + project_id=kwargs.get('project_id', None), + project_name=kwargs.get('project_name', None), + project_domain_id=kwargs.get('project_domain_id', None), + project_domain_name=kwargs.get('project_domain_name', None), + tenant_id=kwargs.get('tenant_id', None), + tenant_name=kwargs.get('tenant_name', None), + domain_id=kwargs.get('domain_id', None), + domain_name=kwargs.get('domain_name', None), + trust_id=kwargs.get('trust_id', None), + default_domain_id=kwargs.get('default_domain_id', None), + default_domain_name=kwargs.get('default_domain_name', None), + reauthenticate=kwargs.get('reauthenticate', True) + ) + sess = session.Session(auth=auth, + app_name='monasca-agent', + app_version=ma_version.version_string, + user_agent='monasca-agent', + timeout=kwargs.get('keystone_timeout', None), + verify=not kwargs.get('insecure', False), + cert=kwargs.get('ca_file', None)) return sess -# Make this a singleton class so we don't get the token every time -# the class is created +def get_client(**kwargs): + """Creates new keystone client. + + Initializes new keystone client. + Method does not assume what version of keystone is used. + That responsibility is delegated to + :py:class:`keystoneauth1.discover.Discover`. + Version of the keystone will be the newest one available. + + There are two ways to call this method: + + using existing session object (:py:class:`keystoneauth1.session.Session` + + .. code-block:: python + + s = session.Session(**args) + c = get_client(session=s) + + initializing new keystone client from credentials + + .. code-block:: python + + c = get_client({'username':'mini-mon', 'password':'test', ...}) + + :param kwargs: list of arguments passed to method + :type kwargs: dict + :return: keystone client instance + :rtype: Union[keystoneclient.v3.client.Client, + keystoneclient.v2_0.client.Client] + """ + + if 'session' not in kwargs: + LOG.debug('Initializing fresh keystone client') + sess = get_session(**kwargs) + else: + LOG.debug('Initializing keystone client from existing session') + sess = kwargs.get('session') + + disc = discover.Discover(session=sess) + LOG.debug('Available keystone versions are %s' % disc.version_data()) + + ks = disc.create_client(**kwargs) + ks.auth_ref = sess.auth.get_auth_ref(session=sess) + LOG.info('Using keystone version %s', ks.version) + + return ks + + +def get_args(config): + """Utility to extract keystone args from agent's config. + + Method retrieves all keystone related settings, from + agent's configuration, that are actually set. + + :param config: agent's config + :type config: dict + :returns: cleaned args + :rtype: dict + + """ + raw_args = { + 'auth_url': config.get('keystone_url', None), + 'username': config.get('username', None), + 'password': config.get('password', None), + 'user_id': config.get('user_id', None), + 'user_domain_id': config.get('user_domain_id', None), + 'user_domain_name': config.get('user_domain_name', None), + 'project_id': config.get('project_id', None), + 'project_name': config.get('project_name', None), + 'project_domain_name': config.get('project_domain_name', None), + 'project_domain_id': config.get('project_domain_id', None), + 'domain_id': config.get('domain_id', None), + 'domain_name': config.get('domain_name', None), + 'tenant_id': config.get('tenant_id', None), + 'tenant_name': config.get('tenant_name', None), + 'trust_id': config.get('trust_id', None), + 'default_domain_id': config.get('default_domain_id', None), + 'default_domain_name': config.get('default_domain_name', None), + 'url': config.get('url', None), # hardcoded monasca-api url + 'service_type': config.get('service_type', _DEFAULT_SERVICE_TYPE), + 'endpoint_type': config.get('endpoint_type', _DEFAULT_ENDPOINT_TYPE), + 'region_name': config.get('region_name', None), + 'keystone_timeout': config.get('keystone_timeout', None), + 'insecure': config.get('insecure', False), + 'ca_file': config.get('ca_file', None), + 'reauthenticate': config.get('reauthenticate', True) + } + clean_args = _sanitize_args(raw_args) + + LOG.debug('Removed %d keys that did not present values in configuration', + len(raw_args) - len(clean_args)) + + return clean_args + + @six.add_metaclass(singleton.Singleton) class Keystone(object): def __init__(self, config): - self.config = config + self._config = get_args(config) self._keystone_client = None - self._token = None - def get_credential_args(self): - auth_url = self.config.get('keystone_url', None) - username = self.config.get('username', None) - password = str(self.config.get('password', None)) - user_domain_id = self.config.get('user_domain_id', None) - user_domain_name = self.config.get('user_domain_name', None) - insecure = self.config.get('insecure', False) - cacert = self.config.get('ca_file', None) - project_id = self.config.get('project_id', None) - project_name = self.config.get('project_name', None) - project_domain_name = self.config.get('project_domain_name', None) - project_domain_id = self.config.get('project_domain_id', None) - keystone_timeout = self.config.get('keystone_timeout', None) - - kc_args = {'auth_url': auth_url, - 'username': username, - 'password': password, - 'keystone_timeout': keystone_timeout} - - if user_domain_id: - kc_args.update({'user_domain_id': user_domain_id}) - elif user_domain_name: - kc_args.update({'user_domain_name': user_domain_name}) - - if insecure: - kc_args.update({'insecure': insecure}) - else: - if cacert: - kc_args.update({'os_cacert': cacert}) - if project_id: - kc_args.update({'project_id': project_id}) - elif project_name: - kc_args.update({'project_name': project_name}) - if project_domain_name: - kc_args.update({'domain_name': project_domain_name}) - if project_domain_id: - kc_args.update({'domain_id': project_domain_id}) - return kc_args - - def _get_ksclient(self): + def _init_client(self): """Get a new keystone client object. - The client provides a monasca_url property whose value is pulled from - Keystone Service Catalog filtering by service_type, endpoint_type - and/or region_name. + For more details see: + + - :py:func:`monasca_agent.common.keystone.get_session(**args)` + - :py:func:`monasca_agent.common.keystone.get_client(**args)` + + Note: + This method initializes client only once on + behalf of its own + + :return: keystone client instance + :rtype: Union[keystoneclient.v3.client.Client, + keystoneclient.v2_0.client.Client] """ - service_type = self.config.get('service_type', None) - endpoint_type = self.config.get('endpoint_type', None) - region_name = self.config.get('region_name', None) - - kc_args = self.get_credential_args() - if service_type: - kc_args.update({'service_type': service_type}) - if endpoint_type: - kc_args.update({'endpoint_type': endpoint_type}) - if region_name: - kc_args.update({'region_name': region_name}) - - return ksclient.KSClient(**kc_args) - - def get_monasca_url(self): - if not self._keystone_client: - self.get_token() if self._keystone_client: - return self._keystone_client.monasca_url + LOG.debug('Keystone client is already initialized') + return self._keystone_client + + ks = get_client(**self._config) + self._keystone_client = ks + + return ks + + def get_credential_args(self): + return self._config + + def get_monasca_url(self): + """Retrieves monasca endpoint url. + + monasca endpoint url can be retrieved from two locations: + + * agent configuration (value must be present under api.url key) + * keystone catalog (requires settings api.service_type, + api.endpoint_type and api.region_name) + + First method tries low-cost approach: checking if url is available + in configuration file. If not, it moves to querying the keystone + catalog + + :return: monasca endpoint url + :rtype: basestring + + """ + if self._config.get('url', None): + endpoint = self._config.get('url') + LOG.debug('Using monasca-api url %s from configuration' % endpoint) else: - return None + # NOTE(trebskit) no need to sanitize these values here + # as we're using already local (clean) copy + args = { + 'service_type': self._config.get('service_type'), + 'interface': self._config.get('endpoint_type'), + 'region_name': self._config.get('region_name', None) # that one has no default + } + catalog = self._init_client().auth_ref.service_catalog + endpoint = catalog.url_for(**args) + LOG.debug('Using monasca-api url %s from catalog[%s]' + % (endpoint, args)) + + return endpoint def get_token(self): """Validate token is project scoped and return it if it is @@ -112,25 +317,4 @@ class Keystone(object): project_id and auth_token were fetched when keystone client was created """ - if not self._token: - if not self._keystone_client: - try: - self._keystone_client = self._get_ksclient() - except Exception as exc: - log.error("Unable to create the Keystone Client. " + - "Error was {0}".format(repr(exc))) - return None - - self._token = self._keystone_client.token - - return self._token - - def refresh_token(self): - """Gets a new keystone client object and token - - This method should be called if the token has expired - - """ - self._token = None - self._keystone_client = None - return self.get_token() + return self._init_client().auth_token diff --git a/monasca_agent/forwarder/api/monasca_api.py b/monasca_agent/forwarder/api/monasca_api.py index 5e416b8d..adc624ea 100644 --- a/monasca_agent/forwarder/api/monasca_api.py +++ b/monasca_agent/forwarder/api/monasca_api.py @@ -1,4 +1,5 @@ # (C) Copyright 2015-2016 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED import collections import copy @@ -7,7 +8,7 @@ import logging import random import time -import monasca_agent.common.keystone as keystone +from monasca_agent.common import keystone import monascaclient.client log = logging.getLogger(__name__) @@ -27,7 +28,7 @@ class MonascaAPI(object): def __init__(self, config): """Initialize Mon api client connection.""" self.config = config - self.url = config['url'] + self.url = config.get('url', None) self.api_version = '2_0' self.keystone = keystone.Keystone(config) self.mon_client = None diff --git a/monasca_setup/main.py b/monasca_setup/main.py index 0b1d7fa1..1c5b3151 100644 --- a/monasca_setup/main.py +++ b/monasca_setup/main.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # (C) Copyright 2015-2017 Hewlett Packard Enterprise Development LP +# Copyright 2017 Fujitsu LIMITED """ Detect running daemons then configure and start the agent. """ @@ -14,6 +15,8 @@ import socket import subprocess import sys +import six + import agent_config import monasca_setup.utils as utils from monasca_setup.utils import write_template @@ -209,18 +212,23 @@ def parse_arguments(parser): '-u', '--username', help="Username used for keystone authentication. Required for basic configuration.") parser.add_argument( '-p', '--password', help="Password used for keystone authentication. Required for basic configuration.") + parser.add_argument('--user_domain_id', help="User domain id for keystone authentication", default='') parser.add_argument('--user_domain_name', help="User domain name for keystone authentication", default='') parser.add_argument('--keystone_url', help="Keystone url. Required for basic configuration.") + parser.add_argument('--project_name', help="Project name for keystone authentication", default='') parser.add_argument('--project_domain_id', help="Project domain id for keystone authentication", default='') parser.add_argument('--project_domain_name', help="Project domain name for keystone authentication", default='') parser.add_argument('--project_id', help="Keystone project id for keystone authentication", default='') + parser.add_argument('--monasca_url', help="Monasca API url, if not defined the url is pulled from keystone", + type=six.text_type, default='') parser.add_argument('--service_type', help="Monasca API url service type in keystone catalog", default='') parser.add_argument('--endpoint_type', help="Monasca API url endpoint type in keystone catalog", default='') parser.add_argument('--region_name', help="Monasca API url region name in keystone catalog", default='') + parser.add_argument('--system_only', help="Setup the service but only configure the base config and system " + "metrics (cpu, disk, load, memory, network).", action="store_true", default=False) diff --git a/requirements.txt b/requirements.txt index 8ce5415a..cad58923 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ psutil>=1.1.1 # BSD pymongo>=3.0.2,!=3.1 python-memcached>=1.56 # PSF python-monascaclient>=1.1.0 # Apache-2.0 +python-keystoneclient>=3.8.0 # Apache-2.0 redis>=2.10.0 # MIT six>=1.9.0 # MIT supervisor>=3.1.3,<3.4 @@ -23,3 +24,4 @@ stevedore>=1.17.1 # Apache-2.0 tornado>=4.3 futures>=2.1.3 eventlet!=0.18.3,>=0.18.2 # MIT +keystoneauth1>=2.21.0 # Apache-2.0 diff --git a/tests/test_keystone.py b/tests/test_keystone.py index 5d1e71ad..bb05b9bc 100644 --- a/tests/test_keystone.py +++ b/tests/test_keystone.py @@ -1,108 +1,183 @@ -import unittest -import mock +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. -from monasca_agent.common.keystone import Keystone +import mock +import random + +from oslotest import base + +from monasca_agent.common import keystone from tests.common import base_config -class TestKeystone(unittest.TestCase): +class TestUtils(base.BaseTestCase): + def test_should_sanitize_config(self): + config_keys = [ + 'keystone_url', 'username', 'password', 'project_name', + 'service_type', 'url', 'endpoint_type', 'region_name' + ] + config = {c: mock.NonCallableMock() for c in config_keys} + random_key_for_no_value = random.choice(config_keys) + config[random_key_for_no_value] = None - default_endpoint_type = 'publicURL' - default_service_type = 'monitoring' + clean = keystone.get_args(config) - def testKeyStoneIsSingleton(self): - keystone_1 = Keystone({}) - keystone_2 = Keystone({}) - keystone_3 = Keystone({}) + self.assertNotIn(random_key_for_no_value, clean) + + @mock.patch('monasca_agent.common.keystone.discover.Discover') + @mock.patch('monasca_agent.common.keystone.get_session') + def test_get_client_should_use_existing_session_if_present(self, + get_session, + _): + sess = mock.Mock() + sess.auth = mock.PropertyMock() + sess.auth.get_auth_ref = mock.Mock() + + config = { + 'session': sess + } + keystone.get_client(**config) + + get_session.assert_not_called() + + @mock.patch('monasca_agent.common.keystone.discover.Discover') + @mock.patch('monasca_agent.common.keystone.get_session') + def test_get_client_should_create_session_if_missing(self, + get_session, + _): + sess = mock.Mock() + sess.auth = mock.PropertyMock() + sess.auth.get_auth_ref = mock.Mock() + + config = { + 'username': __name__, + 'password': str(random.randint(10, 20)) + } + keystone.get_client(**config) + + get_session.assert_called_once_with(**config) + + +class TestKeystone(base.BaseTestCase): + default_endpoint_type = mock.NonCallableMock() + default_service_type = mock.NonCallableMock() + default_region_name = mock.NonCallableMock() + + def test_keystone_should_be_singleton(self): + keystone_1 = keystone.Keystone({}) + keystone_2 = keystone.Keystone({}) + keystone_3 = keystone.Keystone({}) self.assertTrue(keystone_1 is keystone_2) self.assertTrue(keystone_1 is keystone_3) - def testServiceCatalogMonascaUrlUsingDefaults(self): - Keystone.instance = None - with mock.patch('keystoneclient.v3.client.Client') as client: - config = base_config.get_config('Api') - keystone = Keystone(config) - keystone.get_monasca_url() - self.assertTrue(client.called) - self.assertIn('auth_url', client.call_args[client.call_count]) - self.assertNotIn('service_type', client.call_args[client.call_count]) - self.assertNotIn('endpoint_type', client.call_args[client.call_count]) - self.assertNotIn('region_name', client.call_args[client.call_count]) - client.return_value.service_catalog.url_for.assert_has_calls([ - mock.call(endpoint_type=self.default_endpoint_type, service_type=self.default_service_type) - ]) + def test_should_call_service_catalog_for_endpoint(self): + keystone.Keystone.instance = None + with mock.patch('keystoneauth1.identity.Password') as password, \ + mock.patch('keystoneauth1.session.Session') as session, \ + mock.patch('keystoneclient.discover.Discover') as discover: + client = mock.Mock() + discover.return_value = d = mock.Mock() + d.create_client = mock.Mock(return_value=client) - def testServiceCatalogMonascaUrlWithCustomServiceType(self): - Keystone.instance = None - service_type = 'my_service_type' - with mock.patch('keystoneclient.v3.client.Client') as client: config = base_config.get_config('Api') - config.update({'service_type': service_type}) - keystone = Keystone(config) - keystone.get_monasca_url() - self.assertTrue(client.called) - self.assertIn('auth_url', client.call_args[client.call_count]) - self.assertNotIn('service_type', client.call_args[client.call_count]) - self.assertNotIn('endpoint_type', client.call_args[client.call_count]) - self.assertNotIn('region_name', client.call_args[client.call_count]) - client.return_value.service_catalog.url_for.assert_has_calls([ - mock.call(endpoint_type=self.default_endpoint_type, service_type=service_type) - ]) + config.update({ + 'url': None, + 'service_type': self.default_service_type, + 'endpoint_type': self.default_endpoint_type, + 'region_name': self.default_region_name + }) - def testServiceCatalogMonascaUrlWithCustomEndpointType(self): - Keystone.instance = None - endpoint_type = 'internalURL' - with mock.patch('keystoneclient.v3.client.Client') as client: - config = base_config.get_config('Api') - config.update({'endpoint_type': endpoint_type}) - keystone = Keystone(config) - keystone.get_monasca_url() - self.assertTrue(client.called) - self.assertIn('auth_url', client.call_args[client.call_count]) - self.assertNotIn('service_type', client.call_args[client.call_count]) - self.assertNotIn('endpoint_type', client.call_args[client.call_count]) - self.assertNotIn('region_name', client.call_args[client.call_count]) - client.return_value.service_catalog.url_for.assert_has_calls([ - mock.call(endpoint_type=endpoint_type, service_type=self.default_service_type) - ]) + k = keystone.Keystone(config) + k.get_monasca_url() - def testServiceCatalogMonascaUrlWithCustomRegionName(self): - Keystone.instance = None - region_name = 'my_region' - with mock.patch('keystoneclient.v3.client.Client') as client: - config = base_config.get_config('Api') - config.update({'region_name': region_name}) - keystone = Keystone(config) - keystone.get_monasca_url() - self.assertTrue(client.called) - self.assertIn('auth_url', client.call_args[client.call_count]) - self.assertNotIn('service_type', client.call_args[client.call_count]) - self.assertNotIn('endpoint_type', client.call_args[client.call_count]) - self.assertNotIn('region_name', client.call_args[client.call_count]) - client.return_value.service_catalog.url_for.assert_has_calls([ - mock.call(endpoint_type=self.default_endpoint_type, service_type=self.default_service_type, - attr='region', filter_value=region_name) - ]) + password.assert_called_once() + session.assert_called_once() + discover.assert_called_once() + + client.auth_ref.service_catalog.url_for.assert_called_once_with(**{ + 'service_type': self.default_service_type, + 'interface': self.default_endpoint_type, + 'region_name': self.default_region_name + }) + + def test_should_use_url_from_config_catalog_config_present(self): + keystone.Keystone.instance = None + with mock.patch('keystoneauth1.identity.Password') as password, \ + mock.patch('keystoneauth1.session.Session') as session, \ + mock.patch('keystoneclient.discover.Discover') as discover: + client = mock.Mock() + discover.return_value = d = mock.Mock() + d.create_client = mock.Mock(return_value=client) + + monasca_url = mock.NonCallableMock() - def testServiceCatalogMonascaUrlWithThreeCustomParameters(self): - Keystone.instance = None - endpoint_type = 'internalURL' - service_type = 'my_service_type' - region_name = 'my_region' - with mock.patch('keystoneclient.v3.client.Client') as client: config = base_config.get_config('Api') - config.update({'endpoint_type': endpoint_type}) - config.update({'service_type': service_type}) - config.update({'region_name': region_name}) - keystone = Keystone(config) - keystone.get_monasca_url() - self.assertTrue(client.called) - self.assertIn('auth_url', client.call_args[client.call_count]) - self.assertNotIn('service_type', client.call_args[client.call_count]) - self.assertNotIn('endpoint_type', client.call_args[client.call_count]) - self.assertNotIn('region_name', client.call_args[client.call_count]) - client.return_value.service_catalog.url_for.assert_has_calls([ - mock.call(endpoint_type=endpoint_type, service_type=service_type, - attr='region', filter_value=region_name) - ]) + config.update({ + 'url': monasca_url, + 'service_type': self.default_service_type, + 'endpoint_type': self.default_endpoint_type, + 'region_name': self.default_region_name + }) + + k = keystone.Keystone(config) + k.get_monasca_url() + + password.assert_not_called() + session.assert_not_called() + discover.assert_not_called() + client.auth_ref.service_catalog.url_for.assert_not_called() + + def test_should_use_url_from_config_if_catalog_config_missing(self): + keystone.Keystone.instance = None + with mock.patch('keystoneauth1.identity.Password') as password, \ + mock.patch('keystoneauth1.session.Session') as session, \ + mock.patch('keystoneclient.discover.Discover') as discover: + client = mock.Mock() + discover.return_value = d = mock.Mock() + d.create_client = mock.Mock(return_value=client) + + monasca_url = mock.NonCallableMock() + + config = base_config.get_config('Api') + config.update({ + 'url': monasca_url, + 'service_type': None, + 'endpoint_type': None, + 'region_name': None + }) + k = keystone.Keystone(config) + k.get_monasca_url() + + password.assert_not_called() + session.assert_not_called() + discover.assert_not_called() + client.auth_ref.service_catalog.url_for.assert_not_called() + + def test_should_init_client_just_once(self): + keystone.Keystone.instance = None + + k = keystone.Keystone(config=base_config.get_config('Api')) + client = mock.Mock() + + with mock.patch('monasca_agent.common.keystone.get_client') as gc: + gc.return_value = client + + for _ in range(random.randint(5, 50)): + k._init_client() + + self.assertIsNotNone(k._keystone_client) + self.assertEqual(client, k._keystone_client) + + gc.assert_called_once() diff --git a/tox.ini b/tox.ini index 9019836b..7209001a 100644 --- a/tox.ini +++ b/tox.ini @@ -9,14 +9,9 @@ setenv = VIRTUAL_ENV={envdir} DISCOVER_DIRECTORY=tests CLIENT_NAME=monasca-agent -passenv = http_proxy - HTTP_PROXY - https_proxy - HTTPS_PROXY - no_proxy - NO_PROXY -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +passenv = *_proxy + *_PROXY +deps = -r{toxinidir}/test-requirements.txt whitelist_externals = bash find rm