diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/check_openstack_api.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/check_openstack_api.py index 0f5eea9e5..9d2ef59e0 100644 --- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/check_openstack_api.py +++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/check_openstack_api.py @@ -16,7 +16,6 @@ # Collectd plugin for checking the status of OpenStack API services import collectd -import collectd_base as base import collectd_openstack as openstack from urlparse import urlparse @@ -94,8 +93,7 @@ class APICheckPlugin(openstack.CollectdPlugin): 'region': service['region'] } - @base.read_callback_wrapper - def read_callback(self): + def collect(self): for item in self.check_api(): if item['status'] == self.UNKNOWN: # skip if status is UNKNOWN @@ -112,7 +110,7 @@ class APICheckPlugin(openstack.CollectdPlugin): value.dispatch() -plugin = APICheckPlugin(collectd) +plugin = APICheckPlugin(collectd, PLUGIN_NAME) def config_callback(conf): @@ -124,7 +122,7 @@ def notification_callback(notification): def read_callback(): - plugin.read_callback() + plugin.conditional_read_callback() collectd.register_config(config_callback) collectd.register_notification(notification_callback) diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py index ccc8d4c43..55393384f 100644 --- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py +++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py @@ -18,6 +18,7 @@ import dateutil.parser import dateutil.tz import requests import simplejson as json +import traceback import collectd_base as base @@ -29,6 +30,10 @@ from collections import defaultdict INTERVAL = 50 +class KeystoneException(Exception): + pass + + class OSClient(object): """ Base class for querying the OpenStack API endpoints. @@ -58,8 +63,6 @@ class OSClient(object): self.session.mount( 'https://', requests.adapters.HTTPAdapter(max_retries=max_retries)) - self.get_token() - def is_valid_token(self): now = datetime.datetime.now(tz=dateutil.tz.tzutc()) return self.token and self.valid_until and self.valid_until > now @@ -86,14 +89,12 @@ class OSClient(object): '%s/tokens' % self.keystone_url, data=data, token_required=False) if not r: - self.logger.error("Cannot get a valid token from %s" % - self.keystone_url) - return + raise KeystoneException("Cannot get a valid token from %s" % + self.keystone_url) if r.status_code < 200 or r.status_code > 299: - self.logger.error("%s responded with code %d" % - (self.keystone_url, r.status_code)) - return + raise KeystoneException("%s responded with code %d" % + (self.keystone_url, r.status_code)) data = r.json() self.logger.debug("Got response from Keystone: '%s'" % data) @@ -202,37 +203,44 @@ class CollectdPlugin(base.Base): entry = 'services' ost_services_r = self.get(service, endpoint) - r_status = ost_services_r.status_code - try: - r_json = ost_services_r.json() - except ValueError: - r_json = {} - if r_status == 200 and entry in r_json: - for val in r_json[entry]: - data = {'host': val['host'], 'service': val['binary']} - - if service == 'neutron': - if not val['admin_state_up']: - data['state'] = 'disabled' - else: - data['state'] = 'up' if val['alive'] else 'down' - else: - if val['status'] == 'disabled': - data['state'] = 'disabled' - elif val['state'] == 'up' or val['state'] == 'down': - data['state'] = val['state'] - else: - msg = "Unknown state for {} workers:{}".format( - service, val['state']) - self.logger.warning(msg) - continue - - yield data - else: - msg = "Cannot get state of {} workers: Got {} ({})".format( - service, r_status, ost_services_r.content) + msg = "Cannot get state of {} workers".format(service) + if ost_services_r is None: self.logger.warning(msg) + elif ost_services_r.status_code != 200: + msg = "{}: Got {} ({})".format( + msg, ost_services_r.status_code, ost_services_r.content) + self.logger.warning(msg) + else: + try: + r_json = ost_services_r.json() + except ValueError: + r_json = {} + + if entry not in r_json: + msg = "{}: couldn't find '{}' key".format(msg, entry) + self.logger.warning(msg) + else: + for val in r_json[entry]: + data = {'host': val['host'], 'service': val['binary']} + + if service == 'neutron': + if not val['admin_state_up']: + data['state'] = 'disabled' + else: + data['state'] = 'up' if val['alive'] else 'down' + else: + if val['status'] == 'disabled': + data['state'] = 'disabled' + elif val['state'] == 'up' or val['state'] == 'down': + data['state'] = val['state'] + else: + msg = "Unknown state for {} workers:{}".format( + service, val['state']) + self.logger.warning(msg) + continue + + yield data def get(self, service, resource): url = self._build_url(service, resource) @@ -269,11 +277,26 @@ class CollectdPlugin(base.Base): self.max_retries) def read_callback(self): + """ Wrapper method + + This method calls the actual method which performs + collection. + """ + + try: + self.collect() + except Exception as e: + msg = '{}: fail to get metrics: {}: {}'.format( + self.service_name or self.plugin, e, traceback.format_exc()) + self.logger.error(msg) + + def collect(self): """ Read metrics and dispatch values - This method should be overriden by the derived classes. + This method should be overriden by the derived classes. """ - raise "read_callback() method needs to be overriden!" + + raise 'collect() method needs to be overriden!' def get_objects(self, project, object_name, api_version='', params='all_tenants=1'): diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/hypervisor_stats.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/hypervisor_stats.py index ab98e260e..d03f7ae98 100644 --- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/hypervisor_stats.py +++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/hypervisor_stats.py @@ -16,7 +16,6 @@ # Collectd plugin for getting hypervisor statistics from Nova import collectd -import collectd_base as base import collectd_openstack as openstack PLUGIN_NAME = 'hypervisor_stats' @@ -57,8 +56,7 @@ class HypervisorStatsPlugin(openstack.CollectdPlugin): v.host = host v.dispatch() - @base.read_callback_wrapper - def read_callback(self): + def collect(self): r = self.get('nova', 'os-hypervisors/detail') if not r: self.logger.warning("Could not get hypervisor statistics") @@ -83,8 +81,7 @@ class HypervisorStatsPlugin(openstack.CollectdPlugin): for k, v in total_stats.iteritems(): self.dispatch_value('total_{}'.format(k), v) - -plugin = HypervisorStatsPlugin(collectd) +plugin = HypervisorStatsPlugin(collectd, PLUGIN_NAME) def config_callback(conf): @@ -96,7 +93,7 @@ def notification_callback(notification): def read_callback(): - plugin.read_callback() + plugin.conditional_read_callback() collectd.register_config(config_callback) collectd.register_notification(notification_callback) diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py index b14d01823..3885c459f 100644 --- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py +++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py @@ -19,7 +19,6 @@ from collections import Counter from collections import defaultdict import re -import collectd_base as base import collectd_openstack as openstack PLUGIN_NAME = 'cinder' @@ -37,8 +36,7 @@ class CinderStatsPlugin(openstack.CollectdPlugin): states = {'up': 0, 'down': 1, 'disabled': 2} cinder_re = re.compile('^cinder-') - @base.read_callback_wrapper - def read_callback(self): + def collect(self): # Get information of the state per service # State can be: 'up', 'down' or 'disabled' @@ -106,7 +104,7 @@ class CinderStatsPlugin(openstack.CollectdPlugin): ) v.dispatch() -plugin = CinderStatsPlugin(collectd) +plugin = CinderStatsPlugin(collectd, PLUGIN_NAME) def config_callback(conf): @@ -118,7 +116,7 @@ def notification_callback(notification): def read_callback(): - plugin.read_callback() + plugin.conditional_read_callback() collectd.register_config(config_callback) collectd.register_notification(notification_callback) diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py index ff6552aa8..1077083f0 100644 --- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py +++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py @@ -16,7 +16,6 @@ # Collectd plugin for getting resource statistics from Glance import collectd -import collectd_base as base import collectd_openstack as openstack PLUGIN_NAME = 'glance' @@ -30,8 +29,7 @@ class GlanceStatsPlugin(openstack.CollectdPlugin): total size of images usable and in error state """ - @base.read_callback_wrapper - def read_callback(self): + def collect(self): def is_snap(d): return d.get('properties', {}).get('image_type') == 'snapshot' @@ -84,7 +82,7 @@ class GlanceStatsPlugin(openstack.CollectdPlugin): ) v.dispatch() -plugin = GlanceStatsPlugin(collectd) +plugin = GlanceStatsPlugin(collectd, PLUGIN_NAME) def config_callback(conf): @@ -96,7 +94,7 @@ def notification_callback(notification): def read_callback(): - plugin.read_callback() + plugin.conditional_read_callback() collectd.register_config(config_callback) collectd.register_notification(notification_callback) diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_keystone.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_keystone.py index 66d01aaaf..c88c6e275 100644 --- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_keystone.py +++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_keystone.py @@ -16,7 +16,6 @@ # Collectd plugin for getting statistics from Keystone import collectd -import collectd_base as base import collectd_openstack as openstack PLUGIN_NAME = 'keystone' @@ -30,8 +29,7 @@ class KeystoneStatsPlugin(openstack.CollectdPlugin): number of roles """ - @base.read_callback_wrapper - def read_callback(self): + def collect(self): def groupby(d): return 'enabled' if d.get('enabled') else 'disabled' @@ -78,7 +76,7 @@ class KeystoneStatsPlugin(openstack.CollectdPlugin): ) v.dispatch() -plugin = KeystoneStatsPlugin(collectd) +plugin = KeystoneStatsPlugin(collectd, PLUGIN_NAME) def config_callback(conf): @@ -90,7 +88,7 @@ def notification_callback(notification): def read_callback(): - plugin.read_callback() + plugin.conditional_read_callback() collectd.register_config(config_callback) collectd.register_notification(notification_callback) diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py index 3d1c89dbc..6a2cf226a 100644 --- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py +++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py @@ -19,7 +19,6 @@ from collections import Counter from collections import defaultdict import re -import collectd_base as base import collectd_openstack as openstack PLUGIN_NAME = 'neutron' @@ -41,8 +40,7 @@ class NeutronStatsPlugin(openstack.CollectdPlugin): agent_re = re.compile('-agent$') states = {'up': 0, 'down': 1, 'disabled': 2} - @base.read_callback_wrapper - def read_callback(self): + def collect(self): def groupby_network(x): return "networks.%s" % x.get('status', 'unknown').lower() @@ -141,7 +139,7 @@ class NeutronStatsPlugin(openstack.CollectdPlugin): ) v.dispatch() -plugin = NeutronStatsPlugin(collectd) +plugin = NeutronStatsPlugin(collectd, PLUGIN_NAME) def config_callback(conf): @@ -153,7 +151,7 @@ def notification_callback(notification): def read_callback(): - plugin.read_callback() + plugin.conditional_read_callback() collectd.register_config(config_callback) collectd.register_notification(notification_callback) diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py index 37d3f8629..62c55da4a 100644 --- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py +++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py @@ -19,7 +19,6 @@ from collections import Counter from collections import defaultdict import re -import collectd_base as base import collectd_openstack as openstack PLUGIN_NAME = 'nova' @@ -35,8 +34,7 @@ class NovaStatsPlugin(openstack.CollectdPlugin): states = {'up': 0, 'down': 1, 'disabled': 2} nova_re = re.compile('^nova-') - @base.read_callback_wrapper - def read_callback(self): + def collect(self): # Get information of the state per service # State can be: 'up', 'down' or 'disabled' @@ -82,7 +80,7 @@ class NovaStatsPlugin(openstack.CollectdPlugin): ) v.dispatch() -plugin = NovaStatsPlugin(collectd) +plugin = NovaStatsPlugin(collectd, PLUGIN_NAME) def config_callback(conf): @@ -94,7 +92,7 @@ def notification_callback(notification): def read_callback(): - plugin.read_callback() + plugin.conditional_read_callback() collectd.register_config(config_callback) collectd.register_notification(notification_callback)