x
/
fuel-plugin-lma-collector
Code Issues Proposed changes

Rework error handling for OpenStack related collectd plugins 

When the OpenStack services cannot be polled due to authentication
issues (eg invalid username/passwod), collectd logged this message:
AttributeError: 'NoneType' object has no attribute 'status_code'

This change raises a KeystoneException when the plugin can't
authenticate against Keystone.
All other exceptions are logged as well.

Change-Id: I97c527d30b519989e43c1cfc0db4ce9d1dc50ca8
This commit is contained in:
Swann Croiset 2016-07-29 09:46:50 +02:00
parent 1d924661f0
commit a5063ce1d2
8 changed files with 83 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
deployment_scripts/puppet/modules/lma_collector/files/collectd/check_openstack_api.py
View File

@ -16,7 +16,6 @@
# Collectd plugin for checking the status of OpenStack API services
import collectd
import collectd_base as base
import collectd_openstack as openstack
from urlparse import urlparse
@ -94,8 +93,7 @@ class APICheckPlugin(openstack.CollectdPlugin):
'region': service['region']
}
@base.read_callback_wrapper
def read_callback(self):
def collect(self):
for item in self.check_api():
if item['status'] == self.UNKNOWN:
# skip if status is UNKNOWN
@ -112,7 +110,7 @@ class APICheckPlugin(openstack.CollectdPlugin):
value.dispatch()
plugin = APICheckPlugin(collectd)
plugin = APICheckPlugin(collectd, PLUGIN_NAME)
def config_callback(conf):
@ -124,7 +122,7 @@ def notification_callback(notification):
def read_callback():
plugin.read_callback()
plugin.conditional_read_callback()
collectd.register_config(config_callback)
collectd.register_notification(notification_callback)

101
deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py
View File

@ -18,6 +18,7 @@ import dateutil.parser
import dateutil.tz
import requests
import simplejson as json
import traceback
import collectd_base as base
@ -29,6 +30,10 @@ from collections import defaultdict
INTERVAL = 50
class KeystoneException(Exception):
pass
class OSClient(object):
""" Base class for querying the OpenStack API endpoints.
@ -58,8 +63,6 @@ class OSClient(object):
self.session.mount(
'https://', requests.adapters.HTTPAdapter(max_retries=max_retries))
self.get_token()
def is_valid_token(self):
now = datetime.datetime.now(tz=dateutil.tz.tzutc())
return self.token and self.valid_until and self.valid_until > now
@ -86,14 +89,12 @@ class OSClient(object):
'%s/tokens' % self.keystone_url, data=data,
token_required=False)
if not r:
self.logger.error("Cannot get a valid token from %s" %
self.keystone_url)
return
raise KeystoneException("Cannot get a valid token from %s" %
self.keystone_url)
if r.status_code < 200 or r.status_code > 299:
self.logger.error("%s responded with code %d" %
(self.keystone_url, r.status_code))
return
raise KeystoneException("%s responded with code %d" %
(self.keystone_url, r.status_code))
data = r.json()
self.logger.debug("Got response from Keystone: '%s'" % data)
@ -202,37 +203,44 @@ class CollectdPlugin(base.Base):
entry = 'services'
ost_services_r = self.get(service, endpoint)
r_status = ost_services_r.status_code
try:
r_json = ost_services_r.json()
except ValueError:
r_json = {}
if r_status == 200 and entry in r_json:
for val in r_json[entry]:
data = {'host': val['host'], 'service': val['binary']}
if service == 'neutron':
if not val['admin_state_up']:
data['state'] = 'disabled'
else:
data['state'] = 'up' if val['alive'] else 'down'
else:
if val['status'] == 'disabled':
data['state'] = 'disabled'
elif val['state'] == 'up' or val['state'] == 'down':
data['state'] = val['state']
else:
msg = "Unknown state for {} workers:{}".format(
service, val['state'])
self.logger.warning(msg)
continue
yield data
else:
msg = "Cannot get state of {} workers: Got {} ({})".format(
service, r_status, ost_services_r.content)
msg = "Cannot get state of {} workers".format(service)
if ost_services_r is None:
self.logger.warning(msg)
elif ost_services_r.status_code != 200:
msg = "{}: Got {} ({})".format(
msg, ost_services_r.status_code, ost_services_r.content)
self.logger.warning(msg)
else:
try:
r_json = ost_services_r.json()
except ValueError:
r_json = {}
if entry not in r_json:
msg = "{}: couldn't find '{}' key".format(msg, entry)
self.logger.warning(msg)
else:
for val in r_json[entry]:
data = {'host': val['host'], 'service': val['binary']}
if service == 'neutron':
if not val['admin_state_up']:
data['state'] = 'disabled'
else:
data['state'] = 'up' if val['alive'] else 'down'
else:
if val['status'] == 'disabled':
data['state'] = 'disabled'
elif val['state'] == 'up' or val['state'] == 'down':
data['state'] = val['state']
else:
msg = "Unknown state for {} workers:{}".format(
service, val['state'])
self.logger.warning(msg)
continue
yield data
def get(self, service, resource):
url = self._build_url(service, resource)
@ -269,11 +277,26 @@ class CollectdPlugin(base.Base):
self.max_retries)
def read_callback(self):
""" Wrapper method
This method calls the actual method which performs
collection.
"""
try:
self.collect()
except Exception as e:
msg = '{}: fail to get metrics: {}: {}'.format(
self.service_name or self.plugin, e, traceback.format_exc())
self.logger.error(msg)
def collect(self):
""" Read metrics and dispatch values
This method should be overriden by the derived classes.
This method should be overriden by the derived classes.
"""
raise "read_callback() method needs to be overriden!"
raise 'collect() method needs to be overriden!'
def get_objects(self, project, object_name, api_version='',
params='all_tenants=1'):

9
deployment_scripts/puppet/modules/lma_collector/files/collectd/hypervisor_stats.py
View File

@ -16,7 +16,6 @@
# Collectd plugin for getting hypervisor statistics from Nova
import collectd
import collectd_base as base
import collectd_openstack as openstack
PLUGIN_NAME = 'hypervisor_stats'
@ -57,8 +56,7 @@ class HypervisorStatsPlugin(openstack.CollectdPlugin):
v.host = host
v.dispatch()
@base.read_callback_wrapper
def read_callback(self):
def collect(self):
r = self.get('nova', 'os-hypervisors/detail')
if not r:
self.logger.warning("Could not get hypervisor statistics")
@ -83,8 +81,7 @@ class HypervisorStatsPlugin(openstack.CollectdPlugin):
for k, v in total_stats.iteritems():
self.dispatch_value('total_{}'.format(k), v)
plugin = HypervisorStatsPlugin(collectd)
plugin = HypervisorStatsPlugin(collectd, PLUGIN_NAME)
def config_callback(conf):
@ -96,7 +93,7 @@ def notification_callback(notification):
def read_callback():
plugin.read_callback()
plugin.conditional_read_callback()
collectd.register_config(config_callback)
collectd.register_notification(notification_callback)

8
deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py
View File

@ -19,7 +19,6 @@ from collections import Counter
from collections import defaultdict
import re
import collectd_base as base
import collectd_openstack as openstack
PLUGIN_NAME = 'cinder'
@ -37,8 +36,7 @@ class CinderStatsPlugin(openstack.CollectdPlugin):
states = {'up': 0, 'down': 1, 'disabled': 2}
cinder_re = re.compile('^cinder-')
@base.read_callback_wrapper
def read_callback(self):
def collect(self):
# Get information of the state per service
# State can be: 'up', 'down' or 'disabled'
@ -106,7 +104,7 @@ class CinderStatsPlugin(openstack.CollectdPlugin):
)
v.dispatch()
plugin = CinderStatsPlugin(collectd)
plugin = CinderStatsPlugin(collectd, PLUGIN_NAME)
def config_callback(conf):
@ -118,7 +116,7 @@ def notification_callback(notification):
def read_callback():
plugin.read_callback()
plugin.conditional_read_callback()
collectd.register_config(config_callback)
collectd.register_notification(notification_callback)

8
deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py
View File

@ -16,7 +16,6 @@
# Collectd plugin for getting resource statistics from Glance
import collectd
import collectd_base as base
import collectd_openstack as openstack
PLUGIN_NAME = 'glance'
@ -30,8 +29,7 @@ class GlanceStatsPlugin(openstack.CollectdPlugin):
total size of images usable and in error state
"""
@base.read_callback_wrapper
def read_callback(self):
def collect(self):
def is_snap(d):
return d.get('properties', {}).get('image_type') == 'snapshot'
@ -84,7 +82,7 @@ class GlanceStatsPlugin(openstack.CollectdPlugin):
)
v.dispatch()
plugin = GlanceStatsPlugin(collectd)
plugin = GlanceStatsPlugin(collectd, PLUGIN_NAME)
def config_callback(conf):
@ -96,7 +94,7 @@ def notification_callback(notification):
def read_callback():
plugin.read_callback()
plugin.conditional_read_callback()
collectd.register_config(config_callback)
collectd.register_notification(notification_callback)

8
deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_keystone.py
View File

@ -16,7 +16,6 @@
# Collectd plugin for getting statistics from Keystone
import collectd
import collectd_base as base
import collectd_openstack as openstack
PLUGIN_NAME = 'keystone'
@ -30,8 +29,7 @@ class KeystoneStatsPlugin(openstack.CollectdPlugin):
number of roles
"""
@base.read_callback_wrapper
def read_callback(self):
def collect(self):
def groupby(d):
return 'enabled' if d.get('enabled') else 'disabled'
@ -78,7 +76,7 @@ class KeystoneStatsPlugin(openstack.CollectdPlugin):
)
v.dispatch()
plugin = KeystoneStatsPlugin(collectd)
plugin = KeystoneStatsPlugin(collectd, PLUGIN_NAME)
def config_callback(conf):
@ -90,7 +88,7 @@ def notification_callback(notification):
def read_callback():
plugin.read_callback()
plugin.conditional_read_callback()
collectd.register_config(config_callback)
collectd.register_notification(notification_callback)

8
deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py
View File

@ -19,7 +19,6 @@ from collections import Counter
from collections import defaultdict
import re
import collectd_base as base
import collectd_openstack as openstack
PLUGIN_NAME = 'neutron'
@ -41,8 +40,7 @@ class NeutronStatsPlugin(openstack.CollectdPlugin):
agent_re = re.compile('-agent$')
states = {'up': 0, 'down': 1, 'disabled': 2}
@base.read_callback_wrapper
def read_callback(self):
def collect(self):
def groupby_network(x):
return "networks.%s" % x.get('status', 'unknown').lower()
@ -141,7 +139,7 @@ class NeutronStatsPlugin(openstack.CollectdPlugin):
)
v.dispatch()
plugin = NeutronStatsPlugin(collectd)
plugin = NeutronStatsPlugin(collectd, PLUGIN_NAME)
def config_callback(conf):
@ -153,7 +151,7 @@ def notification_callback(notification):
def read_callback():
plugin.read_callback()
plugin.conditional_read_callback()
collectd.register_config(config_callback)
collectd.register_notification(notification_callback)

8
deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py
View File

@ -19,7 +19,6 @@ from collections import Counter
from collections import defaultdict
import re
import collectd_base as base
import collectd_openstack as openstack
PLUGIN_NAME = 'nova'
@ -35,8 +34,7 @@ class NovaStatsPlugin(openstack.CollectdPlugin):
states = {'up': 0, 'down': 1, 'disabled': 2}
nova_re = re.compile('^nova-')
@base.read_callback_wrapper
def read_callback(self):
def collect(self):
# Get information of the state per service
# State can be: 'up', 'down' or 'disabled'
@ -82,7 +80,7 @@ class NovaStatsPlugin(openstack.CollectdPlugin):
)
v.dispatch()
plugin = NovaStatsPlugin(collectd)
plugin = NovaStatsPlugin(collectd, PLUGIN_NAME)
def config_callback(conf):
@ -94,7 +92,7 @@ def notification_callback(notification):
def read_callback():
plugin.read_callback()
plugin.conditional_read_callback()
collectd.register_config(config_callback)
collectd.register_notification(notification_callback)