summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJenkins <jenkins@review.openstack.org>2017-02-07 08:25:37 +0000
committerGerrit Code Review <review@openstack.org>2017-02-07 08:25:37 +0000
commit03113b8a3ab191d0c02206c0d1e37cd2f36a77b3 (patch)
treeeb06cb71923985c5dbb9898120ed1b31545d9cd0
parenta438d9f9271ef6f6f9d20268a6ed3cc981e24c5f (diff)
parent7c248af9fab69e25b1ab84a2ecb28f6bbeb89046 (diff)
Merge "Rework collectd plugins for OpenStack"
-rw-r--r--deployment_scripts/puppet/manifests/collectd.pp13
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/check_openstack_api.py25
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_base.py3
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py23
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/hypervisor_stats.py47
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py81
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder_services.py93
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py33
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_keystone.py36
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py76
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron_agents.py96
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py77
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova_services.py98
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/manifests/collectd/openstack.pp4
14 files changed, 436 insertions, 269 deletions
diff --git a/deployment_scripts/puppet/manifests/collectd.pp b/deployment_scripts/puppet/manifests/collectd.pp
index ae9d0d4..7e6ab37 100644
--- a/deployment_scripts/puppet/manifests/collectd.pp
+++ b/deployment_scripts/puppet/manifests/collectd.pp
@@ -162,11 +162,14 @@ if hiera('lma::collector::influxdb::server', false) {
162 require => Class['lma_collector::collectd::base'], 162 require => Class['lma_collector::collectd::base'],
163 } 163 }
164 $openstack_services = { 164 $openstack_services = {
165 'nova' => $openstack_service_config, 165 'nova' => $openstack_service_config,
166 'cinder' => $openstack_service_config, 166 'nova_services' => $openstack_service_config,
167 'glance' => $openstack_service_config, 167 'cinder' => $openstack_service_config,
168 'keystone' => $openstack_service_config, 168 'cinder_services' => $openstack_service_config,
169 'neutron' => $openstack_service_config, 169 'glance' => $openstack_service_config,
170 'keystone' => $openstack_service_config,
171 'neutron' => $openstack_service_config,
172 'neutron_agents' => $openstack_service_config,
170 } 173 }
171 create_resources(lma_collector::collectd::openstack, $openstack_services) 174 create_resources(lma_collector::collectd::openstack, $openstack_services)
172 175
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/check_openstack_api.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/check_openstack_api.py
index 44a5b2b..f775208 100644
--- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/check_openstack_api.py
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/check_openstack_api.py
@@ -49,6 +49,11 @@ class APICheckPlugin(openstack.CollectdPlugin):
49 'path': 'healthcheck', 'expect': [200], 'name': 'swift-s3-api'}, 49 'path': 'healthcheck', 'expect': [200], 'name': 'swift-s3-api'},
50 } 50 }
51 51
52 def __init__(self, *args, **kwargs):
53 super(APICheckPlugin, self).__init__(*args, **kwargs)
54 self.plugin = PLUGIN_NAME
55 self.interval = INTERVAL
56
52 def _service_url(self, endpoint, path): 57 def _service_url(self, endpoint, path):
53 url = urlparse(endpoint) 58 url = urlparse(endpoint)
54 u = '%s://%s' % (url.scheme, url.netloc) 59 u = '%s://%s' % (url.scheme, url.netloc)
@@ -94,21 +99,15 @@ class APICheckPlugin(openstack.CollectdPlugin):
94 'region': service['region'] 99 'region': service['region']
95 } 100 }
96 101
97 def collect(self): 102 def itermetrics(self):
98 for item in self.check_api(): 103 for item in self.check_api():
99 if item['status'] == self.UNKNOWN: 104 if item['status'] != self.UNKNOWN:
100 # skip if status is UNKNOWN 105 # skip if status is UNKNOWN
101 continue 106 yield {
102 107 'plugin_instance': item['service'],
103 value = collectd.Values( 108 'values': item['status'],
104 plugin=PLUGIN_NAME, 109 'meta': {'region': item['region']},
105 plugin_instance=item['service'], 110 }
106 type='gauge',
107 interval=INTERVAL,
108 values=[item['status']],
109 meta={'region': item['region']}
110 )
111 value.dispatch()
112 111
113 112
114plugin = APICheckPlugin(collectd, PLUGIN_NAME) 113plugin = APICheckPlugin(collectd, PLUGIN_NAME)
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_base.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_base.py
index e869a97..c59149f 100644
--- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_base.py
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_base.py
@@ -141,11 +141,12 @@ class Base(object):
141 (self.plugin, type_instance[:24], len(type_instance), 141 (self.plugin, type_instance[:24], len(type_instance),
142 self.MAX_IDENTIFIER_LENGTH)) 142 self.MAX_IDENTIFIER_LENGTH))
143 143
144 plugin_instance = metric.get('plugin_instance', self.plugin_instance)
144 v = self.collectd.Values( 145 v = self.collectd.Values(
145 plugin=self.plugin, 146 plugin=self.plugin,
146 host=metric.get('hostname', ''), 147 host=metric.get('hostname', ''),
147 type=metric.get('type', 'gauge'), 148 type=metric.get('type', 'gauge'),
148 plugin_instance=self.plugin_instance, 149 plugin_instance=plugin_instance,
149 type_instance=type_instance, 150 type_instance=type_instance,
150 values=values, 151 values=values,
151 # w/a for https://github.com/collectd/collectd/issues/716 152 # w/a for https://github.com/collectd/collectd/issues/716
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py
index 955d780..7108d04 100644
--- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py
@@ -18,7 +18,6 @@ import dateutil.parser
18import dateutil.tz 18import dateutil.tz
19import requests 19import requests
20import simplejson as json 20import simplejson as json
21import traceback
22 21
23import collectd_base as base 22import collectd_base as base
24 23
@@ -276,28 +275,6 @@ class CollectdPlugin(base.Base):
276 keystone_url, self.timeout, self.logger, 275 keystone_url, self.timeout, self.logger,
277 self.max_retries) 276 self.max_retries)
278 277
279 def read_callback(self):
280 """ Wrapper method
281
282 This method calls the actual method which performs
283 collection.
284 """
285
286 try:
287 self.collect()
288 except Exception as e:
289 msg = '{}: fail to get metrics: {}: {}'.format(
290 self.service_name or self.plugin, e, traceback.format_exc())
291 self.logger.error(msg)
292
293 def collect(self):
294 """ Read metrics and dispatch values
295
296 This method should be overriden by the derived classes.
297 """
298
299 raise 'collect() method needs to be overriden!'
300
301 def get_objects(self, project, object_name, api_version='', 278 def get_objects(self, project, object_name, api_version='',
302 params='all_tenants=1'): 279 params='all_tenants=1'):
303 """ Return a list of OpenStack objects 280 """ Return a list of OpenStack objects
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/hypervisor_stats.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/hypervisor_stats.py
index ba6e050..9b61e3c 100644
--- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/hypervisor_stats.py
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/hypervisor_stats.py
@@ -34,6 +34,11 @@ class HypervisorStatsPlugin(openstack.CollectdPlugin):
34 'vcpus_used': 'used_vcpus', 34 'vcpus_used': 'used_vcpus',
35 } 35 }
36 36
37 def __init__(self, *args, **kwargs):
38 super(HypervisorStatsPlugin, self).__init__(*args, **kwargs)
39 self.plugin = PLUGIN_NAME
40 self.interval = INTERVAL
41
37 def config_callback(self, config): 42 def config_callback(self, config):
38 super(HypervisorStatsPlugin, self).config_callback(config) 43 super(HypervisorStatsPlugin, self).config_callback(config)
39 for node in config.children: 44 for node in config.children:
@@ -42,19 +47,7 @@ class HypervisorStatsPlugin(openstack.CollectdPlugin):
42 if 'cpu_ratio' not in self.extra_config: 47 if 'cpu_ratio' not in self.extra_config:
43 self.logger.warning('CpuAllocationRatio parameter not set') 48 self.logger.warning('CpuAllocationRatio parameter not set')
44 49
45 def dispatch_value(self, name, value, meta=None): 50 def itermetrics(self):
46 v = collectd.Values(
47 plugin=PLUGIN_NAME,
48 type='gauge',
49 type_instance=name,
50 interval=INTERVAL,
51 # w/a for https://github.com/collectd/collectd/issues/716
52 meta=meta or {'0': True},
53 values=[value]
54 )
55 v.dispatch()
56
57 def collect(self):
58 nova_aggregates = {} 51 nova_aggregates = {}
59 r = self.get('nova', 'os-aggregates') 52 r = self.get('nova', 'os-aggregates')
60 if not r: 53 if not r:
@@ -84,7 +77,11 @@ class HypervisorStatsPlugin(openstack.CollectdPlugin):
84 host = stats['hypervisor_hostname'].split('.')[0] 77 host = stats['hypervisor_hostname'].split('.')[0]
85 for k, v in self.VALUE_MAP.iteritems(): 78 for k, v in self.VALUE_MAP.iteritems():
86 m_val = stats.get(k, 0) 79 m_val = stats.get(k, 0)
87 self.dispatch_value(v, m_val, {'host': host}) 80 yield {
81 'type_instance': v,
82 'values': m_val,
83 'meta': {'host': host},
84 }
88 total_stats[v] += m_val 85 total_stats[v] += m_val
89 for agg in nova_aggregates.keys(): 86 for agg in nova_aggregates.keys():
90 agg_hosts = nova_aggregates[agg]['hosts'] 87 agg_hosts = nova_aggregates[agg]['hosts']
@@ -95,7 +92,11 @@ class HypervisorStatsPlugin(openstack.CollectdPlugin):
95 m_vcpus_used = stats.get('vcpus_used', 0) 92 m_vcpus_used = stats.get('vcpus_used', 0)
96 free = (int(self.extra_config['cpu_ratio'] * 93 free = (int(self.extra_config['cpu_ratio'] *
97 m_vcpus)) - m_vcpus_used 94 m_vcpus)) - m_vcpus_used
98 self.dispatch_value('free_vcpus', free, {'host': host}) 95 yield {
96 'type_instance': 'free_vcpus',
97 'values': free,
98 'meta': {'host': host},
99 }
99 total_stats['free_vcpus'] += free 100 total_stats['free_vcpus'] += free
100 for agg in nova_aggregates.keys(): 101 for agg in nova_aggregates.keys():
101 agg_hosts = nova_aggregates[agg]['hosts'] 102 agg_hosts = nova_aggregates[agg]['hosts']
@@ -122,12 +123,20 @@ class HypervisorStatsPlugin(openstack.CollectdPlugin):
122 agg_total_free_ram, 123 agg_total_free_ram,
123 2) 124 2)
124 for k, v in nova_aggregates[agg]['metrics'].iteritems(): 125 for k, v in nova_aggregates[agg]['metrics'].iteritems():
125 self.dispatch_value('aggregate_{}'.format(k), v, 126 yield {
126 {'aggregate': agg, 127 'type_instance': 'aggregate_{}'.format(k),
127 'aggregate_id': agg_id}) 128 'values': v,
129 'meta': {
130 'aggregate': agg,
131 'aggregate_id': agg_id,
132 }
133 }
128 # Dispatch the global metrics 134 # Dispatch the global metrics
129 for k, v in total_stats.iteritems(): 135 for k, v in total_stats.iteritems():
130 self.dispatch_value('total_{}'.format(k), v) 136 yield {
137 'type_instance': 'total_{}'.format(k),
138 'values': v,
139 }
131 140
132plugin = HypervisorStatsPlugin(collectd, PLUGIN_NAME) 141plugin = HypervisorStatsPlugin(collectd, PLUGIN_NAME)
133 142
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py
index 9ad209d..373c081 100644
--- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py
@@ -15,9 +15,6 @@
15# 15#
16# Collectd plugin for getting statistics from Cinder 16# Collectd plugin for getting statistics from Cinder
17import collectd 17import collectd
18from collections import Counter
19from collections import defaultdict
20import re
21 18
22import collectd_openstack as openstack 19import collectd_openstack as openstack
23 20
@@ -26,46 +23,18 @@ INTERVAL = openstack.INTERVAL
26 23
27 24
28class CinderStatsPlugin(openstack.CollectdPlugin): 25class CinderStatsPlugin(openstack.CollectdPlugin):
29 """ Class to report the statistics on Cinder service. 26 """ Class to report the statistics on Cinder objects.
30 27
31 state of agents
32 number of volumes broken down by state 28 number of volumes broken down by state
33 total size of volumes usable and in error state 29 total size of volumes usable and in error state
34 """ 30 """
35 31
36 states = {'up': 0, 'down': 1, 'disabled': 2} 32 def __init__(self, *args, **kwargs):
37 cinder_re = re.compile('^cinder-') 33 super(CinderStatsPlugin, self).__init__(*args, **kwargs)
34 self.plugin = PLUGIN_NAME
35 self.interval = INTERVAL
38 36
39 def collect(self): 37 def itermetrics(self):
40
41 # Get information of the state per service
42 # State can be: 'up', 'down' or 'disabled'
43 aggregated_workers = defaultdict(Counter)
44
45 for worker in self.iter_workers('cinder'):
46 host = worker['host'].split('.')[0]
47 service = self.cinder_re.sub('', worker['service'])
48 state = worker['state']
49
50 aggregated_workers[service][state] += 1
51 self.dispatch_value('cinder_service', '',
52 self.states[state],
53 {'host': host,
54 'service': service,
55 'state': state})
56
57 for service in aggregated_workers:
58 totalw = sum(aggregated_workers[service].values())
59
60 for state in self.states:
61 prct = (100.0 * aggregated_workers[service][state]) / totalw
62 self.dispatch_value('cinder_services_percent', '',
63 prct,
64 {'state': state, 'service': service})
65
66 self.dispatch_value('cinder_services', '',
67 aggregated_workers[service][state],
68 {'state': state, 'service': service})
69 38
70 volumes_details = self.get_objects_details('cinder', 'volumes') 39 volumes_details = self.get_objects_details('cinder', 'volumes')
71 40
@@ -78,38 +47,42 @@ class CinderStatsPlugin(openstack.CollectdPlugin):
78 status = self.count_objects_group_by(volumes_details, 47 status = self.count_objects_group_by(volumes_details,
79 group_by_func=groupby) 48 group_by_func=groupby)
80 for s, nb in status.iteritems(): 49 for s, nb in status.iteritems():
81 self.dispatch_value('volumes', s, nb) 50 yield {
51 'plugin_instance': 'volumes',
52 'type_instance': s,
53 'values': nb
54 }
82 55
83 sizes = self.count_objects_group_by(volumes_details, 56 sizes = self.count_objects_group_by(volumes_details,
84 group_by_func=groupby, 57 group_by_func=groupby,
85 count_func=count_size_bytes) 58 count_func=count_size_bytes)
86 for n, size in sizes.iteritems(): 59 for n, size in sizes.iteritems():
87 self.dispatch_value('volumes_size', n, size) 60 yield {
61 'plugin_instance': 'volumes_size',
62 'type_instance': n,
63 'values': size
64 }
88 65
89 snaps_details = self.get_objects_details('cinder', 'snapshots') 66 snaps_details = self.get_objects_details('cinder', 'snapshots')
90 status_snaps = self.count_objects_group_by(snaps_details, 67 status_snaps = self.count_objects_group_by(snaps_details,
91 group_by_func=groupby) 68 group_by_func=groupby)
92 for s, nb in status_snaps.iteritems(): 69 for s, nb in status_snaps.iteritems():
93 self.dispatch_value('snapshots', s, nb) 70 yield {
71 'plugin_instance': 'snapshots',
72 'type_instance': s,
73 'values': nb
74 }
94 75
95 sizes = self.count_objects_group_by(snaps_details, 76 sizes = self.count_objects_group_by(snaps_details,
96 group_by_func=groupby, 77 group_by_func=groupby,
97 count_func=count_size_bytes) 78 count_func=count_size_bytes)
98 for n, size in sizes.iteritems(): 79 for n, size in sizes.iteritems():
99 self.dispatch_value('snapshots_size', n, size) 80 yield {
100 81 'plugin_instance': 'snapshots_size',
101 def dispatch_value(self, plugin_instance, name, value, meta=None): 82 'type_instance': n,
102 v = collectd.Values( 83 'values': size
103 plugin=PLUGIN_NAME, # metric source 84 }
104 plugin_instance=plugin_instance, 85
105 type='gauge',
106 type_instance=name,
107 interval=INTERVAL,
108 # w/a for https://github.com/collectd/collectd/issues/716
109 meta=meta or {'0': True},
110 values=[value]
111 )
112 v.dispatch()
113 86
114plugin = CinderStatsPlugin(collectd, PLUGIN_NAME) 87plugin = CinderStatsPlugin(collectd, PLUGIN_NAME)
115 88
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder_services.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder_services.py
new file mode 100644
index 0000000..df08f28
--- /dev/null
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder_services.py
@@ -0,0 +1,93 @@
1#!/usr/bin/python
2# Copyright 2017 Mirantis, Inc.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16# Collectd plugin for getting statistics from Cinder
17import collectd
18from collections import Counter
19from collections import defaultdict
20import re
21
22import collectd_openstack as openstack
23
24PLUGIN_NAME = 'cinder'
25INTERVAL = openstack.INTERVAL
26
27
28class CinderServiceStatsPlugin(openstack.CollectdPlugin):
29 """ Class to report the statistics on Cinder services.
30
31 state of workers broken down by state
32 """
33
34 states = {'up': 0, 'down': 1, 'disabled': 2}
35 cinder_re = re.compile('^cinder-')
36
37 def __init__(self, *args, **kwargs):
38 super(CinderServiceStatsPlugin, self).__init__(*args, **kwargs)
39 self.plugin = PLUGIN_NAME
40 self.interval = INTERVAL
41
42 def itermetrics(self):
43
44 # Get information of the state per service
45 # State can be: 'up', 'down' or 'disabled'
46 aggregated_workers = defaultdict(Counter)
47
48 for worker in self.iter_workers('cinder'):
49 host = worker['host'].split('.')[0]
50 service = self.cinder_re.sub('', worker['service'])
51 state = worker['state']
52
53 aggregated_workers[service][state] += 1
54 yield {
55 'plugin_instance': 'cinder_service',
56 'values': self.states[state],
57 'meta': {'host': host, 'service': service, 'state': state},
58 }
59
60 for service in aggregated_workers:
61 totalw = sum(aggregated_workers[service].values())
62
63 for state in self.states:
64 prct = (100.0 * aggregated_workers[service][state]) / totalw
65 yield {
66 'plugin_instance': 'cinder_services_percent',
67 'values': prct,
68 'meta': {'state': state, 'service': service}
69 }
70 yield {
71 'plugin_instance': 'cinder_services',
72 'values': aggregated_workers[service][state],
73 'meta': {'state': state, 'service': service},
74 }
75
76
77plugin = CinderServiceStatsPlugin(collectd, PLUGIN_NAME)
78
79
80def config_callback(conf):
81 plugin.config_callback(conf)
82
83
84def notification_callback(notification):
85 plugin.notification_callback(notification)
86
87
88def read_callback():
89 plugin.conditional_read_callback()
90
91collectd.register_config(config_callback)
92collectd.register_notification(notification_callback)
93collectd.register_read(read_callback, INTERVAL)
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py
index 1077083..a6b451f 100644
--- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py
@@ -29,7 +29,12 @@ class GlanceStatsPlugin(openstack.CollectdPlugin):
29 total size of images usable and in error state 29 total size of images usable and in error state
30 """ 30 """
31 31
32 def collect(self): 32 def __init__(self, *args, **kwargs):
33 super(GlanceStatsPlugin, self).__init__(*args, **kwargs)
34 self.plugin = PLUGIN_NAME
35 self.interval = INTERVAL
36
37 def itermetrics(self):
33 38
34 def is_snap(d): 39 def is_snap(d):
35 return d.get('properties', {}).get('image_type') == 'snapshot' 40 return d.get('properties', {}).get('image_type') == 'snapshot'
@@ -48,8 +53,11 @@ class GlanceStatsPlugin(openstack.CollectdPlugin):
48 group_by_func=groupby) 53 group_by_func=groupby)
49 for s, nb in status.iteritems(): 54 for s, nb in status.iteritems():
50 (name, visibility, status) = s.split('.') 55 (name, visibility, status) = s.split('.')
51 self.dispatch_value(name, nb, meta={'visibility': visibility, 56 yield {
52 'status': status}) 57 'type_instance': name,
58 'values': nb,
59 'meta': {'visibility': visibility, 'status': status}
60 }
53 61
54 # sizes 62 # sizes
55 def count_size_bytes(d): 63 def count_size_bytes(d):
@@ -67,20 +75,11 @@ class GlanceStatsPlugin(openstack.CollectdPlugin):
67 count_func=count_size_bytes) 75 count_func=count_size_bytes)
68 for s, nb in sizes.iteritems(): 76 for s, nb in sizes.iteritems():
69 (name, visibility, status) = s.split('.') 77 (name, visibility, status) = s.split('.')
70 self.dispatch_value(name, nb, meta={'visibility': visibility, 78 yield {
71 'status': status}) 79 'type_instance': name,
72 80 'values': nb,
73 def dispatch_value(self, name, value, meta=None): 81 'meta': {'visibility': visibility, 'status': status},
74 v = collectd.Values( 82 }
75 plugin=PLUGIN_NAME, # metric source
76 type='gauge',
77 type_instance=name,
78 interval=INTERVAL,
79 # w/a for https://github.com/collectd/collectd/issues/716
80 meta=meta or {'0': True},
81 values=[value]
82 )
83 v.dispatch()
84 83
85plugin = GlanceStatsPlugin(collectd, PLUGIN_NAME) 84plugin = GlanceStatsPlugin(collectd, PLUGIN_NAME)
86 85
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_keystone.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_keystone.py
index c88c6e2..b84fbc6 100644
--- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_keystone.py
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_keystone.py
@@ -29,7 +29,12 @@ class KeystoneStatsPlugin(openstack.CollectdPlugin):
29 number of roles 29 number of roles
30 """ 30 """
31 31
32 def collect(self): 32 def __init__(self, *args, **kwargs):
33 super(KeystoneStatsPlugin, self).__init__(*args, **kwargs)
34 self.plugin = PLUGIN_NAME
35 self.interval = INTERVAL
36
37 def itermetrics(self):
33 38
34 def groupby(d): 39 def groupby(d):
35 return 'enabled' if d.get('enabled') else 'disabled' 40 return 'enabled' if d.get('enabled') else 'disabled'
@@ -43,7 +48,11 @@ class KeystoneStatsPlugin(openstack.CollectdPlugin):
43 status = self.count_objects_group_by(tenants_details, 48 status = self.count_objects_group_by(tenants_details,
44 group_by_func=groupby) 49 group_by_func=groupby)
45 for s, nb in status.iteritems(): 50 for s, nb in status.iteritems():
46 self.dispatch_value('tenants', nb, meta={'state': s}) 51 yield {
52 'type_instance': 'tenants',
53 'values': nb,
54 'meta': {'state': s},
55 }
47 56
48 # users 57 # users
49 r = self.get('keystone', 'users') 58 r = self.get('keystone', 'users')
@@ -54,7 +63,11 @@ class KeystoneStatsPlugin(openstack.CollectdPlugin):
54 status = self.count_objects_group_by(users_details, 63 status = self.count_objects_group_by(users_details,
55 group_by_func=groupby) 64 group_by_func=groupby)
56 for s, nb in status.iteritems(): 65 for s, nb in status.iteritems():
57 self.dispatch_value('users', nb, meta={'state': s}) 66 yield {
67 'type_instance': 'users',
68 'values': nb,
69 'meta': {'state': s},
70 }
58 71
59 # roles 72 # roles
60 r = self.get('keystone', 'OS-KSADM/roles') 73 r = self.get('keystone', 'OS-KSADM/roles')
@@ -62,19 +75,10 @@ class KeystoneStatsPlugin(openstack.CollectdPlugin):
62 self.logger.warning('Could not find Keystone roles') 75 self.logger.warning('Could not find Keystone roles')
63 return 76 return
64 roles = r.json().get('roles', []) 77 roles = r.json().get('roles', [])
65 self.dispatch_value('roles', len(roles)) 78 yield {
66 79 'type_instance': 'roles',
67 def dispatch_value(self, name, value, meta=None): 80 'values': len(roles),
68 v = collectd.Values( 81 }
69 plugin=PLUGIN_NAME, # metric source
70 type='gauge',
71 type_instance=name,
72 interval=INTERVAL,
73 # w/a for https://github.com/collectd/collectd/issues/716
74 meta=meta or {'0': True},
75 values=[value]
76 )
77 v.dispatch()
78 82
79plugin = KeystoneStatsPlugin(collectd, PLUGIN_NAME) 83plugin = KeystoneStatsPlugin(collectd, PLUGIN_NAME)
80 84
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py
index fdc2005..798f091 100644
--- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py
@@ -15,9 +15,6 @@
15# 15#
16# Collectd plugin for getting resource statistics from Neutron 16# Collectd plugin for getting resource statistics from Neutron
17import collectd 17import collectd
18from collections import Counter
19from collections import defaultdict
20import re
21 18
22import collectd_openstack as openstack 19import collectd_openstack as openstack
23 20
@@ -26,9 +23,8 @@ INTERVAL = openstack.INTERVAL
26 23
27 24
28class NeutronStatsPlugin(openstack.CollectdPlugin): 25class NeutronStatsPlugin(openstack.CollectdPlugin):
29 """ Class to report the statistics on Neutron service. 26 """ Class to report the statistics on Neutron objects.
30 27
31 state of agents
32 number of networks broken down by status 28 number of networks broken down by status
33 number of subnets 29 number of subnets
34 number of ports broken down by owner and status 30 number of ports broken down by owner and status
@@ -36,11 +32,12 @@ class NeutronStatsPlugin(openstack.CollectdPlugin):
36 number of floating IP addresses broken down by free/associated 32 number of floating IP addresses broken down by free/associated
37 """ 33 """
38 34
39 neutron_re = re.compile('^neutron-') 35 def __init__(self, *args, **kwargs):
40 agent_re = re.compile('-agent$') 36 super(NeutronStatsPlugin, self).__init__(*args, **kwargs)
41 states = {'up': 0, 'down': 1, 'disabled': 2} 37 self.plugin = PLUGIN_NAME
38 self.interval = INTERVAL
42 39
43 def collect(self): 40 def itermetrics(self):
44 41
45 def groupby_network(x): 42 def groupby_network(x):
46 return "networks.%s" % x.get('status', 'unknown').lower() 43 return "networks.%s" % x.get('status', 'unknown').lower()
@@ -67,63 +64,33 @@ class NeutronStatsPlugin(openstack.CollectdPlugin):
67 status = 'free' 64 status = 'free'
68 return "floatingips.%s" % status 65 return "floatingips.%s" % status
69 66
70 # Get information of the state per agent
71 # State can be up or down
72 aggregated_agents = defaultdict(Counter)
73
74 for agent in self.iter_workers('neutron'):
75 host = agent['host'].split('.')[0]
76 service = self.agent_re.sub(
77 '', self.neutron_re.sub('', agent['service']))
78 state = agent['state']
79
80 aggregated_agents[service][state] += 1
81 self.dispatch_value('neutron_agent',
82 self.states[state],
83 {'host': host,
84 'service': service,
85 'state': state})
86
87 for service in aggregated_agents:
88 totala = sum(aggregated_agents[service].values())
89
90 for state in self.states:
91 prct = (100.0 * aggregated_agents[service][state]) / totala
92 self.dispatch_value('neutron_agents_percent',
93 prct,
94 {'service': service, 'state': state})
95
96 self.dispatch_value('neutron_agents',
97 aggregated_agents[service][state],
98 {'service': service, 'state': state})
99
100 # Networks 67 # Networks
101 networks = self.get_objects('neutron', 'networks', api_version='v2.0') 68 networks = self.get_objects('neutron', 'networks', api_version='v2.0')
102 status = self.count_objects_group_by(networks, 69 status = self.count_objects_group_by(networks,
103 group_by_func=groupby_network) 70 group_by_func=groupby_network)
104 for s, nb in status.iteritems(): 71 for s, nb in status.iteritems():
105 self.dispatch_value(s, nb) 72 yield {'type_instance': s, 'values': nb}
106 self.dispatch_value('networks', len(networks)) 73 yield {'type_instance': 'networks', 'values': len(networks)}
107 74
108 # Subnets 75 # Subnets
109 subnets = self.get_objects('neutron', 'subnets', api_version='v2.0') 76 subnets = self.get_objects('neutron', 'subnets', api_version='v2.0')
110 self.dispatch_value('subnets', len(subnets)) 77 yield {'type_instance': 'subnets', 'values': len(subnets)}
111 78
112 # Ports 79 # Ports
113 ports = self.get_objects('neutron', 'ports', api_version='v2.0') 80 ports = self.get_objects('neutron', 'ports', api_version='v2.0')
114 status = self.count_objects_group_by(ports, 81 status = self.count_objects_group_by(ports,
115 group_by_func=groupby_port) 82 group_by_func=groupby_port)
116 for s, nb in status.iteritems(): 83 for s, nb in status.iteritems():
117 self.dispatch_value(s, nb) 84 yield {'type_instance': s, 'values': nb}
118 self.dispatch_value('ports', len(ports)) 85 yield {'type_instance': 'ports', 'values': len(ports)}
119 86
120 # Routers 87 # Routers
121 routers = self.get_objects('neutron', 'routers', api_version='v2.0') 88 routers = self.get_objects('neutron', 'routers', api_version='v2.0')
122 status = self.count_objects_group_by(routers, 89 status = self.count_objects_group_by(routers,
123 group_by_func=groupby_router) 90 group_by_func=groupby_router)
124 for s, nb in status.iteritems(): 91 for s, nb in status.iteritems():
125 self.dispatch_value(s, nb) 92 yield {'type_instance': s, 'values': nb}
126 self.dispatch_value('routers', len(routers)) 93 yield {'type_instance': 'routers', 'values': len(routers)}
127 94
128 # Floating IP addresses 95 # Floating IP addresses
129 floatingips = self.get_objects('neutron', 'floatingips', 96 floatingips = self.get_objects('neutron', 'floatingips',
@@ -131,20 +98,9 @@ class NeutronStatsPlugin(openstack.CollectdPlugin):
131 status = self.count_objects_group_by(floatingips, 98 status = self.count_objects_group_by(floatingips,
132 group_by_func=groupby_floating) 99 group_by_func=groupby_floating)
133 for s, nb in status.iteritems(): 100 for s, nb in status.iteritems():
134 self.dispatch_value(s, nb) 101 yield {'type_instance': s, 'values': nb}
135 self.dispatch_value('floatingips', len(floatingips)) 102 yield {'type_instance': 'floatingips', 'values': len(routers)}
136 103
137 def dispatch_value(self, name, value, meta=None):
138 v = collectd.Values(
139 plugin=PLUGIN_NAME, # metric source
140 type='gauge',
141 type_instance=name,
142 interval=INTERVAL,
143 # w/a for https://github.com/collectd/collectd/issues/716
144 meta=meta or {'0': True},
145 values=[value]
146 )
147 v.dispatch()
148 104
149plugin = NeutronStatsPlugin(collectd, PLUGIN_NAME) 105plugin = NeutronStatsPlugin(collectd, PLUGIN_NAME)
150 106
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron_agents.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron_agents.py
new file mode 100644
index 0000000..6116055
--- /dev/null
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron_agents.py
@@ -0,0 +1,96 @@
1#!/usr/bin/python
2# Copyright 2015 Mirantis, Inc.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16# Collectd plugin for getting resource statistics from Neutron
17import collectd
18from collections import Counter
19from collections import defaultdict
20import re
21
22import collectd_openstack as openstack
23
24PLUGIN_NAME = 'neutron'
25INTERVAL = openstack.INTERVAL
26
27
28class NeutronAgentStatsPlugin(openstack.CollectdPlugin):
29 """ Class to report the statistics on Neutron agents.
30
31 state of agents
32 """
33
34 neutron_re = re.compile('^neutron-')
35 agent_re = re.compile('-agent$')
36 states = {'up': 0, 'down': 1, 'disabled': 2}
37
38 def __init__(self, *args, **kwargs):
39 super(NeutronAgentStatsPlugin, self).__init__(*args, **kwargs)
40 self.plugin = PLUGIN_NAME
41 self.interval = INTERVAL
42
43 def itermetrics(self):
44
45 # Get information of the state per agent
46 # State can be up or down
47 aggregated_agents = defaultdict(Counter)
48
49 for agent in self.iter_workers('neutron'):
50 host = agent['host'].split('.')[0]
51 service = self.agent_re.sub(
52 '', self.neutron_re.sub('', agent['service']))
53 state = agent['state']
54
55 aggregated_agents[service][state] += 1
56
57 yield {
58 'type_instance': 'neutron_agent',
59 'values': self.states[state],
60 'meta': {'host': host, 'service': service, 'state': state}
61 }
62
63 for service in aggregated_agents:
64 totala = sum(aggregated_agents[service].values())
65
66 for state in self.states:
67 prct = (100.0 * aggregated_agents[service][state]) / totala
68 yield {
69 'type_instance': 'neutron_agents_percent',
70 'values': prct,
71 'meta': {'service': service, 'state': state},
72 }
73 yield {
74 'type_instance': 'neutron_agents',
75 'values': aggregated_agents[service][state],
76 'meta': {'service': service, 'state': state},
77 }
78
79
80plugin = NeutronAgentStatsPlugin(collectd, PLUGIN_NAME)
81
82
83def config_callback(conf):
84 plugin.config_callback(conf)
85
86
87def notification_callback(notification):
88 plugin.notification_callback(notification)
89
90
91def read_callback():
92 plugin.conditional_read_callback()
93
94collectd.register_config(config_callback)
95collectd.register_notification(notification_callback)
96collectd.register_read(read_callback, INTERVAL)
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py
index 7c4aa0d..455e5d1 100644
--- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py
@@ -15,9 +15,6 @@
15# 15#
16# Collectd plugin for getting statistics from Nova 16# Collectd plugin for getting statistics from Nova
17import collectd 17import collectd
18from collections import Counter
19from collections import defaultdict
20import re
21 18
22import collectd_openstack as openstack 19import collectd_openstack as openstack
23 20
@@ -25,73 +22,33 @@ PLUGIN_NAME = 'nova'
25INTERVAL = openstack.INTERVAL 22INTERVAL = openstack.INTERVAL
26 23
27 24
28class NovaStatsPlugin(openstack.CollectdPlugin): 25class NovaInstanceStatsPlugin(openstack.CollectdPlugin):
29 """ Class to report the statistics on Nova service. 26 """ Class to report the statistics on Nova instances.
30 27
31 status per service and number of instances broken down by state 28 Number of instances broken down by state
32 """ 29 """
30 def __init__(self, *args, **kwargs):
31 super(NovaInstanceStatsPlugin, self).__init__(*args, **kwargs)
32 self.plugin = PLUGIN_NAME
33 self.interval = INTERVAL
33 34
34 states = {'up': 0, 'down': 1, 'disabled': 2} 35 def itermetrics(self):
35 nova_re = re.compile('^nova-')
36
37 def collect(self):
38
39 # Get information of the state per service
40 # State can be: 'up', 'down' or 'disabled'
41 aggregated_workers = defaultdict(Counter)
42
43 for worker in self.iter_workers('nova'):
44 host = worker['host'].split('.')[0]
45 service = self.nova_re.sub('', worker['service'])
46 state = worker['state']
47
48 aggregated_workers[service][state] += 1
49 self.dispatch_value('nova_service', '',
50 self.states[state],
51 {'host': host,
52 'service': service,
53 'state': state})
54
55 for service in set(aggregated_workers.keys()).union(
56 ('compute', 'scheduler', 'conductor', 'cert', 'consoleauth')):
57
58 total = sum(aggregated_workers[service].values())
59
60 for state in self.states:
61 prct = 0
62 if total > 0:
63 prct = (100.0 * aggregated_workers[service][state]) / total
64
65 self.dispatch_value('nova_services_percent', '',
66 prct,
67 {'state': state, 'service': service})
68
69 self.dispatch_value('nova_services', '',
70 aggregated_workers[service][state],
71 {'state': state, 'service': service})
72 servers_details = self.get_objects_details('nova', 'servers') 36 servers_details = self.get_objects_details('nova', 'servers')
73 37
74 def groupby(d): 38 def groupby(d):
75 return d.get('status', 'unknown').lower() 39 return d.get('status', 'unknown').lower()
40
76 status = self.count_objects_group_by(servers_details, 41 status = self.count_objects_group_by(servers_details,
77 group_by_func=groupby) 42 group_by_func=groupby)
78 for s, nb in status.iteritems(): 43 for s, nb in status.iteritems():
79 self.dispatch_value('instances', s, nb) 44 yield {
80 45 'plugin_instance': 'instances',
81 def dispatch_value(self, plugin_instance, name, value, meta=None): 46 'values': nb,
82 v = collectd.Values( 47 'type_instance': s,
83 plugin=PLUGIN_NAME, # metric source 48 }
84 plugin_instance=plugin_instance, 49
85 type='gauge', 50
86 type_instance=name, 51plugin = NovaInstanceStatsPlugin(collectd, PLUGIN_NAME)
87 interval=INTERVAL,
88 # w/a for https://github.com/collectd/collectd/issues/716
89 meta=meta or {'0': True},
90 values=[value]
91 )
92 v.dispatch()
93
94plugin = NovaStatsPlugin(collectd, PLUGIN_NAME)
95 52
96 53
97def config_callback(conf): 54def config_callback(conf):
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova_services.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova_services.py
new file mode 100644
index 0000000..774c28c
--- /dev/null
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova_services.py
@@ -0,0 +1,98 @@
1#!/usr/bin/python
2# Copyright 2017 Mirantis, Inc.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16# Collectd plugin for getting statistics from Nova
17import collectd
18from collections import Counter
19from collections import defaultdict
20import re
21
22import collectd_openstack as openstack
23
24PLUGIN_NAME = 'nova'
25INTERVAL = openstack.INTERVAL
26
27
28class NovaServiceStatsPlugin(openstack.CollectdPlugin):
29 """ Class to report the statistics on Nova services.
30
31 status per service broken down by state
32 """
33
34 states = {'up': 0, 'down': 1, 'disabled': 2}
35 nova_re = re.compile('^nova-')
36
37 def __init__(self, *args, **kwargs):
38 super(NovaServiceStatsPlugin, self).__init__(*args, **kwargs)
39 self.plugin = PLUGIN_NAME
40 self.interval = INTERVAL
41
42 def itermetrics(self):
43
44 # Get information of the state per service
45 # State can be: 'up', 'down' or 'disabled'
46 aggregated_workers = defaultdict(Counter)
47
48 for worker in self.iter_workers('nova'):
49 host = worker['host'].split('.')[0]
50 service = self.nova_re.sub('', worker['service'])
51 state = worker['state']
52
53 aggregated_workers[service][state] += 1
54 yield {
55 'plugin_instance': 'nova_service',
56 'values': self.states[state],
57 'meta': {'host': host, 'service': service, 'state': state}
58 }
59
60 for service in set(aggregated_workers.keys()).union(
61 ('compute', 'scheduler', 'conductor', 'cert', 'consoleauth')):
62
63 total = sum(aggregated_workers[service].values())
64
65 for state in self.states:
66 prct = 0
67 if total > 0:
68 prct = (100.0 * aggregated_workers[service][state]) / total
69
70 yield {
71 'plugin_instance': 'nova_services_percent',
72 'values': prct,
73 'meta': {'state': state, 'service': service},
74 }
75 yield {
76 'plugin_instance': 'nova_services',
77 'values': aggregated_workers[service][state],
78 'meta': {'state': state, 'service': service},
79 }
80
81
82plugin = NovaServiceStatsPlugin(collectd, PLUGIN_NAME)
83
84
85def config_callback(conf):
86 plugin.config_callback(conf)
87
88
89def notification_callback(notification):
90 plugin.notification_callback(notification)
91
92
93def read_callback():
94 plugin.conditional_read_callback()
95
96collectd.register_config(config_callback)
97collectd.register_notification(notification_callback)
98collectd.register_read(read_callback, INTERVAL)
diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/openstack.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/openstack.pp
index 009f696..c14e1bf 100644
--- a/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/openstack.pp
+++ b/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/openstack.pp
@@ -28,7 +28,9 @@ define lma_collector::collectd::openstack (
28 28
29 $service = $title 29 $service = $title
30 30
31 $supported_services = ['nova', 'cinder', 'glance', 'keystone', 'neutron'] 31 $supported_services = ['nova', 'nova_services',
32 'cinder', 'cinder_services', 'glance', 'keystone',
33 'neutron', 'neutron_agents']
32 if ! member($supported_services, $service) { 34 if ! member($supported_services, $service) {
33 fail("service '${service}' is not supported") 35 fail("service '${service}' is not supported")
34 } 36 }