Merge "Fix gnocchi collector metadata collection"

This commit is contained in:
Zuul 2019-03-21 09:16:44 +00:00 committed by Gerrit Code Review
commit 5c6964469b
3 changed files with 66 additions and 8 deletions

View File

@ -98,6 +98,16 @@ GNOCCHI_EXTRA_SCHEMA = {
}
class AssociatedResourceNotFound(Exception):
"""Exception raised when no resource can be associated with a metric."""
def __init__(self, resource_key, resource_id):
super(AssociatedResourceNotFound, self).__init__(
'Resource with {}={} could not be found'.format(
resource_key, resource_id),
)
class GnocchiCollector(collector.BaseCollector):
collector_name = 'gnocchi'
@ -227,12 +237,18 @@ class GnocchiCollector(collector.BaseCollector):
# Get gnocchi specific conf
extra_args = self.conf[metric_name]['extra_args']
# Build query
query_parameters = self._generate_time_filter(start, end)
resource_type = extra_args['resource_type']
scope_key = CONF.collect.scope_key
# Build query
# FIXME(peschk_l): In order not to miss any resource whose metrics may
# contain measures after its destruction, we scan resources over three
# collect periods.
start -= CONF.collect.period
end += CONF.collect.period
query_parameters = self._generate_time_filter(start, end)
if project_id:
kwargs = {scope_key: project_id}
query_parameters.append(self.gen_filter(**kwargs))
@ -312,8 +328,12 @@ class GnocchiCollector(collector.BaseCollector):
# metadata as defined in the conf
metadata = dict()
if resources_info is not None:
resource = resources_info[
groupby[metconf['extra_args']['resource_key']]]
resource_key = metconf['extra_args']['resource_key']
resource_id = groupby[resource_key]
try:
resource = resources_info[resource_id]
except KeyError:
raise AssociatedResourceNotFound(resource_key, resource_id)
for i in metconf['metadata']:
metadata[i] = resource.get(i, '')
qty = data['measures']['measures']['aggregated'][0][2]
@ -348,8 +368,19 @@ class GnocchiCollector(collector.BaseCollector):
for d in data:
# Only if aggregates have been found
if d['measures']['measures']['aggregated']:
metadata, groupby, qty = self._format_data(
met, d, resources_info)
try:
metadata, groupby, qty = self._format_data(
met, d, resources_info)
except AssociatedResourceNotFound as e:
LOG.warning(
'[{}] An error occured during data collection '
'between {} and {}: {}'.format(
project_id,
ck_utils.ts2dt(start),
ck_utils.ts2dt(end),
e),
)
continue
data = self.t_cloudkitty.format_item(
groupby,
metadata,

View File

@ -17,13 +17,33 @@
from cloudkitty.collector import gnocchi
from cloudkitty import tests
from cloudkitty.tests import samples
from cloudkitty import transformer
class GnocchiCollectorTest(tests.TestCase):
def setUp(self):
super(GnocchiCollectorTest, self).setUp()
self._tenant_id = samples.TENANT
self.collector = gnocchi.GnocchiCollector
self.conf.set_override('collector', 'gnocchi', 'collect')
self.conf.set_override(
'gnocchi_auth_type', 'basic', 'collector_gnocchi')
self.collector = gnocchi.GnocchiCollector(
transformer.get_transformers(),
period=3600,
conf=samples.DEFAULT_METRICS_CONF,
)
def test_format_data_raises_exception(self):
metconf = {'extra_args': {'resource_key': 'id'}}
data = {'group': {'id': '281b9dc6-5d02-4610-af2d-10d0d6887f48'}}
self.assertRaises(
gnocchi.AssociatedResourceNotFound,
self.collector._format_data,
metconf,
data,
resources_info={},
)
# Filter generation
def test_generate_one_field_filter(self):

View File

@ -0,0 +1,7 @@
---
fixes:
- |
Metadata collection failures in the gnocchi collector caused by resources
having measures in periods where they are supposed to be deleted have
been fixed. Metadata is now collected over a three-period window in the
gnocchi collector.