From f538fee66e6e3af7e6ec7603b1a59888616eef52 Mon Sep 17 00:00:00 2001 From: akhiljain23 Date: Mon, 27 Aug 2018 17:53:08 +0530 Subject: [PATCH] Normalizing monasca webhook driver tables This commit splits one alarm-notification table to three tables namely alarm-notification, metrics and dimensions. Due to circular dependency making this patch dependent on another patch [0] that disable corresponding test. Also another patch-set [1] doing required test is updated. [0] https://review.openstack.org/#/c/600527/ [1] https://review.openstack.org/#/c/597110/ Depends-on: I7966021c3f9c21d343ffc4412fe5f5b01cffe811 Change-Id: If6138710a0ee29320c3bdbefaffb9cc5df4b90c0 Partially-Implements: blueprint add-monasca-push-driver --- congress/datasources/monasca_driver.py | 85 +++++++++++-------- .../tests/datasources/test_monasca_driver.py | 82 ++++++++++++++---- 2 files changed, 114 insertions(+), 53 deletions(-) diff --git a/congress/datasources/monasca_driver.py b/congress/datasources/monasca_driver.py index a7c450a68..766cb9e62 100644 --- a/congress/datasources/monasca_driver.py +++ b/congress/datasources/monasca_driver.py @@ -167,25 +167,32 @@ class MonascaDriver(datasource_driver.PollingDataSourceDriver, class MonascaWebhookDriver(datasource_driver.PushedDataSourceDriver): - def flatten_alarm_webhook(alarm_obj): - flattened = [] - key_to_sub_dict = 'metrics' - for alarm in alarm_obj: - sub_dict = alarm.pop(key_to_sub_dict)[0] - for k, v in sub_dict.items(): - if isinstance(v, dict): - for key, value in v.items(): - alarm[key_to_sub_dict + '_' + k + '_' + key] = value - else: - alarm[key_to_sub_dict + '_' + k] = v - flattened.append(alarm) - return flattened + METRICS = 'alarms.' + METRICS + DIMENSIONS = METRICS + '.' + DIMENSIONS + + metric_translator = { + 'translation-type': 'HDICT', + 'table-name': METRICS, + 'parent-key': 'alarm_id', + 'parent-col-name': 'alarm_id', + 'parent-key-desc': 'ALARM id', + 'selector-type': 'DICT_SELECTOR', + 'in-list': True, + 'field-translators': + ({'fieldname': 'id', 'translator': value_trans}, + {'fieldname': 'name', 'translator': value_trans}, + {'fieldname': 'dimensions', + 'translator': {'translation-type': 'VDICT', + 'table-name': DIMENSIONS, + 'id-col': 'id', + 'key-col': 'key', 'val-col': 'value', + 'translator': value_trans}}) + } alarm_notification_translator = { 'translation-type': 'HDICT', 'table-name': NOTIFICATIONS, 'selector-type': 'DICT_SELECTOR', - 'objects-extract-fn': flatten_alarm_webhook, 'field-translators': ({'fieldname': 'alarm_id', 'translator': value_trans}, {'fieldname': 'alarm_definition_id', 'translator': value_trans}, @@ -196,16 +203,7 @@ class MonascaWebhookDriver(datasource_driver.PushedDataSourceDriver): {'fieldname': 'old_state', 'translator': value_trans}, {'fieldname': 'message', 'translator': value_trans}, {'fieldname': 'tenant_id', 'translator': value_trans}, - {'fieldname': 'metrics_id', 'col': 'first_metric_id', - 'translator': value_trans}, - {'fieldname': 'metrics_name', 'col': 'first_metric_name', - 'translator': value_trans}, - {'fieldname': 'metrics_dimensions_hostname', - 'col': 'first_metric_hostname', - 'translator': value_trans}, - {'fieldname': 'metrics_dimensions_service', - 'col': 'first_metric_service', - 'translator': value_trans},) + {'fieldname': 'metrics', 'translator': metric_translator},) } TRANSLATORS = [alarm_notification_translator] @@ -231,27 +229,37 @@ class MonascaWebhookDriver(datasource_driver.PushedDataSourceDriver): 'hours_to_keep_alarm': constants.OPTIONAL} return result - def _webhook_handler(self, alarm): - tablename = NOTIFICATIONS - # remove already existing same alarm row - alarm_id = alarm['alarm_id'] - column_index_number_of_alarm_id = 0 + def _delete_rows(self, tablename, column_number, value): to_remove = [row for row in self.state[tablename] - if row[column_index_number_of_alarm_id] == alarm_id] + if row[column_number] == value] for row in to_remove: self.state[tablename].discard(row) + def _webhook_handler(self, alarm): + tablenames = [NOTIFICATIONS, self.METRICS, self.DIMENSIONS] + + # remove already existing same alarm row from alarm_notification + alarm_id = alarm['alarm_id'] + column_index_number_of_alarm_id = 0 + self._delete_rows(NOTIFICATIONS, column_index_number_of_alarm_id, + alarm_id) + + # remove already existing same metric from metrics + self._delete_rows(self.METRICS, column_index_number_of_alarm_id, + alarm_id) + translator = self.alarm_notification_translator row_data = MonascaWebhookDriver.convert_objs([alarm], translator) # add alarm to table for table, row in row_data: - if table == tablename: - self.state[tablename].add(row) - LOG.debug('publish a new state %s in %s', - self.state[tablename], tablename) - self.publish(tablename, self.state[tablename]) - return [tablename] + if table in tablenames: + self.state[table].add(row) + for table in tablenames: + LOG.debug('publish a new state %s in %s', + self.state[table], table) + self.publish(table, self.state[table]) + return tablenames def set_up_periodic_tasks(self): @lockutils.synchronized('congress_monasca_webhook_ds_data') @@ -267,6 +275,11 @@ class MonascaWebhookDriver(datasource_driver.PushedDataSourceDriver): >= timedelta(hours=self.hours_to_keep_alarm))] for row in to_remove: self.state[tablename].discard(row) + # deletes corresponding metrics table rows + col_index_of_alarm_id = 0 + alarm_id = row[col_index_of_alarm_id] + self._delete_rows(self.METRICS, col_index_of_alarm_id, + alarm_id) periodic_task_callables = [ (delete_old_alarms, None, {}), diff --git a/congress/tests/datasources/test_monasca_driver.py b/congress/tests/datasources/test_monasca_driver.py index 028571b00..3aff3f492 100644 --- a/congress/tests/datasources/test_monasca_driver.py +++ b/congress/tests/datasources/test_monasca_driver.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. # + +import copy import mock import sys import time @@ -24,6 +26,11 @@ from congress.tests import base from congress.tests import helper +METRICS = "alarms.metrics" +DIMENSIONS = METRICS + ".dimensions" +NOTIFICATIONS = "alarm_notification" + + class TestMonascaDriver(base.TestCase): def setUp(self): @@ -157,21 +164,60 @@ class TestMonascaWebhookDriver(base.TestCase): 'alarm_name': u'alarmPerHost23'} self.monasca._webhook_handler(test_alarm) - expected_rows = set([(u'3beb4934-053d-4f8f-9704-273bffc2441b', - u'8e5d033f-28cc-459f-91d4-813307e4ca8a', - u'alarmPerHost23', - u'', - 1531821822, - u'ALARM', - u'UNDETERMINED', - u'Thresholds were exceeded for the sub-alarms', - u'3661888238874df098988deab07c599d', - None, - u'load.avg_1_min', - u'openstack-13.local.lan', - u'monitoring')]) - self.assertEqual(self.monasca.state['alarm_notification'], - expected_rows) + expected_alarm_notification = set( + [(u'3beb4934-053d-4f8f-9704-273bffc2441b', + u'8e5d033f-28cc-459f-91d4-813307e4ca8a', + u'alarmPerHost23', + u'', + 1531821822, + u'ALARM', + u'UNDETERMINED', + u'Thresholds were exceeded for the sub-alarms', + u'3661888238874df098988deab07c599d')]) + self.assertEqual(self.monasca.state[NOTIFICATIONS], + expected_alarm_notification) + + dimension_id = (copy.deepcopy(self.monasca.state[METRICS])).pop()[3] + expected_metrics = set([(u'3beb4934-053d-4f8f-9704-273bffc2441b', + None, + u'load.avg_1_min', + dimension_id)]) + self.assertEqual(self.monasca.state[METRICS], expected_metrics) + + expected_dimensions = set( + [(dimension_id, 'hostname', 'openstack-13.local.lan'), + (dimension_id, 'service', 'monitoring')]) + self.assertEqual(self.monasca.state[DIMENSIONS], + expected_dimensions) + + # generate another webhook notification with same alarm_id to check + # if it gets updated + test_alarm['metrics'][0]['name'] = 'modified_name' + test_alarm['state'] = 'OK' + test_alarm['old_state'] = 'ALARM' + self.monasca._webhook_handler(test_alarm) + expected_alarm_notification = set( + [(u'3beb4934-053d-4f8f-9704-273bffc2441b', + u'8e5d033f-28cc-459f-91d4-813307e4ca8a', + u'alarmPerHost23', + u'', + 1531821822, + u'OK', + u'ALARM', + u'Thresholds were exceeded for the sub-alarms', + u'3661888238874df098988deab07c599d')]) + self.assertEqual(self.monasca.state[NOTIFICATIONS], + expected_alarm_notification) + # to check that same alarm is updated rather than creating a new one + self.assertEqual(len(self.monasca.state[NOTIFICATIONS]), 1) + + expected_metrics = set([(u'3beb4934-053d-4f8f-9704-273bffc2441b', + None, + u'modified_name', + dimension_id)]) + self.assertEqual(self.monasca.state[METRICS], expected_metrics) + # to check that same alarm metric is updated rather than creating new + self.assertEqual(len(self.monasca.state[METRICS]), 1) @mock.patch.object(monasca_driver.MonascaWebhookDriver, 'publish') def test_webhook_alarm_cleanup(self, mocked_publish): @@ -197,6 +243,8 @@ class TestMonascaWebhookDriver(base.TestCase): self.monasca._webhook_handler(test_alarm) - self.assertEqual(1, len(self.monasca.state['alarm_notification'])) + self.assertEqual(1, len(self.monasca.state[NOTIFICATIONS])) + self.assertEqual(1, len(self.monasca.state[METRICS])) time.sleep(3) - self.assertEqual(0, len(self.monasca.state['alarm_notification'])) + self.assertEqual(0, len(self.monasca.state[NOTIFICATIONS])) + self.assertEqual(0, len(self.monasca.state[METRICS]))