vitrage driver for accepting webhook notifications
Accepts POST webhook notifications [1] at /v1/data-sources/<data-source-name>/webhook On alarm activate notification from Vitrage, a new alarm row [2] is created in Congress vitrage data source service. On alarm deactivate notification from Vitrage, the corresponding alarm row is deleted in Congress vitrage data source service. A retry policy on the sender side would be helpful to avoid notifications being lost to transient issues (example from zaqar [3]). Nonetheless, it is still possible that an alarm deactivate notification is missed, and an alarm row remains in Congress indefinitely. To clean-up these rows, the optional config 'hours_to_keep_alarm' sets the approximate number of hours before an active alarm row is deleted from Congress. [1] Expected payload format: { "notification": "vitrage.alarm.activate", "payload": { "vitrage_id": "2def31e9-6d9f-4c16-b007-893caa806cd4", "resource": { "vitrage_id": "437f1f4c-ccce-40a4-ac62-1c2f1fd9f6ac", "name": "app-1-server-1-jz6qvznkmnif", "update_timestamp": "2018-01-22 10:00:34.327142+00:00", "vitrage_category": "RESOURCE", "vitrage_operational_state": "OK", "vitrage_type": "nova.instance", "project_id": "8f007e5ba0944e84baa6f2a4f2b5d03a", "id": "9b7d93b9-94ec-41e1-9cec-f28d4f8d702c" }, "update_timestamp": "2018-01-22T10:00:34Z", "vitrage_category": "ALARM", "state": "Active", "vitrage_type": "vitrage", "vitrage_operational_severity": "WARNING", "name": "Instance memory performance degraded" } } https://docs.openstack.org/vitrage/latest/contributor/notifier-webhook-plugin.html [2] alarms table schema: 'name' 'state' 'type' 'operational_severity', 'vitrage_id' 'update_timestamp' 'receive_timestamp' 'resource_name' 'resource_id' 'resource_vitrage_id' 'resource_project_id' 'resource_operational_state' 'resource_type' [3] https://docs.openstack.org/zaqar/pike/user/notification_delivery_policy.html Change-Id: I1943cb81eadb1abba9e62d3d886778a09ec63118
This commit is contained in:
parent
cebb8b18ac
commit
d10c7db6eb
|
@ -0,0 +1,159 @@
|
|||
# Copyright (c) 2018 VMware Inc All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
from __future__ import division
|
||||
from __future__ import absolute_import
|
||||
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
|
||||
import eventlet
|
||||
from futurist import periodics
|
||||
from oslo_concurrency import lockutils
|
||||
from oslo_log import log as logging
|
||||
|
||||
from congress.datasources import constants
|
||||
from congress.datasources import datasource_driver
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
|
||||
|
||||
|
||||
class VitrageDriver(datasource_driver.PushedDataSourceDriver):
|
||||
'''Datasource driver that accepts Vitrage webhook alarm notification.'''
|
||||
|
||||
value_trans = {'translation-type': 'VALUE'}
|
||||
|
||||
def flatten_and_timestamp_alarm_webhook(webhook_alarms_objects):
|
||||
flattened = []
|
||||
key_to_sub_dict = 'resource'
|
||||
for alarm in webhook_alarms_objects:
|
||||
sub_dict = alarm.pop(key_to_sub_dict)
|
||||
for k, v in sub_dict.items():
|
||||
# add prefix to key and move to top level dict
|
||||
alarm[key_to_sub_dict + '_' + k] = v
|
||||
alarm['receive_timestamp'] = datetime.utcnow().strftime(
|
||||
TIMESTAMP_FORMAT)
|
||||
flattened.append(alarm)
|
||||
return flattened
|
||||
|
||||
webhook_alarm_translator = {
|
||||
'translation-type': 'HDICT',
|
||||
'table-name': 'alarms',
|
||||
'selector-type': 'DICT_SELECTOR',
|
||||
'objects-extract-fn': flatten_and_timestamp_alarm_webhook,
|
||||
'field-translators':
|
||||
({'fieldname': 'name', 'translator': value_trans},
|
||||
{'fieldname': 'state', 'translator': value_trans},
|
||||
{'fieldname': 'vitrage_type', 'col': 'type',
|
||||
'translator': value_trans},
|
||||
{'fieldname': 'vitrage_operational_severity',
|
||||
'col': 'operational_severity',
|
||||
'translator': value_trans},
|
||||
{'fieldname': 'vitrage_id', 'translator': value_trans},
|
||||
{'fieldname': 'update_timestamp', 'translator': value_trans},
|
||||
{'fieldname': 'receive_timestamp', 'translator': value_trans},
|
||||
{'fieldname': 'resource_name', 'translator': value_trans},
|
||||
{'fieldname': 'resource_id', 'translator': value_trans},
|
||||
{'fieldname': 'resource_vitrage_id', 'translator': value_trans},
|
||||
{'fieldname': 'resource_project_id', 'translator': value_trans},
|
||||
{'fieldname': 'resource_vitrage_operational_state',
|
||||
'col': 'resource_operational_state',
|
||||
'translator': value_trans},
|
||||
{'fieldname': 'resource_vitrage_type',
|
||||
'col': 'resource_type', 'translator': value_trans},
|
||||
)}
|
||||
|
||||
TRANSLATORS = [webhook_alarm_translator]
|
||||
|
||||
def __init__(self, name='', args=None):
|
||||
super(VitrageDriver, self).__init__(name, args=args)
|
||||
if args is None:
|
||||
args = {}
|
||||
# set default time to 10 days before deleting an active alarm
|
||||
self.hours_to_keep_alarm = int(args.get('hours_to_keep_alarm', 240))
|
||||
self.set_up_periodic_tasks()
|
||||
|
||||
@lockutils.synchronized('congress_vitrage_ds_data')
|
||||
def _webhook_handler(self, payload):
|
||||
tablename = 'alarms'
|
||||
if payload['notification'] == 'vitrage.alarm.activate':
|
||||
# add alarm to table
|
||||
translator = self.webhook_alarm_translator
|
||||
row_data = VitrageDriver.convert_objs(
|
||||
[payload['payload']], translator)
|
||||
for table, row in row_data:
|
||||
if table == tablename:
|
||||
self.state[tablename].add(row)
|
||||
elif payload['notification'] == 'vitrage.alarm.deactivate':
|
||||
# remove alarm from table
|
||||
row_id = payload['payload']['vitrage_id']
|
||||
column_index_number_of_row_id = 4
|
||||
to_remove = [row for row in self.state[tablename]
|
||||
if row[column_index_number_of_row_id] == row_id]
|
||||
for row in to_remove:
|
||||
self.state[tablename].discard(row)
|
||||
else:
|
||||
raise ValueError('Unexpected webhook notification type: '
|
||||
'{0}'.format(payload['notification']))
|
||||
|
||||
LOG.debug('publish a new state %s in %s',
|
||||
self.state[tablename], tablename)
|
||||
# Note (thread-safety): blocking call
|
||||
self.publish(tablename, self.state[tablename])
|
||||
return [tablename]
|
||||
|
||||
def set_up_periodic_tasks(self):
|
||||
@lockutils.synchronized('congress_vitrage_ds_data')
|
||||
@periodics.periodic(spacing=max(self.hours_to_keep_alarm * 3600/10, 1))
|
||||
def delete_old_alarms():
|
||||
tablename = 'alarms'
|
||||
col_index_of_timestamp = 5
|
||||
# find for removal all alarms at least self.hours_to_keep_alarm old
|
||||
to_remove = [
|
||||
row for row in self.state[tablename]
|
||||
if (datetime.utcnow() -
|
||||
datetime.strptime(row[col_index_of_timestamp],
|
||||
TIMESTAMP_FORMAT)
|
||||
>= timedelta(hours=self.hours_to_keep_alarm))]
|
||||
for row in to_remove:
|
||||
self.state[tablename].discard(row)
|
||||
|
||||
periodic_task_callables = [
|
||||
(delete_old_alarms, None, {}),
|
||||
(delete_old_alarms, None, {})]
|
||||
self.periodic_tasks = periodics.PeriodicWorker(periodic_task_callables)
|
||||
self.periodic_tasks_thread = eventlet.spawn_n(
|
||||
self.periodic_tasks.start)
|
||||
|
||||
@staticmethod
|
||||
def get_datasource_info():
|
||||
result = {}
|
||||
result['id'] = 'vitrage'
|
||||
result['description'] = ('Datasource driver that accepts Vitrage '
|
||||
'webhook alarm notifications.')
|
||||
result['config'] = {'persist_data': constants.OPTIONAL,
|
||||
'hours_to_keep_alarm': constants.OPTIONAL}
|
||||
return result
|
||||
|
||||
def __del__(self):
|
||||
if self.periodic_tasks:
|
||||
self.periodic_tasks.stop()
|
||||
self.periodic_tasks.wait()
|
||||
self.periodic_tasks = None
|
||||
if self.periodic_tasks_thread:
|
||||
eventlet.greenthread.kill(self.periodic_tasks_thread)
|
||||
self.periodic_tasks_thread = None
|
|
@ -0,0 +1,153 @@
|
|||
# Copyright (c) 2018 VMware Inc
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
from __future__ import division
|
||||
from __future__ import absolute_import
|
||||
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
import mock
|
||||
import time
|
||||
|
||||
from congress.datasources import vitrage_driver
|
||||
from congress.tests import base
|
||||
|
||||
|
||||
class TestVitrageDriver(base.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestVitrageDriver, self).setUp()
|
||||
self.vitrage = vitrage_driver.VitrageDriver('test-vitrage')
|
||||
|
||||
@mock.patch.object(vitrage_driver.VitrageDriver, 'publish')
|
||||
def test_webhook_alarm_activate(self, mocked_publish):
|
||||
test_payload = {
|
||||
"notification": "vitrage.alarm.activate",
|
||||
"payload": {
|
||||
"vitrage_id": "2def31e9-6d9f-4c16-b007-893caa806cd4",
|
||||
"resource": {
|
||||
"vitrage_id": "437f1f4c-ccce-40a4-ac62-1c2f1fd9f6ac",
|
||||
"name": "app-1-server-1-jz6qvznkmnif",
|
||||
"update_timestamp": "2018-01-22 10:00:34.327142+00:00",
|
||||
"vitrage_category": "RESOURCE",
|
||||
"vitrage_operational_state": "OK",
|
||||
"vitrage_type": "nova.instance",
|
||||
"project_id": "8f007e5ba0944e84baa6f2a4f2b5d03a",
|
||||
"id": "9b7d93b9-94ec-41e1-9cec-f28d4f8d702c"},
|
||||
"update_timestamp": "2018-01-22T10:00:34Z",
|
||||
"vitrage_category": "ALARM",
|
||||
"state": "Active",
|
||||
"vitrage_type": "vitrage",
|
||||
"vitrage_operational_severity": "WARNING",
|
||||
"name": "Instance memory performance degraded"}}
|
||||
|
||||
self.vitrage._webhook_handler(test_payload)
|
||||
|
||||
# check receive_timestamp
|
||||
self.assertTrue((
|
||||
abs(datetime.utcnow() -
|
||||
datetime.strptime(next(iter(self.vitrage.state['alarms']))[6],
|
||||
vitrage_driver.TIMESTAMP_FORMAT))
|
||||
<= timedelta(seconds=1)))
|
||||
|
||||
self.assertEqual(1, len(self.vitrage.state['alarms']))
|
||||
|
||||
expected_rows = set([(u'Instance memory performance degraded',
|
||||
u'Active',
|
||||
u'vitrage',
|
||||
u'WARNING',
|
||||
u'2def31e9-6d9f-4c16-b007-893caa806cd4',
|
||||
u'2018-01-22T10:00:34Z',
|
||||
next(iter(self.vitrage.state['alarms']))[6],
|
||||
u'app-1-server-1-jz6qvznkmnif',
|
||||
u'9b7d93b9-94ec-41e1-9cec-f28d4f8d702c',
|
||||
u'437f1f4c-ccce-40a4-ac62-1c2f1fd9f6ac',
|
||||
u'8f007e5ba0944e84baa6f2a4f2b5d03a',
|
||||
u'OK',
|
||||
u'nova.instance')])
|
||||
self.assertEqual(self.vitrage.state['alarms'], expected_rows)
|
||||
|
||||
@mock.patch.object(vitrage_driver.VitrageDriver, 'publish')
|
||||
def test_webhook_alarm_deactivate(self, mocked_publish):
|
||||
test_payload = {
|
||||
"notification": "vitrage.alarm.deactivate",
|
||||
"payload": {
|
||||
"vitrage_id": "2def31e9-6d9f-4c16-b007-893caa806cd4",
|
||||
"resource": {
|
||||
"vitrage_id": "437f1f4c-ccce-40a4-ac62-1c2f1fd9f6ac",
|
||||
"name": "app-1-server-1-jz6qvznkmnif",
|
||||
"update_timestamp": "2018-01-22 10:00:34.327142+00:00",
|
||||
"vitrage_category": "RESOURCE",
|
||||
"vitrage_operational_state": "OK",
|
||||
"vitrage_type": "nova.instance",
|
||||
"project_id": "8f007e5ba0944e84baa6f2a4f2b5d03a",
|
||||
"id": "9b7d93b9-94ec-41e1-9cec-f28d4f8d702c"},
|
||||
"update_timestamp": "2018-01-22T10:00:34Z",
|
||||
"vitrage_category": "ALARM",
|
||||
"state": "Inactive",
|
||||
"vitrage_type": "vitrage",
|
||||
"vitrage_operational_severity": "OK",
|
||||
"name": "Instance memory performance degraded"}}
|
||||
|
||||
self.vitrage.state['alarms'] = set([(
|
||||
u'Instance memory performance degraded',
|
||||
u'Active',
|
||||
u'vitrage',
|
||||
u'WARNING',
|
||||
u'2def31e9-6d9f-4c16-b007-893caa806cd4',
|
||||
u'2018-01-22T10:00:34Z',
|
||||
u'2018-01-22T10:00:34Z',
|
||||
u'app-1-server-1-jz6qvznkmnif',
|
||||
u'9b7d93b9-94ec-41e1-9cec-f28d4f8d702c',
|
||||
u'437f1f4c-ccce-40a4-ac62-1c2f1fd9f6ac',
|
||||
u'8f007e5ba0944e84baa6f2a4f2b5d03a',
|
||||
u'OK',
|
||||
u'nova.instance')])
|
||||
self.vitrage._webhook_handler(test_payload)
|
||||
|
||||
self.assertEqual(0, len(self.vitrage.state['alarms']))
|
||||
|
||||
@mock.patch.object(vitrage_driver.VitrageDriver, 'publish')
|
||||
def test_webhook_alarm_cleanup(self, mocked_publish):
|
||||
self.vitrage = vitrage_driver.VitrageDriver(
|
||||
'test-vitrage',
|
||||
args={'hours_to_keep_alarm': 1 / 3600}) # set to 1 sec for test
|
||||
|
||||
test_payload = {
|
||||
"notification": "vitrage.alarm.activate",
|
||||
"payload": {
|
||||
"vitrage_id": "2def31e9-6d9f-4c16-b007-893caa806cd4",
|
||||
"resource": {
|
||||
"vitrage_id": "437f1f4c-ccce-40a4-ac62-1c2f1fd9f6ac",
|
||||
"name": "app-1-server-1-jz6qvznkmnif",
|
||||
"update_timestamp": "2018-01-22 10:00:34.327142+00:00",
|
||||
"vitrage_category": "RESOURCE",
|
||||
"vitrage_operational_state": "OK",
|
||||
"vitrage_type": "nova.instance",
|
||||
"project_id": "8f007e5ba0944e84baa6f2a4f2b5d03a",
|
||||
"id": "9b7d93b9-94ec-41e1-9cec-f28d4f8d702c"},
|
||||
"update_timestamp": "2018-01-22T10:00:34Z",
|
||||
"vitrage_category": "ALARM",
|
||||
"state": "Active",
|
||||
"vitrage_type": "vitrage",
|
||||
"vitrage_operational_severity": "WARNING",
|
||||
"name": "Instance memory performance degraded"}}
|
||||
|
||||
self.vitrage._webhook_handler(test_payload)
|
||||
|
||||
self.assertEqual(1, len(self.vitrage.state['alarms']))
|
||||
time.sleep(3)
|
||||
self.assertEqual(0, len(self.vitrage.state['alarms']))
|
|
@ -512,5 +512,8 @@ def supported_drivers():
|
|||
{"id": "keystone",
|
||||
"description": "Datasource driver that interfaces with keystone."},
|
||||
{"id": "mistral",
|
||||
"description": "Datasource driver that interfaces with Mistral."}]
|
||||
"description": "Datasource driver that interfaces with Mistral."},
|
||||
{"id": "vitrage",
|
||||
"description": "Datasource driver that accepts Vitrage "
|
||||
"webhook alarm notifications."}]
|
||||
return results
|
||||
|
|
|
@ -70,6 +70,7 @@ congress.datasource.drivers =
|
|||
plexxi = congress.datasources.plexxi_driver:PlexxiDriver
|
||||
swift = congress.datasources.swift_driver:SwiftDriver
|
||||
vcenter = congress.datasources.vCenter_driver:VCenterDriver
|
||||
vitrage = congress.datasources.vitrage_driver:VitrageDriver
|
||||
|
||||
[compile_catalog]
|
||||
directory = congress/locale
|
||||
|
|
Loading…
Reference in New Issue