From fa0b731093b69d02c2f6981104b7cbd30d29b761 Mon Sep 17 00:00:00 2001 From: Volodymyr Samotiy Date: Mon, 10 Oct 2016 14:43:55 +0300 Subject: [PATCH] Add support for black hole detection Change-Id: I985080d14d2ed927de6fe87ba3f02deefde86310 --- broadview_collector/handlers/bhdhandler.py | 32 +++++ .../plugins/broadviewpublisherbase.py | 4 + broadview_collector/plugins/kafkapublisher.py | 5 + broadview_collector/plugins/logpublisher.py | 5 +- .../plugins/monascapublisher.py | 3 + .../plugins/syslogpublisher.py | 3 + .../serializers/bhd_to_monasca.py | 136 ++++++++++++++++++ .../tools/bhd/bhd_event_report.py | 66 +++++++++ broadview_collector/tools/bst_burst.py | 4 +- 9 files changed, 255 insertions(+), 3 deletions(-) create mode 100644 broadview_collector/handlers/bhdhandler.py create mode 100644 broadview_collector/serializers/bhd_to_monasca.py create mode 100644 broadview_collector/tools/bhd/bhd_event_report.py diff --git a/broadview_collector/handlers/bhdhandler.py b/broadview_collector/handlers/bhdhandler.py new file mode 100644 index 0000000..b053402 --- /dev/null +++ b/broadview_collector/handlers/bhdhandler.py @@ -0,0 +1,32 @@ +# (C) Copyright Broadcom Corporation 2016 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from broadviewhandlerbase import BroadViewHandlerBase +from broadview_lib.bhd.bhd_parser import BHDParser + +class BroadViewHandler(BroadViewHandlerBase): + def __init__(self): + pass + + def handlePOST(self, path, ctype, length, data): + parser = BHDParser() + try: + handled = parser.process(data) + except: + handled = False + return (parser, handled) + + def __repr__(self): + return "BHD Handler" + diff --git a/broadview_collector/plugins/broadviewpublisherbase.py b/broadview_collector/plugins/broadviewpublisherbase.py index 20c34e1..7b4e00d 100644 --- a/broadview_collector/plugins/broadviewpublisherbase.py +++ b/broadview_collector/plugins/broadviewpublisherbase.py @@ -14,6 +14,7 @@ from broadview_lib.pt.pt_parser import PTParser from broadview_lib.bst.bst_parser import BSTParser +from broadview_lib.bhd.bhd_parser import BHDParser class BroadViewPublisherBase(object): def __init__(self): @@ -28,3 +29,6 @@ class BroadViewPublisherBase(object): def isPT(self, parser): return isinstance(parser, PTParser) + def isBHD(self, parser): + return isinstance(parser, BHDParser) + diff --git a/broadview_collector/plugins/kafkapublisher.py b/broadview_collector/plugins/kafkapublisher.py index 707b53d..9d9a839 100644 --- a/broadview_collector/plugins/kafkapublisher.py +++ b/broadview_collector/plugins/kafkapublisher.py @@ -15,6 +15,8 @@ from broadviewpublisherbase import BroadViewPublisherBase import kafka from broadview_collector.serializers.bst_to_monasca import BSTToMonasca +from broadview_collector.serializers.pt_to_monasca import PTToMonasca +from broadview_collector.serializers.bhd_to_monasca import BHDToMonasca import json import ConfigParser import sys @@ -65,6 +67,9 @@ class BroadViewPublisher(BroadViewPublisherBase): elif self.isPT(data): self._topic = "broadview-pt" success, sdata = PTToMonasca().serialize(host, data) + elif self.isBHD(data): + self._topic = "broadview-bhd" + success, sdata = BHDToMonasca().serialize(host, data) else: success = False if success: diff --git a/broadview_collector/plugins/logpublisher.py b/broadview_collector/plugins/logpublisher.py index 1330270..f7eb54b 100644 --- a/broadview_collector/plugins/logpublisher.py +++ b/broadview_collector/plugins/logpublisher.py @@ -19,6 +19,7 @@ from broadviewpublisherbase import BroadViewPublisherBase from broadview_collector.serializers.bst_to_monasca import BSTToMonasca from broadview_collector.serializers.pt_to_monasca import PTToMonasca +from broadview_collector.serializers.bhd_to_monasca import BHDToMonasca import json try: from oslo_log import log @@ -59,8 +60,10 @@ class BroadViewPublisher(BroadViewPublisherBase): success, sdata = BSTToMonasca().serialize(host, data) elif self.isPT(data): success, sdata = PTToMonasca().serialize(host, data) + elif self.isBHD(data): + success, sdata = BHDToMonasca().serialize(host, data) else: - LOG.info("log publisher is not PT or BST") + LOG.info("log publisher is not PT, BHD, or BST") success = False if success: sdata = json.loads(sdata) diff --git a/broadview_collector/plugins/monascapublisher.py b/broadview_collector/plugins/monascapublisher.py index b36593d..5cebb4b 100644 --- a/broadview_collector/plugins/monascapublisher.py +++ b/broadview_collector/plugins/monascapublisher.py @@ -17,6 +17,7 @@ from monascaclient import client import monascaclient.exc as exc from broadview_collector.serializers.bst_to_monasca import BSTToMonasca from broadview_collector.serializers.pt_to_monasca import PTToMonasca +from broadview_collector.serializers.bhd_to_monasca import BHDToMonasca import json import ConfigParser @@ -69,6 +70,8 @@ class BroadViewPublisher(BroadViewPublisherBase): success, sdata = BSTToMonasca().serialize(host, data) elif self.isPT(data): success, sdata = PTToMonasca().serialize(host, data) + elif self.isBHD(data): + success, sdata = BHDToMonasca().serialize(host, data) else: success = False sdata = None diff --git a/broadview_collector/plugins/syslogpublisher.py b/broadview_collector/plugins/syslogpublisher.py index 6797b87..523f47c 100644 --- a/broadview_collector/plugins/syslogpublisher.py +++ b/broadview_collector/plugins/syslogpublisher.py @@ -19,6 +19,7 @@ from broadviewpublisherbase import BroadViewPublisherBase from broadview_collector.serializers.bst_to_monasca import BSTToMonasca from broadview_collector.serializers.pt_to_monasca import PTToMonasca +from broadview_collector.serializers.bhd_to_monasca import BHDToMonasca import json import syslog @@ -36,6 +37,8 @@ class BroadViewPublisher(BroadViewPublisherBase): success, sdata = BSTToMonasca().serialize(host, data) elif self.isPT(data): success, sdata = PTToMonasca().serialize(host, data) + elif self.isBHD(data): + success, sdata = BHDToMonasca().serialize(host, data) else: success = False if success: diff --git a/broadview_collector/serializers/bhd_to_monasca.py b/broadview_collector/serializers/bhd_to_monasca.py new file mode 100644 index 0000000..e2f0100 --- /dev/null +++ b/broadview_collector/serializers/bhd_to_monasca.py @@ -0,0 +1,136 @@ +# (C) Copyright Broadcom Corporation 2016 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from broadviewserializerbase import BroadViewSerializerBase +from broadview_lib.bhd.bhd_parser import BHDParser, ReportTypes +import json +import unittest +import datetime +import time + + +class BHDToMonasca(BroadViewSerializerBase): + ''' + Class that converts Black Hole Detection object model to OpenStack Monasca + metrics. + See broadview-collector/doc/bhd_to_monasca_serializer.md for documentation + ''' + + def __init__(self): + pass + + def __serializeToJSON(self, host, data): + ret = [] + try: + timestamp = time.mktime(data.getTimestamp()) * 1000 + except: + timestamp = int(time.mktime(datetime.datetime.utcnow().timetuple())) \ + * 1000 * 1000 + asic = data.getASICId() + + ''' + Note that monasca requires a value field. Where required, but not + obvious value is present in the data, we use 0 and notate that the + value is to be ignored in the metadata + ''' + + x = data.getBlackHoleEventReport() + + print("{}".format(x)) + m = {} + m["name"] = "broadview.bhd." + repr(x) + m["timestamp"] = timestamp + m["dimensions"] = {} + m["dimensions"]["asic-id"] = asic + m["dimensions"]["bv-agent"] = host + m["dimensions"]["ingress-port"] = x.getIngressPort() + m["dimensions"]["egress-port-list"] = x.getEgressPortList() + m["dimensions"]["sample-packet"] = x.getSamplePacket() + m["dimensions"]["ignore-value"] = 0 + m["value"] = x.getBlackHoledPacketCount() + ret.append(m) + + return json.dumps(ret) + + def serialize(self, host, data): + # serialize a parsed BHD report to Monasca metrics + ret = (False, None) + + s = self.__serializeToJSON(host, data) + + if s: + ret = (True, s) + + return ret + + def __repr__(self): + return "Black Hole Detection To Monasca Serializer" + +class TestSerializer(unittest.TestCase): + + def setUp(self): + self._host = "127.0.0.1" + self.black_hole_event_report_1 = { + "jsonrpc": "2.0", + "method": "get-black-hole-event-report", + "asic-id": "1", + "version": "2", + "time-stamp": "2014-11-18 - 00:15:04 ", + "report": { + "ingress-port": "1", + "egress-port-list": ["2", "3"], + "black-holed-packet-count": 100, + "sample-packet": "0010203232.." + } + } + + def test_black_hole_event_report_1(self): + rep = BHDParser() + rep.process(self.black_hole_event_report_1) + serializer = BHDToMonasca() + ret = serializer.serialize(self._host, rep) + self.assertEqual(ret[0], True) + data = json.loads(ret[1]) + + dim = data[0]["dimensions"] + + self.assertTrue("bv-agent" in dim) + self.assertTrue("asic-id" in dim) + self.assertTrue("timestamp" in data[0]) + self.assertTrue("name" in data[0]) + self.assertEqual(data[0]["name"], "broadview.bhd.get-black-hole-event-report") + self.assertTrue("dimensions" in data[0]) + timestamp = int(data[0]["timestamp"]) / 1000 + t1 = datetime.datetime.fromtimestamp(timestamp) + t1 = t1.strftime("%Y-%m-%d - %H:%M:%S") + t2 = self.black_hole_event_report_1["time-stamp"].strip() + self.assertEqual(t1, t2) + dim = data[0]["dimensions"] + self.assertEqual(dim["asic-id"], self.black_hole_event_report_1["asic-id"]) + self.assertTrue("ingress-port" in dim) + self.assertEqual(dim["ingress-port"], "1") + self.assertTrue("egress-port-list" in dim) + self.assertEqual(len(dim["egress-port-list"]), 2) + self.assertTrue("2" in dim["egress-port-list"]) + self.assertTrue("3" in dim["egress-port-list"]) + self.assertTrue("sample-packet" in dim) + self.assertEqual(dim["sample-packet"], "0010203232..") + self.assertTrue("ignore-value" in dim) + self.assertEqual(dim["ignore-value"], 0) + self.assertTrue("value" in data[0]) + self.assertEqual(data[0]["value"], 100) + +if __name__ == "__main__": + unittest.main() + diff --git a/broadview_collector/tools/bhd/bhd_event_report.py b/broadview_collector/tools/bhd/bhd_event_report.py new file mode 100644 index 0000000..80cad3a --- /dev/null +++ b/broadview_collector/tools/bhd/bhd_event_report.py @@ -0,0 +1,66 @@ +# (C) Copyright Broadcom Corporation 2016 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import requests +import datetime +import time + +# Change these to the host and port the collector is listening on + +host = "127.0.0.1" +port = 8082 + +''' + +Program to generate a black hole event report. + +''' + +class BHDReport(): + def setUp(self): + # convert datetime string to agent timestamp + + print("{} UTC".format(datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"))) + d = str(datetime.datetime.now()).split(" ") + t = d[1].split(".")[0] + d = "{} - {}".format(d[0], t) + print("Setting timestamp to {}".format(d)) + self.black_hole_event_report = { + "jsonrpc": "2.0", + "method": "get-black-hole-event-report", + "asic-id": "1", + "version": "2", + "time-stamp": d, + "report": { + "ingress-port": "1", + "egress-port-list": ["2", "3"], + "black-holed-packet-count": 100, + "sample-packet": "0010203232.." + } + } + + def send(self): + self.setUp() + j = json.dumps(self.black_hole_event_report) + r = requests.post('http://{}:{}'.format(host, port), json=j) + +def main(): + x = BHDReport() + x.send() + +if __name__ == "__main__": + main() + + diff --git a/broadview_collector/tools/bst_burst.py b/broadview_collector/tools/bst_burst.py index ef9d3a2..f02d9ef 100644 --- a/broadview_collector/tools/bst_burst.py +++ b/broadview_collector/tools/bst_burst.py @@ -19,12 +19,12 @@ import time # Change these to the host and port the collector is listening on -host = "10.14.244.106" +host = "127.0.0.1" port = 8082 ''' -Program to generate a similated microburst. +Program to generate a simulated microburst. A great way to use this is from a loop in bash: