From 41800dd195efaf99119b44a6681ec36ea7bcde38 Mon Sep 17 00:00:00 2001 From: Andrea Adams Date: Fri, 2 Jun 2017 13:39:21 -0600 Subject: [PATCH] Add a query language for group, inhibit, and silence rules The new alarm rules will each have an expression in their definition which will need to be parsed by both the Monasca- API and the Monasca-Notification-Engine. Documentation for this will be included in the API along with descriptions of the new rules. Story: 2000939 Task: 4692 Change-Id: I1a98fafae8dfdfa6fdb2eb66f4a4a4f40e518e46 --- .../monasca_query_language/__init__.py | 0 .../monasca_query_language/aql_parser.py | 96 +++++++++ .../monasca_query_language/exceptions.py | 18 ++ .../query_structures.py | 186 ++++++++++++++++++ .../tests/test_monasca_query_language.py | 186 ++++++++++++++++++ requirements.txt | 1 + 6 files changed, 487 insertions(+) create mode 100644 monasca_common/monasca_query_language/__init__.py create mode 100644 monasca_common/monasca_query_language/aql_parser.py create mode 100644 monasca_common/monasca_query_language/exceptions.py create mode 100644 monasca_common/monasca_query_language/query_structures.py create mode 100644 monasca_common/tests/test_monasca_query_language.py diff --git a/monasca_common/monasca_query_language/__init__.py b/monasca_common/monasca_query_language/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/monasca_common/monasca_query_language/aql_parser.py b/monasca_common/monasca_query_language/aql_parser.py new file mode 100644 index 00000000..da741881 --- /dev/null +++ b/monasca_common/monasca_query_language/aql_parser.py @@ -0,0 +1,96 @@ +# (C) Copyright 2017 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import six +import sys +import time + +import pyparsing + +from monasca_common.monasca_query_language import query_structures + +COMMA = pyparsing.Suppress(pyparsing.Literal(",")) +LPAREN = pyparsing.Suppress(pyparsing.Literal("(")) +RPAREN = pyparsing.Suppress(pyparsing.Literal(")")) +LBRACE = pyparsing.Suppress(pyparsing.Literal("{")) +RBRACE = pyparsing.Suppress(pyparsing.Literal("}")) +LBRACKET = pyparsing.Suppress(pyparsing.Literal("[")) +RBRACKET = pyparsing.Suppress(pyparsing.Literal("]")) + +MINUS = pyparsing.Literal("-") + +integer_number = pyparsing.Word(pyparsing.nums) +decimal_number = (pyparsing.Optional(MINUS) + integer_number + + pyparsing.Optional("." + integer_number)) +decimal_number.setParseAction(lambda tokens: float("".join(tokens))) + +# Initialize non-ascii unicode code points in the Basic Multilingual Plane. +unicode_printables = u''.join( + six.unichr(c) for c in range(128, 65536) if not six.unichr(c).isspace()) + +# Does not like comma. No Literals from above allowed. +valid_identifier_chars = ( + (unicode_printables + pyparsing.alphanums + ".-_#$%&'*+/:;?@[\\]^`|")) + +metric_name = ( + pyparsing.Word(pyparsing.alphas, valid_identifier_chars, min=1, max=255)("metric_name")) +dimension_name = pyparsing.Word(valid_identifier_chars + ' ', min=1, max=255) +dimension_value = pyparsing.Word(valid_identifier_chars + ' ', min=1, max=255) + +dim_comparison_op = pyparsing.oneOf("=") + +dimension = dimension_name + dim_comparison_op + dimension_value +dimension.setParseAction(query_structures.Dimension) + +dimension_list = pyparsing.Group((LBRACE + pyparsing.Optional( + pyparsing.delimitedList(dimension)) + + RBRACE)) + +metric = (metric_name + pyparsing.Optional(dimension_list) | + pyparsing.Optional(metric_name) + dimension_list) +metric.addParseAction(query_structures.MetricSelector) + +source = pyparsing.Keyword("source") +source_expression = source + metric +source_expression.addParseAction(query_structures.SourceExpression) + +targets = pyparsing.Keyword("targets") +targets_expression = targets + metric +targets_expression.addParseAction(query_structures.TargetsExpression) + +excludes = pyparsing.Keyword("excluding") +excludes_expression = excludes + metric +excludes_expression.addParseAction(query_structures.ExcludesExpression) + +group_by = pyparsing.Keyword("group by") +group_by_expr = group_by + pyparsing.delimitedList(dimension_name) +group_by_expr.addParseAction(query_structures.GroupByExpression) + +grammar = (pyparsing.Optional(source_expression) + + pyparsing.Optional(targets_expression) + + pyparsing.Optional(excludes_expression) + + pyparsing.Optional(group_by_expr)) +grammar.addParseAction(query_structures.Rule) + + +class RuleExpressionParser(object): + + def __init__(self, expr): + self._expr = expr + + def parse(self): + parse_result = grammar.parseString(self._expr, parseAll=True) + return parse_result diff --git a/monasca_common/monasca_query_language/exceptions.py b/monasca_common/monasca_query_language/exceptions.py new file mode 100644 index 00000000..1da5c25f --- /dev/null +++ b/monasca_common/monasca_query_language/exceptions.py @@ -0,0 +1,18 @@ +# (C) Copyright 2017 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class InvalidExpressionException(Exception): + pass diff --git a/monasca_common/monasca_query_language/query_structures.py b/monasca_common/monasca_query_language/query_structures.py new file mode 100644 index 00000000..2b689d96 --- /dev/null +++ b/monasca_common/monasca_query_language/query_structures.py @@ -0,0 +1,186 @@ +# (C) Copyright 2017 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pyparsing + +from monasca_common.monasca_query_language import exceptions + + +class Dimension(object): + def __init__(self, tokens): + self.args = tokens + self.key = tokens[0] + self.operator = tokens[1] + self.value = tokens[2] + + def __str__(self): + return "Dimension(key={},operator='{}',value={})".format( + self.key, self.operator, self.value) + + +class MetricSelector(object): + def __init__(self, tokens): + self.args = tokens + self.name = None + self.dimensions = {} + _dimensions = [] + for token in tokens: + if isinstance(token, str): + self.name = token + elif isinstance(token, pyparsing.ParseResults): + _dimensions = token + + for dim in _dimensions: + self.dimensions[dim.key] = dim.value + if self.name is not None: + self.dimensions["__metricName__"] = self.name + + def get_filters(self): + return self.dimensions + + def __repr__(self): + return "MetricSelector(name={},dimensions={})".format( + self.name, self.dimensions) + + def __str__(self): + return self.__repr__() + + +class LogicalExpression(object): + def __init__(self, tokens): + self.args = tokens + self.left_operand = tokens[0][0] + self.operator = None + self.right_operand = None + if len(tokens[0]) > 1: + self.operator = tokens[0][1] + if len(tokens[0]) > 2: + self.right_operand = tokens[0][2] + + @property + def normalized_operator(self): + if self.operator == '&&': + result = 'and' + elif self.operator == '||': + result = 'or' + else: + result = self.operator + return result + + def get_filters(self): + left_filters = self.left_operand.get_filters() + right_filters = self.right_operand.get_filters() + for key, value in right_filters.items(): + if key in left_filters and left_filters[key] != value: + raise exceptions.InvalidExpressionException( + "Duplicate keys specified ".format(key)) + left_filters[key] = value + return left_filters + + def __str__(self): + return "LogicalExpression(left={},operator='{}',right={})".format( + self.left_operand, self.operator, self.right_operand) + return self.__repr__() + + +class SourceExpression(object): + def __init__(self, tokens): + self.args = tokens + self.source = tokens[1] + + def get_filters(self): + return self.source.get_filters() + + def __str__(self): + return "SourceExpression(source={})".format(self.source) + + +class TargetsExpression(object): + def __init__(self, tokens): + self.args = tokens + self.target = tokens[1] + + def get_filters(self): + return self.target.get_filters() + + def __str__(self): + return "TargetExpression(target={})".format(self.target) + + +class ExcludesExpression(object): + def __init__(self, tokens): + self.args = tokens + self.exclude = tokens[1] + + def get_filters(self): + return self.exclude.get_filters() + + def __str__(self): + return "ExcludesExpression(exclude={})".format(self.exclude) + + +class GroupByExpression(object): + def __init__(self, tokens): + self.args = tokens + self.group_keys = tokens[1:] + + def get_filters(self): + return self.group_keys + + def __str__(self): + return "GroupByExpression({})".format(self.group_keys) + + +class Rule(object): + def __init__(self, tokens): + self.source = None + self.target = None + self.excludes = None + self.group_by = None + for token in tokens: + if isinstance(token, SourceExpression): + self.source = token + elif isinstance(token, TargetsExpression): + self.target = token + elif isinstance(token, ExcludesExpression): + self.excludes = token + elif isinstance(token, GroupByExpression): + self.group_by = token + + def get_struct(self, _type): + result = {} + if _type == "silence": + result['matchers'] = self.target.get_filters() if self.target is not None else {} + if any([self.source, self.group_by, self.excludes]): + raise exceptions.InvalidExpressionException( + "Silence rule contains unexpected elements") + elif _type == "inhibit": + result['source_match'] = self.source.get_filters() if self.source is not None else {} + result['target_match'] = self.target.get_filters() if self.target is not None else {} + result['equal'] = self.group_by.get_filters() if self.group_by is not None else [] + result['exclusions'] = self.excludes.get_filters() if self.excludes is not None else {} + elif _type == "group": + result['matchers'] = self.group_by.get_filters() if self.group_by is not None else [] + result['exclusions'] = self.excludes.get_filters() if self.excludes is not None else {} + if any([self.source, self.target]): + raise exceptions.InvalidExpressionException( + "Group rule contains unexpected elements") + else: + raise exceptions.InvalidExpressionException("Unknown type for expression") + return result + + def __str__(self): + return "Rule(source={},target={},excludes={},group_by={})".format( + self.source, self.target, self.excludes, self.group_by) diff --git a/monasca_common/tests/test_monasca_query_language.py b/monasca_common/tests/test_monasca_query_language.py new file mode 100644 index 00000000..c2df3442 --- /dev/null +++ b/monasca_common/tests/test_monasca_query_language.py @@ -0,0 +1,186 @@ +# (C) Copyright 2017 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslotest import base +import pyparsing + +from monasca_common.monasca_query_language import aql_parser +from monasca_common.monasca_query_language import exceptions +from monasca_common.monasca_query_language import query_structures + + +class TestMonascaQueryLanguage(base.BaseTestCase): + + def test_parse_group_expression(self): + expressions = [ + "", + "excluding metric_two", + "group by hostname, service", + "excluding metric_two group by hostname, service", + "group by __severity__", + "excluding {__severity__=HIGH} group by __severity__", + "excluding {__severity__=HIGH, hostname=host1} group by __severity__, hostname", + "group by excluding" # excluding is an acceptable metric name + ] + negative_expressions = [ + "group by hostname excluding {__metricName__=metric_two}", + "excluding metric_one excluding metric_two", + "targets metric_one", + ] + matchers = [ + [], + [], + ["hostname", "service"], + ["hostname", "service"], + ["__severity__"], + ["__severity__"], + ["__severity__", "hostname"], + ["excluding"] + ] + exclusions = [ + {}, + {"__metricName__": "metric_two"}, + {}, + {"__metricName__": "metric_two"}, + {}, + {"__severity__": "HIGH"}, + {"__severity__": "HIGH", "hostname": "host1"}, + {}, + ] + for i in range(len(expressions)): + result = aql_parser.RuleExpressionParser(expressions[i]).parse() + result = result[0].get_struct("group") + self.assertEqual(result['matchers'], matchers[i]) + self.assertEqual(result['exclusions'], exclusions[i]) + for negative_expression in negative_expressions: + try: + result = aql_parser.RuleExpressionParser(negative_expression) + self.assertRaises(exceptions.InvalidExpressionException, + result.parse()) + except TypeError: + pass + except pyparsing.ParseException: + pass + + def test_parse_inhibit_rule(self): + expressions = [ + "", + "source metric_one", + "targets metric_two", + "source metric_one targets metric_two", + "source metric_one targets metric_two excluding metric_three", + "source metric_one targets metric_two excluding metric_three group by hostname", + "source metric_one targets metric_two group by hostname", + "source metric_one group by hostname", + "source {__severity__=HIGH} targets {__severity__=LOW} excluding " + "{__alarmName__=alarm_one} group by __alarmName__" + ] + negative_expressions = [ + "targets metric_two source_metric_one" + ] + source = [ + {}, + {"__metricName__": "metric_one"}, + {}, + {"__metricName__": "metric_one"}, + {"__metricName__": "metric_one"}, + {"__metricName__": "metric_one"}, + {"__metricName__": "metric_one"}, + {"__metricName__": "metric_one"}, + {"__severity__": "HIGH"}, + ] + target = [ + {}, + {}, + {"__metricName__": "metric_two"}, + {"__metricName__": "metric_two"}, + {"__metricName__": "metric_two"}, + {"__metricName__": "metric_two"}, + {"__metricName__": "metric_two"}, + {}, + {"__severity__": "LOW"} + ] + equals = [ + [], + [], + [], + [], + [], + ["hostname"], + ["hostname"], + ["hostname"], + ["__alarmName__"] + ] + exclusions = [ + {}, + {}, + {}, + {}, + {"__metricName__": "metric_three"}, + {"__metricName__": "metric_three"}, + {}, + {}, + {"__alarmName__": "alarm_one"} + ] + for i in range(len(expressions)): + result = aql_parser.RuleExpressionParser(expressions[i]).parse() + result = result[0].get_struct("inhibit") + self.assertEqual(result['source_match'], source[i]) + self.assertEqual(result['target_match'], target[i]) + self.assertEqual(result['equal'], equals[i]) + self.assertEqual(result['exclusions'], exclusions[i]) + + for expression in negative_expressions: + try: + result = aql_parser.RuleExpressionParser(expression) + self.assertRaises(exceptions.InvalidExpressionException, + result.parse()) + except pyparsing.ParseException: + pass + + def test_parse_silence_rule(self): + expressions = [ + "", + "targets metric_one", + "targets metric_one{}", + "targets metric_one{hostname=host_one}", + "targets metric_one{hostname=host_one, region=region_one}", + ] + negative_expressions = [ + "excludes metric_one", + "source metric_one", + "group by hostname", + "targets metric_one, {hostname=host_one}", + ] + matchers = [ + {}, + {"__metricName__": "metric_one"}, + {"__metricName__": "metric_one"}, + {"__metricName__": "metric_one", "hostname": "host_one"}, + {"__metricName__": "metric_one", "hostname": "host_one", "region": "region_one"}, + ] + for i in range(len(expressions)): + result = aql_parser.RuleExpressionParser(expressions[i]).parse() + result = result[0].get_struct("silence") + self.assertEqual(result['matchers'], matchers[i]) + + for expression in negative_expressions: + try: + self.assertRaises(exceptions.InvalidExpressionException, + aql_parser.RuleExpressionParser(expression).parse()) + except TypeError: + pass + except pyparsing.ParseException: + pass diff --git a/requirements.txt b/requirements.txt index 648d9e06..2dedcee5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ pykafka>=2.5.0 # Apache 2.0 License PyMySQL>=0.7.6 # MIT License oslo.config>=4.0.0 # Apache-2.0 pbr!=2.1.0,>=2.0.0 # Apache-2.0 +pyparsing>=2.1.0 # MIT ujson>=1.35 # BSD