From 5812bd84297aec114b11520a47c2d237ce5d40f3 Mon Sep 17 00:00:00 2001 From: Joan Varvenne Date: Thu, 14 Jul 2016 17:15:55 +0100 Subject: [PATCH] This commit introduces the first version of Banana configuration language. As of this commit, to change the configuration using Banana, we need to make an HTTP POST request to `/banana` REST API. This API is temporary and is likely to be changed later. The implementation is done entirely in the `banana` module. Under this module there are: * `typeck` module contains the type checker, * `grammar` module contains the parser and the AST and, * `eval` module contains the interpreter. Additionally, a test framework has been created to ease the test of particular conditions of the language. Within the banana module, there is a README.md file for each associated sub-module explaining further the details of the language. Once this commit is merged, there's still a lot that can be improved: - All components should be tested in Banana. - The 'deadpathck' pass could be improved (see TODO) - We don't support generated JSON ingestors yet. - Imports will be key for reusability (not implemented). Change-Id: I1305bdfa0606f30619b31404afbe0acf111c029f --- config/iptables_anomalies.banana | 16 + config/iptables_anomalies.json | 8 +- config/markov_source_config.banana | 57 ++ config/markov_source_config.json | 20 +- config/metric_experiments.banana | 30 + config/metric_experiments.json | 32 +- config/monasca_aggregate_all_fn.banana | 13 + config/monasca_aggregate_all_fn.json | 36 +- config_dsl.py | 9 +- doc/banana.md | 141 ++++ doc/cli.md | 187 +++++ doc/dsl.md | 129 ---- doc/getting_started.md | 2 +- monasca_analytics/banana/README.md | 40 ++ .../{dsl => banana/cli}/__init__.py | 0 .../{dsl => banana/cli}/const.py | 0 monasca_analytics/{dsl => banana/cli}/dsl.py | 4 + .../{dsl => banana/cli}/interpreter.py | 6 +- .../{dsl => banana/cli}/parser.py | 11 +- monasca_analytics/banana/deadpathck/README.md | 22 + .../banana/deadpathck}/__init__.py | 0 monasca_analytics/banana/deadpathck/config.py | 109 +++ monasca_analytics/banana/deadpathck/dag.py | 56 ++ monasca_analytics/banana/emitter.py | 90 +++ monasca_analytics/banana/eval/README.md | 50 ++ monasca_analytics/banana/eval/__init__.py | 0 monasca_analytics/banana/eval/config.py | 262 +++++++ monasca_analytics/banana/eval/ctx.py | 119 ++++ monasca_analytics/banana/eval/old_style.py | 56 ++ monasca_analytics/banana/grammar/README.md | 63 ++ monasca_analytics/banana/grammar/__init__.py | 0 monasca_analytics/banana/grammar/ast.py | 674 ++++++++++++++++++ monasca_analytics/banana/grammar/base_ast.py | 145 ++++ monasca_analytics/banana/grammar/config.py | 285 ++++++++ monasca_analytics/banana/grammar/const.py | 31 + monasca_analytics/banana/pass_manager.py | 91 +++ monasca_analytics/banana/typeck/README.md | 85 +++ monasca_analytics/banana/typeck/__init__.py | 0 monasca_analytics/banana/typeck/config.py | 281 ++++++++ .../banana/typeck/connections.py | 59 ++ monasca_analytics/banana/typeck/type_table.py | 199 ++++++ monasca_analytics/banana/typeck/type_util.py | 559 +++++++++++++++ monasca_analytics/component/base.py | 32 + monasca_analytics/component/params.py | 47 ++ monasca_analytics/config/connection.py | 3 +- monasca_analytics/config/creation.py | 4 +- monasca_analytics/exception/banana.py | 312 +++++++- monasca_analytics/ingestor/cloud.py | 4 + monasca_analytics/ingestor/iptables.py | 4 + monasca_analytics/ldp/cloud_causality.py | 4 + monasca_analytics/ldp/iptables_ldp.py | 4 + monasca_analytics/ldp/monasca_aggregate.py | 44 +- monasca_analytics/ldp/monasca_combine.py | 202 +++--- monasca_analytics/ldp/monasca_derivative.py | 24 +- monasca_analytics/monanas.py | 23 +- monasca_analytics/parsing/__init__.py | 0 monasca_analytics/{banana => parsing}/api.py | 8 +- .../{banana => parsing}/expression.py | 5 +- .../{banana => parsing}/private.py | 6 +- monasca_analytics/sink/file.py | 20 +- monasca_analytics/sink/iptables_sqlite.py | 14 +- monasca_analytics/sink/kafkas.py | 30 +- .../sink/sink_config_validator.py | 15 +- monasca_analytics/sink/stdout_sink.py | 4 + monasca_analytics/sml/lingam.py | 19 +- monasca_analytics/sml/svm_one_class.py | 4 + .../source/cloud_markov_chain.py | 42 +- .../source/iptables_markov_chain.py | 22 +- monasca_analytics/source/kafka.py | 16 + monasca_analytics/source/markov_chain/base.py | 3 +- .../source/monasca_markov_chain.py | 20 +- monasca_analytics/source/randoms.py | 19 + monasca_analytics/spark/driver.py | 27 +- monasca_analytics/util/common_util.py | 3 - monasca_analytics/util/string_util.py | 123 ++++ monasca_analytics/voter/pick_index.py | 23 +- .../web_service/request_handler.py | 53 +- monasca_analytics/web_service/web_service.py | 4 +- .../web_service/web_service_model.py | 9 + test/banana/deadpathck/__init__.py | 0 .../test_at_least_one_sink_or_source.py | 95 +++ test/banana/deadpathck/test_deadpathck.py | 111 +++ test/banana/eval/__init__.py | 0 .../cloud_markov_chain_source.banana | 3 + .../eval_each_stmt/calc_expressions.banana | 8 + .../eval_each_stmt/json_expressions.banana | 4 + test/banana/eval/test_eval_config.py | 77 ++ test/banana/grammar/__init__.py | 0 .../component_cant_be_in_object.banana | 2 + .../json_obj_props_cant_shadow.banana | 4 + .../wrong_symbol_in_connection.banana | 3 + .../should_fail/wrong_symbol_in_expr.banana | 2 + .../should_pass/component_creation.banana | 12 + .../grammar/should_pass/connection.banana | 13 + .../grammar/should_pass/dot_accesses.banana | 10 + .../grammar/should_pass/eval_json_obj.banana | 6 + .../should_pass/eval_simple_expression.banana | 8 + .../grammar/should_pass/full_example.banana | 14 + .../grammar/should_pass/json_obj.banana | 8 + .../should_pass/jsonlike_objects.banana | 6 + .../grammar/should_pass/scary_json_ops.banana | 4 + .../should_pass/valid_expression.banana | 5 + .../various_literal_assignments.banana | 5 + test/banana/grammar/test_config.py | 126 ++++ test/banana/test_api.py | 8 +- test/banana/test_config_examples.py | 104 +++ test/banana/typeck/__init__.py | 0 .../comp_var_cant_be_shadowed.banana | 3 + .../component_cant_be_in_object.banana | 4 + .../component_unkown_param_in_arg.banana | 2 + .../component_unkown_param_in_stmt.banana | 9 + .../should_fail/impossible_connection.banana | 5 + .../incompatible_json_props_types.banana | 8 + .../should_fail/invalid_str_concat.banana | 14 + .../type_has_no_nested_properties.banana | 9 + .../should_fail/type_is_not_a_subtype.banana | 18 + .../should_fail/wrong_param_type.banana | 7 + .../wrong_parameter_name_throw.banana | 9 + .../compatible_json_props_types.banana | 3 + .../component_param_has_a_type.banana | 3 + .../should_pass/expression_shenanigan.banana | 19 + .../typeck/should_pass/monasca_combine.banana | 22 + .../nested_value_in_component_params.banana | 9 + .../number_can_be_shadowed_by_object.banana | 4 + .../object_can_be_shadowed_by_number.banana | 4 + .../should_pass/simple_connection.banana | 7 + .../typeck/should_pass/simple_literal.banana | 4 + .../should_pass/type_is_a_subtype.banana | 20 + .../should_pass/valid_str_concat.banana | 4 + .../various_object_use_case.banana | 7 + test/banana/typeck/test_typeck_config.py | 155 ++++ test/cli/__init__.py | 0 test/{dsl => cli}/test_dsl.py | 7 +- test/{dsl => cli}/test_interpreter.py | 11 +- test/{dsl => cli}/test_parser.py | 4 +- test/ldp/test_monasca_aggregator.py | 4 +- test/ldp/test_monasca_combine.py | 2 +- test/mocks/sml_mocks.py | 42 +- test/mocks/sources.py | 6 +- test/resources/test_json.json | 4 +- test/sink/test_base_sqlite.py | 12 + test/sink/test_kafka.py | 8 +- test/sink/test_sink_config_validator.py | 14 +- test/source/test_iptables_markov_chain.py | 15 +- test/source/test_markov_chain.py | 10 +- test/spark/test_driver.py | 2 - test/voter/test_base_voter.py | 6 +- 147 files changed, 5818 insertions(+), 489 deletions(-) create mode 100644 config/iptables_anomalies.banana create mode 100644 config/markov_source_config.banana create mode 100644 config/metric_experiments.banana create mode 100644 config/monasca_aggregate_all_fn.banana create mode 100644 doc/banana.md create mode 100644 doc/cli.md delete mode 100644 doc/dsl.md create mode 100644 monasca_analytics/banana/README.md rename monasca_analytics/{dsl => banana/cli}/__init__.py (100%) rename monasca_analytics/{dsl => banana/cli}/const.py (100%) rename monasca_analytics/{dsl => banana/cli}/dsl.py (99%) rename monasca_analytics/{dsl => banana/cli}/interpreter.py (98%) rename monasca_analytics/{dsl => banana/cli}/parser.py (90%) create mode 100644 monasca_analytics/banana/deadpathck/README.md rename {test/dsl => monasca_analytics/banana/deadpathck}/__init__.py (100%) create mode 100644 monasca_analytics/banana/deadpathck/config.py create mode 100644 monasca_analytics/banana/deadpathck/dag.py create mode 100644 monasca_analytics/banana/emitter.py create mode 100644 monasca_analytics/banana/eval/README.md create mode 100644 monasca_analytics/banana/eval/__init__.py create mode 100644 monasca_analytics/banana/eval/config.py create mode 100644 monasca_analytics/banana/eval/ctx.py create mode 100644 monasca_analytics/banana/eval/old_style.py create mode 100644 monasca_analytics/banana/grammar/README.md create mode 100644 monasca_analytics/banana/grammar/__init__.py create mode 100644 monasca_analytics/banana/grammar/ast.py create mode 100644 monasca_analytics/banana/grammar/base_ast.py create mode 100644 monasca_analytics/banana/grammar/config.py create mode 100644 monasca_analytics/banana/grammar/const.py create mode 100644 monasca_analytics/banana/pass_manager.py create mode 100644 monasca_analytics/banana/typeck/README.md create mode 100644 monasca_analytics/banana/typeck/__init__.py create mode 100644 monasca_analytics/banana/typeck/config.py create mode 100644 monasca_analytics/banana/typeck/connections.py create mode 100644 monasca_analytics/banana/typeck/type_table.py create mode 100644 monasca_analytics/banana/typeck/type_util.py create mode 100644 monasca_analytics/component/params.py create mode 100644 monasca_analytics/parsing/__init__.py rename monasca_analytics/{banana => parsing}/api.py (74%) rename monasca_analytics/{banana => parsing}/expression.py (99%) rename monasca_analytics/{banana => parsing}/private.py (83%) create mode 100644 monasca_analytics/util/string_util.py create mode 100644 test/banana/deadpathck/__init__.py create mode 100644 test/banana/deadpathck/test_at_least_one_sink_or_source.py create mode 100644 test/banana/deadpathck/test_deadpathck.py create mode 100644 test/banana/eval/__init__.py create mode 100644 test/banana/eval/should_pass/eval_components_only/cloud_markov_chain_source.banana create mode 100644 test/banana/eval/should_pass/eval_each_stmt/calc_expressions.banana create mode 100644 test/banana/eval/should_pass/eval_each_stmt/json_expressions.banana create mode 100644 test/banana/eval/test_eval_config.py create mode 100644 test/banana/grammar/__init__.py create mode 100644 test/banana/grammar/should_fail/component_cant_be_in_object.banana create mode 100644 test/banana/grammar/should_fail/json_obj_props_cant_shadow.banana create mode 100644 test/banana/grammar/should_fail/wrong_symbol_in_connection.banana create mode 100644 test/banana/grammar/should_fail/wrong_symbol_in_expr.banana create mode 100644 test/banana/grammar/should_pass/component_creation.banana create mode 100644 test/banana/grammar/should_pass/connection.banana create mode 100644 test/banana/grammar/should_pass/dot_accesses.banana create mode 100644 test/banana/grammar/should_pass/eval_json_obj.banana create mode 100644 test/banana/grammar/should_pass/eval_simple_expression.banana create mode 100644 test/banana/grammar/should_pass/full_example.banana create mode 100644 test/banana/grammar/should_pass/json_obj.banana create mode 100644 test/banana/grammar/should_pass/jsonlike_objects.banana create mode 100644 test/banana/grammar/should_pass/scary_json_ops.banana create mode 100644 test/banana/grammar/should_pass/valid_expression.banana create mode 100644 test/banana/grammar/should_pass/various_literal_assignments.banana create mode 100644 test/banana/grammar/test_config.py create mode 100644 test/banana/test_config_examples.py create mode 100644 test/banana/typeck/__init__.py create mode 100644 test/banana/typeck/should_fail/comp_var_cant_be_shadowed.banana create mode 100644 test/banana/typeck/should_fail/component_cant_be_in_object.banana create mode 100644 test/banana/typeck/should_fail/component_unkown_param_in_arg.banana create mode 100644 test/banana/typeck/should_fail/component_unkown_param_in_stmt.banana create mode 100644 test/banana/typeck/should_fail/impossible_connection.banana create mode 100644 test/banana/typeck/should_fail/incompatible_json_props_types.banana create mode 100644 test/banana/typeck/should_fail/invalid_str_concat.banana create mode 100644 test/banana/typeck/should_fail/type_has_no_nested_properties.banana create mode 100644 test/banana/typeck/should_fail/type_is_not_a_subtype.banana create mode 100644 test/banana/typeck/should_fail/wrong_param_type.banana create mode 100644 test/banana/typeck/should_fail/wrong_parameter_name_throw.banana create mode 100644 test/banana/typeck/should_pass/compatible_json_props_types.banana create mode 100644 test/banana/typeck/should_pass/component_param_has_a_type.banana create mode 100644 test/banana/typeck/should_pass/expression_shenanigan.banana create mode 100644 test/banana/typeck/should_pass/monasca_combine.banana create mode 100644 test/banana/typeck/should_pass/nested_value_in_component_params.banana create mode 100644 test/banana/typeck/should_pass/number_can_be_shadowed_by_object.banana create mode 100644 test/banana/typeck/should_pass/object_can_be_shadowed_by_number.banana create mode 100644 test/banana/typeck/should_pass/simple_connection.banana create mode 100644 test/banana/typeck/should_pass/simple_literal.banana create mode 100644 test/banana/typeck/should_pass/type_is_a_subtype.banana create mode 100644 test/banana/typeck/should_pass/valid_str_concat.banana create mode 100644 test/banana/typeck/should_pass/various_object_use_case.banana create mode 100644 test/banana/typeck/test_typeck_config.py create mode 100644 test/cli/__init__.py rename test/{dsl => cli}/test_dsl.py (98%) rename test/{dsl => cli}/test_interpreter.py (96%) rename test/{dsl => cli}/test_parser.py (98%) diff --git a/config/iptables_anomalies.banana b/config/iptables_anomalies.banana new file mode 100644 index 0000000..1677f5f --- /dev/null +++ b/config/iptables_anomalies.banana @@ -0,0 +1,16 @@ +############################## +# IP Tables anomalies +# + +sleep = 0.01 +src = IPTablesSource(sleep=sleep) +ing1 = IptablesIngestor() +svm = SvmOneClass() +voter = PickIndexVoter(0) +ldp1 = IptablesLDP() +stdout = StdoutSink() +sqlite = IptablesSQLiteSink() + +src -> [ing1, ldp1] +ing1 -> svm -> voter -> ldp1 +ldp1 -> [sqlite, stdout] diff --git a/config/iptables_anomalies.json b/config/iptables_anomalies.json index 99d5f52..d89102b 100644 --- a/config/iptables_anomalies.json +++ b/config/iptables_anomalies.json @@ -12,9 +12,7 @@ "sources": { "src1": { "module": "IPTablesSource", - "params": { - "server_sleep_in_seconds": 0.01 - } + "sleep": 0.01 } }, "ingestors": { @@ -30,9 +28,7 @@ "voters": { "vot1": { "module": "PickIndexVoter", - "params": { - "index": 0 - } + "index": 0 } }, "sinks": { diff --git a/config/markov_source_config.banana b/config/markov_source_config.banana new file mode 100644 index 0000000..a58f05f --- /dev/null +++ b/config/markov_source_config.banana @@ -0,0 +1,57 @@ +############################## +# Markov source config example +# (cloud-like data model) +# + +src = CloudMarkovChainSource(sleep=0.01) + +src.transitions.web_service = { + "run=>slow": { + "0": 0.001, + "8": 0.02, + "12": 0.07, + "14": 0.07, + "22": 0.03, + "24": 0.00 + }, + "slow=>run": { + "0": 0.99, + "8": 0.7, + "12": 0.1, + "14": 0.1, + "22": 0.8, + "24": 0.9 + }, + "stop=>run": 0.7 +} + +src.transitions.host = { + "on=>off": 0.005, + "off=>on": 0.5 +} + +src.transitions.switch = { + "on=>off": 0.01, + "off=>on": 0.7 +} + +src.triggers.support = { + "get_called" : { + "0": 0.1, + "8": 0.2, + "12": 0.8, + "14": 0.8, + "22": 0.5, + "24": 0.0 + } +} + +ing1 = CloudIngestor() +ling = LiNGAM(threshold=0.5) +voter = PickIndexVoter(0) +sink = KafkaSink(host="localhost", port=9092, topic="transformed_alerts") +ldp = CloudCausalityLDP() + +# Connections +src -> [ing1 -> ling, ldp] +ling -> voter -> ldp -> sink diff --git a/config/markov_source_config.json b/config/markov_source_config.json index 2effaf2..8cdb1b1 100644 --- a/config/markov_source_config.json +++ b/config/markov_source_config.json @@ -12,9 +12,7 @@ "sources": { "src1": { "module": "CloudMarkovChainSource", - "params": { - "server_sleep_in_seconds": 0.01 - }, + "sleep": 0.01, "transitions": { "web_service": { "run=>slow": { @@ -74,27 +72,21 @@ "smls": { "sml1": { "module": "LiNGAM", - "params": { - "threshold": 0.5 - } + "threshold": 0.5 } }, "voters": { "vot1": { "module": "PickIndexVoter", - "params": { - "index": 0 - } + "index": 0 } }, "sinks": { "snk1": { "module": "KafkaSink", - "params": { - "host": "localhost", - "port": 9092, - "topic": "transformed_alerts" - } + "host": "localhost", + "port": 9092, + "topic": "transformed_alerts" } }, "ldps": { diff --git a/config/metric_experiments.banana b/config/metric_experiments.banana new file mode 100644 index 0000000..fa34e9c --- /dev/null +++ b/config/metric_experiments.banana @@ -0,0 +1,30 @@ +####################### +# Metric experiments +# + +# Sources +src = MonascaMarkovChainSource(sleep=0.01) + +# Sinks +stdout = StdoutSink() +file = FileSink(path="~/monasca-aggregate.log") + +# Live data processors +period = 0.1 * 2 +aggregator = MonascaAggregateLDP(func="cnt", period=period) +combiner = MonascaCombineLDP( + metric= "cpu.logical_cores_actives", + bindings= { + a: "cpu.idle_perc", + b: "cpu.total_logical_cores", + }, + lambda= "a * b", + period= period +) +derivator = MonascaDerivativeLDP(period=period) + + +# Connections +src -> aggregator -> stdout +src -> [combiner, derivator] -> stdout +[combiner, derivator] -> file diff --git a/config/metric_experiments.json b/config/metric_experiments.json index daed41c..63c564d 100644 --- a/config/metric_experiments.json +++ b/config/metric_experiments.json @@ -12,9 +12,7 @@ "sources": { "src1": { "module": "MonascaMarkovChainSource", - "params": { - "server_sleep_in_seconds": 0.01 - } + "sleep": 0.01 } }, "ingestors": {}, @@ -26,36 +24,28 @@ }, "snk3": { "module": "FileSink", - "params": { - "path": "~/monasca-aggregate.log" - } + "path": "~/monasca-aggregate.log" } }, "ldps": { "ldp3": { "module": "MonascaAggregateLDP", - "params": { - "aggregation_period": 2, - "aggregation_function": "cnt" - } + "period": 2, + "func": "cnt" }, "ldp4": { "module": "MonascaCombineLDP", - "params": { - "metric_name": "cpu.logical_cores_actives", - "combine_period": 1, - "lambda": "a * b", - "metric_names_binding": { - "a": "cpu.idle_perc", - "b": "cpu.total_logical_cores" - } + "metric": "cpu.logical_cores_actives", + "period": 1, + "lambda": "a * b", + "bindings": { + "a": "cpu.idle_perc", + "b": "cpu.total_logical_cores" } }, "ldp5": { "module": "MonascaDerivativeLDP", - "params": { - "derivative_period": 1 - } + "period": 1 } }, "connections": { diff --git a/config/monasca_aggregate_all_fn.banana b/config/monasca_aggregate_all_fn.banana new file mode 100644 index 0000000..9f4303a --- /dev/null +++ b/config/monasca_aggregate_all_fn.banana @@ -0,0 +1,13 @@ +############################## +# Monasca aggregate all functions + +src = MonascaMarkovChainSource(sleep=0.01) +snk = KafkaSink(host="localhost", port=9092, topic="experiments") +stdout = StdoutSink() + +ldp1 = MonascaAggregateLDP(func="cnt") +ldp2 = MonascaAggregateLDP(func="max") +ldp3 = MonascaAggregateLDP(func="min") +ldp4 = MonascaAggregateLDP(func="avg") + +src -> [ldp1, ldp2, ldp3, ldp4] -> [snk, stdout] diff --git a/config/monasca_aggregate_all_fn.json b/config/monasca_aggregate_all_fn.json index 2d2bd21..c81754f 100644 --- a/config/monasca_aggregate_all_fn.json +++ b/config/monasca_aggregate_all_fn.json @@ -12,9 +12,7 @@ "sources": { "src1": { "module": "MonascaMarkovChainSource", - "params": { - "server_sleep_in_seconds": 0.01 - } + "sleep": 0.01 } }, "ingestors": {}, @@ -23,11 +21,9 @@ "sinks": { "snk1": { "module": "KafkaSink", - "params": { - "host": "localhost", - "port": 9092, - "topic": "monasca_experiments" - } + "host": "localhost", + "port": 9092, + "topic": "monasca_experiments" }, "snk2": { "module": "StdoutSink" @@ -36,31 +32,23 @@ "ldps": { "ldp1": { "module": "MonascaAggregateLDP", - "params": { - "aggregation_period": 2, - "aggregation_function": "max" - } + "period": 2, + "func": "max" }, "ldp2": { "module": "MonascaAggregateLDP", - "params": { - "aggregation_period": 2, - "aggregation_function": "min" - } + "period": 2, + "func": "min" }, "ldp3": { "module": "MonascaAggregateLDP", - "params": { - "aggregation_period": 2, - "aggregation_function": "avg" - } + "period": 2, + "func": "avg" }, "ldp4": { "module": "MonascaAggregateLDP", - "params": { - "aggregation_period": 2, - "aggregation_function": "sum" - } + "period": 2, + "func": "sum" } }, "connections": { diff --git a/config_dsl.py b/config_dsl.py index aa42e9a..b3efea7 100644 --- a/config_dsl.py +++ b/config_dsl.py @@ -15,11 +15,11 @@ # under the License. import json -from logging import config as log_conf import os import six +from logging import config as log_conf -from monasca_analytics.dsl import interpreter +from monasca_analytics.banana.cli import interpreter DEFAULT_LOGGING_CONFIG_FILE = "config/logging.json" @@ -35,11 +35,14 @@ def setup_logging(): def main(): setup_logging() + print(">>>>> DEPRECATED TOOL <<<<<") + print(">>>>> Use the banana language instead <<<<<") + print("") print("Welcome to Monanas config command line") print("Type help for help about commands") inter = interpreter.DSLInterpreter() cmd = "" - while("exit" != cmd.lower()): + while "exit" != cmd.lower(): cmd = six.moves.input(">> ") if cmd != "": try: diff --git a/doc/banana.md b/doc/banana.md new file mode 100644 index 0000000..be1b0d1 --- /dev/null +++ b/doc/banana.md @@ -0,0 +1,141 @@ +# Banana: a configuration language for Monanas + +Welcome to Banana, a configuration language for Monanas. The language is the +key of "recipes" allowing users to reuse it to tailor solutions for their +use-cases. In other words, Banana allows us to write a recipe(s) that will be +ingested by Monanas. + +The language is fully typed. It uses type inference to avoid having to add any +typing annotation. It is still in its early stages, so more features will be +added to solve common problems. + +> Note: a valid `banana` recipe (or file) might still contain errors that +> can only be discovered at run-time. The type system, will remove +> most of them though. + +To get you started, we provide an example below, which would allow us to +understand most parts of the language. + +> TODO: Once we have a specified interface, we should use it instead as it will +> provide syntax highlighting, in-editor error indication as well as other +> features such as autocompletion. + +## Part 1: Creating components + +Here is how we create a component: + +```python +# Create an IPTablesSource with sleep parameter set to 0.01 +src = IPTablesSource(sleep=0.01) +``` + +We could also create a component without any parameter. In that case, each +parameter is initialized with a default value. + +In order to get something interesting we first create the following components: + +```python +src = IPTablesSource(sleep=0.01) +ing1 = IptablesIngestor() +svm = SvmOneClass() +voter = PickIndexVoter(0) +ldp1 = IptablesLDP() +stdout = StdoutSink() +sqlite = IptablesSQLiteSink() +``` + +## Part 2: Connecting components + +Connections can be placed anywhere in the file. They will always be processed +after everything else. + +We have created five components so far, but note that, some components can only +connected to certain types of components). For instance, a source can only +be connected to an ingestor or a live data processor. However, you can't +connect it to a statistical or machine learning algorithms as those need to get +curated data only. Try to add the following line: + +```py +src -> alg +``` + +You should see an error, saying that this is not possible: + +``` +Error: Can't connect src to alg, src can only be connected to Ingestor. +``` + +What we want is to have those connections: + +``` ++---------+ +---------+ +---------+ +---------+ +---------+ +------------+ +| src +-----> | ing1 +------> | alg +-----> | vot +---> | ldp +-----> | stdout | ++----+----+ +---------+ +---------+ +---------+ +----+----+ +------------+ + | ^ + | | + +----------------------------------------------------------------------+ +``` + +Here is how we can achieve that: + +``` +src -> [ing1 -> alg -> vot -> ldp, ldp] +ldp -> stdout +``` + +We could also write it like this: + +``` +src -> ing1 -> alg -> vot -> ldp +src -> ldp -> stdout +``` + +Or like this: + +``` +src -> ing1 +src -> ldp +ing1 -> alg +alg -> vot +vot -> ldp +ldp -> stdout +``` + +The main difference is readability and this is subjective. Use the version that +you think is more readable. + +Banana will treat all of them as being semantically identical. + +## Part 3: Sharing settings between components + +From what we described above, it is possible that we could end up with many +similar parameters across components. It would be great if we could share them. +In Banana we can declare a variable not only for components, but also for +`string`, `number` and json-like `object`. + +For instance, this is valid in banana: + +```python +sleep = 0.01 +# ... +``` + +You can also make use of arithmetic operators to perform any computation you +might require with previously declared variables or anything else. + +Some global variables, defined by the execution environment are also available. +For instance, you could define `sleep` like this: + +```python +sleep = 0.01 / $spark.BATCH_INTERVAL +# ... +``` + +> TODO: the above feature has not yet been implemented. + +Finally, Banana supports string interpolation to mix many types together: + +```python +port = 9645 +host = "localhost:" + port +``` diff --git a/doc/cli.md b/doc/cli.md new file mode 100644 index 0000000..9f19e74 --- /dev/null +++ b/doc/cli.md @@ -0,0 +1,187 @@ +# Command line interface to generate a JSON configuration + +A simple command line tool has been implemented in order to manage +configurations in an easier way. It is not expected to be maintained in the +long term. It is introduced to experiment with the creation of `banana`, the +configuration language of Monanas. + +This section explains what operations are currently available and how to use +them. + +> NOTE: Please refer to [Monasca/Configuration](configuration.md) for the +> structure of the JSON configuration. + +> NOTE: This tool is DEPRECATED, use [BANANA](banana.md) configuration language +> instead. + +## Usage + +* Start the Monanas cli +```bash +python $MONANAS_HOME/config_dsl.py +``` +After running this command, a simple empty configuration will be created. You +can then modify it by running the commands listed below: + +## Available commands + +You can run the following operations from the CLI in order to create, remove, connect, +disconnect and modify components from the configuration. + +### Create component + +Create a component and assign it to a variable: + +```python +>> A = IPTablesSource +``` + +This command adds a new source of type `IPTablesSource` to the configuration, +assigning it with a default configuration. It links the source component to +variable A and returns a unique ID. You can either use the variable or the ID +to refer to the instance of the `IPTablesSource` created. + +### Remove component + +Remove a component using an ID or a variable name: + +```python +>> rm A +``` + +This command removes the component referenced by `A` from the configuration. +The parameter can either be a variable or an ID associated to a component +in the configuration. The component will only be removed if it is not connected +to any other component. + +### Connect components + +Connect two components in the configuration: + +```python +>> A -> B +``` + +This command connects the component referenced by `A` with the component +referenced by `B`. Both `A` and `B` can be variables or IDs, and the connection +is directional from `A` to `B`. The connection is valid and considered only if +the associated components exist and allowed for connection. For example, +connecting a source with an ingestor is allowed, but connecting a source with +a voter is not. + +### Disconnect components + +Disconnect two components in the configuration: + +```python +>> A !-> B +``` + +This command disconnects the component `A` from component `B`. Both `A` and `B` +can be variables or IDs and the connection is directional from `A` to `B`. If +the connection doesn't exist, nothing will happen. + +### Modify component + +Modify values of the configuration of a component: + +```python +>> A.params.subparam1.subparam2 = value +``` + +This command modifies the value of the configuration parameter at the end of +the path defined by a dot notation. The configuration is validated before being +modified; hence, if the modification results in an invalid configuration, it +will not be executed. `A` can either be a variable or an ID. + +## Config presentation operations + +The following operations can be run using the tool in order to view the +current configuration, sub-configurations, and available components that can be +instantiated. + +### Print + +Print the full configuration: + +```python +>> print +``` + +This command displays the full configuration in JSON format on the screen. + +Print component type sub-configuration: + +```python +>> print connections +``` + +If a parameter is passed to the print command that corresponds to a component +type, or in general, a first level key of the configuration, only the relevant +sub-configuration that is selected will be displayed on the screen. + +Print a particular component sub-configuration: + +```python +>> print A +``` + +If a parameter is passed to the print command that corresponds to a variable +or an ID associated to a particular component, only its configuration will be +displayed on the screen. + +### List + +Print all available components: + +```python +>> list +``` + +This command displays all available components that can be add to the +configuration, organized by type. + +Print all available components of a particular type: + +```python +>> list smls +``` + +If a type is passed as a parameter to the list command, only the available +components of that type will be listed. + +## Config storage operations + +The following operations can be run from the tool in order to load and save +configurations from/to files. + +### Load + +Load a configuration from a file: + +```python +>> load filename +``` + +This command loads the configuration stored in the file 'filename', overriding +the configuration currently being manipulated in memory. + +### Save + +Save a configuration to a file: + +```python +>> save filename +``` + +This command saves the configuration being currently handled to the file +'filename', overriding the file if it existed previously. + +Save a configuration to the last file: + +```python +>> save +``` + +If no parameter is provided, the save operation saves the current +configuration being handled to the last file loaded from or saved to. diff --git a/doc/dsl.md b/doc/dsl.md deleted file mode 100644 index c9b9cd2..0000000 --- a/doc/dsl.md +++ /dev/null @@ -1,129 +0,0 @@ -# Domain Specific Language (DSL) for configuration handling - -A simple DSL language, accessible from a command line tool, has been implemented -in order to manage configurations in an easier way. This section explains what -operations are available, and how to use them. - -> Note: Please refer to [Monasca/Configuration](configuration.md) to understand the structure of the configuration. - -## Usage - -* Start Mananas DSL tool -```bash -python $MONANAS_HOME/config_dsl.py -``` -After running this command, a simple empty orchestration configuration will be created, -which you can then modify running operations according to your needs. - -## Config manipulation operations -You can run the following operations from the DSL command line in order to create, remove, connect, disconnect and modify components in the configuration. - -###Create component -* Create a component and assign it to a variable -```bash ->> A = IPTablesSource -``` -This command adds a new source of type IPTablesSource to the configuration, assigning it -its default configuration. It links the source component to variable A, and returns a unique ID. -You can either use the variable or the ID in order to reference the instance of IPTablesSource -you just created. - -### Remove component -* Remove a component using its ID or variable name -```bash ->> rm A -``` -This command removes the component referenced by A from the configuration. -The parameter can either be a variable or an ID associated to a component -in the configuration. -The component will only be removed if it is not connected to any other component. - -### Connect components -* Connect two components in the configuration -```bash ->> A -> B -``` -This command connects the component referenced by A with the component referenced -by B. Both A and B can be variables or IDs, and the connection is directional from A to B. -The connection will only be performed if the components exist and their connection is allowed. -For example, connecting a source with an ingestor is allowed, but connecting a source with a voter is not. - -### Disconnect components -* Disconnect two components in the configuration -```bash ->> A !-> B -``` -This command disconnects the component A from component B. Both A and B can be variables or IDs, and the connection is directional from A to B. -If the connection didn't exist, nothing will happen. - -### Modify component -* Modify values of the configuration of a component -```bash ->> A.params.subparam1.subparam2 = value -``` -This command modifies the value of the configuration parameter at the end of the path defined by the dot notation. -The configuration is validated before being modified, hence if the modification results in an invalid configuration, -it will not be performed. -A can either be a variable or an ID. - -## Config presentation operations -You can run the following operations from the DSL command line in order to view the current configuration, sub-configurations, and available components that you can instantiate. - -### Print - -* Print the full configuration -```bash ->> print -``` -This command displays the full configuration in json format to your screen. - -* Print component type sub-configuration -```bash ->> print connections -``` -If you pass a parameter to the print command that corresponds to a component type, or, in general, a first level key of the configuration, only the relevant sub-configuration that you selected will be displayed to your screen. - -* Print a particular component sub-configuration -```bash ->> print A -``` -If you pass a parameter to the print command that corresponds to a variable or an ID associated to a particular component, only its configuration will be displayed to your screen. - -### List - -* Print all available components -```bash ->> list -``` -This command displays all available components that you can add to your configuration, organized by type. - -* Print all available components of a particular type -```bash ->> list smls -``` -If you pass the type as a parameter to the list command, only the available components of that type will be listed. - -## Config storage operations -You can run the following operations from the DSL command line in order to load and save configurations from/to files. - -### Load - -* Load a configuration from a file -```bash ->> load filename -``` -This command loads the configuration stored in the file 'filename', overriding the existing configuration you were handling before. - -### Save - -* Save a configuration to a file -```bash ->> save filename -``` -This command saves the configuration being currently handled to the file 'filename', overriding the file if it existed before. - -* Save a configuration to the last file -```bash ->> save -``` -If no parameter is provided, the save operation saves the current configuration being handled to the last file you loaded from or saved to. diff --git a/doc/getting_started.md b/doc/getting_started.md index 5962af9..cb3670f 100644 --- a/doc/getting_started.md +++ b/doc/getting_started.md @@ -61,7 +61,7 @@ python $HOME/monanas/run.py -p $HOME/spark -c $HOME/monanas/config/config.json \ MoNanas consumes two configuration files, one for orchestrating data execution and the other for logging. A Domain Specific Language (DSL) has been implemented in order to manipulate configurations. Please, see [MoNanas/Configuration](configuration.md) for more details on MoNanas configuration; -and [MoNanas/Dsl](dsl.md) for more details on MoNanas DSL. +and [MoNanas/Banana](banana.md) for more details on `banana` the language to write recipes. ## Examples diff --git a/monasca_analytics/banana/README.md b/monasca_analytics/banana/README.md new file mode 100644 index 0000000..89f2028 --- /dev/null +++ b/monasca_analytics/banana/README.md @@ -0,0 +1,40 @@ +## Banana configuration language + +This module contains everything related to Banana. In each +sub-module (sub-folder) you will find a `README.md` file +that describes: + + * Purpose of the module. + * The current status of the implementation. + * How testing is done. + +The compiler is split in passes. Each pass performs some +transformations and / or generates more data. Only the last +step has side-effects on the Monanas instance. + +Each sub-module roughly maps to one pass run by the compiler. + +### Passes + +The Banana compiler runs the following passes: + + * `parse`, parse the input and build an [AST](./grammar/README.md). + * `typeck`, type check the input. + * `deadpathck`, remove dead path in the connections. + * `eval`, evaluate the AST generated. + +Each pass makes some assumptions about the state of +the data, and in particular that the previous passes +have run successfully. While this is made obvious by +the arguments required to run some passes, it is less +so for others. + +Generally, things to remember: + + * Changing the ordering of passes is more likely to + break things. + * New passes are free to modify the AST / TypeTable. + * New passes should not break invariants. + +For more information on passes, have a look in their +specific `README.md` file. \ No newline at end of file diff --git a/monasca_analytics/dsl/__init__.py b/monasca_analytics/banana/cli/__init__.py similarity index 100% rename from monasca_analytics/dsl/__init__.py rename to monasca_analytics/banana/cli/__init__.py diff --git a/monasca_analytics/dsl/const.py b/monasca_analytics/banana/cli/const.py similarity index 100% rename from monasca_analytics/dsl/const.py rename to monasca_analytics/banana/cli/const.py diff --git a/monasca_analytics/dsl/dsl.py b/monasca_analytics/banana/cli/dsl.py similarity index 99% rename from monasca_analytics/dsl/dsl.py rename to monasca_analytics/banana/cli/dsl.py index 355ac6e..545ba3d 100644 --- a/monasca_analytics/dsl/dsl.py +++ b/monasca_analytics/banana/cli/dsl.py @@ -31,6 +31,10 @@ logger = logging.getLogger(__name__) MODULE = "module" +""" +DEPRECATED: Preferred way is to now use the Banana language instead. +""" + class MonanasDSL(object): diff --git a/monasca_analytics/dsl/interpreter.py b/monasca_analytics/banana/cli/interpreter.py similarity index 98% rename from monasca_analytics/dsl/interpreter.py rename to monasca_analytics/banana/cli/interpreter.py index 2871d3a..bf72c11 100644 --- a/monasca_analytics/dsl/interpreter.py +++ b/monasca_analytics/banana/cli/interpreter.py @@ -18,10 +18,10 @@ import copy import json import logging +from monasca_analytics.banana.cli import const as dsl_const +from monasca_analytics.banana.cli import dsl +from monasca_analytics.banana.cli import parser from monasca_analytics.config import const as config_const -from monasca_analytics.dsl import const as dsl_const -from monasca_analytics.dsl import dsl -from monasca_analytics.dsl import parser from monasca_analytics.exception import dsl as err import monasca_analytics.util.common_util as cu diff --git a/monasca_analytics/dsl/parser.py b/monasca_analytics/banana/cli/parser.py similarity index 90% rename from monasca_analytics/dsl/parser.py rename to monasca_analytics/banana/cli/parser.py index e6a3175..50f50ad 100644 --- a/monasca_analytics/dsl/parser.py +++ b/monasca_analytics/banana/cli/parser.py @@ -16,8 +16,7 @@ import pyparsing as p -from monasca_analytics.dsl import const - +from monasca_analytics.banana.cli import const EQUALS = p.Literal("=").suppress() CONNECT = p.Literal("->").suppress() @@ -51,10 +50,10 @@ cmd_list = (LIST + p.Optional(LPAREN) + p.Optional(VARNAME) + p.Optional(RPAREN)) cmd_help = (HELP + p.Optional(LPAREN) + p.Optional(RPAREN)) -bnfLine = cmd_create(const.CREATE) | cmd_connect(const.CONNECT) |\ - cmd_disconnect(const.DISCONNECT) | cmd_load(const.LOAD) |\ - cmd_save_as(const.SAVE_AS) | cmd_save(const.SAVE) |\ - cmd_remove(const.REMOVE) | cmd_modify(const.MODIFY) |\ +bnfLine = cmd_create(const.CREATE) | cmd_connect(const.CONNECT) | \ + cmd_disconnect(const.DISCONNECT) | cmd_load(const.LOAD) | \ + cmd_save_as(const.SAVE_AS) | cmd_save(const.SAVE) | \ + cmd_remove(const.REMOVE) | cmd_modify(const.MODIFY) | \ cmd_print(const.PRINT) | cmd_list(const.LIST) | cmd_help(const.HELP) bnfComment = "#" + p.restOfLine diff --git a/monasca_analytics/banana/deadpathck/README.md b/monasca_analytics/banana/deadpathck/README.md new file mode 100644 index 0000000..0326592 --- /dev/null +++ b/monasca_analytics/banana/deadpathck/README.md @@ -0,0 +1,22 @@ +## Dead path checker + +Dead path checking is about removing paths in the pipeline that +lead to nothing. For instance, if there's no source or no sink +in a path. This pass is the only one that modifies the AST. + +This is the third step of the pipeline: +``` + +---------------------+ +---------------------+ + | | | | + ---> | AST & TypeTable | ---- deadpathck ---> | AST' & TypeTable' | ---> + | | | | + +---------------------+ +---------------------+ +``` + +### Current status: + +* [x] Remove branches that are dead from the list of connections. +* [x] Remove the components from the collected list of components. +* [ ] Remove statements that are dead code: + - [ ] Do not instantiate components. + - [ ] Do not compute expressions for unused variables. diff --git a/test/dsl/__init__.py b/monasca_analytics/banana/deadpathck/__init__.py similarity index 100% rename from test/dsl/__init__.py rename to monasca_analytics/banana/deadpathck/__init__.py diff --git a/monasca_analytics/banana/deadpathck/config.py b/monasca_analytics/banana/deadpathck/config.py new file mode 100644 index 0000000..6d8244a --- /dev/null +++ b/monasca_analytics/banana/deadpathck/config.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import monasca_analytics.banana.deadpathck.dag as dag +import monasca_analytics.banana.emitter as emit +import monasca_analytics.banana.grammar.ast as ast +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.exception.banana as exception + + +def deadpathck(banana_file, type_table, emitter=emit.PrintEmitter()): + """ + Perform dead path elimination on the provided AST. + This allow to remove branches and components that + are not connected to a Sink. + :type banana_file: ast.BananaFile + :param banana_file: The AST tree we will clean. + :type type_table monasca_analytics.banana.typeck.type_table.TypeTable + :param type_table: The TypeTable of the provided AST. + :type emitter: emit.Emitter + :param emitter: Emitter for reporting warnings. + """ + # Check that first argument is a banana file. Mainly + # an excuse to remove the F401 warning. + if not isinstance(banana_file, ast.BananaFile): + raise Exception("Expected BananaFile as first argument.") + + # Look first for all branch that are "dead" + connections = banana_file.connections # type: ast.Connection + + # Collect the nodes and connect them. + dag_nodes = {} + # Create all the nodes + for ident in banana_file.components.keys(): + dag_nodes[ident] = dag.DagNode(type_table.get_type(ident)) + # Connect them + for ident_from, ident_to in connections.connections: + dag_from = dag_nodes[ident_from] + dag_to = dag_nodes[ident_to] + dag_from.children.append(dag_to) + dag_to.parents.append(dag_from) + + # Start from every sources and for each, check if the path is dead + for node in dag_nodes.values(): + if isinstance(node.typec, type_util.Source): + node.visit() + + # We can now remove all the components that are "dead" + # from the list of connections + for ident, node in dag_nodes.iteritems(): + if not node.is_alive(): + emitter.emit_warning( + ident.span, + "Dead code found, this component is not in a path " + "starting from a 'Source' and ending with a 'Sink'." + ) + banana_file.components.pop(ident) + connections.connections = filter( + lambda edge: edge[0] != ident and edge[1] != ident, + connections.connections + ) + + # TODO(Joan): We could also remove them from the statements. + # TODO(Joan): But for this we need a dependency graph between + # TODO(Joan): statements to make sure we don't break the code. + + +def contains_at_least_one_path_to_a_sink(banana_file, type_table): + """ + Check that there's at least one path to a sink in the list + of components. + To run this pass, you need to make sure you + have eliminated all dead path first. + + :type banana_file: ast.BananaFile + :param banana_file: The AST to check. + :type type_table monasca_analytics.banana.typeck.type_table.TypeTable + :param type_table: The TypeTable of the provided AST. + :raise: Raise an exception if there's no Sink. + """ + def is_sink(comp): + type_comp = type_table.get_type(comp) + return isinstance(type_comp, type_util.Sink) + + def is_src(comp): + type_comp = type_table.get_type(comp) + return isinstance(type_comp, type_util.Source) + + comp_vars = banana_file.components.keys() + at_least_one_sink = len(filter(is_sink, comp_vars)) > 0 + at_least_one_source = len(filter(is_src, comp_vars)) > 0 + + if not at_least_one_sink: + raise exception.BananaNoFullPath("Sink") + if not at_least_one_source: + raise exception.BananaNoFullPath("Source") diff --git a/monasca_analytics/banana/deadpathck/dag.py b/monasca_analytics/banana/deadpathck/dag.py new file mode 100644 index 0000000..6df8dd0 --- /dev/null +++ b/monasca_analytics/banana/deadpathck/dag.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import monasca_analytics.banana.typeck.type_util as type_util + + +class DagNode(object): + + def __init__(self, typec): + """ + Create a DAGNode. + + :param typec: The type of the node. + """ + self.parents = [] + self.children = [] + self.typec = typec + self._visited = False + self._seen_sink = False + + def visit(self): + """ + Visit this nodes and all of its connections. + """ + if not self._visited: + self._visited = True + if isinstance(self.typec, type_util.Sink): + self.visit_parents() + return + for child in self.children: + child.visit() + + def visit_parents(self): + """ + Visit the parent to tell them that we've seen a Sink. + """ + if not self._seen_sink: + self._seen_sink = True + for parent in self.parents: + parent.visit_parents() + + def is_alive(self): + return self._visited and self._seen_sink diff --git a/monasca_analytics/banana/emitter.py b/monasca_analytics/banana/emitter.py new file mode 100644 index 0000000..2a96695 --- /dev/null +++ b/monasca_analytics/banana/emitter.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class Emitter(object): + """ + Emitter base class to emit errors and warnings. + Typically errors will be collected and then send + over the network as an http response but for tests + and debugging, a `PrintEmitter` can be used instead. + """ + + @abc.abstractmethod + def emit_warning(self, span, message): + """ + Emit a warning. + :type span: monasca_analytics.banana.grammar.base_ast.Span + :param span: Span associated with the message. + :type message: str + :param message: message to emit with the warning level. + """ + pass + + @abc.abstractmethod + def emit_error(self, span, message): + """ + Emit an error + :type span: monasca_analytics.banana.grammar.base_ast.Span + :param span: Span associated with the message. + :type message: str + :param message: message to emit with the error level. + """ + pass + + +class PrintEmitter(Emitter): + """ + Print warnings and errors to the console. + """ + def emit_warning(self, span, message): + print("WARNING at line:{}".format(span.get_lineno())) + print("WARNING: {}".format(message)) + + def emit_error(self, span, message): + print("ERROR at line:{}".format(span.get_lineno())) + print("ERROR: {}".format(message)) + + +class JsonEmitter(Emitter): + """ + Print warnings and errors in a Json object. + """ + def __init__(self): + self.result = { + "errors": [], + "warnings": [], + } + + def emit_error(self, span, message): + self.result["errors"].append({ + "line": span.get_lineno(), + "col": 0, + "byteRange": [span.lo, span.hi], + "message": message + }) + + def emit_warning(self, span, message): + self.result["warnings"].append({ + "line": span.get_lineno(), + "col": 0, + "byteRange": [span.lo, span.hi], + "message": message + }) diff --git a/monasca_analytics/banana/eval/README.md b/monasca_analytics/banana/eval/README.md new file mode 100644 index 0000000..fc5c25e --- /dev/null +++ b/monasca_analytics/banana/eval/README.md @@ -0,0 +1,50 @@ +## Interpreter / Evaluator + +This folder contains everything related to the evaluation +of banana files. + +This pass makes some assumptions: it is valid to create +all the components and connecting them won't throw any errors. + +Some components might need to be created in order to +check if they are valid. For instance, when a DNS lookup is +involved. In such cases, an error will be thrown during +the interpretation. However, the general intention is to move +the checks out of the evaluation as much as possible. We want to +avoid at all cost an half-working pipeline as it could have +side-effects on external data sources by corrupting them or +feeding them with incorrect data. + +The execution environment (e.g Spark) might also reject the +pipeline during an evaluation for some reason. However, this is +less likely to happen as the `deadpathck` pass removes +components and paths that would lead to errors. + +This is the last step of the pipeline: +``` + +---------------------+ + | | + ---> | AST & TypeTable | ---- interpret ---> Done + | | + +---------------------+ +``` + +### Current status + +* [x] Evaluate expressions +* [x] Create components +* [x] Connect components +* [x] Restart the pipeline + +### Tests + +All tests are located in `test/banana/eval`. We only try +to evaluate valid files, so for this pass there's only a +`should_pass` directory. + +#### Available instruction + +* `# LHS_EQ `: This instruction + compares the evaluation of the left hand side of the previous + expression with the provided string. If they are not equal, + the test will fail. \ No newline at end of file diff --git a/monasca_analytics/banana/eval/__init__.py b/monasca_analytics/banana/eval/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/monasca_analytics/banana/eval/config.py b/monasca_analytics/banana/eval/config.py new file mode 100644 index 0000000..eb36d07 --- /dev/null +++ b/monasca_analytics/banana/eval/config.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import operator + +import monasca_analytics.banana.eval.ctx as ctx +import monasca_analytics.banana.eval.old_style as into +import monasca_analytics.banana.grammar.ast as ast +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.config.connection as connection +import monasca_analytics.config.const as conf_const +import monasca_analytics.exception.banana as exception +import monasca_analytics.util.common_util as introspect + +logger = logging.getLogger(__name__) + + +def eval_ast(ast_root, type_table, driver): + """ + Evaluate the provided AST by instantiating + the appropriate components and connecting them + together using the Driver interface. + :type ast_root: ast.BananaFile + :param ast_root: AST to evaluate. + :type type_table: monasca_analytics.banana.typeck.type_table.TypeTable + :param type_table: the TypeTable (used to create configurations) + :type driver: monasca_analytics.spark.driver.DriverExecutor + :param driver: Driver that will manage the created + components and connect them together. + """ + logger.debug("Creating the config dictionary from the AST...") + _config = conf_const.get_default_base_config() + + try: + logger.debug("Creating components according to banana config...") + components = eval_create_components(ast_root.statements, type_table) + convert_connections(ast_root.connections, _config) + logger.debug("Done creating components. Creating link data...") + # Pre-process to convert to old style components + components_old_style = into.into_old_conf_dict(components) + links = connection.connect_components(components_old_style, _config) + logger.debug("Done connecting components. Successful instantiation") + except Exception as ex: + logger.error("Failed to instantiate components") + logger.error("Reason : " + str(ex)) + return + + # Restart Spark using the new config + logger.debug("Stop pipeline") + driver.stop_pipeline() + logger.debug("Set new links") + driver.set_links(links) + logger.debug("Start pipeline") + driver.start_pipeline() + + +def convert_connections(connections, output_config): + """ + Augment the output_config object with the list of + connections + :type connections: ast.Connection + :param connections: The list of connections. + :type output_config: dict + :param output_config: Config where the links will be written. + """ + output_config[conf_const.CONNECTIONS] = connections.connections_cache + + +def eval_create_components(statements, type_table): + """ + Convert the provided AST into the old dict configuration. + :type statements: list[(ast.ASTNode, ast.ASTNode)] + :param statements: The AST to process + :type type_table: monasca_analytics.banana.typeck.type_table.TypeTable + :param type_table: the type table. + :rtype: dict[str, Component] + :return: Returns the component keyed by name. + """ + context = ctx.EvaluationContext() + + eval_statements_generic( + statements, + type_table, + context + ) + return context.get_components() + + +def eval_statements_generic( + statements, + type_table, + context, + cb=lambda *a, **k: None): + """ + Eval the list of statements, and call the cb after evaluating + each statement providing it with the type of the value, the + left hand side ast node, and the computed value. + :type statements: list[(ast.ASTNode, ast.ASTNode)] + :param statements: The AST to process + :type type_table: monasca_analytics.banana.typeck.type_table.TypeTable + :param type_table: the type table. + :type context: ctx.EvaluationContext + :param context: evaluation context that will collect + all intermediary results. + :type cb: (type_util.IsType, ast.ASTNode, object) -> None + :param cb: Callback called after each statement evaluation. + """ + stmt_index = 0 + for stmt in statements: + lhs, rhs = stmt + expected_type = type_table.get_type(lhs, stmt_index + 1) + stmt_index += 1 + # Provide the expected type + value = eval_rhs(context, rhs, expected_type) + # Call the cb with the expected_type of the value + # The lhs node and the value + cb(expected_type, lhs, value) + # Store result if referenced later. + context.set_variable(lhs, value) + + +def eval_rhs(context, ast_node, expected_type): + """ + Eval the right hand side node. + :type context: ctx.EvaluationContext + :param context: Evaluation context. + :type ast_node: ast.ASTNode + :param ast_node: the node to evaluate. + :type expected_type: type_util.IsType + :param expected_type: The expected type of this computation. + :return: Returns the result of this evaluation. + """ + if isinstance(ast_node, ast.StringLit): + return ast_node.inner_val() + if isinstance(ast_node, ast.Ident): + return context.get_variable(ast_node.inner_val()) + if isinstance(ast_node, ast.JsonObj): + return eval_object(context, ast_node, expected_type) + if isinstance(ast_node, ast.Number): + return ast_node.val + if isinstance(ast_node, ast.DotPath): + variable_name = ast_node.varname.inner_val() + prop = map(lambda x: x.inner_val(), ast_node.properties) + return context.get_prop_of_variable(variable_name, prop) + if isinstance(ast_node, ast.Expr): + return eval_expr(context, ast_node, expected_type) + if isinstance(ast_node, ast.Component): + return eval_comp(context, ast_node, expected_type) + raise Exception("Unhandled ast value type {}!!".format(ast_node)) + + +def eval_comp(context, comp, expected_type): + """ + Instantiate the given component, computing + the required config. + :type context: ctx.EvaluationContext + :param context: Evaluation context. + :type comp: ast.Component + :param comp: the node to evaluate. + :type expected_type: type_util.IsType + :param expected_type: The expected type of this computation. + :return: Returns the instantiated component. + """ + arguments = {} + # Compute arguments + for arg in comp.args: + arg_name = ast.DotPath(arg.arg_name.span, arg.arg_name, []) + arg_value = eval_rhs(context, arg.value, expected_type[arg_name]) + arguments[arg.arg_name.inner_val()] = arg_value + # Lookup component + component_type = introspect.get_class_by_name(comp.type_name.val) + # Get default config for the component + conf = component_type.get_default_config() + # Update modified params + for k, val in arguments.iteritems(): + conf[k] = val + # Delay evaluation until we do the assign + return component_type, conf + + +def eval_object(context, obj, expected_type): + """ + Evaluate the provided object + :type context: ctx.EvaluationContext + :param context: Evaluation context. + :type obj: ast.JsonObj + :param obj: The expression to evaluate + :type expected_type: type_util.IsType + :param expected_type: The expected type of this computation. + :return: Returns the computed value + """ + result = expected_type.default_value() + for name, val in obj.props.iteritems(): + subtype = expected_type[name] + ctx.set_property(result, name, eval_rhs(context, val, subtype)) + return result + + +def eval_expr(context, expr, expected_type): + """ + Eval the provided expression + :type context: ctx.EvaluationContext + :param context: Evaluation context. + :type expr: ast.Expr + :param expr: The expression to evaluate + :type expected_type: type_util.IsType + :param expected_type: The expected type of this computation. + :rtype: str | float + :return: Returns the computed value + """ + if len(expr.expr_tree) == 1: + return eval_rhs(context, expr.expr_tree[0], expected_type) + + if isinstance(expected_type, type_util.Number): + result = 0 + cast_func = float + elif isinstance(expected_type, type_util.String): + result = "" + cast_func = str + else: + raise exception.BananaEvalBug( + "Expected type for an expression can only be " + "'TypeNumber' or 'TypeString', got '{}'".format( + str(expected_type)) + ) + current_operator = operator.add + for el in expr.expr_tree: + if isinstance(el, basestring) and el in ['+', '-', '*', '/']: + current_operator = get_op_func(el) + else: + value = eval_rhs(context, el, expected_type) + value = cast_func(value) + result = current_operator(result, value) + return result + + +def get_op_func(op_str): + if op_str == '+': + return operator.add + if op_str == '-': + return operator.sub + if op_str == '*': + return operator.mul + if op_str == '/': + return operator.div + raise exception.BananaEvalBug( + "Unknown operator '{}'".format(op_str) + ) diff --git a/monasca_analytics/banana/eval/ctx.py b/monasca_analytics/banana/eval/ctx.py new file mode 100644 index 0000000..b6bff2e --- /dev/null +++ b/monasca_analytics/banana/eval/ctx.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import monasca_analytics.banana.grammar.ast as ast +import monasca_analytics.exception.banana as exception + + +class EvaluationContext(object): + """Evalutation context for an AST evaluation""" + + def __init__(self): + """ + Construct an evaluation context using the type table + as based for the variable needed to be created. + """ + self._variables = {} + self._components = {} + + def set_variable(self, name, value): + """Set the variable value.""" + if isinstance(value, tuple) and len(value) == 2: + comp_type, config = value + comp_name = name.varname.inner_val() + self._components[comp_name] = comp_type( + comp_name, + config + ) + self._variables[comp_name] = config + elif not set_property(self._variables, name, value): + raise exception.BananaEvalBug( + "set_variable can only be used with DotPath or Ident." + ) + + def get_components(self): + return self._components + + def get_variable(self, name): + """Returns the variable value.""" + return self._variables[name] + + def get_prop_of_variable(self, name, prop): + """Returns the sub property of the given variable name.""" + variable = self._variables[name] + for el in prop: + variable = variable[el] + return variable + + +def set_property(root_object, name, value): + """ + Set the property name of the root_object to value. + :type root_object: dict + :param root_object: The root object + :type name: ast.DotPath | ast.Ident | ast.StringLit + :param name: Name of + :param value: + :return: Returns true if succeeded. + """ + if isinstance(name, ast.Ident) or isinstance(name, ast.StringLit): + root_object[name.inner_val()] = value + return True + elif isinstance(name, ast.DotPath): + _create_as_many_object_as_needed(root_object, name, value) + return True + return False + + +def _create_as_many_object_as_needed(root_object, dot_path, value): + """ + Create as many object as needed to be able to access the + nested property. + + :type root_object: dict + :param root_object: The root object + :type dot_path: ast.DotPath + :param dot_path: Dot Path to use. + :type value: object + :param value: Any value to set. + """ + name = dot_path.varname.inner_val() + + if len(dot_path.properties) == 0: + root_object[name] = value + return + + if name in root_object: + variable = root_object[name] + else: + variable = {} + + current_var = variable + last_index = len(dot_path.properties) - 1 + + for index, subpath in enumerate(dot_path.properties): + subpath_name = subpath.inner_val() + if index != last_index: + if subpath_name in current_var: + current_var = current_var[subpath_name] + else: + new_object = {} + current_var[subpath_name] = new_object + current_var = new_object + else: + current_var[subpath_name] = value + + root_object[name] = variable diff --git a/monasca_analytics/banana/eval/old_style.py b/monasca_analytics/banana/eval/old_style.py new file mode 100644 index 0000000..d1a981a --- /dev/null +++ b/monasca_analytics/banana/eval/old_style.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import monasca_analytics.config.const as conf_const +import monasca_analytics.ingestor.base as ingestor +import monasca_analytics.ldp.base as ldp +import monasca_analytics.sink.base as sink +import monasca_analytics.sml.base as sml +import monasca_analytics.source.base as source +import monasca_analytics.voter.base as voter + + +def into_old_conf_dict(components): + """ + Convert the provided dict of components + keyed by ids into a dict keyed by component + type. This is the data structure used to do + the validation of JSON configuration (the old format). + + :type components: dict[str, object] + :param components: The dictionary of components. + :return: Returns the old conf dictionary. + """ + return { + conf_const.INGESTORS: + dict(filter(lambda x: isinstance(x[1], ingestor.BaseIngestor), + components.iteritems())), + conf_const.VOTERS: + dict(filter(lambda x: isinstance(x[1], voter.BaseVoter), + components.iteritems())), + conf_const.SINKS: + dict(filter(lambda x: isinstance(x[1], sink.BaseSink), + components.iteritems())), + conf_const.LDPS: + dict(filter(lambda x: isinstance(x[1], ldp.BaseLDP), + components.iteritems())), + conf_const.SOURCES: + dict(filter(lambda x: isinstance(x[1], source.BaseSource), + components.iteritems())), + conf_const.SMLS: + dict(filter(lambda x: isinstance(x[1], sml.BaseSML), + components.iteritems())), + } diff --git a/monasca_analytics/banana/grammar/README.md b/monasca_analytics/banana/grammar/README.md new file mode 100644 index 0000000..deeff47 --- /dev/null +++ b/monasca_analytics/banana/grammar/README.md @@ -0,0 +1,63 @@ +## Grammar + +This folder is all about the definition of the `banana` grammar. +The grammar purpose is to convert the input, text, into an +abstract syntax tree (AST). + +This is the first step of the pipeline: +``` + +--------+ +---------+ + | | | | + | Text | --- grammar ---> | AST | ---> + | | | | + +--------+ +---------+ +``` + +The module `ast.py` contains all the possible `ASTNode` which +itself is defined in `base_ast.py`. + +### Current status + +* [x] Parsing connections such as `a -> b`, `a -> [b, c]`, + `[a, b] -> [c, d]` +* [x] Parsing numbers +* [x] Parsing string literals +* [ ] Parsing booleans +* [x] Parsing assignments where the left hand side can be a property + or an identifier. +* [x] Parsing assignments where the right hand side is a number, a + string literal, a property or an identifier. +* [x] Parsing components arguments using a constructor-like syntax. +* [ ] Parsing ingestors generators (for JSON dialect) +* [ ] Parsing imports such as `from ldp.monasca import *` +* [ ] Parsing disconnections such as `a !-> b` *(requires imports)* + +### Tests + +All test regarding the grammar (i.e. the syntax and the way +the AST is built) is defined in `test/banana/grammar`. + +This folder looks like this: + +``` +test/banana/grammar +├── should_fail +│   ├── ... +│   └── file.banana +├── should_pass +│   ├── ... +│   └── file.banana +└── test_config.py +``` + +The `test_config` generates one test for each file in the +`should_pass` and `should_fail` directories. + +Test can assert various things using instructions below. + +#### Available instruction + +* `# RAISE `: Check that `exception-name` is raised. +* `# STMT_EQ ` Check the AST of statements. +* `# AST_EQ ` Check the full AST. +* `# CONN_EQ ` Check the AST of connections. \ No newline at end of file diff --git a/monasca_analytics/banana/grammar/__init__.py b/monasca_analytics/banana/grammar/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/monasca_analytics/banana/grammar/ast.py b/monasca_analytics/banana/grammar/ast.py new file mode 100644 index 0000000..09ab617 --- /dev/null +++ b/monasca_analytics/banana/grammar/ast.py @@ -0,0 +1,674 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import monasca_analytics.banana.emitter as emit +import monasca_analytics.banana.grammar.base_ast as base +import monasca_analytics.exception.banana as exception +import monasca_analytics.util.string_util as strut +import pyparsing as p + + +ASTNode = base.ASTNode +Span = base.Span + + +class BananaFile(object): + + def __init__(self, emitter=emit.PrintEmitter()): + self.imports = [] + self._emitter = emitter + # Components is a dict where keys + # are the name of the var and + # values are of type Component + self.components = dict() + # Statement are component + # creation or variable creation + self.statements = [] + self.connections = None + + def add_component_ctor(self, dot_path, component): + """ + Add a component if the component + does not already exists. If it does exists, + then this raises an Error. + :type dot_path: DotPath + :param dot_path: Name of the variable or property path + :type component: Component + :param component: AST part that will contains all properties + """ + if not isinstance(dot_path, DotPath): + raise exception.BananaAssignmentError( + dot_path.span, component.span) + if not len(dot_path.properties) == 0: + raise exception.BananaAssignmentError( + dot_path.span, component.span) + if dot_path.varname in self.components: + other_dot_path = filter( + lambda x: x == dot_path, + self.components.keys())[0] + no_effect_str = dot_path.span.str_from_to(component.span) + collision_str = other_dot_path.span.str_from_to( + self.components[other_dot_path].span) + self._emitter.emit_warning( + dot_path.span, + "Statement has no effect: '{}'".format(no_effect_str) + ) + self._emitter.emit_warning( + other_dot_path.span, + "It collides with: '{}'".format(collision_str) + ) + else: + self.components[dot_path.varname] = component + self.statements.append((dot_path, component)) + + def add_assignment(self, dot_path, ast_value): + """ + Add an assignment to a property or a variable. + We don't check at this point whether or not the variable + has collision with a component. + This will be done during the name resolution pass. + :type dot_path: DotPath + :param dot_path: Name of the variable or property + :type ast_value: Ident | JsonObj | StringLit | Number | DotPath + :param ast_value: Ast node this variable is assigned to. + """ + if not isinstance(dot_path, DotPath): + raise exception.BananaAssignmentError( + dot_path.span, + ast_value.span + ) + self.statements.append((dot_path, ast_value)) + + def add_connections(self, connections): + """ + Add a new set of connections between components + This function performs the same checks as the one + performs when components are connected. It warns + on redundant connections. + :type connections: Connection + :param connections: AST node that contains the collected connections. + """ + if self.connections is None: + self.connections = connections + else: + self.connections.merge_and_reset_inputs_outputs( + connections, + self._emitter + ) + + def statements_to_str(self): + return "{ " + ', '.join( + map(lambda x: '{} = {}'.format(x[0], x[1]), self.statements) + ) + ' }' + + def __str__(self): + res = "BananaFile { " + res += 'components: { ' + res += strut.dict_to_str(self.components) + res += ' }, ' + res += 'statements: ' + self.statements_to_str() + res += 'connections: { ' + res += str(self.connections) + res += " } }" + return res + + +def make_span(s, l, t): + if len(t) > 0: + if isinstance(t[0], ASTNode): + return Span(s, l, t[0].span.hi) + return Span(s, l, len(t[0]) + l) + else: + return Span(s, l, 2) + + +class Number(ASTNode): + def __init__(self, span, val): + """ + Construct a Number ast node. + :type span: Span + :param span: Span for this number + :type val: str + :param val: Value for this number + """ + super(Number, self).__init__(span) + self.val = float(val) + + def __str__(self): + return "Number< {} >".format(self.val) + + def into_unmodified_str(self): + return str(self.val) + + +class StringLit(ASTNode): + def __init__(self, span, val): + """ + Construct a StringLit ast node. + :type span: Span + :param span: Span for this string + :type val: str + :param val: Value for this string + """ + super(StringLit, self).__init__(span) + self.val = val + + def __hash__(self): + return hash(self.inner_val()) + + def __eq__(self, other): + return (isinstance(other, StringLit) or isinstance(other, Ident))\ + and self.inner_val() == other.inner_val() + + def __str__(self): + return "StringLit< {} >".format(self.val) + + def into_unmodified_str(self): + return self.val + + def inner_val(self): + return self.val.strip()[1:-1] + + +class Ident(ASTNode): + def __init__(self, span, val): + """ + Construct an Ident ast node. + :type span: Span + :param span: Span for this identifier + :type val: str + :param val: Value of this identifier + """ + super(Ident, self).__init__(span) + self.val = val + + def __hash__(self): + return hash(self.val) + + def __eq__(self, other): + return (isinstance(other, StringLit) or isinstance(other, Ident))\ + and self.val == other.inner_val() + + def __str__(self): + return "Ident< {} >".format(self.val) + + def into_unmodified_str(self): + return self.val + + def inner_val(self): + return self.val + + +class DotPath(ASTNode): + def __init__(self, span, varname, properties): + """ + :type span: Span + :param span: Span for this dotpath. + :type varname: Ident | StringLit + :param varname: Name of the variable being changed. + :type properties: list[StringLit | Ident] + :param properties: Properties being accessed. + """ + super(DotPath, self).__init__(span) + self.varname = varname + self.properties = properties + + def next_dot_path(self): + """ + Assuming the properties length is more than zero. + This returns the dot path where the varname has been + dropped. So given 'a.b.c' the returned dot path will + be 'b.c'. + + :rtype: DotPath + :return: Returns the next dot path. + """ + return DotPath( + self.span.new_with_offset(len(self.varname.val)), + self.properties[0], + self.properties[1:] + ) + + def into_unmodified_str(self): + arr = [self.varname.into_unmodified_str()] + arr.extend(map(lambda x: x.into_unmodified_str(), self.properties)) + return '.'.join(arr) + + def __str__(self): + arr = [str(self.varname)] + arr.extend(map(lambda x: str(x), self.properties)) + return 'DotPath< {} >'.format('.'.join(arr)) + + def __key(self): + return self.into_unmodified_str().replace('"', '') + + def __eq__(self, other): + return self.__key() == other.__key() + + def __hash__(self): + return hash(self.__key()) + + +class Expr(ASTNode): + def __init__(self, span, expr_tree): + """ + Construct an expression + :type span: Span + :param span: Span for the expression. + :type expr_tree: p.ParseResults + ;:param expr_tree: The tree generated by pyparsing.infixNotation + """ + super(Expr, self).__init__(span) + # We don't use this tree at this point. + # During typecheck we will make sure + # that the expression can evaluate + # Finally during evaluation, we will evaluate + # the final result. + if isinstance(expr_tree, p.ParseResults): + expr_tree = expr_tree.asList() + if isinstance(expr_tree, list): + for i in xrange(0, len(expr_tree)): + if isinstance(expr_tree[i], list): + expr_tree[i] = Expr(span, expr_tree[i]) + self.expr_tree = expr_tree + else: + self.expr_tree = [expr_tree] + + def into_unmodified_str(self): + # TODO(Joan): reconstruct the original expression + return 'expression' + + def __str__(self): + return "Expr< {} >".format(strut.array_to_str(self.expr_tree)) + + +class JsonObj(ASTNode): + def __init__(self, span, tokens): + super(JsonObj, self).__init__(span) + self.props = {} + last_prop = None + if len(tokens) > 0: + for toks in tokens: + for tok in toks: + if isinstance(tok, DotPath): + if last_prop is None: + last_prop = tok + else: + self._set_new_prop(last_prop, tok) + last_prop = None + elif isinstance(tok, StringLit): + if last_prop is None: + last_prop = tok + else: + self._set_new_prop(last_prop, tok) + last_prop = None + elif isinstance(tok, list): + if last_prop is None: + raise p.ParseFatalException( + "Bug Found in JsonObj!" + ) + self._set_new_prop( + last_prop, + JsonObj.dictify_array(tok) + ) + last_prop = None + else: + if last_prop is None: + raise p.ParseFatalException( + "Bug Found in JsonObj!" + ) + self._set_new_prop(last_prop, tok) + last_prop = None + + def _set_new_prop(self, prop, token): + if prop in self.props: + raise exception.BananaJsonObjShadowingError(self.span, prop.span) + else: + self.props[prop] = token + + def into_unmodified_str(self): + # TODO(Joan): improve this for error reporting + return str(self.props) + + def __str__(self): + return "JsonObj< {} >".format(strut.dict_to_str(self.props)) + + @staticmethod + def dictify_array(tok): + new_array = [] + for el in tok: + if isinstance(el, list): + new_array.append(JsonObj.dictify_array(el)) + else: + new_array.append(el) + return new_array + + +def into_connection(ast_node): + """ + Convert an ast node into a Connection node. + :type ast_node: Connection | Ident + :param ast_node: The ast node to convert. + :rtype: Connection + :return: Returns a Connection node + """ + if isinstance(ast_node, Connection): + return ast_node + elif isinstance(ast_node, Ident): + return Connection( + ast_node.span, + [ast_node], + [ast_node] + ) + else: + raise p.ParseFatalException("Bug found!") + + +class Connection(ASTNode): + + def __init__(self, span, inputs=None, outputs=None, connections=None): + """ + Create a connection object. + :type span: Span + :param span: Span for this connection + :type inputs: list[Ident] + :param inputs: Input ast nodes of the connection + :type outputs: list[Ident] + :param outputs: Outputs nodes + :type connections: list[(Ident, Ident)] + :param connections: The list of connections aggregated so far. + """ + super(Connection, self).__init__(span) + if inputs is None: + inputs = [] + if outputs is None: + outputs = [] + if connections is None: + connections = [] + self.inputs = inputs + self.outputs = outputs + self.connections = connections + self.connections_cache = {} + self._build_connection_cache() + + def connect_to(self, other_con, emitter): + """ + Connect this connection to the other one. + After this function has been executed, the other_con + object can be dropped. + :type other_con: Connection + :param other_con: Other connection to connect to. + :type emitter: emit.Emitter + :param emitter: Emitter. + """ + old_outputs = self.outputs + self.outputs = other_con.outputs + + # Generate new connections + for old_output in old_outputs: + for other_input in other_con.inputs: + self._check_and_connect(old_output, other_input, emitter) + + # Preserve old connections + self._merge_connections(other_con, emitter) + + def merge_all(self, tokens, emitter): + """ + Merge all the tokens with this class + :type tokens: list[list[Connection | Ident]] + :param tokens: List of list of tokens + :type emitter: emit.Emitter + :param emitter: Emitter to report errors + """ + if len(tokens) == 1: + if len(tokens[0]) > 0: + for tok in tokens[0]: + other_con = into_connection(tok) + self.merge_with(other_con, emitter) + + def merge_and_reset_inputs_outputs(self, other_con, emitter): + """ + Merge this connection with other_con and reset inputs / outputs + as they're no longer necessary. + :type other_con: Connection + :param other_con: the other connection we are gonna merge with. + :type emitter: emit.Emitter + :param emitter: Emitter to report errors + """ + self.inputs = [] + self.outputs = [] + self._merge_connections(other_con, emitter) + + def merge_with(self, other_con, emitter): + """ + Merge the provided connection with this one. + :type other_con: Connection + :param other_con: Connection to merge with self. + :type emitter: emit.Emitter + :param emitter: Emitter to report errors + """ + def extend(into, iterable, what): + for other_thing in iterable: + if len(filter(lambda x: x.val == other_thing.val, into)) > 0: + emitter.emit_warning( + other_thing.span, + "{} {} already present".format( + what, other_thing.val + ) + ) + else: + into.append(other_thing) + extend(self.inputs, other_con.inputs, 'Input') + extend(self.outputs, other_con.outputs, 'Output') + self._merge_connections(other_con, emitter) + + def _merge_connections(self, other_con, emitter): + """ + Merge only the connections field from other_con into self. + :type other_con: Connection + :param other_con: Connection to merge with self. + :type emitter: emit.Emitter + :param emitter: Emitter to report errors + """ + for ident_from, ident_to in other_con.connections: + self._check_and_connect(ident_from, ident_to, emitter) + + def _check_and_connect(self, ident_from, ident_to, emitter): + """ + Check if the connection does not already exists and if it does not, + add it to the list of connections. Otherwise report a warning and + do nothing. + + :type ident_from: Ident + :param ident_from: The 'From' node of the directed edge. + :type ident_to: Ident + :param ident_to: The 'To' node of the directed edge we are creating. + :type emitter: emit.Emitter + :param emitter: Emitter to report errors. + """ + if ident_from.val in self.connections_cache: + if ident_to.val in self.connections_cache[ident_from.val]: + emitter.emit_warning( + ident_to.span, + "Connection from '{}' to '{}'" + " is already present" + .format(ident_from.val, ident_to.val) + ) + return + self.connections_cache[ident_from.val].append(ident_to.val) + else: + self.connections_cache[ident_from.val] = [ident_to.val] + self.connections.append((ident_from, ident_to)) + + def _build_connection_cache(self): + """ + Build a cache of connections keyed by where they start from. + """ + for ident_from, ident_to in self.connections: + if ident_from.val not in self.connections_cache: + self.connections_cache[ident_from.val] = [] + if ident_to.val not in self.connections_cache: + self.connections_cache[ident_to.val] = [] + self.connections_cache[ident_from.val].append(ident_to.val) + # Sanity check + for _, vals in self.connections_cache: + if len(set(vals)) != len(vals): + raise p.ParseFatalException("Bug found in Connection!!") + + def into_unmodified_str(self): + # TODO(Joan): improve this + return "connection" + + def __str__(self): + res = "Connection<" + res += " {} ".format(map(lambda (x, y): (str(x), str(y)), + self.connections)) + res += ">" + return res + + +class Assignment(ASTNode): + + def __init__(self, span, dot_path, value): + """ + Construct an assignment AST node. + :type span: Span + :param span: the span of the assignment. + :type dot_path: DotPath + :param dot_path: the left hand side of the assignment. + :type value: Component | Number | StringLit | JsonObj | DotPath | Expr + :param value: the right hand side of the assignment. + """ + super(Assignment, self).__init__(span) + if (isinstance(value, Component) or + isinstance(value, JsonObj) or + isinstance(value, Number) or + isinstance(value, StringLit) or + isinstance(value, Ident) or + isinstance(value, DotPath) or + isinstance(value, Expr)) and\ + isinstance(dot_path, DotPath): + self.lhs = dot_path + self.rhs = value + else: + raise exception.BananaGrammarBug( + 'Impossible assignment found with' + ' left hand side: {} and' + ' right hand side: {}' + .format(type(dot_path), type(value)) + ) + + def into_unmodified_str(self): + return "{} = {}".format(self.lhs.into_unmodified_str(), + self.rhs.into_unmodified_str()) + + def __str__(self): + return "{} = {}".format(str(self.lhs), str(self.rhs)) + + +class ComponentCtorArg(ASTNode): + + def __init__(self, span, value, arg_name=None): + """ + Construct an argument for a component ctor + :type span: Span + :param span: Span of the argument. + :type value: Number | StringLit | JsonObj | DotPath | Expr + :param value: Value for the argument + :type arg_name: Ident + :param arg_name: Name of the argument + """ + super(ComponentCtorArg, self).__init__(span) + if (isinstance(value, JsonObj) or + isinstance(value, Number) or + isinstance(value, StringLit) or + isinstance(value, Ident) or + isinstance(value, DotPath) or + isinstance(value, Expr)) and ( + isinstance(arg_name, Ident) or + arg_name is None): + self.arg_name = arg_name + self.value = value + else: + # This code should be unreachable. + # The grammar as defined should prevent us from + # seeing an arg value or a value of the incorrect type + raise exception.BananaGrammarBug( + 'Impossible constructor argument found with' + ' left hand side: {} and' + ' right hand side: {}' + .format(type(arg_name), type(value)) + ) + + def into_unmodified_str(self): + return "{} = {}".format(self.arg_name.into_unmodified_str(), + self.value.into_unmodified_str()) + + def __str__(self): + if self.arg_name is not None: + return "{} = {}".format(self.arg_name, self.value) + else: + return "{}".format(self.value) + + +class Component(ASTNode): + + def __init__(self, span, type_name=None, args=None): + """ + Construct a component + :type span: Span + :param span: Span of this component + :type type_name: Ident + :param type_name: Name of this component + :type args: list[ComponentCtorArg] + :param args: List of arguments + """ + super(Component, self).__init__(span) + if args is None: + args = [] + self.type_name = type_name + self.args = args + + def set_ctor(self, type_name): + """ + Set the constructor name of that component + :type type_name: Ident + :param type_name: Name of that constructor + """ + self.type_name = type_name + + def add_arg(self, arg): + """ + Add an argument to that component constructor. + :type arg: ComponentCtorArg + :param arg: Argument to add to that component. + """ + self.args.append(arg) + + def into_unmodified_str(self): + return self.type_name.into_unmodified_str() + "(" + \ + ', '.join(map(lambda x: x.into_unmodified_str(), self.args)) +\ + ")" + + def __str__(self): + res = "" + res += "Component {" + res += " type_name: {},".format(self.type_name) + arr = ', '.join(map(lambda x: str(x), self.args)) + res += " args: {}".format("[" + arr + "]") + res += "}" + return res diff --git a/monasca_analytics/banana/grammar/base_ast.py b/monasca_analytics/banana/grammar/base_ast.py new file mode 100644 index 0000000..f11044f --- /dev/null +++ b/monasca_analytics/banana/grammar/base_ast.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class ASTNode(object): + """ + Base class of all ast nodes + """ + def __init__(self, span): + """ + Construct an ASTNode + :type span: Span + :param span: span for this AST node. + """ + self.span = span + + @abc.abstractmethod + def into_unmodified_str(self): + """ + Returns a simple name for this ASTNode. It should be minimalist + and user oriented. No span info, no debug info. + :rtype: str + :returns: A simple name for that ast node. + """ + pass + + def __ne__(self, other): + return not self.__eq__(other) + + +class Span(object): + """ + Represent a region of code, used for error reporting. + Position are absolute within the file. + """ + + def __init__(self, text, lo, hi): + """ + :type text: str | None + :param text: Full text of the file + :type lo: int + :param lo: position of the beginning of the region + :type hi: int + :param hi: position of the end of the region + """ + self._text = text + self.lo = lo + self.hi = hi + + def __str__(self): + if self._text is not None: + return self._text[self.lo:self.hi] + else: + return '?SPAN?' + + def new_with_offset(self, offset): + """ + Construct a new Span with an offset applied + to lo. + + :type offset: int + :param offset: Offset to apply to lo. + :rtype: Span + :return: Returns a new span + """ + return Span(self._text, self.lo + offset, self.hi) + + def str_from_to(self, to_span): + """ + Returns a string that start at self and stops at to_span. + :type to_span: Span + :param to_span: Span to stop at. + :rtype: basestring + :return: Returns the string encapsulating both + """ + return self._text[self.lo:to_span.hi] + + def get_line(self): + """ + Returns the line for associated with this span. + """ + if self._text is not None: + splitted = self._text.splitlines() + current_pos = 0 + for line in splitted: + if current_pos < self.lo < len(line) + current_pos: + return line.strip() + else: + current_pos += len(line) + else: + return '?LINE?' + + def get_lineno(self): + """ + Returns the line number of this span. + """ + if self._text is not None: + splitted = self._text.splitlines() + current_pos = 0 + lineno = 0 + for _ in xrange(0, len(splitted)): + line = splitted[lineno] + if current_pos < self.lo < len(line) + current_pos: + return lineno + 1 + else: + current_pos += len(line) + lineno += 1 + return lineno + else: + return -1 + +DUMMY_SPAN = Span(None, 0, 0) + + +def from_parse_fatal(parse_fatal_exception): + """ + Convert the provided ParseFatalException into a Span. + + :type parse_fatal_exception: pyparsing.ParseFatalException + :param parse_fatal_exception: Exception to convert. + :rtype: Span + :return: Returns the span mapping to that fatal exception. + """ + return Span( + parse_fatal_exception.pstr, + parse_fatal_exception.loc, + parse_fatal_exception.loc + 1 + ) diff --git a/monasca_analytics/banana/grammar/config.py b/monasca_analytics/banana/grammar/config.py new file mode 100644 index 0000000..5b564d4 --- /dev/null +++ b/monasca_analytics/banana/grammar/config.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pyparsing as p + +import monasca_analytics.banana.emitter as emit +import monasca_analytics.banana.grammar.ast as ast +import monasca_analytics.banana.grammar.const as const +import monasca_analytics.exception.banana as exception + + +# This file describe the grammar for the banana config file. +# It make use of one sub grammar for certain configuration +# that requires expressions (see expression.py file) + + +def banana_grammar(emitter=emit.PrintEmitter()): + """ + Generate a banana parser that can be then used to + parse a banana content. It build an AST on which + operation can then be applied. + :return: Return a banana parser + :rtype: BananaScopeParser + """ + # Should debug + debug_grammar = False + + # Actions + def action_str_lit(s, l, t): + return ast.StringLit(ast.make_span(s, l, t), t[0]) + + def action_num_lit(s, l, t): + return ast.Number(ast.make_span(s, l, t), t[0]) + + def action_ident(s, l, t): + return ast.Ident(ast.make_span(s, l, t), t[0]) + + def action_expr(s, l, t): + if len(t) != 1: + raise exception.BananaGrammarBug( + 'Bug found in the grammar for expression,' + ' Please report this bug.' + ) + if isinstance(t[0], ast.Expr): + return t[0] + return ast.Expr(ast.make_span(s, l, t), t[0]) + + def action_dot_path(s, l, t): + # First token is the name of the variable + # The rest is the property path + if isinstance(t[0], ast.StringLit) and len(t[1:]) == 0: + return t[0] + return ast.DotPath(ast.make_span(s, l, t), t[0], t[1:]) + + def action_json_obj(s, l, t): + return ast.JsonObj(ast.make_span(s, l, t), t) + + def action_parse_ctor_arg(s, l, t): + if len(t) > 1: + return ast.ComponentCtorArg(ast.make_span(s, l, t), t[1], t[0]) + else: + return ast.ComponentCtorArg(ast.make_span(s, l, t), t[0]) + + def action_parse_comp_ctor(s, l, tokens): + comp = ast.Component(ast.make_span(s, l, tokens)) + for tok in tokens: + if isinstance(tok, ast.Ident): + comp.set_ctor(tok) + elif isinstance(tok, ast.ComponentCtorArg): + comp.add_arg(tok) + else: + raise exception.BananaGrammarBug( + 'Bug found in the grammar, Please report this bug' + ) + return comp + + def action_assignment(s, l, t): + return ast.Assignment(ast.make_span(s, l, t), t[0], t[1]) + + def action_create_connections(s, l, t): + ast_conn = ast.into_connection(t[0]) + ast_conn.span = ast.make_span(s, l, t) + for i in xrange(1, len(t)): + next_conn = ast.into_connection(t[i]) + ast_conn.connect_to(next_conn, emitter) + return ast_conn + + def action_merge_connections(s, l, t): + ast_conn = ast.Connection(ast.make_span(s, l, t)) + ast_conn.merge_all(t, emitter) + return ast_conn + + def action_root_ast(s, l, tokens): + root = ast.BananaFile(emitter) + for tok in tokens: + if isinstance(tok, ast.Assignment): + if isinstance(tok.rhs, ast.Component): + root.add_component_ctor(tok.lhs, tok.rhs) + else: + root.add_assignment(tok.lhs, tok.rhs) + elif isinstance(tok, ast.Connection): + root.add_connections(tok) + else: + raise exception.BananaGrammarBug( + 'Bug found in the grammar, Please report this bug.' + ) + return root + + # TODO(Joan): Remove once it is no longer needed + def print_stmt(s, l, t): + print("\nPRINT AST") + print(l, map(lambda x: str(x), t)) + print("END PRINT AST\n") + + def action_unimplemented(s, l, t): + raise exception.BananaGrammarBug("unimplemented code reached") + + # Tokens + equals = p.Literal("=").suppress().setName('"="').setDebug(debug_grammar) + arrow = p.Literal("->").suppress().setName('"->"').setDebug(debug_grammar) + lbra = p.Literal("[").suppress().setName('"["').setDebug(debug_grammar) + rbra = p.Literal("]").suppress().setName('"]"').setDebug(debug_grammar) + colon = p.Literal(":").suppress().setName('":"') + comma = p.Literal(",").suppress().setName(",") + less = p.Literal("<").suppress().setName('"<"') + greater = p.Literal(">").suppress().setName('">"') + lbrace = p.Literal("{").suppress().setName('"{"').setDebug(debug_grammar) + rbrace = p.Literal("}").suppress().setName('"}"').setDebug(debug_grammar) + lpar = p.Literal("(").suppress().setName('"("') + rpar = p.Literal(")").suppress().setName('")"') + + # Keywords + ing = p.Literal("ing").suppress() + imp = p.Literal("import").suppress() + fro = p.Literal("from").suppress() + + # String Literal, Numbers, Identifiers + string_lit = p.quotedString()\ + .setParseAction(action_str_lit)\ + .setName(const.STRING_LIT) + number_lit = p.Regex(r'\d+(\.\d*)?([eE]\d+)?')\ + .setParseAction(action_num_lit)\ + .setName(const.NUMBER) + ident = p.Word(p.alphas + "_", p.alphanums + "_")\ + .setParseAction(action_ident)\ + .setName(const.IDENT) + + # Path for properties + dot_prop = ident | string_lit + dot_path = p.delimitedList(dot_prop, ".")\ + .setParseAction(action_dot_path)\ + .setName(const.DOT_PATH)\ + .setDebug(debug_grammar) + + # Expressions + + # Here to simplify the logic, we can match directly + # against ident and string_lit to avoid having to deal + # only with dot_path. It also allow to remove the confusion + # where '"a"' could be interpreted as a dot_path and would thus + # be the same as 'a'. With the following, the first we + # always be type-checked as a String whereas the latter will + # be as the type of the variable. + expr = p.infixNotation(number_lit | dot_path, [ + (p.oneOf('* /'), 2, p.opAssoc.LEFT), + (p.oneOf('+ -'), 2, p.opAssoc.LEFT), + ], lpar=lpar, rpar=rpar) + expr.setParseAction(action_expr)\ + .setName(const.EXPR)\ + .setDebug(debug_grammar) + + # Json-like object (value are much more) + json_obj = p.Forward() + json_value = p.Forward() + json_array = p.Group( + lbra + p.Optional(p.delimitedList(json_value)) + rbra + ) + json_array.setDebug(debug_grammar) + json_array.setName(const.JSON_ARRAY) + json_value <<= expr | json_obj | json_array + json_value.setDebug(debug_grammar)\ + .setName(const.JSON_VALUE) + json_members = p.delimitedList(p.Group(dot_path + colon - json_value)) +\ + p.Optional(comma) + json_members.setDebug(debug_grammar)\ + .setName(const.JSON_MEMBERS) + json_obj <<= p.Dict(lbrace + p.Optional(json_members) - rbrace) + json_obj.setParseAction(action_json_obj)\ + .setName(const.JSON_OBJ)\ + .setDebug(debug_grammar) + + # Component constructor + arg = (ident + equals - (expr | json_obj)) | expr | json_obj + arg.setParseAction(action_parse_ctor_arg) + params = p.delimitedList(arg) + comp_ctor = ident + lpar - p.Optional(params) + rpar + comp_ctor.setParseAction(action_parse_comp_ctor)\ + .setName(const.COMP_CTOR)\ + .setDebug(debug_grammar) + + # Assignments + assignment = dot_path + equals - (comp_ctor | expr | json_obj) + assignment.setParseAction(action_assignment) + + # Connections + connection = p.Forward() + array_of_connection = p.Group( + lbra + p.Optional(p.delimitedList(connection)) + rbra + ) + array_of_connection.setParseAction(action_merge_connections) + last_expr = ident | array_of_connection + this_expr = p.Forward() + match_expr = p.FollowedBy(last_expr + arrow - last_expr) + \ + (last_expr + p.OneOrMore(arrow - last_expr)) + this_expr <<= match_expr | last_expr + connection <<= this_expr + + match_expr.setDebug(debug_grammar)\ + .setName(const.CONNECTION) \ + .setParseAction(action_create_connections) + + # Definitions + definition = ing - less - string_lit - greater - ident - lbrace - rbrace + definition.setDebug(debug_grammar)\ + .setName(const.DEFINITION)\ + .setParseAction(action_unimplemented) + + # Import directive + module_def = (imp - ident) | fro - ident - imp - ident + module_def.setDebug(debug_grammar)\ + .setName(const.MOD_IMPORT)\ + .setParseAction(action_unimplemented) + + # Comments + comments = "#" + p.restOfLine + + statement = assignment | \ + match_expr | \ + definition | \ + module_def + + statement.setName(const.STATEMENT) + statement.setDebug(debug_grammar) + statement.setParseAction(print_stmt) + + # Grammar + grammar = p.OneOrMore(statement).ignore(comments) + grammar.setParseAction(action_root_ast) + + return BananaScopeParser(grammar) + + +class BananaScopeParser(object): + """ + Aggregate and resolve conflicts as everything was define + within the same scope. Usefull for have cpp "include"-like + functionality when importing another file. + """ + + def __init__(self, grammar): + self._grammar = grammar + + def parse(self, string): + """ + Parse the given input string. + :type string: str + :param string: Input string. + :rtype: ast.BananaFile + :return: Returns the ast root. + """ + tree = self._grammar.parseString(string, parseAll=True)[0] + return tree diff --git a/monasca_analytics/banana/grammar/const.py b/monasca_analytics/banana/grammar/const.py new file mode 100644 index 0000000..167196a --- /dev/null +++ b/monasca_analytics/banana/grammar/const.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +ASSIGNMENT = "assignment" +CONNECTION = "connection" +DEFINITION = "definition" +MOD_IMPORT = "mod_import" +DOT_PATH = "path" +NUMBER = "number" +STRING_LIT = "string_lit" +IDENT = "ident" +EXPR = "expression" +JSON_OBJ = "jsonlike_obj" +STATEMENT = "statement" +JSON_ARRAY = "jsonlike_array" +JSON_VALUE = "jsonlike_value" +JSON_MEMBERS = "jsonlike_members" +COMP_CTOR = "component_ctor" diff --git a/monasca_analytics/banana/pass_manager.py b/monasca_analytics/banana/pass_manager.py new file mode 100644 index 0000000..7eb7ba3 --- /dev/null +++ b/monasca_analytics/banana/pass_manager.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pyparsing as p + +import monasca_analytics.banana.deadpathck.config as deadpathck +import monasca_analytics.banana.emitter as emit +import monasca_analytics.banana.eval.config as ev +import monasca_analytics.banana.eval.ctx as ctx +import monasca_analytics.banana.grammar.base_ast as span_util +import monasca_analytics.banana.grammar.config as grammar +import monasca_analytics.banana.typeck.config as typeck +import monasca_analytics.exception.banana as exception + + +def execute_banana_string(banana_str, driver, emitter=emit.PrintEmitter()): + """ + Execute the provided banana string. + It will run the parse phase, and the typechecker. + :type banana_str: str + :param banana_str: The string to parse and type check. + :type driver: monasca_analytics.spark.driver.DriverExecutor + :param driver: Driver that will manage the created + components and connect them together. + :type emitter: emit.Emitter + :param emitter: Emitter for reporting errors/warning. + """ + try: + # Convert the grammar into an AST + parser = grammar.banana_grammar(emitter) + ast = parser.parse(banana_str) + # Compute the type table for the given AST + type_table = typeck.typeck(ast) + # Remove from the tree path that are "dead" + deadpathck.deadpathck(ast, type_table, emitter) + # Check that there's at least one path to be executed + deadpathck.contains_at_least_one_path_to_a_sink(ast, type_table) + # Evaluate the script + ev.eval_ast(ast, type_table, driver) + except exception.BananaException as err: + emitter.emit_error(err.get_span(), str(err)) + except p.ParseSyntaxException as err: + emitter.emit_error(span_util.from_parse_fatal(err), err.msg) + except p.ParseFatalException as err: + emitter.emit_error(span_util.from_parse_fatal(err), err.msg) + + +def compute_type_table(banana_str): + """ + Compute the type table for the provided banana string + if possible. + :type banana_str: str + :param banana_str: The string to parse and type check. + """ + # Convert the grammar into an AST + parser = grammar.banana_grammar() + ast = parser.parse(banana_str) + # Compute the type table for the given AST + return typeck.typeck(ast) + + +def compute_evaluation_context(banana_str, cb=lambda *a, **k: None): + """ + Compute the evaluation context for the provided + banana string. + :type banana_str: str + :param banana_str: The string to parse and type check. + :param cb: Callback called after each statement + """ + parser = grammar.banana_grammar() + ast = parser.parse(banana_str) + type_table = typeck.typeck(ast) + context = ctx.EvaluationContext() + + def custom_cb(_type, lhs, value): + cb(context, _type, lhs, value) + + ev.eval_statements_generic(ast.statements, type_table, context, custom_cb) diff --git a/monasca_analytics/banana/typeck/README.md b/monasca_analytics/banana/typeck/README.md new file mode 100644 index 0000000..226b1ed --- /dev/null +++ b/monasca_analytics/banana/typeck/README.md @@ -0,0 +1,85 @@ +## Type-checker + +This folder is all about the type checking of `banana` files. +The type checker purpose is to verify that components exist, +the type of local variable matches the requirements of components +parameters and assignments between them are correct. It also +checks that connections between components are valid. + +The biggest difference between the old `validation` of the JSON +format is that we have more information available. We can warn +users when they make mistakes and point at the exact locations +using `Span`. Also, the type table generated is used by other passes +to perform other static analyses. + +This is the second step of the pipeline: +``` + +-------+ +---------------------+ + | | | | + ---> | AST | ---- typeck ---> | AST & TypeTable | ---> + | | | | + +-------+ +---------------------+ +``` + +The module `type_util.py` contains all the possible types that are +known by the type checker. The `TypeTable` built lives in the +`type_table.py` module. + +### Current status + +* [x] Type check numbers +* [x] Type check string literals +* [x] Type check variable assignments +* [x] Type check component assignments +* [x] Type check component parameters +* [x] Type check connections +* [x] Resolve variable names +* [ ] Resolve imports +* [ ] Type check disconnections + +### Tests + +All tests for the type checker (i.e. making sure that +inferred types are correct and that errors are raised in +appropriate situation) lives in `test/banana/typeck`. + +This folder looks like this: + +``` +test/banana/typeck +├── should_fail +│   ├── ... +│   └── file.banana +├── should_pass +│   ├── ... +│   └── file.banana +└── test_typeck_config.py +``` + +The `test_typeck_config`, generates one test for each file +in the `should_pass` and `should_fail` directories. + +For each generated test, we basically run the following passes: + +* `grammar`: convert the input text into an AST. +* `typeck`: run the type checker. + +Tests can assert various things in `banana` comments: + + - In the `should_fail` directory, a test is expected to use + the `RAISE` instruction to specify a type of exceptions + that should be raised by the test. + - In the `should_pass` directory, a test is expected to not + raised any exception **and** to specify the state of the + `TypeTable` when the type checker has type checked everything + in the file. This is done with the `TYPE_TABLE_EQ` instruction. + + +#### Available instruction + +* `# RAISE `: Check that `exception-name` is raised. +* `# TYPE_TABLE_EQ ` +* `# NEW_TEST`: This instruction splits the file into two tests. However, + the expected exception or type table should still be the same. It + allows us to verify what should be semantically equivalent in the + grammar. \ No newline at end of file diff --git a/monasca_analytics/banana/typeck/__init__.py b/monasca_analytics/banana/typeck/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/monasca_analytics/banana/typeck/config.py b/monasca_analytics/banana/typeck/config.py new file mode 100644 index 0000000..f63d4fe --- /dev/null +++ b/monasca_analytics/banana/typeck/config.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import monasca_analytics.ingestor.base as ingestor +import monasca_analytics.ldp.base as ldp +import monasca_analytics.sink.base as sink +import monasca_analytics.sml.base as sml +import monasca_analytics.source.base as source +import monasca_analytics.voter.base as voter + +import monasca_analytics.banana.grammar.ast as ast +import monasca_analytics.banana.typeck.connections as conn +import monasca_analytics.banana.typeck.type_table as typetbl +import monasca_analytics.banana.typeck.type_util as u +import monasca_analytics.exception.banana as exception +import monasca_analytics.exception.monanas as exception_monanas +import monasca_analytics.util.common_util as introspect + + +def typeck(banana_file): + """ + Type-check the provided BananaFile instance. + If it type check, it returns the associated TypeTable. + :type banana_file: ast.BananaFile + :param banana_file: The file to typecheck. + :rtype: typetbl.TypeTable + :return: Returns the TypeTable for this BananaFile + """ + type_table = typetbl.TypeTable() + statement_index = 0 + for stmt in banana_file.statements: + lhs, rhs = stmt + type_computed = typeck_rhs(rhs, type_table) + type_table.set_type(lhs, type_computed, statement_index) + statement_index += 1 + conn.typeck_connections(banana_file.connections, type_table) + return type_table + + +def typeck_rhs(ast_value, type_table): + """ + Type-check the provided ast value. And returns its type. + This function does not support assignment, + :type ast_value: ast.ASTNode + :param ast_value: The ast_value to type check. + :type type_table: typetbl.TypeTable + :param type_table: The type table. Used for type lookup. + :rtype: u.Component | u.Object | u.Number | u.String + :return: Returns the computed type. + """ + if isinstance(ast_value, ast.Number): + return u.Number() + if isinstance(ast_value, ast.StringLit): + return u.String() + if isinstance(ast_value, ast.Ident): + return type_table.get_type(ast_value) + if isinstance(ast_value, ast.DotPath): + return type_table.get_type(ast_value) + if isinstance(ast_value, ast.Expr): + return typeck_expr(ast_value, type_table) + if isinstance(ast_value, ast.JsonObj): + return typeck_jsonobj(ast_value, type_table) + if isinstance(ast_value, ast.Component): + return typeck_component(ast_value, type_table) + raise Exception("Unhandled ast value type {}!!".format(ast_value)) + + +def typeck_jsonobj(json_obj, type_table): + """ + Type-check a json-like object. If it succeeds + it return the appropriate type describing this + json like object. Raise an exception otherwise. + :type json_obj: ast.JsonObj + :param json_obj: The JsonObj ast node. + :type type_table: typetbl.TypeTable + :param type_table: The type table. + :rtype: u.Object + :return: Returns an instance of util.Object describing + the full type of this json object. + """ + root_type = u.Object(strict_checking=False) + + for k, v in json_obj.props.iteritems(): + sub_type = u.create_object_tree(k, typeck_rhs(v, type_table)) + u.attach_to_root(root_type, sub_type, json_obj.span) + + return root_type + + +def typeck_expr(expr, type_table): + """ + Type-check the given expression. If the typecheck + pass, the resulting type will be used for the strategy + to use when evaluating this expression. + :type expr: ast.Expr + :param expr: The expression to typecheck. + :type type_table: typetbl.TypeTable + :param type_table: Type of the table + :rtype: u.Number | u.String + :return: Returns the type of the expression if possible + :raise: Raise an exception + """ + # In the case where we are just wrapping around + # only one expression, the logic below + # needs to be skipped. + if len(expr.expr_tree) == 1: + return typeck_rhs(expr.expr_tree[0], type_table) + + _type = None + must_be_number = False + + def check_type(old_type, new_type): + if new_type == old_type: + return old_type + elif new_type == u.String(): + if must_be_number: + raise exception.BananaTypeError( + expected_type=u.Number, + found_type=new_type + ) + if old_type is None: + return new_type + elif u.can_to_str(old_type): + return new_type + else: + raise exception.BananaTypeError( + expected_type=old_type, + found_type=new_type + ) + elif new_type == u.Number(): + if old_type is None: + return new_type + elif old_type == u.String(): + return old_type + elif not old_type == u.Number(): + raise exception.BananaTypeError( + expected_type=old_type, + found_type=new_type + ) + else: + raise exception.BananaTypeError( + expected_type=old_type, + found_type=new_type + ) + + def allowed_symbol(current_type): + if current_type == u.String(): + return ['+'] + else: + return ['+', '-', '*', '/'] + + for el in expr.expr_tree: + if isinstance(el, ast.StringLit): + _type = check_type(_type, u.String()) + elif isinstance(el, ast.Number): + _type = check_type(_type, u.Number()) + elif isinstance(el, ast.Ident): + ident_type = type_table.get_type(el) + _type = check_type(_type, ident_type) + elif isinstance(el, ast.DotPath): + dotpath_type = type_table.get_type(el) + _type = check_type(_type, dotpath_type) + elif isinstance(el, ast.Expr): + _type = check_type(_type, typeck_expr(el, type_table)) + elif isinstance(el, basestring): + if el not in allowed_symbol(_type): + raise exception.BananaUnknownOperator(expr.span, el, _type) + if el in ['-', '*', '/']: + must_be_number = True + else: + raise exception.BananaTypeError( + expected_type=[u.Number.__name__, u.String.__name__, + u.Object.__name__], + ) + + # The final type if we made until here! + return _type + + +def typeck_component(component, type_table): + """ + Type-check the provided component. Returns + the appropriate subclass of util.Component if + successful, or raise an exception if there's + an error. + :type component: ast.Component + :param component: The component ast node. + :type type_table: typetbl.TypeTable + :param type_table: the type table. + :rtype: u.Source | u.Sink | u.Voter | u.Ldp | u.Sml | u.Ingestor + :return: Returns the appropriate type for the component. + """ + # TODO(Joan): This wont't work for type that are defined + # TODO(Joan): at the language level. We need a registration service + # TODO(Joan): to manage the Types of component that we can create + # TODO(Joan): instead of this hacky function call. + try: + component_type = introspect.get_class_by_name(component.type_name.val) + comp_params = component_type.get_params() + except exception_monanas.MonanasNoSuchClassError: + raise exception.BananaUnknown( + component + ) + + # Compute the type of the component + if issubclass(component_type, source.BaseSource): + comp_type = u.Source(component_type.__name__, comp_params) + elif issubclass(component_type, sink.BaseSink): + comp_type = u.Sink(component_type.__name__, comp_params) + elif issubclass(component_type, sml.BaseSML): + comp_type = u.Sml(component_type.__name__, comp_params) + elif issubclass(component_type, voter.BaseVoter): + comp_type = u.Voter(component_type.__name__, comp_params) + elif issubclass(component_type, ldp.BaseLDP): + comp_type = u.Ldp(component_type.__name__, comp_params) + elif issubclass(component_type, ingestor.BaseIngestor): + comp_type = u.Ingestor(component_type.__name__, comp_params) + else: + raise exception.BananaTypeCheckerBug("Couldn't find a type for '{}'" + .format(component.type_name.val)) + + # Type check the parameters + if len(component.args) > len(comp_params): + raise exception.BananaComponentTooManyParams(component.span) + + # Does saying that parameter should either all have a name + # or non at all satisfying? -> Yes + # Are parameter all named? + all_named = -1 + for arg in component.args: + if arg.arg_name is not None: + if all_named == 0: + raise exception.BananaComponentMixingParams(arg.span, False) + all_named = 1 + else: + if all_named == 1: + raise exception.BananaComponentMixingParams(arg.span, True) + all_named = 0 + + if all_named == 1: + for arg in component.args: + param = filter(lambda x: x.param_name == arg.arg_name.inner_val(), + comp_params) + if len(param) != 1: + raise exception.BananaComponentIncorrectParamName( + component=component.type_name, + found=arg.arg_name + ) + param = param[0] + expr_type = typeck_rhs(arg.value, type_table) + if not u.can_be_cast_to(expr_type, param.param_type): + raise exception.BananaArgumentTypeError( + where=arg, + expected_type=param.param_type, + received_type=expr_type + ) + else: + for arg, param in zip(component.args, comp_params): + arg.arg_name = ast.Ident(arg.span, param.param_name) + expr_type = typeck_rhs(arg.value, type_table) + if not u.can_be_cast_to(expr_type, param.param_type): + raise exception.BananaArgumentTypeError( + where=arg, + expected_type=param.param_type, + received_type=expr_type + ) + + return comp_type diff --git a/monasca_analytics/banana/typeck/connections.py b/monasca_analytics/banana/typeck/connections.py new file mode 100644 index 0000000..6d77521 --- /dev/null +++ b/monasca_analytics/banana/typeck/connections.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import monasca_analytics.banana.typeck.type_util as util +import monasca_analytics.exception.banana as exception + +valid_connections_types = { + util.Source: [util.Ingestor, util.Ldp], + util.Ingestor: [util.Sml, util.Sink], + util.Sml: [util.Voter, util.Sink], + util.Voter: [util.Ldp, util.Sink], + util.Ldp: [util.Sink], + util.Sink: [] +} + + +def typeck_connections(connection, type_table): + """ + Once all variable have been type-checked, we can + try to type-check connections. + :type connection: monasca_analytics.banana.grammar.ast.Connection + :param connection: The connection to type-check + :type type_table: monasca_analytics.banana.typeck.type_table.TypeTable + :param type_table: The table with all variable already type-checked. + :raise Raise an exception if there's a type error in connections. + """ + if connection is not None: + for ident_from, ident_to in connection.connections: + type_from = type_table.get_type(ident_from) + type_to = type_table.get_type(ident_to) + if not util.is_comp(type_from): + raise exception.BananaTypeError( + expected_type=util.Component(), + found_type=type_from + ) + if not util.is_comp(type_to): + raise exception.BananaTypeError( + expected_type=util.Component(), + found_type=type_to + ) + if type(type_to) not in valid_connections_types[type(type_from)]: + possible_types = map(lambda x: x.__name__, + valid_connections_types[type(type_from)]) + raise exception.BananaConnectionError( + ident_from, ident_to, type_from, possible_types + ) diff --git a/monasca_analytics/banana/typeck/type_table.py b/monasca_analytics/banana/typeck/type_table.py new file mode 100644 index 0000000..55a756a --- /dev/null +++ b/monasca_analytics/banana/typeck/type_table.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy + +import monasca_analytics.banana.grammar.ast as ast +import monasca_analytics.banana.typeck.type_util as util +import monasca_analytics.exception.banana as exception +import monasca_analytics.util.string_util as strut + + +class TypeTable(object): + """ + Type table. Support lookup for JsonLike object. + Json-like object have properties that needs to be + type-checked. The TypeTable allows to store + that information as well. All type values are + rooted by their variable name. + + Every-time a variable type is erased, we create a new + snapshot of the variables types. This allow to have + variable where the type change as the statement are + being executed. + """ + def __init__(self): + self._variables_snapshots = [(0, {})] + self._variables = self._variables_snapshots[0][1] + + def get_type(self, var, statement_index=None): + variables = self.get_variables(statement_index) + if isinstance(var, ast.Ident): + if var in variables: + return variables[var] + else: + raise exception.BananaUnknown(var) + # If we encounter a dot path: + if isinstance(var, ast.DotPath): + if var.varname in variables: + if len(var.properties) > 0: + return variables[var.varname][var.next_dot_path()] + else: + return variables[var.varname] + else: + raise exception.BananaUnknown(var.varname) + raise exception.BananaTypeCheckerBug("Unkown type for {}".format(var)) + + def set_type(self, var, _type, statement_index): + """ + Set the type for the given var to _type. + + :type var: ast.Ident | ast.DotPath + :param var: The var to set a type. + :type _type: util.Object | util.Component | util.String | util.Number + :param _type: The type for the var. + :type statement_index: int + :param statement_index: The statement at which this assignment was + made. + """ + if _type is None: + raise exception.BananaTypeCheckerBug( + "'None' is not a valid banana type" + ) + + if isinstance(var, ast.Ident): + self._check_needs_for_snapshot(var, _type, statement_index) + self._variables[var] = _type + return + + if isinstance(var, ast.DotPath): + if util.is_comp(_type) and len(var.properties) > 0: + raise exception.BananaAssignCompError(var.span) + + if len(var.properties) == 0: + self._check_needs_for_snapshot( + var.varname, + _type, + statement_index + ) + self._variables[var.varname] = _type + else: + if var.varname in self._variables: + var_type = self._variables[var.varname] + if isinstance(var_type, util.Object): + new_type = util.create_object_tree( + var.next_dot_path(), _type) + util.attach_to_root(var_type, new_type, var.span, + erase_existing=True) + elif isinstance(var_type, util.Component): + var_type[var.next_dot_path()] = _type + else: + raise exception.BananaTypeError( + expected_type=util.Object, + found_type=type(var) + ) + # Var undeclared, declare its own type + else: + new_type = util.create_object_tree(var.next_dot_path(), + _type) + self._variables[var.varname] = new_type + return + raise exception.BananaTypeCheckerBug("Unreachable code reached.") + + def get_variables(self, statement_index=None): + """ + Returns the list of variables with their associated type. + + :type statement_index: int + :param: Statement index. + :rtype: dict[str, util.Object|util.Component|util.String|util.Number] + """ + if statement_index is None: + return self._variables + + variables = {} + for created_at, snap in self._variables_snapshots: + if created_at < statement_index: + variables = snap + else: + break + + return variables + + def get_variables_snapshots(self): + return self._variables_snapshots + + def _check_needs_for_snapshot(self, var, _type, statement_index): + if var in self._variables: + # If we shadow a component, we need to raise an error + if util.is_comp(self._variables[var]): + raise exception.BananaShadowingComponentError( + where=var.span, + comp=self._variables[var].class_name + ) + + # If we change the type of the variable, we create a new snapshot: + # This is very strict but will allow to know exactly how + # the type of a variable (or a property) changed. + if self._variables[var] != _type: + self._create_snapshot(statement_index) + + def _create_snapshot(self, statement_index): + """ + Create a new snapshot of the variables. + :type statement_index: int + :param statement_index: index of the statement + (should be strictly positive) + """ + new_snapshot = copy.deepcopy(self._variables) + self._variables_snapshots.append(( + statement_index, new_snapshot + )) + self._variables = new_snapshot + + def __contains__(self, key): + """ + Test if the type table contains or not the provided + path. This function is more permissive than the other two. + It will never raise any exception (or should aim not to). + :type key: basestring | ast.Ident | ast.DothPath + :param key: The key to test. + :return: Returns True if the TypeTable contains a type for the + given path or identifier. + """ + if isinstance(key, basestring): + return key in self._variables + + if isinstance(key, ast.Ident): + return key.val in self._variables + + if isinstance(key, ast.DotPath): + res = key.varname in self._variables + if not res: + return False + val = self._variables[key.varname] + for prop in key.properties: + if isinstance(val, util.Object): + if prop in val.props: + val = val[prop] + else: + return False + return True + + return False + + def __str__(self): + return strut.dict_to_str(self._variables) diff --git a/monasca_analytics/banana/typeck/type_util.py b/monasca_analytics/banana/typeck/type_util.py new file mode 100644 index 0000000..b5e6444 --- /dev/null +++ b/monasca_analytics/banana/typeck/type_util.py @@ -0,0 +1,559 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Util files to manipulates banana types. + +The list of possible types is as follow: + + * `Number` + * `Boolean` + * `String` + * `Object` (Json-like object) + * `Component.Source.` + * `Component.Ingestor.` + * `Component.Sink.` + * `Component.Voter.` + * `Component.Ldp.` + * `Component.Sml.` + +where will be the component class name defined +in the code base. + +For type defined in banana such as Json parsers, +refers the name they are defined with. +""" +import abc +import six + +import monasca_analytics.banana.grammar.ast as ast +import monasca_analytics.exception.banana as exception +import monasca_analytics.util.string_util as strut + + +@six.add_metaclass(abc.ABCMeta) +class IsType(object): + """ + Any class that represents a Banana type should inherit + from this class. + """ + + def __ne__(self, other): + # Dispatch to eq function + return not self.__eq__(other) + + @abc.abstractmethod + def default_value(self): + pass + + +class Any(IsType): + """ + Any type. This type should be used by component's writer when + they have a complex handling of parameters. This is not + recommended though as it move the error handling to + the component writer. + """ + + def __str__(self): + return "TypeAny" + + def __eq__(self, _): + # Any type is equal to nothing not even itself. + return False + + def __getitem__(self, _): + return Any() + + def __hash__(self): + raise Exception("Any type should not be used in dictionaries.") + + def default_value(self): + return {} + + +class String(IsType): + """ + String Type. + """ + + def __str__(self): + return "TypeString" + + def __eq__(self, other): + return isinstance(other, String) + + def __hash__(self): + return hash(str(self)) + + def default_value(self): + return "" + + +class Number(String): + """ + Number type. Banana has only floating point value. + """ + + def __str__(self): + return "TypeNumber" + + def __eq__(self, other): + return isinstance(other, Number) + + def __hash__(self): + return hash(str(self)) + + def default_value(self): + return 0 + + +class Enum(String): + """ + Enum type. This type is a way to constraint a string or number, + to a specific set of values. + """ + + def __init__(self, variants): + self.variants = variants + + def __eq__(self, other): + return isinstance(other, Enum) and self.variants == other.variants + + def __hash__(self): + return hash(self.variants) + + def __str__(self): + return "TypeEnum < {} >".format(','.join(self.variants)) + + def default_value(self): + return "" + + +def attach_to_root(root_obj, obj1, span, erase_existing=False): + """ + Attach the object obj1 to the root_obj object type. + + :type root_obj: Object + :param root_obj: The root object + :type obj1: Object + :param obj1: The object to attach. + :type span: Span + :param span: The span for this change. + :type erase_existing: bool + :param erase_existing: Set to true if the root type should + always be erased. + """ + for key, child_type in obj1.props.iteritems(): + if key in root_obj.props: + root_sub_type = root_obj.props[key] + # Both are object -> recurse + if isinstance(root_sub_type, Object) and\ + isinstance(child_type, Object): + attach_to_root(root_sub_type, child_type, span, erase_existing) + elif erase_existing: + root_obj.props[key] = child_type + else: + raise exception.BananaTypeError( + expected_type=root_sub_type, + found_type=child_type, + span=span + ) + else: + # We can simply attach the new type! + root_obj.props[key] = child_type + + +def create_object_tree(dot_path, value): + """ + Create a linear tree of object type from the dot_path. + Also work when dot_path is an Ident or StringLit. + + :type dot_path: ast.DotPath | ast.Ident | ast.StringLit + :param dot_path: The ast node that forms a linear tree of type. + :type value: Object | String | Number + :param value: the value to set at the end of the linear tree. + :rtype: Object + :return: Returns the created object + """ + if is_comp(value): + raise exception.BananaAssignCompError(dot_path.span) + + # {a.b.c: value} + root_object = Object(strict_checking=False) + if isinstance(dot_path, ast.DotPath): + # {a: value} + if len(dot_path.properties) == 0: + root_object.props[dot_path.varname.inner_val()] = value + else: + # {a: } + root_object.props[dot_path.varname.inner_val()] = \ + Object(strict_checking=False) + # {b.c: value} + current_obj = root_object.props[dot_path.varname.inner_val()] + last_index = len(dot_path.properties) - 1 + for index, sub_prop in enumerate(dot_path.properties): + sub_prop_name = sub_prop.inner_val() + if index != last_index: + current_obj.props[sub_prop_name] = \ + Object(strict_checking=False) + current_obj = current_obj.props[sub_prop_name] + else: + current_obj.props[sub_prop_name] = value + else: + # Ident and StringLit are captured here. + root_object.props[dot_path.inner_val()] = value + return root_object + + +class Object(String): + """ + Object Type. The value that are dictionary-like have this type. + """ + + def __init__(self, props=None, strict_checking=True): + if props is None: + props = {} + self.props = props + # Strict checking is off for all objects defined within the banana + # language. It is on by default for components so that they can + # force the type checker to throw errors when we try to access + # or to modify unknown properties + self.strict_checking = strict_checking + + def __getitem__(self, key): + # a.b or a."b" + if isinstance(key, ast.Ident) or isinstance(key, ast.StringLit): + if key.inner_val() not in self.props: + raise exception.BananaPropertyDoesNotExists(key, + on_type=self) + return self.props[key.inner_val()] + + # a.b.c + if isinstance(key, ast.DotPath): + if key.varname.inner_val() not in self.props: + raise exception.BananaPropertyDoesNotExists(key.varname, + on_type=self) + sub_object = self.props[key.varname.inner_val()] + if len(key.properties) == 0: + return sub_object + # Recurse + if isinstance(sub_object, Object): + return sub_object[key.next_dot_path()] + if isinstance(sub_object, Any): + return sub_object + + raise exception.BananaPropertyDoesNotExists(key.next_dot_path(), + on_type=sub_object) + + raise exception.BananaTypeCheckerBug( + "Unreachable code in Object.__getitem__ reached." + ) + + def __str__(self): + if self.strict_checking: + return "TypeStruct < {} >".format(strut.dict_to_str(self.props)) + else: + return "TypeObject < {} >".format(strut.dict_to_str(self.props)) + + def __eq__(self, other): + return self.props == other + + def __hash__(self): + return hash(self.props) + + def default_value(self): + default_value = {} + for key, val in self.props.iteritems(): + default_value[key] = val.default_value() + return default_value + + +class Component(IsType): + """ + Type of all components. While not strictly used directly, it + is very useful to performs checks on variable that are supposed + to be any of the available components. + """ + + def __init__(self, ctor_properties=None, class_name=None): + """ + Component type + + :type ctor_properties: + list[monasca_analytics.component.params.ParamDescriptor] + :param ctor_properties: + :type class_name: str + :param class_name: Name of the class if there's any. + """ + self.ctor_properties = ctor_properties + self.class_name = class_name + + def __str__(self): + if self.class_name is None: + return "TypeComponent" + else: + return self.class_name + "(" +\ + ",".join(map(lambda x: x.param_name + "=" + str(x.param_type), + self.ctor_properties))\ + + ")" + + def __setitem__(self, dot_path, value): + """ + Attempt to set the value at 'dot_path' to 'value'. + + :type dot_path: ast.DotPath + :param dot_path: The path of the property + :type value: String | Enum | Object | Number + :param value: The new type to set. + """ + if self.ctor_properties is None: + raise exception.BananaTypeCheckerBug( + "Component type can't have properties" + ) + + if len(dot_path.properties) == 0: + for arg in self.ctor_properties: + if arg.param_name == dot_path.varname.inner_val(): + if not can_be_cast_to(value, arg.param_type): + raise exception.BananaArgumentTypeError( + expected_type=arg.param_type, + received_type=value, + where=dot_path.span + ) + else: + return + else: + for arg in self.ctor_properties: + if arg.param_name == dot_path.varname.inner_val(): + if isinstance(arg.param_type, Any): + return + elif isinstance(arg.param_type, Object): + next_dot_path = dot_path.next_dot_path() + sub_arg_type = arg.param_type[next_dot_path] + if not can_be_cast_to(value, sub_arg_type): + raise exception.BananaArgumentTypeError( + expected_type=sub_arg_type, + received_type=value, + where=next_dot_path.span + ) + else: + return + else: + raise exception.BananaPropertyDoesNotExists( + dot_path.next_dot_path(), + arg.param_type + ) + + raise exception.BananaPropertyDoesNotExists(dot_path, on_type=self) + + def __getitem__(self, dot_path): + """ + Return the type of the given item. + + :type dot_path: ast.DotPath + :param dot_path: The path to follow + :return: + """ + if self.ctor_properties is None: + raise exception.BananaTypeCheckerBug( + "Component type can't have properties" + ) + + if len(dot_path.properties) == 0: + for arg in self.ctor_properties: + if arg.param_name == dot_path.varname.inner_val(): + return arg.param_type + else: + for arg in self.ctor_properties: + if arg.param_name == dot_path.varname.inner_val(): + if isinstance(arg.param_type, Object): + return arg.param_type[dot_path.next_dot_path()] + else: + raise exception.BananaPropertyDoesNotExists( + dot_path.next_dot_path(), + arg.param_type + ) + + raise exception.BananaPropertyDoesNotExists(dot_path, on_type=self) + + def __eq__(self, other): + return isinstance(other, Component) + + def __hash__(self): + return hash(str(self)) + + def default_value(self): + return {} + + +class Source(Component): + """ + Source type. All component that inherits from BaseSource have + this type in Banana. + """ + def __init__(self, class_name, ctor_properties): + super(Source, self).__init__(ctor_properties, class_name) + + def __eq__(self, other): + return self.class_name == other.class_name + + def __hash__(self): + return hash(self.class_name) + + +class Ingestor(Component): + """ + Ingestor type. All component that inherits from BaseIngestor have + this type in Banana. + """ + def __init__(self, class_name, ctor_properties): + super(Ingestor, self).__init__(ctor_properties, class_name) + + def __eq__(self, other): + return self.class_name == other.class_name + + def __hash__(self): + return hash(self.class_name) + + +class Sink(Component): + """ + Sink type. All component that inherits from BaseSink have + this type in Banana. + """ + def __init__(self, class_name, ctor_properties): + super(Sink, self).__init__(ctor_properties, class_name) + + def __eq__(self, other): + return self.class_name == other.class_name + + def __hash__(self): + return hash(self.class_name) + + +class Voter(Component): + """ + Voter type. All component that inherits from BaseVoter have + this type in Banana. + """ + def __init__(self, class_name, ctor_properties): + super(Voter, self).__init__(ctor_properties, class_name) + + def __eq__(self, other): + return self.class_name == other.class_name + + def __hash__(self): + return hash(self.class_name) + + +class Ldp(Component): + """ + Ldp type. All component that inherits from BaseLdp have + this type in Banana. + """ + def __init__(self, class_name, ctor_properties): + super(Ldp, self).__init__(ctor_properties, class_name) + + def __eq__(self, other): + return self.class_name == other.class_name + + def __hash__(self): + return hash(self.class_name) + + +class Sml(Component): + """ + Sml type. All component that inherits from BaseSml have + this type in Banana. + """ + def __init__(self, class_name, ctor_properties): + super(Sml, self).__init__(ctor_properties, class_name) + + def __eq__(self, other): + return self.class_name == other.class_name + + def __hash__(self): + return hash(self.class_name) + + +def get_type(ast_node): + """ + Returns the type for the given ast node. + This function only works for literal node such as + Number, StringLit and JsonObj. + :type ast_node: ast.Number | ast.StringLit | ast.JsonObj | ast.Component + :param ast_node: the node. + :return: Returns the appropriate type. + """ + if isinstance(ast_node, ast.Number): + return Number() + if isinstance(ast_node, ast.StringLit): + return String() + if isinstance(ast_node, ast.JsonObj): + return Object(strict_checking=False) + if isinstance(ast_node, ast.Component): + return Component() + return None + + +def can_to_str(_type): + """ + Check if we the type can be cast to str. + :param _type: Type to check + :return: Returns True if it can be casted + """ + return isinstance(_type, String) + + +def is_comp(_type): + """ + :type _type: String | Number | Object | Component + :param _type: Type to check. + :rtype: bool + :return: Returns True if the provided _type is a component + """ + return isinstance(_type, Component) + + +def can_be_cast_to(_type1, _type2): + """ + Check if the given type `_type1` can be cast into `_type2`. + :type _type1: String | Number | Enum | Object + :param _type1: Type to try to change into _type2 + :type _type2: String | Number | Enum | Object + :param _type2: Type reference. + :return: Returns true if the conversion can be done. + """ + if isinstance(_type2, Any): + return True + elif _type1 == _type2: + return True + elif _type2 == String(): + return can_to_str(_type1) + elif isinstance(_type2, Enum): + return isinstance(_type1, String) or isinstance(_type2, Enum) + elif isinstance(_type1, Object) and isinstance(_type2, Object): + if not _type2.strict_checking: + return True + else: + for prop_name, prop_type in _type2.props.iteritems(): + if prop_name not in _type1.props: + return False + if not can_be_cast_to(_type1.props[prop_name], prop_type): + return False + return True + return False diff --git a/monasca_analytics/component/base.py b/monasca_analytics/component/base.py index e649c3a..2a4ad91 100644 --- a/monasca_analytics/component/base.py +++ b/monasca_analytics/component/base.py @@ -78,6 +78,38 @@ class BaseComponent(object): """ pass + @abstractstatic + def get_params(): # @NoSelf + """Abstract static method that returns the description of the params. + + To be implemented by BaseComponent children. It has to return + a list of the params description such as: + + return [ + ParamDescriptor('param1', type_util.String(), 'default value'), + ParamDescriptor('param2', type_util.Object({ + 'a': type_util.Number() + ), {'a': 123}), + ... + ] + + This function must be kept in sync with `get_default_config` and + `validate_config`, otherwise banana scripts using this component + will get runtime errors when being evaluated. + + The order in the list maps to the order the parameter must be + passed when the component would be created, in banana: + + a = MyComponent(param1, param2) + + `param1` and `param2` would be type-checked respectively against + the first and the second element of the returned list. + + :rtype: list[monasca_analytics.component.params.ParamDescriptor] + :return: Returns the list of parameters accepted by this component. + """ + pass + def id(self): return self._id diff --git a/monasca_analytics/component/params.py b/monasca_analytics/component/params.py new file mode 100644 index 0000000..b445f7d --- /dev/null +++ b/monasca_analytics/component/params.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import monasca_analytics.banana.typeck.type_util as u + + +class ParamDescriptor(object): + """ + Description of a component parameter. This object contains + information such as the name of the parameter, the type, + the default value and a validator that will be evaluated + when the component is instantiated. + """ + def __init__(self, name, _type, default=None, validator=None): + """ + Construct a parameter descriptor. + :type name: str + :param name: The name of the parameter + :type _type: u.String | u.Number | u.Object | u.Enum | u.Any + :param _type: The type of the parameter + :type default: str | float | int | dict + :param default: The default value for the parameter. + :param validator: Additional validator for the parameter. + """ + if not isinstance(_type, u.String) and\ + not isinstance(_type, u.Number) and\ + not isinstance(_type, u.Object) and\ + not isinstance(_type, u.Enum) and\ + not isinstance(_type, u.Any): + raise Exception("ParamDescriptor incorrectly defined") + self.param_name = name + self.default_value = default + self.param_type = _type + self.validator = validator diff --git a/monasca_analytics/config/connection.py b/monasca_analytics/config/connection.py index 9914709..9f8ed17 100644 --- a/monasca_analytics/config/connection.py +++ b/monasca_analytics/config/connection.py @@ -17,7 +17,6 @@ import logging import monasca_analytics.config.const as const -import monasca_analytics.config.validation as validation import monasca_analytics.exception.monanas as err @@ -54,7 +53,7 @@ def _perform_all_connections(connection_kind, _config, components): """ links = {} for origin_id in _config[connection_kind].keys(): - for comp_type in validation.valid_connection_types.keys(): + for comp_type in const.components_types: if origin_id in components[comp_type]: component = components[comp_type][origin_id] connections_list = _config[connection_kind][origin_id] diff --git a/monasca_analytics/config/creation.py b/monasca_analytics/config/creation.py index 274db27..ad8cf40 100644 --- a/monasca_analytics/config/creation.py +++ b/monasca_analytics/config/creation.py @@ -16,7 +16,7 @@ import logging -from monasca_analytics.config import validation +from monasca_analytics.config import const from monasca_analytics.util import common_util logger = logging.getLogger(__name__) @@ -31,7 +31,7 @@ def create_components(_config): :returns: Created components indexed by type and ID """ components = {} - for component_type in validation.valid_connection_types.keys(): + for component_type in const.components_types: components[component_type] = \ _create_comps_by_module(component_type, _config) return components diff --git a/monasca_analytics/exception/banana.py b/monasca_analytics/exception/banana.py index 42948d0..3ad6341 100644 --- a/monasca_analytics/exception/banana.py +++ b/monasca_analytics/exception/banana.py @@ -17,8 +17,11 @@ """Banana Error classes.""" import abc +import pyparsing as p import six +import monasca_analytics.banana.grammar.base_ast as ast + @six.add_metaclass(abc.ABCMeta) class BananaException(Exception): @@ -27,6 +30,14 @@ class BananaException(Exception): def __str__(self): pass + @abc.abstractmethod + def get_span(self): + """ + :rtype: ast.Span + :return: Returns the span where the error occured if appropriate + """ + pass + class BananaInvalidExpression(BananaException): def __init__(self, value): @@ -35,6 +46,9 @@ class BananaInvalidExpression(BananaException): def __str__(self): return repr(self._value) + def get_span(self): + return ast.DUMMY_SPAN + class BananaEnvironmentError(BananaException): def __init__(self, value): @@ -43,11 +57,299 @@ class BananaEnvironmentError(BananaException): def __str__(self): return repr(self._value) + def get_span(self): + return ast.DUMMY_SPAN -class BananaArgumentTypeError(BananaException): - def __init__(self, expected_type, received_type): - self._value = "Wrong type of argument: expected '{}' got '{}'".\ - format(expected_type.__name__, received_type.__name__) + +class BananaNoFullPath(BananaException): + def __init__(self, missing): + self._value = "None of the paths can be executed. Missing at least" \ + " one {}.".format(missing) def __str__(self): - return repr(self._value) + return self._value + + def get_span(self): + return ast.DUMMY_SPAN + + +class BananaArgumentTypeError(BananaException): + def __init__(self, where, expected_type, received_type): + if isinstance(where, ast.ASTNode): + self._span = where.span + where = where.span + else: + self._span = where + self._value = "'{}': Wrong type of argument. Expected '{}' got '{}'"\ + .format(where.get_line(), expected_type, received_type) + + def __str__(self): + return self._value + + def get_span(self): + return self._span + + +class BananaComponentTooManyParams(BananaException): + def __init__(self, span): + self._span = span + self._value = "Too many params provided to '{}' (line {})".format( + span, span.get_lineno() + ) + + def __str__(self): + return self._value + + def get_span(self): + return self._span + + +class BananaComponentMixingParams(BananaException): + def __init__(self, span, named_is_wrong): + self._span = span + if named_is_wrong: + self._value = "'{}' should be named as " \ + "previous parameters are.".format(span) + else: + self._value = "'{}' should not be named as " \ + "previous parameters are.".format(span) + + def __str__(self): + return self._value + + def get_span(self): + return self._span + + +class BananaComponentIncorrectParamName(BananaException): + def __init__(self, found, component): + if isinstance(component, ast.ASTNode): + component = component.span + if isinstance(found, ast.ASTNode): + self._span = found.span + found = found.span + else: + self._span = found + self._value = "Incorrect parameter name. Parameter '{}' " \ + "does not exists on component {}."\ + .format(found, component) + + def __str__(self): + return self._value + + def get_span(self): + return self._span + + +class BananaComponentAlreadyDefined(BananaException): + def __init__(self, first_def, second_def): + self._value = "Component already defined!\n" \ + " First definition: '{}'\n" \ + " Second definition: '{}'"\ + .format(first_def, second_def) + + def __str__(self): + return self._value + + def get_span(self): + # TODO(Joan): This could be a real span instead of this one. + return ast.DUMMY_SPAN + + +class BananaShadowingComponentError(BananaException): + def __init__(self, where, comp): + self._span = where + self._value = "Shadowing component '{}'. " \ + "Please use another variable name.".format(comp) + + def __str__(self): + return self._value + + def get_span(self): + return self._span + + +class BananaAssignmentError(BananaException): + def __init__(self, lhs, rhs): + self._value = "You can't assign '{}' to '{}'".format(lhs, rhs) + + def __str__(self): + return self._value + + def get_span(self): + return ast.DUMMY_SPAN + + +class BananaGrammarBug(BananaException, p.ParseFatalException): + def __init__(self, error): + super(BananaGrammarBug, self).__init__(pstr=error) + self._value = "Bug found in the grammar!" \ + " Please report this error: {}".format(error) + + def __str__(self): + return self._value + + def get_span(self): + return ast.DUMMY_SPAN + + +class BananaJsonObjShadowingError(BananaException, p.ParseFatalException): + def __init__(self, span, error): + self._span = span + error = "Can't shadow property already defined in {}".format(error) + super(BananaJsonObjShadowingError, self).__init__(pstr=error) + + def __str__(self): + return self.msg + + def get_span(self): + return self._span + + +class BananaTypeCheckerBug(BananaException): + def __init__(self, error): + self._value = "Bug found in the TypeChecker!" \ + " Please report this error: {}".format(error) + + def __str__(self): + return self._value + + def get_span(self): + return ast.DUMMY_SPAN + + +class BananaEvalBug(BananaException): + def __init__(self, error): + self._value = "Bug found in the evaluator!" \ + " Please report this error: {}".format(error) + + def __str__(self): + return self._value + + def get_span(self): + return ast.DUMMY_SPAN + + +class BananaUnknown(BananaException): + def __init__(self, ident): + self._span = ident.span + self._value = "Unknown '{}'".format( + ident.into_unmodified_str() + ) + + def __str__(self): + return self._value + + def get_span(self): + return self._span + + +class BananaUnknownOperator(BananaException): + def __init__(self, span, operator, for_type): + self._span = span + self._value = "Unknown operator '{}' for type '{}'".format( + operator, + for_type + ) + + def __str__(self): + return self._value + + def get_span(self): + return self._span + + +class BananaPropertyDoesNotExists(BananaException): + def __init__(self, dotpath, on_type=None): + self._span = dotpath.span + if on_type is None: + self._value = "Error at '{}': Property '{}' " \ + "does not exists"\ + .format( + dotpath.span.get_line(), + dotpath.into_unmodified_str() + ) + else: + self._value = "Error at '{}': Property '{}' " \ + "does not exists on type '{}'"\ + .format( + dotpath.span.get_line(), + dotpath.into_unmodified_str(), + str(on_type) + ) + + def __str__(self): + return self._value + + def get_span(self): + return self._span + + +class BananaTypeError(BananaException): + def __init__(self, expected_type, found_type=None, span=None): + self._span = span + if expected_type is None: + class DummyType(object): + def __str__(self): + return "_" + expected_type = DummyType + if found_type is None: + if isinstance(expected_type, list): + self._value = "Type error found. Expected" \ + " one among '{}'"\ + .format(', '.join(map(lambda x: str(x), expected_type))) + else: + self._value = "Type error found. Expected '{}'".format( + str(expected_type) + ) + else: + if isinstance(expected_type, list): + self._value = "Type error found. Expected" \ + " one among '{}', found '{}'"\ + .format(', '.join(map(lambda x: str(x), expected_type)), + str(found_type)) + else: + self._value = "Type error found. Expected" \ + " '{}', found '{}'"\ + .format(str(expected_type), str(found_type)) + + def __str__(self): + return self._value + + def get_span(self): + if self._span is None: + return ast.DUMMY_SPAN + return self._span + + +class BananaAssignCompError(BananaException): + def __init__(self, span): + self._span = span + self._value = "Component objects " \ + "can't be assigned to " \ + "properties of other objects" + + def __str__(self): + return self._value + + def get_span(self): + return self._span + + +class BananaConnectionError(BananaException): + + def __init__(self, ident_from, ident_to, type_from, possible_connections): + self._span = ident_to.span + self._value = "Can't connect '{}' (line:{})" \ + " to '{}' (line:{})," \ + " '{}' can only be connected to {}"\ + .format( + ident_from.val, ident_from.span.get_lineno(), + ident_to.val, ident_to.span.get_lineno(), + type_from, possible_connections) + + def __str__(self): + return self._value + + def get_span(self): + return self._span diff --git a/monasca_analytics/ingestor/cloud.py b/monasca_analytics/ingestor/cloud.py index 5cde734..38ab41e 100644 --- a/monasca_analytics/ingestor/cloud.py +++ b/monasca_analytics/ingestor/cloud.py @@ -40,6 +40,10 @@ class CloudIngestor(base.BaseIngestor): }, required=True) return cloud_schema(_config) + @staticmethod + def get_params(): + return [] + def map_dstream(self, dstream): features_list = list(self._features) return dstream.map(fn.from_json)\ diff --git a/monasca_analytics/ingestor/iptables.py b/monasca_analytics/ingestor/iptables.py index 81ce13a..0265ea3 100644 --- a/monasca_analytics/ingestor/iptables.py +++ b/monasca_analytics/ingestor/iptables.py @@ -53,6 +53,10 @@ class IptablesIngestor(base.BaseIngestor): def get_default_config(): return {"module": IptablesIngestor.__name__} + @staticmethod + def get_params(): + return [] + def map_dstream(self, dstream): features_list = list(self._features) return dstream.map(fn.from_json)\ diff --git a/monasca_analytics/ldp/cloud_causality.py b/monasca_analytics/ldp/cloud_causality.py index 2b98e40..37b4d3f 100644 --- a/monasca_analytics/ldp/cloud_causality.py +++ b/monasca_analytics/ldp/cloud_causality.py @@ -40,6 +40,10 @@ class CloudCausalityLDP(bt.BaseLDP): def get_default_config(): return {"module": CloudCausalityLDP.__name__} + @staticmethod + def get_params(): + return [] + def map_dstream(self, dstream): """Executes _aggregate for each RDD in the dstream diff --git a/monasca_analytics/ldp/iptables_ldp.py b/monasca_analytics/ldp/iptables_ldp.py index b33f96a..8aa9dd1 100644 --- a/monasca_analytics/ldp/iptables_ldp.py +++ b/monasca_analytics/ldp/iptables_ldp.py @@ -44,6 +44,10 @@ class IptablesLDP(bt.BaseLDP): def get_default_config(): return {"module": IptablesLDP.__name__} + @staticmethod + def get_params(): + return [] + def map_dstream(self, dstream): """Detect anomalies in a dstream using the learned classifier diff --git a/monasca_analytics/ldp/monasca_aggregate.py b/monasca_analytics/ldp/monasca_aggregate.py index 0cae899..1cb4753 100644 --- a/monasca_analytics/ldp/monasca_aggregate.py +++ b/monasca_analytics/ldp/monasca_aggregate.py @@ -17,6 +17,8 @@ import logging import voluptuous +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.component.params as params import monasca_analytics.ldp.base as bt import monasca_analytics.ldp.monasca.helpers as helpers @@ -31,24 +33,22 @@ class MonascaAggregateLDP(bt.BaseLDP): def __init__(self, _id, _config): super(MonascaAggregateLDP, self).__init__(_id, _config) - self._aggregation_period = _config["params"]["aggregation_period"] + self._aggregation_period = _config["period"] self._reducer_func = MonascaAggregateLDP.select_reducer(_config) - self._suffix = "_" + _config["params"]["aggregation_function"] + self._suffix = "_" + _config["func"] @staticmethod def validate_config(_config): monasca_ag_schema = voluptuous.Schema({ "module": voluptuous.And(basestring, vu.NoSpaceCharacter()), - "params": { - "aggregation_period": int, - "aggregation_function": voluptuous.Or( - "avg", - "max", - "sum", - "min", - "cnt" - ) - } + "period": voluptuous.Or(float, int), + "func": voluptuous.Or( + "avg", + "max", + "sum", + "min", + "cnt" + ) }, required=True) return monasca_ag_schema(_config) @@ -56,13 +56,21 @@ class MonascaAggregateLDP(bt.BaseLDP): def get_default_config(): return { "module": MonascaAggregateLDP.__name__, - "params": { - # One hour - "aggregation_period": 60 * 60, - "aggregation_function": "avg" - } + "period": 60.0 * 60.0, + "func": "avg" } + @staticmethod + def get_params(): + return [ + params.ParamDescriptor('period', type_util.Number(), 60 * 60), + params.ParamDescriptor( + 'func', + type_util.Enum(['avg', 'max', 'sum', 'min', 'cnt']), + 'avg' + ) + ] + def map_dstream(self, dstream): """ Map the given DStream into a new DStream where metrics @@ -185,4 +193,4 @@ class MonascaAggregateLDP(bt.BaseLDP): lambda m, cnt: m, lambda acc, m, cnt: cnt, ), - }[_config["params"]["aggregation_function"]] + }[_config["func"]] diff --git a/monasca_analytics/ldp/monasca_combine.py b/monasca_analytics/ldp/monasca_combine.py index 34ed7e3..371f8fa 100644 --- a/monasca_analytics/ldp/monasca_combine.py +++ b/monasca_analytics/ldp/monasca_combine.py @@ -15,11 +15,15 @@ # under the License. import logging +import math import voluptuous -import monasca_analytics.banana.api as banana +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.component.params as params + import monasca_analytics.ldp.base as bt import monasca_analytics.ldp.monasca.helpers as helpers +import monasca_analytics.parsing.api as parsing import monasca_analytics.util.spark_func as fn from monasca_analytics.util import validation_utils as vu @@ -31,16 +35,15 @@ class MonascaCombineLDP(bt.BaseLDP): def __init__(self, _id, _config): super(MonascaCombineLDP, self).__init__(_id, _config) - logger.debug(_config["params"]["metric_names_binding"]) - logger.debug(_config["params"]["lambda"]) - self._combine_function = banana.create_fn_with_config( - env=_config["params"]["metric_names_binding"], - expr_string=_config["params"]["lambda"] + logger.debug(_config["bindings"]) + logger.debug(_config["lambda"]) + self._combine_function = parsing.create_fn_with_config( + env=_config["bindings"], + expr_string=_config["lambda"] ) - self._combine_period = _config["params"]["combine_period"] - self._combine_metric_name = _config["params"]["metric_name"] - self._metrics_of_interest = _config["params"][ - "metric_names_binding"].values() + self._combine_period = _config["period"] + self._combine_metric_name = _config["metric"] + self._metrics_of_interest = _config["bindings"].values() def map_dstream(self, dstream): """ @@ -143,92 +146,115 @@ class MonascaCombineLDP(bt.BaseLDP): def validate_config(_config): monasca_comb_schema = voluptuous.Schema({ "module": voluptuous.And(basestring, vu.NoSpaceCharacter()), - "params": { - "metric_name": basestring, - "combine_period": int, - "lambda": basestring, - "metric_names_binding": { - basestring: voluptuous.Or( - "apache.net.kbytes_sec", - "apache.net.requests_sec", - "apache.performance.cpu_load_perc", - "cpu.idle_perc", - "cpu.stolen_perc", - "cpu.system_perc", - "cpu.total_logical_cores", - "cpu.user_perc", - "cpu.wait_perc", - "disk.allocation", - "disk.inode_used_perc", - "disk.space_used_perc", - "disk.total_space_mb", - "disk.total_used_space_mb", - "host_alive_status", - "io.read_kbytes_sec", - "io.read_req_sec", - "io.write_time_sec", - "kafka.consumer_lag", - "load.avg_1_min", - "load.avg_5_min", - "mem.free_mb", - "mem.swap_free_mb", - "mem.swap_total_mb", - "mem.total_mb", - "mem.usable_mb", - "mem.used_cache", - "metrics-added-to-batch-counter[0]", - "mysql.innodb.buffer_pool_free", - "mysql.innodb.buffer_pool_used", - "mysql.innodb.data_reads", - "mysql.innodb.mutex_spin_rounds", - "mysql.performance.com_delete_multi", - "mysql.performance.com_insert", - "mysql.performance.com_insert_select", - "mysql.performance.com_select", - "mysql.performance.com_update", - "mysql.performance.created_tmp_disk_tables", - "mysql.performance.created_tmp_files", - "mysql.performance.open_files", - "mysql.performance.questions", - "mysql.performance.user_time", - "net.in_bytes_sec", - "net.in_errors_sec", - "net.in_packets_dropped_sec", - "net.in_packets_sec", - "net.out_bytes_sec", - "net.out_errors_sec", - "net.out_packets_dropped_sec", - "net.out_packets_sec", - "nova.vm.disk.total_allocated_gb", - "process.pid_count", - "raw-sql.time.max", - "vcpus", - "vm.cpu.utilization_perc", - "vm.host_alive_status", - "vm.mem.total_mb", - "zookeeper.out_bytes", - "zookeeper.outstanding_bytes" - ) - } + "metric": basestring, + "period": voluptuous.And( + voluptuous.Or(float, int), + lambda i: i >= 0 and math.floor(i) == math.ceil(i)), + "lambda": basestring, + "bindings": { + basestring: voluptuous.Or( + "apache.net.kbytes_sec", + "apache.net.requests_sec", + "apache.performance.cpu_load_perc", + "cpu.idle_perc", + "cpu.stolen_perc", + "cpu.system_perc", + "cpu.total_logical_cores", + "cpu.user_perc", + "cpu.wait_perc", + "disk.allocation", + "disk.inode_used_perc", + "disk.space_used_perc", + "disk.total_space_mb", + "disk.total_used_space_mb", + "host_alive_status", + "io.read_kbytes_sec", + "io.read_req_sec", + "io.write_time_sec", + "kafka.consumer_lag", + "load.avg_1_min", + "load.avg_5_min", + "mem.free_mb", + "mem.swap_free_mb", + "mem.swap_total_mb", + "mem.total_mb", + "mem.usable_mb", + "mem.used_cache", + "metrics-added-to-batch-counter[0]", + "mysql.innodb.buffer_pool_free", + "mysql.innodb.buffer_pool_used", + "mysql.innodb.data_reads", + "mysql.innodb.mutex_spin_rounds", + "mysql.performance.com_delete_multi", + "mysql.performance.com_insert", + "mysql.performance.com_insert_select", + "mysql.performance.com_select", + "mysql.performance.com_update", + "mysql.performance.created_tmp_disk_tables", + "mysql.performance.created_tmp_files", + "mysql.performance.open_files", + "mysql.performance.questions", + "mysql.performance.user_time", + "net.in_bytes_sec", + "net.in_errors_sec", + "net.in_packets_dropped_sec", + "net.in_packets_sec", + "net.out_bytes_sec", + "net.out_errors_sec", + "net.out_packets_dropped_sec", + "net.out_packets_sec", + "nova.vm.disk.total_allocated_gb", + "process.pid_count", + "raw-sql.time.max", + "vcpus", + "vm.cpu.utilization_perc", + "vm.host_alive_status", + "vm.mem.total_mb", + "zookeeper.out_bytes", + "zookeeper.outstanding_bytes" + ) } }, required=True) monasca_comb_schema(_config) # Checks the expression and the environment - handle = banana.validate_expression(_config["params"]["lambda"]) - banana.validate_name_binding(handle, - _config["params"]["metric_names_binding"]) + handle = parsing.validate_expression(_config["lambda"]) + parsing.validate_name_binding(handle, + _config["bindings"]) @staticmethod def get_default_config(): return { "module": MonascaCombineLDP.__name__, - "params": { - "metric_name": "cpu.logical_cores_actives", - "combine_period": 1, - "lambda": "a * b", - "metric_names_binding": { - "a": "cpu.idle_perc", - "b": "cpu.total_logical_cores" - } + "metric": "cpu.logical_cores_actives", + "period": 1, + "lambda": "a * b", + "bindings": { + "a": "cpu.idle_perc", + "b": "cpu.total_logical_cores" } } + + @staticmethod + def get_params(): + return [ + params.ParamDescriptor( + 'metric', + type_util.String(), + 'cpu.logcal_cores_actives' + ), + params.ParamDescriptor( + 'period', + type_util.Number(), + 1 + ), + params.ParamDescriptor( + 'lambda', + type_util.String(), + 'a * b' + ), + params.ParamDescriptor( + 'bindings', + type_util.Any(), + {'a': 'cpu.ilde_perc', 'b': 'cpu.total_logical_cores'} + ) + ] diff --git a/monasca_analytics/ldp/monasca_derivative.py b/monasca_analytics/ldp/monasca_derivative.py index 63ba33f..c93a7c2 100644 --- a/monasca_analytics/ldp/monasca_derivative.py +++ b/monasca_analytics/ldp/monasca_derivative.py @@ -15,8 +15,12 @@ # under the License. import logging +import math import voluptuous +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.component.params as params + import monasca_analytics.ldp.base as bt import monasca_analytics.ldp.monasca.helpers as helpers import monasca_analytics.util.spark_func as fn @@ -38,16 +42,16 @@ class MonascaDerivativeLDP(bt.BaseLDP): def __init__(self, _id, _config): super(MonascaDerivativeLDP, self).__init__(_id, _config) - self._period = _config["params"]["derivative_period"] + self._period = _config["period"] @staticmethod def validate_config(_config): monasca_der_schema = voluptuous.Schema({ "module": voluptuous.And(basestring, vu.NoSpaceCharacter()), - "params": { - # Derivative period in multiple of batch interval - "derivative_period": int - } + # Derivative period in multiple of batch interval + "period": voluptuous.And( + voluptuous.Or(float, int), + lambda i: i >= 0 and math.floor(i) == math.ceil(i)) }, required=True) return monasca_der_schema(_config) @@ -55,11 +59,15 @@ class MonascaDerivativeLDP(bt.BaseLDP): def get_default_config(): return { "module": MonascaDerivativeLDP.__name__, - "params": { - "derivative_period": 1 - } + "period": 1 } + @staticmethod + def get_params(): + return [ + params.ParamDescriptor('period', type_util.Number(), 1) + ] + def map_dstream(self, dstream): """ Map the given DStream into a new DStream where metrics diff --git a/monasca_analytics/monanas.py b/monasca_analytics/monanas.py index 45ecced..2ddbf1f 100644 --- a/monasca_analytics/monanas.py +++ b/monasca_analytics/monanas.py @@ -21,10 +21,12 @@ import sys from tornado import ioloop import voluptuous -from monasca_analytics.exception import monanas as err -from monasca_analytics.spark import driver -from monasca_analytics.util import common_util as cu -from monasca_analytics.web_service import web_service as ws +import monasca_analytics.banana.emitter as emit +import monasca_analytics.banana.pass_manager as executor +import monasca_analytics.exception.monanas as err +import monasca_analytics.spark.driver as driver +import monasca_analytics.util.common_util as cu +import monasca_analytics.web_service.web_service as ws logger = logging.getLogger(__name__) @@ -61,6 +63,19 @@ class Monanas(object): """ return self._is_streaming + def try_change_configuration(self, banana_str, emitter): + """Try to change the configuration to the provided one. + + :type banana_str: str + :param banana_str: New configuration. + :type emitter: emit.JsonEmitter + :param emitter: a Json emitter instance + """ + if not isinstance(emitter, emit.JsonEmitter): + raise err.MonanasException() + # Try to change the configuration. + executor.execute_banana_string(banana_str, self._driver, emitter) + def start_streaming(self): """Starts streaming data. diff --git a/monasca_analytics/parsing/__init__.py b/monasca_analytics/parsing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/monasca_analytics/banana/api.py b/monasca_analytics/parsing/api.py similarity index 74% rename from monasca_analytics/banana/api.py rename to monasca_analytics/parsing/api.py index 26edc95..a0caee3 100644 --- a/monasca_analytics/banana/api.py +++ b/monasca_analytics/parsing/api.py @@ -14,10 +14,10 @@ # License for the specific language governing permissions and limitations # under the License. -from expression import create_fn_with_config -from expression import validate_environment -from expression import validate_expression -from expression import validate_name_binding +from monasca_analytics.parsing.expression import create_fn_with_config +from monasca_analytics.parsing.expression import validate_environment +from monasca_analytics.parsing.expression import validate_expression +from monasca_analytics.parsing.expression import validate_name_binding create_fn_with_config = create_fn_with_config validate_expression = validate_expression diff --git a/monasca_analytics/banana/expression.py b/monasca_analytics/parsing/expression.py similarity index 99% rename from monasca_analytics/banana/expression.py rename to monasca_analytics/parsing/expression.py index 368c647..881e633 100644 --- a/monasca_analytics/banana/expression.py +++ b/monasca_analytics/parsing/expression.py @@ -15,13 +15,12 @@ # under the License. import logging +import pyparsing as p import types import monasca_analytics.banana.bytecode.assembler as asbl -import monasca_analytics.banana.private as priv import monasca_analytics.exception.banana as exception -import pyparsing as p - +import monasca_analytics.parsing.private as priv logger = logging.getLogger(__name__) diff --git a/monasca_analytics/banana/private.py b/monasca_analytics/parsing/private.py similarity index 83% rename from monasca_analytics/banana/private.py rename to monasca_analytics/parsing/private.py index 3018ecf..c77f822 100644 --- a/monasca_analytics/banana/private.py +++ b/monasca_analytics/parsing/private.py @@ -14,12 +14,12 @@ # License for the specific language governing permissions and limitations # under the License. -BANANA_OPERATOR_LIST = ['+', '-', '*', '/'] +EXPRESSION_OPERATOR_LIST = ['+', '-', '*', '/'] def is_op(x): - return x in BANANA_OPERATOR_LIST + return x in EXPRESSION_OPERATOR_LIST def is_not_op(x): - return x not in BANANA_OPERATOR_LIST + return x not in EXPRESSION_OPERATOR_LIST diff --git a/monasca_analytics/sink/file.py b/monasca_analytics/sink/file.py index 646dbe4..573ec09 100644 --- a/monasca_analytics/sink/file.py +++ b/monasca_analytics/sink/file.py @@ -20,6 +20,8 @@ import tempfile import time import voluptuous +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.component.params as params import monasca_analytics.sink.base as base from monasca_analytics.util import validation_utils as vu @@ -32,8 +34,8 @@ class FileSink(base.BaseSink): def __init__(self, _id, _config): super(FileSink, self).__init__(_id, _config) - if "params" in _config: - _path = path.expanduser(_config["params"]["path"]) + if _config["path"] is not None: + _path = path.expanduser(_config["path"]) if path.isdir(_path): _path = path.join(_path, time.time() + '.log') self._file_path = _path @@ -63,17 +65,19 @@ class FileSink(base.BaseSink): def get_default_config(): return { "module": FileSink.__name__, - "params": { - "path": None - } + "path": None } + @staticmethod + def get_params(): + return [params.ParamDescriptor('path', type_util.String())] + @staticmethod def validate_config(_config): file_schema = voluptuous.Schema({ "module": voluptuous.And(basestring, vu.NoSpaceCharacter()), - voluptuous.Optional("params"): { - "path": voluptuous.And(basestring, vu.ExistingPath()) - } + "path": voluptuous.Or( + voluptuous.And(basestring, vu.ExistingPath()), + None) }, required=True) return file_schema(_config) diff --git a/monasca_analytics/sink/iptables_sqlite.py b/monasca_analytics/sink/iptables_sqlite.py index 5b4cf06..85adc10 100644 --- a/monasca_analytics/sink/iptables_sqlite.py +++ b/monasca_analytics/sink/iptables_sqlite.py @@ -16,6 +16,8 @@ import voluptuous +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.component.params as params import monasca_analytics.sink.base_sqlite as base from monasca_analytics.util import validation_utils as vu @@ -36,11 +38,17 @@ class IptablesSQLiteSink(base.BaseSQLiteSink): def get_default_config(): return { "module": IptablesSQLiteSink.__name__, - "params": { - "db_name": "sqlite_sink.db" - } + "db_name": "sqlite_sink.db" } + @staticmethod + def get_params(): + return [ + params.ParamDescriptor('db_name', + type_util.String(), + 'sqlite_sink.db') + ] + @staticmethod def validate_config(_config): iptables_sql_schema = voluptuous.Schema({ diff --git a/monasca_analytics/sink/kafkas.py b/monasca_analytics/sink/kafkas.py index a33360b..4d31f95 100644 --- a/monasca_analytics/sink/kafkas.py +++ b/monasca_analytics/sink/kafkas.py @@ -19,6 +19,8 @@ import time import kafka +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.component.params as params from monasca_analytics.sink import base import monasca_analytics.sink.sink_config_validator as validator @@ -30,13 +32,14 @@ class KafkaSink(base.BaseSink): self._topic = None self._producer = None super(KafkaSink, self).__init__(_id, _config) - host = _config["params"]["host"] - port = _config["params"]["port"] - self._topic = _config["params"]["topic"] - self._producer = kafka.KafkaProducer(bootstrap_servers="{0}:{1}" - .format(host, port)) + self._host = _config["host"] + self._port = int(_config["port"]) + self._topic = _config["topic"] def sink_dstream(self, dstream): + if self._producer is None: + self._producer = kafka.KafkaProducer( + bootstrap_servers="{0}:{1}".format(self._host, self._port)) dstream.foreachRDD(self._persist) def _persist(self, _, rdd): @@ -64,9 +67,16 @@ class KafkaSink(base.BaseSink): def get_default_config(): return { "module": KafkaSink.__name__, - "params": { - "host": "localhost", - "port": 9092, - "topic": "transformed_alerts" - } + "host": "localhost", + "port": 9092, + "topic": "transformed_alerts" } + + @staticmethod + def get_params(): + return [ + params.ParamDescriptor('host', type_util.String(), 'localhost'), + params.ParamDescriptor('port', type_util.Number(), 9092), + params.ParamDescriptor('topic', type_util.String(), + 'transformed_alerts') + ] diff --git a/monasca_analytics/sink/sink_config_validator.py b/monasca_analytics/sink/sink_config_validator.py index 1705f02..5aa6e65 100644 --- a/monasca_analytics/sink/sink_config_validator.py +++ b/monasca_analytics/sink/sink_config_validator.py @@ -16,6 +16,7 @@ """A list of functions for validating sink configs.""" +import math import voluptuous from monasca_analytics.util import validation_utils as vu @@ -26,12 +27,12 @@ def validate_kafka_sink_config(config): config_schema = voluptuous.Schema({ "module": voluptuous.And(basestring, vu.AvailableSink()), - "params": { - "host": voluptuous.And( - basestring, vu.NoSpaceCharacter()), - "port": int, - "topic": voluptuous.And( - basestring, vu.NoSpaceCharacter()) - } + "host": voluptuous.And( + basestring, vu.NoSpaceCharacter()), + "port": voluptuous.And( + voluptuous.Or(float, int), + lambda i: i >= 0 and math.floor(i) == math.ceil(i)), + "topic": voluptuous.And( + basestring, vu.NoSpaceCharacter()) }, required=True) return config_schema(config) diff --git a/monasca_analytics/sink/stdout_sink.py b/monasca_analytics/sink/stdout_sink.py index 22663e7..913c03d 100644 --- a/monasca_analytics/sink/stdout_sink.py +++ b/monasca_analytics/sink/stdout_sink.py @@ -39,3 +39,7 @@ class StdoutSink(base.BaseSink): "module": voluptuous.And(basestring, vu.NoSpaceCharacter()) }, required=True) stdout_schema(_config) + + @staticmethod + def get_params(): + return [] diff --git a/monasca_analytics/sml/lingam.py b/monasca_analytics/sml/lingam.py index fdce40d..87d3c77 100644 --- a/monasca_analytics/sml/lingam.py +++ b/monasca_analytics/sml/lingam.py @@ -21,6 +21,9 @@ import numpy as np from sklearn import decomposition import voluptuous +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.component.params as params + from monasca_analytics.sml import base from monasca_analytics.util import validation_utils as vu @@ -32,16 +35,14 @@ class LiNGAM(base.BaseSML): def __init__(self, _id, _config): super(LiNGAM, self).__init__(_id, _config) - self._threshold = _config["params"]["threshold"] + self._threshold = _config["threshold"] self._threshold = 0.1 @staticmethod def validate_config(_config): lingam_schema = voluptuous.Schema({ "module": voluptuous.And(basestring, vu.NoSpaceCharacter()), - "params": { - "threshold": float - } + "threshold": float }, required=True) return lingam_schema(_config) @@ -49,11 +50,15 @@ class LiNGAM(base.BaseSML): def get_default_config(): return { "module": LiNGAM.__name__, - "params": { - "threshold": 0.1 - } + "threshold": 0.1 } + @staticmethod + def get_params(): + return [ + params.ParamDescriptor('threshold', type_util.Number(), 0.1) + ] + def number_of_samples_required(self): return 5000 diff --git a/monasca_analytics/sml/svm_one_class.py b/monasca_analytics/sml/svm_one_class.py index 3069cd7..941b10a 100644 --- a/monasca_analytics/sml/svm_one_class.py +++ b/monasca_analytics/sml/svm_one_class.py @@ -48,6 +48,10 @@ class SvmOneClass(base.BaseSML): def get_default_config(): return {"module": SvmOneClass.__name__} + @staticmethod + def get_params(): + return [] + def number_of_samples_required(self): return N_SAMPLES diff --git a/monasca_analytics/source/cloud_markov_chain.py b/monasca_analytics/source/cloud_markov_chain.py index 039679f..a5603d1 100644 --- a/monasca_analytics/source/cloud_markov_chain.py +++ b/monasca_analytics/source/cloud_markov_chain.py @@ -18,6 +18,9 @@ import logging import voluptuous +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.component.params as params + from monasca_analytics.source.markov_chain import base from monasca_analytics.source.markov_chain import events as ev import monasca_analytics.source.markov_chain.prob_checks as pck @@ -35,11 +38,9 @@ class CloudMarkovChainSource(base.MarkovChainSource): source_schema = voluptuous.Schema({ "module": voluptuous.And( basestring, vu.NoSpaceCharacter()), - "params": { - "server_sleep_in_seconds": voluptuous.And( - float, voluptuous.Range( - min=0, max=1, min_included=False, max_included=False)) - }, + "sleep": voluptuous.And( + float, voluptuous.Range( + min=0, max=1, min_included=False, max_included=False)), "transitions": { "web_service": { "run=>slow": { @@ -88,9 +89,7 @@ class CloudMarkovChainSource(base.MarkovChainSource): def get_default_config(): return { "module": CloudMarkovChainSource.__name__, - "params": { - "server_sleep_in_seconds": 0.01 - }, + "sleep": 0.01, "transitions": { "web_service": { "run=>slow": { @@ -141,6 +140,33 @@ class CloudMarkovChainSource(base.MarkovChainSource): } } + @staticmethod + def get_params(): + return [ + params.ParamDescriptor('sleep', type_util.Number(), 0.01), + params.ParamDescriptor('transitions', type_util.Object({ + 'web_service': type_util.Object({ + 'run=>slow': type_util.Any(), + 'slow=>run': type_util.Any(), + 'stop=>run': type_util.Any(), + }), + 'switch': type_util.Object({ + 'on=>off': type_util.Number(), + 'off=>on': type_util.Number(), + }), + 'host': type_util.Object({ + 'on=>off': type_util.Number(), + 'off=>on': type_util.Number(), + }) + })), + params.ParamDescriptor('triggers', type_util.Object({ + 'support': type_util.Object({ + 'get_called': type_util.Any() + }) + })), + params.ParamDescriptor('graph', type_util.Any()) + ] + def get_feature_list(self): node_names = [k.split(":")[0] for k in dict(self._config["graph"]).keys()] diff --git a/monasca_analytics/source/iptables_markov_chain.py b/monasca_analytics/source/iptables_markov_chain.py index 153afd4..dc53a0e 100644 --- a/monasca_analytics/source/iptables_markov_chain.py +++ b/monasca_analytics/source/iptables_markov_chain.py @@ -18,6 +18,9 @@ import logging import voluptuous +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.component.params as params + from monasca_analytics.source.markov_chain import base from monasca_analytics.source.markov_chain import events from monasca_analytics.source.markov_chain import prob_checks as pck @@ -65,11 +68,10 @@ class IPTablesSource(base.MarkovChainSource): def validate_config(_config): source_schema = voluptuous.Schema({ "module": voluptuous.And(basestring, vu.NoSpaceCharacter()), - "params": { - "server_sleep_in_seconds": voluptuous.And( - float, voluptuous.Range( - min=0, max=1, min_included=False, max_included=False)) - } + "sleep": voluptuous.And( + float, + voluptuous.Range( + min=0, max=1, min_included=False, max_included=False)), }, required=True) return source_schema(_config) @@ -77,11 +79,15 @@ class IPTablesSource(base.MarkovChainSource): def get_default_config(): return { "module": IPTablesSource.__name__, - "params": { - "server_sleep_in_seconds": 0.01 - } + "sleep": 0.01, } + @staticmethod + def get_params(): + return [ + params.ParamDescriptor('sleep', type_util.Number(), 0.01) + ] + def get_feature_list(self): return iptable_types diff --git a/monasca_analytics/source/kafka.py b/monasca_analytics/source/kafka.py index bd110c4..ba508af 100644 --- a/monasca_analytics/source/kafka.py +++ b/monasca_analytics/source/kafka.py @@ -19,6 +19,9 @@ import logging from pyspark.streaming import kafka import voluptuous +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.component.params as params + from monasca_analytics.source import base from monasca_analytics.util import validation_utils as vu @@ -58,6 +61,19 @@ class KafkaSource(base.BaseSource): } } + @staticmethod + def get_params(): + return [ + params.ParamDescriptor('zk_host', type_util.String(), + 'localhost'), + params.ParamDescriptor('zk_port', type_util.Number(), + 2181), + params.ParamDescriptor('group_id', type_util.String(), + 'my_group_id'), + params.ParamDescriptor('topics', + type_util.Object(strict_checking=False)) + ] + def create_dstream(self, ssc): """Dstream creation diff --git a/monasca_analytics/source/markov_chain/base.py b/monasca_analytics/source/markov_chain/base.py index 313c445..d37d78f 100644 --- a/monasca_analytics/source/markov_chain/base.py +++ b/monasca_analytics/source/markov_chain/base.py @@ -83,8 +83,7 @@ class MarkovChainSource(base.BaseSource): self._server.server_activate() self._server.terminate = False self._server.system = system - self._server.sleep_in_seconds = self._config[ - "params"]["server_sleep_in_seconds"] + self._server.sleep_in_seconds = self._config["sleep"] self._server_thread = threading.Thread(target=self._serve_forever) self._server_thread.start() diff --git a/monasca_analytics/source/monasca_markov_chain.py b/monasca_analytics/source/monasca_markov_chain.py index f610d89..a0bda7f 100644 --- a/monasca_analytics/source/monasca_markov_chain.py +++ b/monasca_analytics/source/monasca_markov_chain.py @@ -18,6 +18,8 @@ import logging import random import voluptuous +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.component.params as params import monasca_analytics.source.markov_chain.base as base import monasca_analytics.source.markov_chain.events as ev @@ -37,11 +39,9 @@ class MonascaMarkovChainSource(base.MarkovChainSource): def validate_config(_config): markov_schema = voluptuous.Schema({ "module": voluptuous.And(basestring, vu.NoSpaceCharacter()), - "params": { - "server_sleep_in_seconds": voluptuous.And( - float, voluptuous.Range( - min=0, max=1, min_included=False, max_included=False)) - }, + "sleep": voluptuous.And( + float, voluptuous.Range( + min=0, max=1, min_included=False, max_included=False)), }, required=True) return markov_schema(_config) @@ -49,11 +49,15 @@ class MonascaMarkovChainSource(base.MarkovChainSource): def get_default_config(): return { "module": MonascaMarkovChainSource.__name__, - "params": { - "server_sleep_in_seconds": 0.01 - } + "sleep": 0.01, } + @staticmethod + def get_params(): + return [ + params.ParamDescriptor('sleep', type_util.Number(), 0.01) + ] + def get_feature_list(self): return ["vm1", "vm2", "host1", "host2"] diff --git a/monasca_analytics/source/randoms.py b/monasca_analytics/source/randoms.py index 356c092..1153926 100644 --- a/monasca_analytics/source/randoms.py +++ b/monasca_analytics/source/randoms.py @@ -26,6 +26,9 @@ import time import uuid import voluptuous +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.component.params as params + import monasca_analytics.exception.monanas as err from monasca_analytics.source import base from monasca_analytics.util import validation_utils as vu @@ -126,6 +129,22 @@ class RandomSource(base.BaseSource): } } + @staticmethod + def get_params(): + return [ + params.ParamDescriptor('host', type_util.String(), 'localhost'), + params.ParamDescriptor('port', type_util.Number(), 1010), + params.ParamDescriptor('model', type_util.Object({ + 'name': type_util.String(), + 'params': type_util.Object({ + 'origin_types': type_util.Object(strict_checking=False) + }) + })), + params.ParamDescriptor('alert_per_burst', type_util.Number(), 1), + params.ParamDescriptor('idle_time_between_bursts', + type_util.Number(), 1.0), + ] + def _start_server(self): if not self._is_server_running: self._server_thread.start() diff --git a/monasca_analytics/spark/driver.py b/monasca_analytics/spark/driver.py index e46df54..e55a6dd 100644 --- a/monasca_analytics/spark/driver.py +++ b/monasca_analytics/spark/driver.py @@ -39,9 +39,10 @@ class DriverExecutor(object): """ def __init__(self, _config): - self._links = config.instantiate_components(_config) - self._sources = config.collect_sources(self._links) - self._orchestrator = agg.Aggregator(self) + self._links = None + self._sources = None + self._orchestrator = None + self.set_links(config.instantiate_components(_config)) def restart_spark(): self._ssc = streamingctx.create_streaming_context( @@ -52,6 +53,18 @@ class DriverExecutor(object): self._sc = pyspark.SparkContext( appName=_config["spark_config"]["appName"]) self._ssc = streamingctx.create_streaming_context(self._sc, _config) + + def set_links(self, links): + """Set new set of links + + This function has no effect on the current pipeline. + In order to use them, you need to restart the pipeline. + """ + self._links = links + logger.debug("Collect sources...") + self._sources = config.collect_sources(self._links) + logger.debug("New list of sources: {}".format(self._sources)) + self._orchestrator = agg.Aggregator(self) logger.debug("Propagating feature list...") self._propagate_feature_list() @@ -81,10 +94,12 @@ class DriverExecutor(object): self._ssc.start() def stop_pipeline(self): + logger.debug("Stop spark context.") + self._ssc.stop(False, False) + logger.debug("Terminate sources.") self._terminate_sources() - self._ssc.awaitTermination() - self._ssc = None - self._sc = None + logger.debug("Restart spark context.") + self._restart_spark() def move_to_phase2(self): if self._ssc is not None: diff --git a/monasca_analytics/util/common_util.py b/monasca_analytics/util/common_util.py index 31976a1..6226e58 100644 --- a/monasca_analytics/util/common_util.py +++ b/monasca_analytics/util/common_util.py @@ -36,9 +36,6 @@ from monasca_analytics import voter logger = logging.getLogger(__name__) -available_classes = None - - def parse_json_file(filename): """Parses json and return a dict. diff --git a/monasca_analytics/util/string_util.py b/monasca_analytics/util/string_util.py new file mode 100644 index 0000000..a81bee7 --- /dev/null +++ b/monasca_analytics/util/string_util.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def array_to_str(array, multiline=False, indent=None): + """ + Convert the provided dictionary into a readable str, by calling + str on both the keys and the values. + :type array: list + :param array: the dictionary to convert. + :type multiline: bool + :param multiline: If each key value pair should be on its own line. + :type indent: int + :param indent: Indentation if multiline is True. + :rtype: str + :return: Returns the converted dict. + """ + if len(array) == 0: + return "[]" + + multiline = multiline or indent is not None + + def dispatch(value): + if isinstance(value, list): + return array_to_str(value, multiline, indent) + if isinstance(value, dict): + return dict_to_str(value, multiline, indent) + return str(value) + + res = "[" + if multiline: + res += "\n" + join_str = "," + if multiline: + join_str += "\n" + else: + join_str += " " + if indent is not None: + join_str = " " * indent + join_str + res += join_str.join(map(dispatch, array)) + if multiline: + res += "\n" + res += "]" + + return res + + +def dict_to_str(dictionary, multiline=False, indent=None): + """ + Convert the provided dictionary into a readable str, by calling + str on both the keys and the values. + :type dictionary: dict + :param dictionary: the dictionary to convert. + :type multiline: bool + :param multiline: If each key value pair should be on its own line. + :type indent: int + :param indent: Indentation if multiline is True. + :rtype: str + :return: Returns the converted dict. + """ + if len(dictionary) == 0: + return "{}" + res = "{" + if multiline: + res += "\n" + multiline = multiline or indent is not None + for k, v in sorted(dictionary.iteritems(), key=lambda ke: str(ke[0])): + if indent is not None: + res += " " * indent + if isinstance(v, dict): + res += "{}: {}, ".format(str(k), + dict_to_str(v, multiline, indent)) + elif isinstance(v, list): + res += "{}: {}, ".format(str(k), + array_to_str(v, multiline, indent)) + else: + res += "{}: {}, ".format(str(k), str(v)) + if multiline: + res += '\n' + res = res[0:-2] + res += "}" + return res + + +def stable_repr(obj): + """ + Convert the provided dictionary into a 'repr' str, by calling + repr on both the keys and the values. + :type obj: dict | str | float + :param obj: the dictionary to convert. + :rtype: str + :return: Returns the converted dict. + """ + + if isinstance(obj, list): + return "[" + ", ".join(map(stable_repr, obj)) + "]" + elif not isinstance(obj, dict): + return repr(obj) + + if len(obj) == 0: + return "{}" + + res = "{" + + for k, v in sorted(obj.iteritems(), key=lambda ke: str(ke[0])): + res += "{}: {}, ".format(repr(k), stable_repr(v)) + + res = res[0:-2] + res += "}" + return res diff --git a/monasca_analytics/voter/pick_index.py b/monasca_analytics/voter/pick_index.py index a72a563..725078d 100644 --- a/monasca_analytics/voter/pick_index.py +++ b/monasca_analytics/voter/pick_index.py @@ -15,9 +15,11 @@ # under the License. import logging - +import math import voluptuous +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.component.params as params from monasca_analytics.voter import base logger = logging.getLogger(__name__) @@ -27,7 +29,7 @@ class PickIndexVoter(base.BaseVoter): def __init__(self, _id, _config): super(PickIndexVoter, self).__init__(_id, _config) - self._index = _config["params"]["index"] + self._index = _config["index"] self._index = 0 @staticmethod @@ -35,9 +37,10 @@ class PickIndexVoter(base.BaseVoter): pick_schema = voluptuous.Schema({ "module": voluptuous.And( basestring, lambda i: not any(c.isspace() for c in i)), - "params": { - "index": voluptuous.And(int, lambda i: i >= 0) - } + "index": voluptuous.And( + voluptuous.Or(float, int), + lambda i: i >= 0 and math.ceil(i) == math.floor(i) + ) }, required=True) return pick_schema(_config) @@ -45,11 +48,15 @@ class PickIndexVoter(base.BaseVoter): def get_default_config(): return { "module": PickIndexVoter.__name__, - "params": { - "index": 0 - } + "index": 0 } + @staticmethod + def get_params(): + return [ + params.ParamDescriptor('index', type_util.Number(), 0), + ] + def elect_structure(self, structures): return structures[ min(len(structures) - 1, diff --git a/monasca_analytics/web_service/request_handler.py b/monasca_analytics/web_service/request_handler.py index 770a48e..175ae60 100644 --- a/monasca_analytics/web_service/request_handler.py +++ b/monasca_analytics/web_service/request_handler.py @@ -17,10 +17,12 @@ import json import logging import sys +import traceback from tornado import web import voluptuous +import monasca_analytics.banana.emitter as emit import monasca_analytics.exception.monanas as err from monasca_analytics.web_service import web_service_model @@ -45,7 +47,7 @@ class MonanasHandler(web.RequestHandler): try: body = json.loads(self.request.body) - getattr(web_service_model, "action_model")(body) + web_service_model.action_model(body) getattr(self._monanas, body["action"])() except (AttributeError, voluptuous.Invalid, ValueError): self.set_status(400, "The request body was malformed.") @@ -67,3 +69,52 @@ class MonanasHandler(web.RequestHandler): if terminate[0]: logger.error(terminate[1]) self._monanas.stop_streaming_and_terminate() + + +class BananaHandler(web.RequestHandler): + """ + Request handler to manage the active config using + the banana configuration language. + """ + + def initialize(self, monanas): + """Initialize the handler. + + :param monanas: A Monana's instance. + """ + self._monanas = monanas + + @web.asynchronous + def post(self): + """Performs a Monanas's action.""" + terminate = (False, "") + + try: + body = json.loads(self.request.body) + web_service_model.banana_model(body) + emitter = emit.JsonEmitter() + # TODO(Joan): Change that + self._monanas.try_change_configuration(body["content"], emitter) + self.write(emitter.result) + except (AttributeError, voluptuous.Invalid, ValueError): + self.set_status(400, "The request body was malformed.") + except (err.MonanasBindSourcesError, + err.MonanasAlreadyStartedStreaming, + err.MonanasAlreadyStoppedStreaming) as e: + self.set_status(400, e.__str__()) + except err.MonanasStreamingError as e: + self.set_status(500, e.__str__()) + terminate = (True, e.__str__()) + except Exception as e: + tb = traceback.format_exc() + print(tb) + logger.error("Unexpected error: {0}. {1}". + format(sys.exc_info()[0], e)) + self.set_status(500, "Internal server error.") + + self.flush() + self.finish() + + if terminate[0]: + logger.error(terminate[1]) + self._monanas.stop_streaming_and_terminate() diff --git a/monasca_analytics/web_service/web_service.py b/monasca_analytics/web_service/web_service.py index 1ca37c6..e9947df 100644 --- a/monasca_analytics/web_service/web_service.py +++ b/monasca_analytics/web_service/web_service.py @@ -26,8 +26,10 @@ class WebService(web.Application): """WebService constructor.""" self._monanas = monanas self._config = config + params = {"monanas": self._monanas} handlers = [ - (r"/", request_handler.MonanasHandler, {"monanas": self._monanas}) + (r"/", request_handler.MonanasHandler, params), + (r"/banana", request_handler.BananaHandler, params), ] settings = {} diff --git a/monasca_analytics/web_service/web_service_model.py b/monasca_analytics/web_service/web_service_model.py index 0b36639..a81590f 100644 --- a/monasca_analytics/web_service/web_service_model.py +++ b/monasca_analytics/web_service/web_service_model.py @@ -26,3 +26,12 @@ def action_model(value): }, required=True) return action_model_schema(value) + + +def banana_model(value): + """Validates the data against the banana_model schema.""" + banana_model_schema = voluptuous.Schema({ + "content": basestring + }, required=True) + + return banana_model_schema(value) diff --git a/test/banana/deadpathck/__init__.py b/test/banana/deadpathck/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/banana/deadpathck/test_at_least_one_sink_or_source.py b/test/banana/deadpathck/test_at_least_one_sink_or_source.py new file mode 100644 index 0000000..a89ef46 --- /dev/null +++ b/test/banana/deadpathck/test_at_least_one_sink_or_source.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +import monasca_analytics.banana.deadpathck.config as deadpathck +import monasca_analytics.banana.grammar.config as grammar +import monasca_analytics.banana.typeck.config as typeck +import monasca_analytics.exception.banana as exception + +from test.util_for_testing import MonanasTestCase + +logger = logging.getLogger(__name__) + + +class PassOneSinkSourceTestCase(MonanasTestCase): + + def setUp(self): + super(PassOneSinkSourceTestCase, self).setUp() + + def tearDown(self): + super(PassOneSinkSourceTestCase, self).tearDown() + + def test_banana_should_fail_when_no_source(self): + banana_str = "" +\ + "a = CloudMarkovChainSource()\n" +\ + "b = StdoutSink()\n" +\ + "c = CloudIngestor()\n" +\ + "d = LiNGAM()\n" +\ + "c -> d -> b" + # Convert the grammar into an AST + parser = grammar.banana_grammar() + ast = parser.parse(banana_str) + # Compute the type table for the given AST + type_table = typeck.typeck(ast) + # Remove from the tree path that are "dead" + deadpathck.deadpathck(ast, type_table) + self.assertRaises( + exception.BananaNoFullPath, + deadpathck.contains_at_least_one_path_to_a_sink, + ast, + type_table + ) + + def test_banana_should_fail_when_no_sink(self): + banana_str = "" +\ + "a = CloudMarkovChainSource()\n" +\ + "b = StdoutSink()\n" +\ + "c = CloudIngestor()\n" +\ + "d = LiNGAM()\n" +\ + "a -> c -> d" + # Convert the grammar into an AST + parser = grammar.banana_grammar() + ast = parser.parse(banana_str) + # Compute the type table for the given AST + type_table = typeck.typeck(ast) + # Remove from the tree path that are "dead" + deadpathck.deadpathck(ast, type_table) + self.assertRaises( + exception.BananaNoFullPath, + deadpathck.contains_at_least_one_path_to_a_sink, + ast, + type_table + ) + + def test_banana_should_pass_when_more_source_sink(self): + banana_str = "" +\ + "a = CloudMarkovChainSource()\n" +\ + "b = StdoutSink()\n" +\ + "c = CloudIngestor()\n" +\ + "d = LiNGAM()\n" +\ + "a -> c -> d -> b" + # Convert the grammar into an AST + parser = grammar.banana_grammar() + ast = parser.parse(banana_str) + # Compute the type table for the given AST + type_table = typeck.typeck(ast) + # Remove from the tree path that are "dead" + deadpathck.deadpathck(ast, type_table) + deadpathck.contains_at_least_one_path_to_a_sink(ast, type_table) + # We should reach this line. + self.assertTrue(True) diff --git a/test/banana/deadpathck/test_deadpathck.py b/test/banana/deadpathck/test_deadpathck.py new file mode 100644 index 0000000..f7d48ca --- /dev/null +++ b/test/banana/deadpathck/test_deadpathck.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +import monasca_analytics.banana.deadpathck.config as deadpathck +import monasca_analytics.banana.emitter as emit +import monasca_analytics.banana.grammar.config as grammar +import monasca_analytics.banana.typeck.config as typeck + +from test.util_for_testing import MonanasTestCase + +logger = logging.getLogger(__name__) + + +class DeadPathTestCase(MonanasTestCase): + + def setUp(self): + super(DeadPathTestCase, self).setUp() + + def tearDown(self): + super(DeadPathTestCase, self).tearDown() + + def test_banana_should_remove_everything(self): + banana_str = "" +\ + "a = CloudMarkovChainSource()\n" +\ + "b = StdoutSink()\n" +\ + "c = CloudIngestor()\n" +\ + "d = LiNGAM()\n" +\ + "a -> c -> d" + emitter = CustomEmitter() + # Convert the grammar into an AST + parser = grammar.banana_grammar(emitter) + ast = parser.parse(banana_str) + # Compute the type table for the given AST + type_table = typeck.typeck(ast) + # Remove from the tree path that are "dead" + deadpathck.deadpathck(ast, type_table, emitter) + self.assertEqual(emitter.nb_errors, 0) + self.assertEqual(emitter.nb_warnings, 4) + self.assertEqual(len(ast.components), 0) + self.assertEqual(len(ast.connections.connections), 0) + + def test_banana_should_remove_one(self): + banana_str = "" +\ + "a = CloudMarkovChainSource()\n" +\ + "b = StdoutSink()\n" +\ + "c = CloudIngestor()\n" +\ + "d = LiNGAM()\n" +\ + "a -> c -> [d, b]" + emitter = CustomEmitter() + # Convert the grammar into an AST + parser = grammar.banana_grammar(emitter) + ast = parser.parse(banana_str) + # Compute the type table for the given AST + type_table = typeck.typeck(ast) + # Remove from the tree path that are "dead" + deadpathck.deadpathck(ast, type_table, emitter) + self.assertEqual(emitter.nb_errors, 0) + self.assertEqual(emitter.nb_warnings, 1) + self.assertEqual(len(ast.components), 3) + self.assertEqual(len(ast.connections.connections), 2) + + def test_banana_should_not_remove_anything(self): + banana_str = "" +\ + "a = CloudMarkovChainSource()\n" +\ + "b = StdoutSink()\n" +\ + "c = CloudIngestor()\n" +\ + "d = LiNGAM()\n" +\ + "a -> c -> d -> b" + emitter = CustomEmitter() + # Convert the grammar into an AST + parser = grammar.banana_grammar(emitter) + ast = parser.parse(banana_str) + # Compute the type table for the given AST + type_table = typeck.typeck(ast) + # Remove from the tree path that are "dead" + deadpathck.deadpathck(ast, type_table, emitter) + self.assertEqual(emitter.nb_errors, 0) + self.assertEqual(emitter.nb_warnings, 0) + self.assertEqual(len(ast.components), 4) + self.assertEqual(len(ast.connections.connections), 3) + + +class CustomEmitter(emit.Emitter): + + def __init__(self): + super(CustomEmitter, self).__init__() + self.nb_warnings = 0 + self.nb_errors = 0 + + def emit_warning(self, span, message): + print(span.get_line(), str(span), message) + self.nb_warnings += 1 + + def emit_error(self, span, message): + print(span.get_line(), str(span), message) + self.nb_errors += 1 diff --git a/test/banana/eval/__init__.py b/test/banana/eval/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/banana/eval/should_pass/eval_components_only/cloud_markov_chain_source.banana b/test/banana/eval/should_pass/eval_components_only/cloud_markov_chain_source.banana new file mode 100644 index 0000000..e725ac9 --- /dev/null +++ b/test/banana/eval/should_pass/eval_components_only/cloud_markov_chain_source.banana @@ -0,0 +1,3 @@ +a = CloudMarkovChainSource() # LHS_EQ IGNORE +c = a.sleep # LHS_EQ 0.01 +c = a.transitions.web_service."run=>slow"."0" # LHS_EQ 0.001 \ No newline at end of file diff --git a/test/banana/eval/should_pass/eval_each_stmt/calc_expressions.banana b/test/banana/eval/should_pass/eval_each_stmt/calc_expressions.banana new file mode 100644 index 0000000..97596da --- /dev/null +++ b/test/banana/eval/should_pass/eval_each_stmt/calc_expressions.banana @@ -0,0 +1,8 @@ +a = 23 # LHS_EQ 23.0 +a = a - a # LHS_EQ 0.0 +a = 12 - 12 - 12 # LHS_EQ -12.0 +a = 3 * 4 - 2 # LHS_EQ 10.0 +a = 2 - 3 * 4 # LHS_EQ -10.0 +b = 21 # LHS_EQ 21.0 +d = 12 # LHS_EQ 12.0 +e = b * (d + a) # LHS_EQ 42.0 diff --git a/test/banana/eval/should_pass/eval_each_stmt/json_expressions.banana b/test/banana/eval/should_pass/eval_each_stmt/json_expressions.banana new file mode 100644 index 0000000..59e635b --- /dev/null +++ b/test/banana/eval/should_pass/eval_each_stmt/json_expressions.banana @@ -0,0 +1,4 @@ +a = {a.b.c: 12} # LHS_EQ {'a': {'b': {'c': 12.0}}} +b = a.a.b.c # LHS_EQ 12.0 +a = {a.b: 21, a.c: "test"} # LHS_EQ {'a': {'b': 21.0, 'c': 'test'}} +d = a.a.c + " " + a.a.b # LHS_EQ 'test 21.0' \ No newline at end of file diff --git a/test/banana/eval/test_eval_config.py b/test/banana/eval/test_eval_config.py new file mode 100644 index 0000000..9e5c285 --- /dev/null +++ b/test/banana/eval/test_eval_config.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import re + +from monasca_analytics.banana.pass_manager import compute_evaluation_context +from monasca_analytics.util.string_util import stable_repr +from test.util_for_testing import MonanasTestCase + + +logger = logging.getLogger(__name__) + +_has_some_file_that_should = dict() +_has_some_file_that_should["pass_eval_stmt"] = False + + +class EvalTestCase(MonanasTestCase): + + def setUp(self): + super(EvalTestCase, self).setUp() + + def tearDown(self): + super(EvalTestCase, self).tearDown() + + def test_files_have_been_found(self): + self.assertTrue(_has_some_file_that_should["pass_eval_stmt"]) + + +def upgrade_test_case(): + regex_var_eq = re.compile("#(?: )*LHS_EQ(?: )+([^\n]+)") + + for root, dirs, files in os.walk('./banana/eval/should_pass/'): + for filename in files: + name_no_ext, _ = os.path.splitext(filename) + _has_some_file_that_should["pass_eval_stmt"] = True + with open(os.path.join(root, filename), 'r') as f: + content = f.read() + + expected_values = regex_var_eq.findall(content) + + def create_test(test_str, expect_values): + def should_pass(self): + # Custom checks runned after each statement + box = {"counter": 0} + + def custom_check(ctx, stmt_type, lhs_node, rhs_value): + if expect_values[box["counter"]] != "IGNORE": + self.assertEqual( + expect_values[box["counter"]], + stable_repr(rhs_value)) + box["counter"] += 1 + # Evaluate the file. + compute_evaluation_context(test_str, custom_check) + + should_pass.__name__ = "test_banana_eval_" + name_no_ext + return should_pass + + setattr(EvalTestCase, "test_banana_eval_" + name_no_ext, + create_test(content, expected_values)) + +# Fill the test case with generated test case from banana files +upgrade_test_case() diff --git a/test/banana/grammar/__init__.py b/test/banana/grammar/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/banana/grammar/should_fail/component_cant_be_in_object.banana b/test/banana/grammar/should_fail/component_cant_be_in_object.banana new file mode 100644 index 0000000..74899f5 --- /dev/null +++ b/test/banana/grammar/should_fail/component_cant_be_in_object.banana @@ -0,0 +1,2 @@ +a = { b: MyComponent() } +# RAISE ParseSyntaxException \ No newline at end of file diff --git a/test/banana/grammar/should_fail/json_obj_props_cant_shadow.banana b/test/banana/grammar/should_fail/json_obj_props_cant_shadow.banana new file mode 100644 index 0000000..973c008 --- /dev/null +++ b/test/banana/grammar/should_fail/json_obj_props_cant_shadow.banana @@ -0,0 +1,4 @@ +# This one shadow the a.b, so this is still an error. +a = { a.b: "test", a.b: 23 } + +# RAISE ParseSyntaxException \ No newline at end of file diff --git a/test/banana/grammar/should_fail/wrong_symbol_in_connection.banana b/test/banana/grammar/should_fail/wrong_symbol_in_connection.banana new file mode 100644 index 0000000..27977ab --- /dev/null +++ b/test/banana/grammar/should_fail/wrong_symbol_in_connection.banana @@ -0,0 +1,3 @@ +# This should fail +a -> b * c +# RAISE ParseException \ No newline at end of file diff --git a/test/banana/grammar/should_fail/wrong_symbol_in_expr.banana b/test/banana/grammar/should_fail/wrong_symbol_in_expr.banana new file mode 100644 index 0000000..42a5362 --- /dev/null +++ b/test/banana/grammar/should_fail/wrong_symbol_in_expr.banana @@ -0,0 +1,2 @@ +a = b -> c +# RAISE ParseException \ No newline at end of file diff --git a/test/banana/grammar/should_pass/component_creation.banana b/test/banana/grammar/should_pass/component_creation.banana new file mode 100644 index 0000000..3e261ed --- /dev/null +++ b/test/banana/grammar/should_pass/component_creation.banana @@ -0,0 +1,12 @@ +# Creating some components +a = Test() +b = Test(param={}) +c = Test(param="test") +d = Test(param=234) + +# Assigning parameters +b.param = {} +c.param = "test" +d.param = 234 + +# STMT_EQ { DotPath< Ident< a > > = Component { type_name: Ident< Test >, args: []}, DotPath< Ident< b > > = Component { type_name: Ident< Test >, args: [Ident< param > = JsonObj< {} >]}, DotPath< Ident< c > > = Component { type_name: Ident< Test >, args: [Ident< param > = Expr< [StringLit< "test" >] >]}, DotPath< Ident< d > > = Component { type_name: Ident< Test >, args: [Ident< param > = Expr< [Number< 234.0 >] >]}, DotPath< Ident< b >.Ident< param > > = JsonObj< {} >, DotPath< Ident< c >.Ident< param > > = Expr< [StringLit< "test" >] >, DotPath< Ident< d >.Ident< param > > = Expr< [Number< 234.0 >] > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/connection.banana b/test/banana/grammar/should_pass/connection.banana new file mode 100644 index 0000000..e4a02f9 --- /dev/null +++ b/test/banana/grammar/should_pass/connection.banana @@ -0,0 +1,13 @@ +# This example would not pass later pass +# but exercise the grammar + +a = Test() +b = Test() + +a -> [b, c] +a -> b -> c +a -> [a -> b, b] -> e +a -> b +[[a]] -> b + +# CONN_EQ Connection< [('Ident< a >', 'Ident< b >'), ('Ident< a >', 'Ident< c >'), ('Ident< b >', 'Ident< c >'), ('Ident< a >', 'Ident< a >'), ('Ident< b >', 'Ident< e >')] > \ No newline at end of file diff --git a/test/banana/grammar/should_pass/dot_accesses.banana b/test/banana/grammar/should_pass/dot_accesses.banana new file mode 100644 index 0000000..1e1430f --- /dev/null +++ b/test/banana/grammar/should_pass/dot_accesses.banana @@ -0,0 +1,10 @@ +a = {} +a.param1 = 23 +a."param3" = 1 +a."param4"."para mmm" = a."param3" +a.param2 = { + "a": 34 + 12, + a.b.c: a.param1 +} + +# STMT_EQ { DotPath< Ident< a > > = JsonObj< {} >, DotPath< Ident< a >.Ident< param1 > > = Expr< [Number< 23.0 >] >, DotPath< Ident< a >.StringLit< "param3" > > = Expr< [Number< 1.0 >] >, DotPath< Ident< a >.StringLit< "param4" >.StringLit< "para mmm" > > = Expr< [DotPath< Ident< a >.StringLit< "param3" > >] >, DotPath< Ident< a >.Ident< param2 > > = JsonObj< {DotPath< Ident< a >.Ident< b >.Ident< c > >: Expr< [DotPath< Ident< a >.Ident< param1 > >] >, StringLit< "a" >: Expr< [Number< 34.0 >, +, Number< 12.0 >] >} > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/eval_json_obj.banana b/test/banana/grammar/should_pass/eval_json_obj.banana new file mode 100644 index 0000000..55d05ca --- /dev/null +++ b/test/banana/grammar/should_pass/eval_json_obj.banana @@ -0,0 +1,6 @@ +a = { a.b: "test" } +c = { e: a } +# VAR a EQ_JSON { "a": {"b": "test"} } +# VAR c EQ_JSON { "e": {"a": {"b": "test"}} } + +# STMT_EQ { DotPath< Ident< a > > = JsonObj< {DotPath< Ident< a >.Ident< b > >: Expr< [StringLit< "test" >] >} >, DotPath< Ident< c > > = JsonObj< {DotPath< Ident< e > >: Expr< [DotPath< Ident< a > >] >} > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/eval_simple_expression.banana b/test/banana/grammar/should_pass/eval_simple_expression.banana new file mode 100644 index 0000000..ea8082c --- /dev/null +++ b/test/banana/grammar/should_pass/eval_simple_expression.banana @@ -0,0 +1,8 @@ +a = 23 + 2 +b = "test " + a +c = a + 4.0 * 2 +# VAR a EQ_NUM 25 +# VAR b EQ_STR "test 25" +# VAR c EQ_NUM 33 + +# STMT_EQ { DotPath< Ident< a > > = Expr< [Number< 23.0 >, +, Number< 2.0 >] >, DotPath< Ident< b > > = Expr< [StringLit< "test " >, +, DotPath< Ident< a > >] >, DotPath< Ident< c > > = Expr< [DotPath< Ident< a > >, +, Expr< [Number< 4.0 >, *, Number< 2.0 >] >] > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/full_example.banana b/test/banana/grammar/should_pass/full_example.banana new file mode 100644 index 0000000..706164c --- /dev/null +++ b/test/banana/grammar/should_pass/full_example.banana @@ -0,0 +1,14 @@ +# Connections +a -> b + +# Some vars +d = 23 +e = d - 23 +c = 23 + 1.5 * 3 - d + +# Components +a = Test(a=23, b=c+10) +b = Test(23, 10) + +# STMT_EQ { DotPath< Ident< d > > = Expr< [Number< 23.0 >] >, DotPath< Ident< e > > = Expr< [DotPath< Ident< d > >, -, Number< 23.0 >] >, DotPath< Ident< c > > = Expr< [Number< 23.0 >, +, Expr< [Number< 1.5 >, *, Number< 3.0 >] >, -, DotPath< Ident< d > >] >, DotPath< Ident< a > > = Component { type_name: Ident< Test >, args: [Ident< a > = Expr< [Number< 23.0 >] >, Ident< b > = Expr< [DotPath< Ident< c > >, +, Number< 10.0 >] >]}, DotPath< Ident< b > > = Component { type_name: Ident< Test >, args: [Expr< [Number< 23.0 >] >, Expr< [Number< 10.0 >] >]} } +# CONN_EQ Connection< [('Ident< a >', 'Ident< b >')] > \ No newline at end of file diff --git a/test/banana/grammar/should_pass/json_obj.banana b/test/banana/grammar/should_pass/json_obj.banana new file mode 100644 index 0000000..562449f --- /dev/null +++ b/test/banana/grammar/should_pass/json_obj.banana @@ -0,0 +1,8 @@ +# Various check for edge cases +a = {} +b = { a: {} } +c = { "a": {} } +d = { "": {} } +e = { "": "" } + +# STMT_EQ { DotPath< Ident< a > > = JsonObj< {} >, DotPath< Ident< b > > = JsonObj< {DotPath< Ident< a > >: JsonObj< {} >} >, DotPath< Ident< c > > = JsonObj< {StringLit< "a" >: JsonObj< {} >} >, DotPath< Ident< d > > = JsonObj< {StringLit< "" >: JsonObj< {} >} >, DotPath< Ident< e > > = JsonObj< {StringLit< "" >: Expr< [StringLit< "" >] >} > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/jsonlike_objects.banana b/test/banana/grammar/should_pass/jsonlike_objects.banana new file mode 100644 index 0000000..de15920 --- /dev/null +++ b/test/banana/grammar/should_pass/jsonlike_objects.banana @@ -0,0 +1,6 @@ +a = {} +b = { "a": a } +c = { a.b: "b" } +d = { a.v."t t": 12 + 23 } + +# STMT_EQ { DotPath< Ident< a > > = JsonObj< {} >, DotPath< Ident< b > > = JsonObj< {StringLit< "a" >: Expr< [DotPath< Ident< a > >] >} >, DotPath< Ident< c > > = JsonObj< {DotPath< Ident< a >.Ident< b > >: Expr< [StringLit< "b" >] >} >, DotPath< Ident< d > > = JsonObj< {DotPath< Ident< a >.Ident< v >.StringLit< "t t" > >: Expr< [Number< 12.0 >, +, Number< 23.0 >] >} > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/scary_json_ops.banana b/test/banana/grammar/should_pass/scary_json_ops.banana new file mode 100644 index 0000000..2621d07 --- /dev/null +++ b/test/banana/grammar/should_pass/scary_json_ops.banana @@ -0,0 +1,4 @@ +a = { b.c: 21 } +b = { a: { a.b: { a.b.c: 10 }}, b.c: 20 } + +# STMT_EQ { DotPath< Ident< a > > = JsonObj< {DotPath< Ident< b >.Ident< c > >: Expr< [Number< 21.0 >] >} >, DotPath< Ident< b > > = JsonObj< {DotPath< Ident< a > >: JsonObj< {DotPath< Ident< a >.Ident< b > >: JsonObj< {DotPath< Ident< a >.Ident< b >.Ident< c > >: Expr< [Number< 10.0 >] >} >} >, DotPath< Ident< b >.Ident< c > >: Expr< [Number< 20.0 >] >} > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/valid_expression.banana b/test/banana/grammar/should_pass/valid_expression.banana new file mode 100644 index 0000000..3c262f4 --- /dev/null +++ b/test/banana/grammar/should_pass/valid_expression.banana @@ -0,0 +1,5 @@ +c = SomeComponent() +b = 12 + c.param1."foo bar constant" +a = b * 12 + +# STMT_EQ { DotPath< Ident< c > > = Component { type_name: Ident< SomeComponent >, args: []}, DotPath< Ident< b > > = Expr< [Number< 12.0 >, +, DotPath< Ident< c >.Ident< param1 >.StringLit< "foo bar constant" > >] >, DotPath< Ident< a > > = Expr< [DotPath< Ident< b > >, *, Number< 12.0 >] > } \ No newline at end of file diff --git a/test/banana/grammar/should_pass/various_literal_assignments.banana b/test/banana/grammar/should_pass/various_literal_assignments.banana new file mode 100644 index 0000000..a82c461 --- /dev/null +++ b/test/banana/grammar/should_pass/various_literal_assignments.banana @@ -0,0 +1,5 @@ +a = "some string" +b = 12.0 +c = {"a": b} + +# STMT_EQ { DotPath< Ident< a > > = Expr< [StringLit< "some string" >] >, DotPath< Ident< b > > = Expr< [Number< 12.0 >] >, DotPath< Ident< c > > = JsonObj< {StringLit< "a" >: Expr< [DotPath< Ident< b > >] >} > } \ No newline at end of file diff --git a/test/banana/grammar/test_config.py b/test/banana/grammar/test_config.py new file mode 100644 index 0000000..3fa49ae --- /dev/null +++ b/test/banana/grammar/test_config.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import pyparsing as p +import re + +from monasca_analytics.banana.grammar.config import banana_grammar +from test.util_for_testing import MonanasTestCase + + +logger = logging.getLogger(__name__) + +_has_some_file_that_should = dict() +_has_some_file_that_should["pass"] = False +_has_some_file_that_should["fail"] = False + + +class GrammarTestCase(MonanasTestCase): + + def setUp(self): + super(GrammarTestCase, self).setUp() + + def tearDown(self): + super(GrammarTestCase, self).tearDown() + + def test_files_have_been_found(self): + self.assertTrue(_has_some_file_that_should["pass"]) + self.assertTrue(_has_some_file_that_should["fail"]) + + +def upgrade_test_case(): + grammar = banana_grammar() + regex_raise = re.compile("#(?: )*RAISE(?: )*([^\n]+)") + regex_ast_eq = re.compile("#(?: )*AST_EQ(?: )(?: )*([^\n]+)") + regex_stmt_eq = re.compile("#(?: )*STMT_EQ(?: )(?: )*([^\n]+)") + regex_conn_eq = re.compile("#(?: )*CONN_EQ(?: )(?: )*([^\n]+)") + + for root, dirs, files in os.walk('./banana/grammar/should_pass'): + for filename in files: + name_no_ext, _ = os.path.splitext(filename) + _has_some_file_that_should["pass"] = True + with open(os.path.join(root, filename), 'r') as f: + content = f.read() + + expected_ast = regex_ast_eq.search(content) + if expected_ast is not None: + expected_ast = expected_ast.group(1) + expected_stmt = regex_stmt_eq.search(content) + if expected_stmt is not None: + expected_stmt = expected_stmt.group(1) + expected_conn = regex_conn_eq.search(content) + if expected_conn is not None: + expected_conn = expected_conn.group(1) + + def create_test(test_str, expect_ast, expect_stmt, exp_conn): + def should_pass(self): + tree = grammar.parse(test_str) + if expect_ast is not None: + self.assertEqual(str(tree), + expect_ast) + if expect_stmt is not None: + self.assertEqual(tree.statements_to_str(), + expect_stmt) + if exp_conn is not None: + self.assertEqual(str(tree.connections), + exp_conn) + if exp_conn is None and expect_ast is None and\ + expect_stmt is None: + raise Exception("Expected at least one check!") + + should_pass.__name__ = "test_banana_pass_" + name_no_ext + return should_pass + + setattr(GrammarTestCase, "test_banana_pass_" + name_no_ext, + create_test(content, expected_ast, expected_stmt, + expected_conn)) + + for root, dirs, files in os.walk('./banana/grammar/should_fail'): + for filename in files: + name_no_ext, _ = os.path.splitext(filename) + _has_some_file_that_should["fail"] = True + with open(os.path.join(root, filename), 'r') as f: + content = f.read() + + def create_test(test_str): + def should_fail(self): + expected_error = regex_raise.search(test_str).group(1) + expected_exception = get_exception_from_str( + expected_error) + self.assertRaises( + expected_exception, + grammar.parse, + test_str) + should_fail.__name__ = "test_banana_fail_" + name_no_ext + return should_fail + + setattr(GrammarTestCase, "test_banana_fail_" + name_no_ext, + create_test(content)) + +# Fill the test case with generated test case from banana files +upgrade_test_case() + + +def get_exception_from_str(string): + if string == p.ParseSyntaxException.__name__: + return p.ParseSyntaxException + if string == p.ParseFatalException.__name__: + return p.ParseFatalException + if string == p.ParseException.__name__: + return p.ParseException + raise Exception("Invalid exception name: '{}'".format(string)) diff --git a/test/banana/test_api.py b/test/banana/test_api.py index f4b931a..f859010 100644 --- a/test/banana/test_api.py +++ b/test/banana/test_api.py @@ -14,12 +14,12 @@ # License for the specific language governing permissions and limitations # under the License. -from monasca_analytics.banana.api import create_fn_with_config -from monasca_analytics.banana.api import validate_environment -from monasca_analytics.banana.api import validate_expression -from monasca_analytics.banana.api import validate_name_binding from monasca_analytics.exception.banana import BananaEnvironmentError from monasca_analytics.exception.banana import BananaInvalidExpression +from monasca_analytics.parsing.api import create_fn_with_config +from monasca_analytics.parsing.api import validate_environment +from monasca_analytics.parsing.api import validate_expression +from monasca_analytics.parsing.api import validate_name_binding from test.util_for_testing import MonanasTestCase diff --git a/test/banana/test_config_examples.py b/test/banana/test_config_examples.py new file mode 100644 index 0000000..215f984 --- /dev/null +++ b/test/banana/test_config_examples.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os + +import monasca_analytics.banana.emitter as emit +import monasca_analytics.banana.pass_manager as p + +from test.util_for_testing import MonanasTestCase + + +logger = logging.getLogger(__name__) + +_has_some_examples_been_found = {0: False} + + +class ConfigExamplesTestCase(MonanasTestCase): + + def setUp(self): + super(ConfigExamplesTestCase, self).setUp() + + def tearDown(self): + super(ConfigExamplesTestCase, self).tearDown() + + def test_files_has_been_found(self): + self.assertTrue(_has_some_examples_been_found[0]) + + +def upgrade_test_case(): + for root, dirs, files in os.walk('../config'): + for filename in files: + name_no_ext, ext = os.path.splitext(filename) + if ext == '.banana': + _has_some_examples_been_found[0] = True + with open(os.path.join(root, filename), 'r') as f: + content = f.read() + + def create_test(test_str): + def should_pass(self): + fake_driver = FakeDriver() + emitter = CustomEmitter() + p.execute_banana_string( + test_str, fake_driver, emitter) + self.assertEqual(emitter.nb_errors, 0) + self.assertEqual(emitter.nb_warnings, 0) + self.assertTrue(fake_driver.set_links_called) + self.assertTrue(fake_driver.start_pipeline_called) + self.assertTrue(fake_driver.stop_pipeline_called) + + should_pass.__name__ = "test_banana_examples_config" + return should_pass + + setattr(ConfigExamplesTestCase, + "test_banana_examples_config_" + name_no_ext, + create_test(content)) + +upgrade_test_case() + + +class FakeDriver(object): + + def __init__(self): + self.start_pipeline_called = False + self.stop_pipeline_called = False + self.set_links_called = False + + def stop_pipeline(self): + self.stop_pipeline_called = True + + def start_pipeline(self): + self.start_pipeline_called = True + + def set_links(self, _): + self.set_links_called = True + + +class CustomEmitter(emit.Emitter): + + def __init__(self): + super(CustomEmitter, self).__init__() + self.nb_warnings = 0 + self.nb_errors = 0 + + def emit_warning(self, span, message): + print(span.get_line(), str(span), message) + self.nb_warnings += 1 + + def emit_error(self, span, message): + print(span.get_line(), str(span), message) + self.nb_errors += 1 diff --git a/test/banana/typeck/__init__.py b/test/banana/typeck/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/banana/typeck/should_fail/comp_var_cant_be_shadowed.banana b/test/banana/typeck/should_fail/comp_var_cant_be_shadowed.banana new file mode 100644 index 0000000..610c34e --- /dev/null +++ b/test/banana/typeck/should_fail/comp_var_cant_be_shadowed.banana @@ -0,0 +1,3 @@ +c = IptablesIngestor() +c = 23 +# RAISE BananaShadowingComponentError \ No newline at end of file diff --git a/test/banana/typeck/should_fail/component_cant_be_in_object.banana b/test/banana/typeck/should_fail/component_cant_be_in_object.banana new file mode 100644 index 0000000..fb8fcb1 --- /dev/null +++ b/test/banana/typeck/should_fail/component_cant_be_in_object.banana @@ -0,0 +1,4 @@ +c = IptablesIngestor() +a = { b: c } +# TODO(Joan): rename Comp to Components +# RAISE BananaAssignCompError \ No newline at end of file diff --git a/test/banana/typeck/should_fail/component_unkown_param_in_arg.banana b/test/banana/typeck/should_fail/component_unkown_param_in_arg.banana new file mode 100644 index 0000000..8ed0288 --- /dev/null +++ b/test/banana/typeck/should_fail/component_unkown_param_in_arg.banana @@ -0,0 +1,2 @@ +a = IPTablesSource(sleep=0.01, test="foobar") +# RAISE BananaComponentTooManyParams \ No newline at end of file diff --git a/test/banana/typeck/should_fail/component_unkown_param_in_stmt.banana b/test/banana/typeck/should_fail/component_unkown_param_in_stmt.banana new file mode 100644 index 0000000..939ad99 --- /dev/null +++ b/test/banana/typeck/should_fail/component_unkown_param_in_stmt.banana @@ -0,0 +1,9 @@ +a = IPTablesSource(sleep=0.01) +a.test = "foobar" + +# NEW_TEST + +a = CloudMarkovChainSource() +a.transitions.web_servic = {} + +# RAISE BananaPropertyDoesNotExists \ No newline at end of file diff --git a/test/banana/typeck/should_fail/impossible_connection.banana b/test/banana/typeck/should_fail/impossible_connection.banana new file mode 100644 index 0000000..218018e --- /dev/null +++ b/test/banana/typeck/should_fail/impossible_connection.banana @@ -0,0 +1,5 @@ +a -> b + +a = IPTablesSource(sleep=0.01) +b = IPTablesSource(sleep=0.01) +# RAISE BananaConnectionError \ No newline at end of file diff --git a/test/banana/typeck/should_fail/incompatible_json_props_types.banana b/test/banana/typeck/should_fail/incompatible_json_props_types.banana new file mode 100644 index 0000000..7e97243 --- /dev/null +++ b/test/banana/typeck/should_fail/incompatible_json_props_types.banana @@ -0,0 +1,8 @@ +# Incompatible type between string and object required by "a.b" +a = { a: "test", a.b: 23 } + +# NEW_TEST + +a = { "a"."b": "test", a.b.c: 23 } + +# RAISE BananaTypeError \ No newline at end of file diff --git a/test/banana/typeck/should_fail/invalid_str_concat.banana b/test/banana/typeck/should_fail/invalid_str_concat.banana new file mode 100644 index 0000000..352f8ef --- /dev/null +++ b/test/banana/typeck/should_fail/invalid_str_concat.banana @@ -0,0 +1,14 @@ +a = "test" +a = a - a + +# NEW_TEST + +a = "test" +a = a * a + +# NEW_TEST + +a = "test" +a = a / a + +# RAISE BananaUnknownOperator \ No newline at end of file diff --git a/test/banana/typeck/should_fail/type_has_no_nested_properties.banana b/test/banana/typeck/should_fail/type_has_no_nested_properties.banana new file mode 100644 index 0000000..835dd5f --- /dev/null +++ b/test/banana/typeck/should_fail/type_has_no_nested_properties.banana @@ -0,0 +1,9 @@ +a = IPTablesSource(sleep=0.01) +b = a.sleep.test + +# NEW_TEST +a = CloudMarkovChainSource() +# Typo: runs -> run +b = a.transitions.web_service."runs=>slow" + +# RAISE BananaPropertyDoesNotExists \ No newline at end of file diff --git a/test/banana/typeck/should_fail/type_is_not_a_subtype.banana b/test/banana/typeck/should_fail/type_is_not_a_subtype.banana new file mode 100644 index 0000000..cccb3de --- /dev/null +++ b/test/banana/typeck/should_fail/type_is_not_a_subtype.banana @@ -0,0 +1,18 @@ +a = RandomSource() +a.model = { + params: { + "origin_types": {} + }, + extra_param: 42 +} + +# NEW_TEST + +a = RandomSource(model = { + params: { + "origin_types": {} + }, + extra_param: 42 +}) + +# RAISE BananaArgumentTypeError \ No newline at end of file diff --git a/test/banana/typeck/should_fail/wrong_param_type.banana b/test/banana/typeck/should_fail/wrong_param_type.banana new file mode 100644 index 0000000..63fd975 --- /dev/null +++ b/test/banana/typeck/should_fail/wrong_param_type.banana @@ -0,0 +1,7 @@ +a = IPTablesSource(sleep="foobar") + +# NEW_TEST + +a = IPTablesSource(sleep={}) + +# RAISE BananaArgumentTypeError \ No newline at end of file diff --git a/test/banana/typeck/should_fail/wrong_parameter_name_throw.banana b/test/banana/typeck/should_fail/wrong_parameter_name_throw.banana new file mode 100644 index 0000000..b58752b --- /dev/null +++ b/test/banana/typeck/should_fail/wrong_parameter_name_throw.banana @@ -0,0 +1,9 @@ +# slep instead of sleep +a = IPTablesSource(slep=0.01) + +# NEW_TEST + +# priod instead of period +a = MonascaDerivativeLDP(priod=0.01) + +# RAISE BananaComponentIncorrectParamName \ No newline at end of file diff --git a/test/banana/typeck/should_pass/compatible_json_props_types.banana b/test/banana/typeck/should_pass/compatible_json_props_types.banana new file mode 100644 index 0000000..cfb9fd0 --- /dev/null +++ b/test/banana/typeck/should_pass/compatible_json_props_types.banana @@ -0,0 +1,3 @@ +a = { a.b: "test", a.c: "test" } + +# TYPE_TABLE_EQ {Ident< a >: TypeObject < {a: TypeObject < {b: TypeString, c: TypeString} >} >} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/component_param_has_a_type.banana b/test/banana/typeck/should_pass/component_param_has_a_type.banana new file mode 100644 index 0000000..cb87f0c --- /dev/null +++ b/test/banana/typeck/should_pass/component_param_has_a_type.banana @@ -0,0 +1,3 @@ +a = IPTablesSource(sleep=0.01) +b = a.sleep +# TYPE_TABLE_EQ {Ident< a >: IPTablesSource(sleep=TypeNumber), Ident< b >: TypeNumber} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/expression_shenanigan.banana b/test/banana/typeck/should_pass/expression_shenanigan.banana new file mode 100644 index 0000000..7cd8eb0 --- /dev/null +++ b/test/banana/typeck/should_pass/expression_shenanigan.banana @@ -0,0 +1,19 @@ +a = 12 * 13 + 1 + +# NEW_TEST + +a = 1 + 12 * 13 + +# NEW_TEST + +a = ((12 * 13) + 1) + +# NEW_TEST + +a = (1 + 3) * 3 + +# NEW_TEST + +a = 4 -5 + 22 - 22 + +# TYPE_TABLE_EQ {Ident< a >: TypeNumber} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/monasca_combine.banana b/test/banana/typeck/should_pass/monasca_combine.banana new file mode 100644 index 0000000..d713dca --- /dev/null +++ b/test/banana/typeck/should_pass/monasca_combine.banana @@ -0,0 +1,22 @@ +# Checking that parameters are working as expected + +a = MonascaCombineLDP() +a.metric = "cpu.logical_cores_actives" +a.lambda = "a * b - b - a + c" +a.bindings = { + a: "cpu.idle_perc", + b: "cpu.total_logical_cores", + c: "some_val" +} + +# NEW_TEST + +# Same test as above + +a = MonascaCombineLDP( + metric = "cpu.logical_cores_actives", + lambda = "a * b", + bindings = {a:"cpu.idle_perc", b:"cpu.total_logical_cores", c:"some_val"} +) + +# TYPE_TABLE_EQ {Ident< a >: MonascaCombineLDP(metric=TypeString,period=TypeNumber,lambda=TypeString,bindings=TypeAny)} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/nested_value_in_component_params.banana b/test/banana/typeck/should_pass/nested_value_in_component_params.banana new file mode 100644 index 0000000..29f2d04 --- /dev/null +++ b/test/banana/typeck/should_pass/nested_value_in_component_params.banana @@ -0,0 +1,9 @@ +a = CloudMarkovChainSource() +b = a.transitions.web_service + +# NEW_TEST + +a = CloudMarkovChainSource() +b = a."transitions"."web_service" + +# TYPE_TABLE_EQ {Ident< a >: CloudMarkovChainSource(sleep=TypeNumber,transitions=TypeStruct < {host: TypeStruct < {off=>on: TypeNumber, on=>off: TypeNumber} >, switch: TypeStruct < {off=>on: TypeNumber, on=>off: TypeNumber} >, web_service: TypeStruct < {run=>slow: TypeAny, slow=>run: TypeAny, stop=>run: TypeAny} >} >,triggers=TypeStruct < {support: TypeStruct < {get_called: TypeAny} >} >,graph=TypeAny), Ident< b >: TypeStruct < {run=>slow: TypeAny, slow=>run: TypeAny, stop=>run: TypeAny} >} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/number_can_be_shadowed_by_object.banana b/test/banana/typeck/should_pass/number_can_be_shadowed_by_object.banana new file mode 100644 index 0000000..3d78957 --- /dev/null +++ b/test/banana/typeck/should_pass/number_can_be_shadowed_by_object.banana @@ -0,0 +1,4 @@ +a = {a: "test"} +a = 23 + +# TYPE_TABLE_EQ {Ident< a >: TypeNumber} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/object_can_be_shadowed_by_number.banana b/test/banana/typeck/should_pass/object_can_be_shadowed_by_number.banana new file mode 100644 index 0000000..6d478c4 --- /dev/null +++ b/test/banana/typeck/should_pass/object_can_be_shadowed_by_number.banana @@ -0,0 +1,4 @@ +a = 23 +a = {} + +# TYPE_TABLE_EQ {Ident< a >: TypeObject < {} >} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/simple_connection.banana b/test/banana/typeck/should_pass/simple_connection.banana new file mode 100644 index 0000000..bb96895 --- /dev/null +++ b/test/banana/typeck/should_pass/simple_connection.banana @@ -0,0 +1,7 @@ +# Simple connection +a -> b + +a = IPTablesSource(sleep=0.01) +b = IptablesIngestor() + +# TYPE_TABLE_EQ {Ident< a >: IPTablesSource(sleep=TypeNumber), Ident< b >: IptablesIngestor()} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/simple_literal.banana b/test/banana/typeck/should_pass/simple_literal.banana new file mode 100644 index 0000000..bfe87a2 --- /dev/null +++ b/test/banana/typeck/should_pass/simple_literal.banana @@ -0,0 +1,4 @@ +a = "test" +b = 213 +c = {} +# TYPE_TABLE_EQ {Ident< a >: TypeString, Ident< b >: TypeNumber, Ident< c >: TypeObject < {} >} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/type_is_a_subtype.banana b/test/banana/typeck/should_pass/type_is_a_subtype.banana new file mode 100644 index 0000000..3ae970c --- /dev/null +++ b/test/banana/typeck/should_pass/type_is_a_subtype.banana @@ -0,0 +1,20 @@ +a = RandomSource() +a.model = { + name: "test", + params: { + "origin_types": {} + }, + extra_param: 42 +} + +# NEW_TEST + +a = RandomSource(model = { + name: "test", + params: { + "origin_types": {} + }, + extra_param: 42 +}) + +# TYPE_TABLE_EQ {Ident< a >: RandomSource(host=TypeString,port=TypeNumber,model=TypeStruct < {name: TypeString, params: TypeStruct < {origin_types: TypeObject < {} >} >} >,alert_per_burst=TypeNumber,idle_time_between_bursts=TypeNumber)} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/valid_str_concat.banana b/test/banana/typeck/should_pass/valid_str_concat.banana new file mode 100644 index 0000000..fb6799c --- /dev/null +++ b/test/banana/typeck/should_pass/valid_str_concat.banana @@ -0,0 +1,4 @@ +a = "test" +a = a + a + +# TYPE_TABLE_EQ {Ident< a >: TypeString} \ No newline at end of file diff --git a/test/banana/typeck/should_pass/various_object_use_case.banana b/test/banana/typeck/should_pass/various_object_use_case.banana new file mode 100644 index 0000000..3bea1bc --- /dev/null +++ b/test/banana/typeck/should_pass/various_object_use_case.banana @@ -0,0 +1,7 @@ +a = {} +b = {a: a} +c.d.e = {} +a.test = 12 +e = a +a.test = "test" +# TYPE_TABLE_EQ {Ident< a >: TypeObject < {test: TypeString} >, Ident< b >: TypeObject < {a: TypeObject < {test: TypeString} >} >, Ident< c >: TypeObject < {d: TypeObject < {e: TypeObject < {} >} >} >, Ident< e >: TypeObject < {test: TypeString} >} \ No newline at end of file diff --git a/test/banana/typeck/test_typeck_config.py b/test/banana/typeck/test_typeck_config.py new file mode 100644 index 0000000..f7551d3 --- /dev/null +++ b/test/banana/typeck/test_typeck_config.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Hewlett Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import re + +import monasca_analytics.banana.pass_manager as p +import monasca_analytics.exception.banana as exception +from test.util_for_testing import MonanasTestCase + + +logger = logging.getLogger(__name__) + +_has_some_file_that_should = dict() +_has_some_file_that_should["pass"] = False +_has_some_file_that_should["fail"] = False + + +class TypecheckTestCase(MonanasTestCase): + + def setUp(self): + super(TypecheckTestCase, self).setUp() + + def tearDown(self): + super(TypecheckTestCase, self).tearDown() + + def test_files_have_been_found(self): + self.assertTrue(_has_some_file_that_should["pass"]) + self.assertTrue(_has_some_file_that_should["fail"]) + + +def upgrade_test_case(): + regex_pass = re.compile("#(?: )*TYPE_TABLE_EQ(?: )*([^\n]+)") + regex_fail = re.compile("#(?: )*RAISE(?: )*([^\n]+)") + regex_split = re.compile("#(?: )*NEW_TEST(?:[^\n]*)\n") + + for root, dirs, files in os.walk('./banana/typeck/should_pass'): + for filename in files: + name_no_ext, _ = os.path.splitext(filename) + _has_some_file_that_should["pass"] = True + with open(os.path.join(root, filename), 'r') as f: + content = f.read() + expect_type_table = regex_pass.search(content) + if expect_type_table is not None: + expect_type_table = expect_type_table.group(1) + + def create_test(test_str, expected_type_table): + def should_pass(self): + type_table = p.compute_type_table(test_str) + if expected_type_table is not None: + self.assertEqual(str(type_table), + expected_type_table) + else: + raise Exception( + "Missing # TYPE_TABLE_EQ <...> in test." + ) + + should_pass.__name__ = "test_banana_pass_" + name_no_ext + return should_pass + + many_tests = regex_split.split(content) + if len(many_tests) == 1: + setattr(TypecheckTestCase, + "test_banana_pass_" + name_no_ext, + create_test(many_tests[0], expect_type_table)) + else: + suffix = 0 + for test_case in many_tests: + suffix += 1 + setattr(TypecheckTestCase, + "test_banana_pass_" + name_no_ext + + str(suffix), + create_test(test_case, expect_type_table)) + + for root, dirs, files in os.walk('./banana/typeck/should_fail'): + for filename in files: + name_no_ext, _ = os.path.splitext(filename) + _has_some_file_that_should["fail"] = True + with open(os.path.join(root, filename), 'r') as f: + content = f.read() + expect_error = regex_fail.search(content) + if expect_error is not None: + expect_error = expect_error.group(1) + + def create_test(s, test_str, expected_error): + def should_fail(self): + if expected_error is not None: + expected_exception = get_exception_from_str( + expected_error) + self.assertRaises( + expected_exception, + p.compute_type_table, + test_str) + else: + raise Exception("Missing # RAISE in test") + should_fail.__name__ = "test_banana_fail_" + name_no_ext +\ + str(s) + return should_fail + + many_tests = regex_split.split(content) + if len(many_tests) == 1: + setattr(TypecheckTestCase, + "test_banana_fail_" + name_no_ext, + create_test("", many_tests[0], expect_error)) + else: + suffix = 0 + for test_case in many_tests: + suffix += 1 + setattr(TypecheckTestCase, + "test_banana_fail_" + name_no_ext + + str(suffix), + create_test(suffix, test_case, expect_error)) + +# Fill the test case with generated test case from banana files +upgrade_test_case() + + +def get_exception_from_str(string): + if string == exception.BananaAssignCompError.__name__: + return exception.BananaAssignCompError + if string == exception.BananaConnectionError.__name__: + return exception.BananaConnectionError + if string == exception.BananaAssignmentError.__name__: + return exception.BananaAssignmentError + if string == exception.BananaTypeError.__name__: + return exception.BananaTypeError + if string == exception.BananaArgumentTypeError.__name__: + return exception.BananaArgumentTypeError + if string == exception.BananaUnknown.__name__: + return exception.BananaUnknown + if string == exception.BananaPropertyDoesNotExists.__name__: + return exception.BananaPropertyDoesNotExists + if string == exception.BananaComponentIncorrectParamName.__name__: + return exception.BananaComponentIncorrectParamName + if string == exception.BananaComponentTooManyParams.__name__: + return exception.BananaComponentTooManyParams + if string == exception.BananaShadowingComponentError.__name__: + return exception.BananaShadowingComponentError + if string == exception.BananaUnknownOperator.__name__: + return exception.BananaUnknownOperator + raise Exception("Invalid exception name: '{}'".format(string)) diff --git a/test/cli/__init__.py b/test/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/dsl/test_dsl.py b/test/cli/test_dsl.py similarity index 98% rename from test/dsl/test_dsl.py rename to test/cli/test_dsl.py index 0e952d3..bec0af9 100644 --- a/test/dsl/test_dsl.py +++ b/test/cli/test_dsl.py @@ -22,8 +22,8 @@ import unittest import voluptuous +from monasca_analytics.banana.cli.dsl import MonanasDSL from monasca_analytics.config import const -from monasca_analytics.dsl.dsl import MonanasDSL from monasca_analytics.exception import dsl as dsl_err from monasca_analytics.exception import monanas as mon_err @@ -140,10 +140,9 @@ class TestMonanasDSL(unittest.TestCase): def test_modify_component(self): self.assertTrue(self.dsl.modify_component( - "src3", ["params", "server_sleep_in_seconds"], 0.02)) + "src3", ["sleep"], 0.02)) expected_config = self.original_config - expected_config[const.SOURCES]["src3"]["params"][ - "server_sleep_in_seconds"] = 0.02 + expected_config[const.SOURCES]["src3"]["sleep"] = 0.02 self.assertEqual(expected_config, self.dsl._config) def test_modify_component_inexistent(self): diff --git a/test/dsl/test_interpreter.py b/test/cli/test_interpreter.py similarity index 96% rename from test/dsl/test_interpreter.py rename to test/cli/test_interpreter.py index 98eaab0..9cc9808 100644 --- a/test/dsl/test_interpreter.py +++ b/test/cli/test_interpreter.py @@ -20,8 +20,8 @@ import logging.config import os import unittest +from monasca_analytics.banana.cli import interpreter from monasca_analytics.config import const -from monasca_analytics.dsl import interpreter from monasca_analytics.exception import dsl as dsl_err from monasca_analytics.exception import monanas as mon_err from monasca_analytics.source import iptables_markov_chain as ipt_src @@ -142,17 +142,16 @@ class TestDSLInterpreter(unittest.TestCase): def test_cmd_modify_to_valid_float(self): self.inter.execute_string("A = IPTablesSource") self.assertTrue(self.inter.execute_string( - "A.params.server_sleep_in_seconds = 0.02")) + "A.sleep = 0.02")) self.assertEqual({ "module": "IPTablesSource", - "params": { - "server_sleep_in_seconds": 0.02 - }}, self.inter.dsl._config[const.SOURCES]["src1"]) + "sleep": 0.02 + }, self.inter.dsl._config[const.SOURCES]["src1"]) def test_cmd_modify_to_invalid_float(self): self.inter.execute_string("A = IPTablesSource") self.assertFalse(self.inter.execute_string( - "A.params.server_sleep_in_seconds = 1.2")) + "A.sleep = 1.2")) def test_cmd_modify_int(self): self.inter.execute_string("A = KafkaSource") diff --git a/test/dsl/test_parser.py b/test/cli/test_parser.py similarity index 98% rename from test/dsl/test_parser.py rename to test/cli/test_parser.py index ac83081..dce7352 100644 --- a/test/dsl/test_parser.py +++ b/test/cli/test_parser.py @@ -19,8 +19,8 @@ import logging.config import os import unittest -from monasca_analytics.dsl import const -from monasca_analytics.dsl import parser +from monasca_analytics.banana.cli import const +from monasca_analytics.banana.cli import parser class TestMonanasDSL(unittest.TestCase): diff --git a/test/ldp/test_monasca_aggregator.py b/test/ldp/test_monasca_aggregator.py index 5b9d07f..9f8b9bb 100644 --- a/test/ldp/test_monasca_aggregator.py +++ b/test/ldp/test_monasca_aggregator.py @@ -43,9 +43,7 @@ class TestMonascaAggregateLDP(MonanasTestCase): def _conf(self, fn_name): return { - "params": { - "aggregation_function": fn_name - } + "func": fn_name } def test_reducer_avg(self): diff --git a/test/ldp/test_monasca_combine.py b/test/ldp/test_monasca_combine.py index 8d5e700..8e74c94 100644 --- a/test/ldp/test_monasca_combine.py +++ b/test/ldp/test_monasca_combine.py @@ -14,8 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. -from monasca_analytics.banana.api import create_fn_with_config from monasca_analytics.ldp.monasca_combine import MonascaCombineLDP +from monasca_analytics.parsing.api import create_fn_with_config from test.util_for_testing import gen_metric from test.util_for_testing import MonanasTestCase diff --git a/test/mocks/sml_mocks.py b/test/mocks/sml_mocks.py index c4b2e4c..baf6401 100644 --- a/test/mocks/sml_mocks.py +++ b/test/mocks/sml_mocks.py @@ -82,7 +82,11 @@ class MockClass_src(base_src.BaseSource): @staticmethod def get_default_config(): - {"module": MockClass_src.__name__} + return {"module": MockClass_src.__name__} + + @staticmethod + def get_params(): + return [] def get_feature_list(self): self.get_feature_list_cnt += 1 @@ -130,7 +134,11 @@ class MockClass_ingestor_module(base_ing.BaseIngestor): @staticmethod def get_default_config(): - {"module": MockClass_ingestor_module.__name__} + return {"module": MockClass_ingestor_module.__name__} + + @staticmethod + def get_params(): + return [] def reset_vars(self): self.map_dstream_cnt = 0 @@ -152,7 +160,11 @@ class MockClass_aggr_module(aggregator.Aggregator): @staticmethod def get_default_config(): - {"module": MockClass_aggr_module.__name__} + return {"module": MockClass_aggr_module.__name__} + + @staticmethod + def get_params(): + return [] def reset_vars(self): self.accumulate_dstream_samples_cnt = 0 @@ -183,7 +195,11 @@ class MockClass_sml_module(base_sml.BaseSML): @staticmethod def get_default_config(): - {"module": MockClass_sml_module.__name__} + return {"module": MockClass_sml_module.__name__} + + @staticmethod + def get_params(): + return [] def reset_vars(self): self._voter = None @@ -209,7 +225,11 @@ class MockClass_voter_module(base_voter.BaseVoter): @staticmethod def get_default_config(): - {"module": MockClass_voter_module.__name__} + return {"module": MockClass_voter_module.__name__} + + @staticmethod + def get_params(): + return [] def reset_vars(self): self.elect_structure_cnt = 0 @@ -234,7 +254,11 @@ class MockClass_sink(base_snk.BaseSink): @staticmethod def get_default_config(): - {"module": MockClass_sink.__name__} + return {"module": MockClass_sink.__name__} + + @staticmethod + def get_params(): + return [] def sink_dstream(self, _): self.sink_dstream_cnt += 1 @@ -270,7 +294,11 @@ class MockClass_ldp_module1(base_ldp.BaseLDP): @staticmethod def get_default_config(): - {"module": MockClass_ldp_module1.__name__} + return {"module": MockClass_ldp_module1.__name__} + + @staticmethod + def get_params(): + return [] def map_dstream(self, dstream): self.map_dstream_cnt += 1 diff --git a/test/mocks/sources.py b/test/mocks/sources.py index 347d405..1cc7103 100644 --- a/test/mocks/sources.py +++ b/test/mocks/sources.py @@ -35,7 +35,11 @@ class MockBaseSource(base.BaseSource): @staticmethod def get_default_config(): - {"module": MockBaseSource.__name__} + return {"module": MockBaseSource.__name__} + + @staticmethod + def get_params(): + return [] def create_dstream(self, ssc): self.before_binding_cnt += 1 diff --git a/test/resources/test_json.json b/test/resources/test_json.json index 88c1b69..af7fbff 100644 --- a/test/resources/test_json.json +++ b/test/resources/test_json.json @@ -23,9 +23,7 @@ }, "src3": { "module": "IPTablesSource", - "params": { - "server_sleep_in_seconds": 0.01 - } + "sleep": 0.01 } }, "ingestors": { diff --git a/test/sink/test_base_sqlite.py b/test/sink/test_base_sqlite.py index 8e26228..434c74a 100644 --- a/test/sink/test_base_sqlite.py +++ b/test/sink/test_base_sqlite.py @@ -22,6 +22,8 @@ import cPickle import numpy as np import voluptuous +import monasca_analytics.banana.typeck.type_util as type_util +import monasca_analytics.component.params as params from monasca_analytics.sink import base_sqlite as bsql from test.mocks import spark_mocks @@ -45,6 +47,16 @@ class BaseSQLiteSinkDummyExtension(bsql.BaseSQLiteSink): } } + @staticmethod + def get_params(): + return [ + params.ParamDescriptor( + "db_name", + type_util.String(), + "sqlite_sink.db" + ) + ] + @staticmethod def validate_config(_config): base_schema = voluptuous.Schema({ diff --git a/test/sink/test_kafka.py b/test/sink/test_kafka.py index 17d6ff0..fec0d69 100644 --- a/test/sink/test_kafka.py +++ b/test/sink/test_kafka.py @@ -39,11 +39,9 @@ class KafkaSinkTest(unittest.TestCase): def testKafkaSinkInit(self): kf.KafkaSink("id", { "module": "KafkaSink", - "params": { - "host": "localhost", - "port": 00, - "topic": "boom", - } + "host": "localhost", + "port": 00, + "topic": "boom", }) self.assertTrue(self.mock_kafka) diff --git a/test/sink/test_sink_config_validator.py b/test/sink/test_sink_config_validator.py index 8d93bc8..514b82b 100644 --- a/test/sink/test_sink_config_validator.py +++ b/test/sink/test_sink_config_validator.py @@ -27,11 +27,9 @@ class SinkConfigValidatorTest(unittest.TestCase): unittest.TestCase.setUp(self) self._valid_config = { "module": "KafkaSink", - "params": { - "host": "127.0.0.1", - "port": 9092, - "topic": "transformed_data" - } + "host": "127.0.0.1", + "port": 9092, + "topic": "transformed_data" } def test_validate_kafka_sink_valid_config(self): @@ -47,17 +45,17 @@ class SinkConfigValidatorTest(unittest.TestCase): def test_validate_kafka_sink_invalid_host(self): invalid_config = self._valid_config - invalid_config["params"]["host"] = "invalid host" + invalid_config["host"] = "invalid host" self.assertRaises(voluptuous.Invalid, kafka, invalid_config) def test_validate_kafka_sink_invalid_port(self): invalid_config = self._valid_config - invalid_config["params"]["port"] = "invalid_port" + invalid_config["port"] = "invalid_port" self.assertRaises(voluptuous.Invalid, kafka, invalid_config) def test_validate_kafka_sink_invalid_topic(self): invalid_config = self._valid_config - invalid_config["params"]["topic"] = "invalid topic" + invalid_config["topic"] = "invalid topic" self.assertRaises(voluptuous.Invalid, kafka, invalid_config) def tearDown(self): diff --git a/test/source/test_iptables_markov_chain.py b/test/source/test_iptables_markov_chain.py index a383fbd..a981450 100644 --- a/test/source/test_iptables_markov_chain.py +++ b/test/source/test_iptables_markov_chain.py @@ -26,26 +26,19 @@ class TestIPTablesSource(MonanasTestCase): super(TestIPTablesSource, self).setUp() self.valid_config = { "module": "IPTablesSource", - "params": { - "server_sleep_in_seconds": 0.01 - } + "sleep": 0.01 } self.config_extra_param = { "module": "IPTablesSource", - "params": { - "server_sleep_in_seconds": 0.01, - "infiltrated": "wrong_param" - } + "sleep": 0.01, + "infiltrated": "wrong_param" } self.config_missing_param = { "module": "IPTablesSource", - "params": {} } self.config_wrong_type = { "module": "IPTablesSource", - "params": { - "server_sleep_in_seconds": "I should be an integer" - } + "sleep": "I should be an integer" } self.config_missing_params = {"module": "IPTablesSource"} self.ips = iptables_markov_chain.IPTablesSource("fake_id", diff --git a/test/source/test_markov_chain.py b/test/source/test_markov_chain.py index fdd4d13..d877884 100644 --- a/test/source/test_markov_chain.py +++ b/test/source/test_markov_chain.py @@ -25,9 +25,6 @@ class MarkovChainSourceTest(MonanasTestCase): def setUp(self): super(MarkovChainSourceTest, self).setUp() - params = { - "server_sleep_in_seconds": 0.1 - } transitions = { "web_service": { "run=>slow": { @@ -72,7 +69,7 @@ class MarkovChainSourceTest(MonanasTestCase): } } }, - "params": dict(params), + "sleep": 0.1, "graph": { "h1:host": ["s1"], "h2:host": ["s1"], @@ -82,15 +79,14 @@ class MarkovChainSourceTest(MonanasTestCase): } } self.config_extra_param = dict(self.valid_config) - self.config_extra_param["params"] = dict(params) - self.config_extra_param["params"]["extra_param"] = "john doe" + self.config_extra_param["extra_param"] = "john doe" self.config_missing_param = dict(self.valid_config) self.config_missing_param["transitions"] = dict(transitions) self.config_missing_param["transitions"].pop("host") self.config_wrong_type = { "module": 123, "transitions": dict(self.valid_config["transitions"]), - "params": dict(params), + "sleep": 0.1, "graph": {} } self.mcs = cloud.CloudMarkovChainSource("fake_id", self.valid_config) diff --git a/test/spark/test_driver.py b/test/spark/test_driver.py index 7c43f9a..6bf5a89 100644 --- a/test/spark/test_driver.py +++ b/test/spark/test_driver.py @@ -189,8 +189,6 @@ class DriverExecutorTest(MonanasTestCase): def assert_stopped_streaming_state(self, ssc=None): if ssc: self.assertEqual(1, ssc.stopped_cnt) - self.assertEqual(None, self.mlf._sc) - self.assertEqual(None, self.mlf._ssc) def test_stop_pipeline(self): self.mlf.start_pipeline() diff --git a/test/voter/test_base_voter.py b/test/voter/test_base_voter.py index 664f803..8b20707 100644 --- a/test/voter/test_base_voter.py +++ b/test/voter/test_base_voter.py @@ -46,4 +46,8 @@ class VoterBasicChild(base.BaseVoter): @staticmethod def get_default_config(): - {"module": VoterBasicChild.__name__} + return {"module": VoterBasicChild.__name__} + + @staticmethod + def get_params(): + return []