Add Banana specific APIs to typecheck and get list of components.
This commit add two APIs: * POST /banana/typeck { "content": "<banana>" } This API type check the provided banana content but does not evaluate it. * POST /banana/metadata { "content": "<banana>" } This API returns the associated TypeTable of the provided banana content. This is useful to provide auto-completion in an editor. * GET /banana/metadata This API returns the list of components availables. This is also useful to provide auto-completion on existing components in an editor for banana. Change-Id: I31c1de86fe420458ac98aad9d9c8ae6ca73fcc81
This commit is contained in:
parent
ecfc9d6b95
commit
716a72d9b9
|
@ -47,7 +47,7 @@ SPARK_DIR="/opt/spark"
|
||||||
SPARK_DOWNLOAD="$SPARK_DIR/download"
|
SPARK_DOWNLOAD="$SPARK_DIR/download"
|
||||||
SPARK_VERSION=${SPARK_VERSION:-1.6.1}
|
SPARK_VERSION=${SPARK_VERSION:-1.6.1}
|
||||||
SPARK_TARBALL_NAME="spark-${SPARK_VERSION}.tgz"
|
SPARK_TARBALL_NAME="spark-${SPARK_VERSION}.tgz"
|
||||||
SPARK_URL="http://apache.claz.org/spark/spark-$SPARK_VERSION/$SPARK_TARBALL_NAME"
|
SPARK_URL="http://archive.apache.org/dist/spark/spark-$SPARK_VERSION/$SPARK_TARBALL_NAME"
|
||||||
|
|
||||||
BASE_KAFKA_VERSION=${BASE_KAFKA_VERSION:-0.9.0.0}
|
BASE_KAFKA_VERSION=${BASE_KAFKA_VERSION:-0.9.0.0}
|
||||||
KAFKA_DIR="/opt/kafka"
|
KAFKA_DIR="/opt/kafka"
|
||||||
|
|
|
@ -129,11 +129,11 @@ class Span(object):
|
||||||
DUMMY_SPAN = Span(None, 0, 0)
|
DUMMY_SPAN = Span(None, 0, 0)
|
||||||
|
|
||||||
|
|
||||||
def from_parse_fatal(parse_fatal_exception):
|
def from_pyparsing_exception(parse_fatal_exception):
|
||||||
"""
|
"""
|
||||||
Convert the provided ParseFatalException into a Span.
|
Convert the provided ParseFatalException into a Span.
|
||||||
|
|
||||||
:type parse_fatal_exception: pyparsing.ParseFatalException
|
:type parse_fatal_exception: pyparsing.ParseBaseException
|
||||||
:param parse_fatal_exception: Exception to convert.
|
:param parse_fatal_exception: Exception to convert.
|
||||||
:rtype: Span
|
:rtype: Span
|
||||||
:return: Returns the span mapping to that fatal exception.
|
:return: Returns the span mapping to that fatal exception.
|
||||||
|
|
|
@ -26,13 +26,13 @@ import monasca_analytics.banana.typeck.config as typeck
|
||||||
import monasca_analytics.exception.banana as exception
|
import monasca_analytics.exception.banana as exception
|
||||||
|
|
||||||
|
|
||||||
def execute_banana_string(banana_str, driver, emitter=emit.PrintEmitter()):
|
def execute_banana_string(banana, driver=None, emitter=emit.PrintEmitter()):
|
||||||
"""
|
"""
|
||||||
Execute the provided banana string.
|
Execute the provided banana string.
|
||||||
It will run the parse phase, and the typechecker.
|
It will run the parse phase, and the typechecker.
|
||||||
:type banana_str: str
|
:type banana: str
|
||||||
:param banana_str: The string to parse and type check.
|
:param banana: The string to parse and type check.
|
||||||
:type driver: monasca_analytics.spark.driver.DriverExecutor
|
:type driver: monasca_analytics.spark.driver.DriverExecutor | None
|
||||||
:param driver: Driver that will manage the created
|
:param driver: Driver that will manage the created
|
||||||
components and connect them together.
|
components and connect them together.
|
||||||
:type emitter: emit.Emitter
|
:type emitter: emit.Emitter
|
||||||
|
@ -41,7 +41,7 @@ def execute_banana_string(banana_str, driver, emitter=emit.PrintEmitter()):
|
||||||
try:
|
try:
|
||||||
# Convert the grammar into an AST
|
# Convert the grammar into an AST
|
||||||
parser = grammar.banana_grammar(emitter)
|
parser = grammar.banana_grammar(emitter)
|
||||||
ast = parser.parse(banana_str)
|
ast = parser.parse(banana)
|
||||||
# Compute the type table for the given AST
|
# Compute the type table for the given AST
|
||||||
type_table = typeck.typeck(ast)
|
type_table = typeck.typeck(ast)
|
||||||
# Remove from the tree path that are "dead"
|
# Remove from the tree path that are "dead"
|
||||||
|
@ -49,30 +49,56 @@ def execute_banana_string(banana_str, driver, emitter=emit.PrintEmitter()):
|
||||||
# Check that there's at least one path to be executed
|
# Check that there's at least one path to be executed
|
||||||
deadpathck.contains_at_least_one_path_to_a_sink(ast, type_table)
|
deadpathck.contains_at_least_one_path_to_a_sink(ast, type_table)
|
||||||
# Evaluate the script
|
# Evaluate the script
|
||||||
|
if driver is not None:
|
||||||
ev.eval_ast(ast, type_table, driver)
|
ev.eval_ast(ast, type_table, driver)
|
||||||
except exception.BananaException as err:
|
except exception.BananaException as err:
|
||||||
emitter.emit_error(err.get_span(), str(err))
|
emitter.emit_error(err.get_span(), str(err))
|
||||||
except p.ParseSyntaxException as err:
|
except p.ParseSyntaxException as err:
|
||||||
emitter.emit_error(span_util.from_parse_fatal(err), err.msg)
|
emitter.emit_error(span_util.from_pyparsing_exception(err), err.msg)
|
||||||
except p.ParseFatalException as err:
|
except p.ParseFatalException as err:
|
||||||
emitter.emit_error(span_util.from_parse_fatal(err), err.msg)
|
emitter.emit_error(span_util.from_pyparsing_exception(err), err.msg)
|
||||||
|
except p.ParseException as err:
|
||||||
|
emitter.emit_error(span_util.from_pyparsing_exception(err), err.msg)
|
||||||
|
|
||||||
|
|
||||||
def compute_type_table(banana_str):
|
def try_compute_type_table(banana):
|
||||||
|
"""
|
||||||
|
Compute the type table for the provided banana string
|
||||||
|
if possible. Does not throw any exception if it fails.
|
||||||
|
:type banana: str
|
||||||
|
:param banana: The string to parse and type check.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Convert the grammar into an AST
|
||||||
|
parser = grammar.banana_grammar()
|
||||||
|
ast = parser.parse(banana)
|
||||||
|
# Compute the type table for the given AST
|
||||||
|
return typeck.typeck(ast)
|
||||||
|
except exception.BananaException:
|
||||||
|
return None
|
||||||
|
except p.ParseSyntaxException:
|
||||||
|
return None
|
||||||
|
except p.ParseFatalException:
|
||||||
|
return None
|
||||||
|
except p.ParseException:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def compute_type_table(banana):
|
||||||
"""
|
"""
|
||||||
Compute the type table for the provided banana string
|
Compute the type table for the provided banana string
|
||||||
if possible.
|
if possible.
|
||||||
:type banana_str: str
|
:type banana: str
|
||||||
:param banana_str: The string to parse and type check.
|
:param banana: The string to parse and type check.
|
||||||
"""
|
"""
|
||||||
# Convert the grammar into an AST
|
# Convert the grammar into an AST
|
||||||
parser = grammar.banana_grammar()
|
parser = grammar.banana_grammar()
|
||||||
ast = parser.parse(banana_str)
|
ast = parser.parse(banana)
|
||||||
# Compute the type table for the given AST
|
# Compute the type table for the given AST
|
||||||
return typeck.typeck(ast)
|
return typeck.typeck(ast)
|
||||||
|
|
||||||
|
|
||||||
def compute_evaluation_context(banana_str, cb=lambda *a, **k: None):
|
def compute_evaluation_context(banana, cb=lambda *a, **k: None):
|
||||||
"""
|
"""
|
||||||
Compute the evaluation context for the provided
|
Compute the evaluation context for the provided
|
||||||
banana string.
|
banana string.
|
||||||
|
@ -81,7 +107,7 @@ def compute_evaluation_context(banana_str, cb=lambda *a, **k: None):
|
||||||
:param cb: Callback called after each statement
|
:param cb: Callback called after each statement
|
||||||
"""
|
"""
|
||||||
parser = grammar.banana_grammar()
|
parser = grammar.banana_grammar()
|
||||||
ast = parser.parse(banana_str)
|
ast = parser.parse(banana)
|
||||||
type_table = typeck.typeck(ast)
|
type_table = typeck.typeck(ast)
|
||||||
context = ctx.EvaluationContext()
|
context = ctx.EvaluationContext()
|
||||||
|
|
||||||
|
|
|
@ -164,6 +164,19 @@ class TypeTable(object):
|
||||||
))
|
))
|
||||||
self._variables = new_snapshot
|
self._variables = new_snapshot
|
||||||
|
|
||||||
|
def to_json(self):
|
||||||
|
"""
|
||||||
|
Convert this type table into a dictionary.
|
||||||
|
Useful to serialize the type table.
|
||||||
|
|
||||||
|
:rtype: dict
|
||||||
|
:return: Returns this type table as a dict.
|
||||||
|
"""
|
||||||
|
res = {}
|
||||||
|
for key, val in self._variables.iteritems():
|
||||||
|
res[key.inner_val()] = val.to_json()
|
||||||
|
return res
|
||||||
|
|
||||||
def __contains__(self, key):
|
def __contains__(self, key):
|
||||||
"""
|
"""
|
||||||
Test if the type table contains or not the provided
|
Test if the type table contains or not the provided
|
||||||
|
|
|
@ -59,6 +59,10 @@ class IsType(object):
|
||||||
def default_value(self):
|
def default_value(self):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def to_json(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
class Any(IsType):
|
class Any(IsType):
|
||||||
"""
|
"""
|
||||||
|
@ -84,6 +88,9 @@ class Any(IsType):
|
||||||
def default_value(self):
|
def default_value(self):
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
def to_json(self):
|
||||||
|
return {"id": "any"}
|
||||||
|
|
||||||
|
|
||||||
class String(IsType):
|
class String(IsType):
|
||||||
"""
|
"""
|
||||||
|
@ -102,6 +109,9 @@ class String(IsType):
|
||||||
def default_value(self):
|
def default_value(self):
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
def to_json(self):
|
||||||
|
return {"id": "string"}
|
||||||
|
|
||||||
|
|
||||||
class Number(String):
|
class Number(String):
|
||||||
"""
|
"""
|
||||||
|
@ -120,6 +130,9 @@ class Number(String):
|
||||||
def default_value(self):
|
def default_value(self):
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
def to_json(self):
|
||||||
|
return {"id": "number"}
|
||||||
|
|
||||||
|
|
||||||
class Enum(String):
|
class Enum(String):
|
||||||
"""
|
"""
|
||||||
|
@ -142,6 +155,12 @@ class Enum(String):
|
||||||
def default_value(self):
|
def default_value(self):
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
def to_json(self):
|
||||||
|
return {
|
||||||
|
"id": "enum",
|
||||||
|
"variants": self.variants
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def attach_to_root(root_obj, obj1, span, erase_existing=False):
|
def attach_to_root(root_obj, obj1, span, erase_existing=False):
|
||||||
"""
|
"""
|
||||||
|
@ -281,6 +300,12 @@ class Object(String):
|
||||||
default_value[key] = val.default_value()
|
default_value[key] = val.default_value()
|
||||||
return default_value
|
return default_value
|
||||||
|
|
||||||
|
def to_json(self):
|
||||||
|
res = {"id": "object", "props": {}}
|
||||||
|
for key, val in self.props.iteritems():
|
||||||
|
res["props"][key] = val.to_json()
|
||||||
|
return res
|
||||||
|
|
||||||
|
|
||||||
class Component(IsType):
|
class Component(IsType):
|
||||||
"""
|
"""
|
||||||
|
@ -397,7 +422,13 @@ class Component(IsType):
|
||||||
return hash(str(self))
|
return hash(str(self))
|
||||||
|
|
||||||
def default_value(self):
|
def default_value(self):
|
||||||
return {}
|
return None
|
||||||
|
|
||||||
|
def to_json(self):
|
||||||
|
res = {"id": "component", "name": self.class_name, "args": []}
|
||||||
|
for arg in self.ctor_properties:
|
||||||
|
res["args"].append(arg.to_json())
|
||||||
|
return res
|
||||||
|
|
||||||
|
|
||||||
class Source(Component):
|
class Source(Component):
|
||||||
|
|
|
@ -45,3 +45,10 @@ class ParamDescriptor(object):
|
||||||
self.default_value = default
|
self.default_value = default
|
||||||
self.param_type = _type
|
self.param_type = _type
|
||||||
self.validator = validator
|
self.validator = validator
|
||||||
|
|
||||||
|
def to_json(self):
|
||||||
|
return {
|
||||||
|
"name": self.param_name,
|
||||||
|
"default_value": self.default_value,
|
||||||
|
"type": self.param_type.to_json(),
|
||||||
|
}
|
||||||
|
|
|
@ -76,6 +76,29 @@ class Monanas(object):
|
||||||
# Try to change the configuration.
|
# Try to change the configuration.
|
||||||
executor.execute_banana_string(banana_str, self._driver, emitter)
|
executor.execute_banana_string(banana_str, self._driver, emitter)
|
||||||
|
|
||||||
|
def typeck_configuration(self, banana_str, emitter):
|
||||||
|
"""Only type check the provided configuration.
|
||||||
|
|
||||||
|
:type banana_str: str
|
||||||
|
:param banana_str: New configuration.
|
||||||
|
:type emitter: emit.JsonEmitter
|
||||||
|
:param emitter: a Json emitter instance
|
||||||
|
"""
|
||||||
|
executor.execute_banana_string(banana_str, None, emitter)
|
||||||
|
|
||||||
|
def compute_type_table(self, banana_str):
|
||||||
|
"""Compute the type table for the provided configuration.
|
||||||
|
|
||||||
|
:type banana_str: str
|
||||||
|
:param banana_str: Configuration to test.
|
||||||
|
:rtype: dict
|
||||||
|
:return: Returns the type table
|
||||||
|
"""
|
||||||
|
type_table = executor.try_compute_type_table(banana_str)
|
||||||
|
if type_table is not None:
|
||||||
|
return type_table.to_json()
|
||||||
|
return {}
|
||||||
|
|
||||||
def start_streaming(self):
|
def start_streaming(self):
|
||||||
"""Starts streaming data.
|
"""Starts streaming data.
|
||||||
|
|
||||||
|
|
|
@ -24,6 +24,7 @@ import voluptuous
|
||||||
|
|
||||||
import monasca_analytics.banana.emitter as emit
|
import monasca_analytics.banana.emitter as emit
|
||||||
import monasca_analytics.exception.monanas as err
|
import monasca_analytics.exception.monanas as err
|
||||||
|
import monasca_analytics.util.common_util as introspect
|
||||||
from monasca_analytics.web_service import web_service_model
|
from monasca_analytics.web_service import web_service_model
|
||||||
|
|
||||||
|
|
||||||
|
@ -77,12 +78,13 @@ class BananaHandler(web.RequestHandler):
|
||||||
the banana configuration language.
|
the banana configuration language.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def initialize(self, monanas):
|
def initialize(self, monanas, typeck_only):
|
||||||
"""Initialize the handler.
|
"""Initialize the handler.
|
||||||
|
|
||||||
:param monanas: A Monana's instance.
|
:param monanas: A Monana's instance.
|
||||||
"""
|
"""
|
||||||
self._monanas = monanas
|
self._monanas = monanas
|
||||||
|
self._typeck_only = typeck_only
|
||||||
|
|
||||||
@web.asynchronous
|
@web.asynchronous
|
||||||
def post(self):
|
def post(self):
|
||||||
|
@ -93,18 +95,15 @@ class BananaHandler(web.RequestHandler):
|
||||||
body = json.loads(self.request.body)
|
body = json.loads(self.request.body)
|
||||||
web_service_model.banana_model(body)
|
web_service_model.banana_model(body)
|
||||||
emitter = emit.JsonEmitter()
|
emitter = emit.JsonEmitter()
|
||||||
# TODO(Joan): Change that
|
if self._typeck_only:
|
||||||
self._monanas.try_change_configuration(body["content"], emitter)
|
self._monanas.typeck_configuration(body["content"],
|
||||||
|
emitter)
|
||||||
|
else:
|
||||||
|
self._monanas.try_change_configuration(body["content"],
|
||||||
|
emitter)
|
||||||
self.write(emitter.result)
|
self.write(emitter.result)
|
||||||
except (AttributeError, voluptuous.Invalid, ValueError):
|
except (AttributeError, voluptuous.Invalid, ValueError) as e:
|
||||||
self.set_status(400, "The request body was malformed.")
|
self.set_status(400, "The request body was malformed.")
|
||||||
except (err.MonanasBindSourcesError,
|
|
||||||
err.MonanasAlreadyStartedStreaming,
|
|
||||||
err.MonanasAlreadyStoppedStreaming) as e:
|
|
||||||
self.set_status(400, e.__str__())
|
|
||||||
except err.MonanasStreamingError as e:
|
|
||||||
self.set_status(500, e.__str__())
|
|
||||||
terminate = (True, e.__str__())
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
tb = traceback.format_exc()
|
tb = traceback.format_exc()
|
||||||
print(tb)
|
print(tb)
|
||||||
|
@ -118,3 +117,51 @@ class BananaHandler(web.RequestHandler):
|
||||||
if terminate[0]:
|
if terminate[0]:
|
||||||
logger.error(terminate[1])
|
logger.error(terminate[1])
|
||||||
self._monanas.stop_streaming_and_terminate()
|
self._monanas.stop_streaming_and_terminate()
|
||||||
|
|
||||||
|
|
||||||
|
class BananaMetaDataHandler(web.RequestHandler):
|
||||||
|
|
||||||
|
def initialize(self, monanas):
|
||||||
|
"""Initializes the handler.
|
||||||
|
|
||||||
|
:param monanas: Monanas -- A Monanas's instance.
|
||||||
|
"""
|
||||||
|
self._monanas = monanas
|
||||||
|
|
||||||
|
@web.asynchronous
|
||||||
|
def get(self):
|
||||||
|
all_components = introspect.get_available_classes()
|
||||||
|
result = {"components": []}
|
||||||
|
for kind, components in all_components.iteritems():
|
||||||
|
for component in components:
|
||||||
|
result["components"].append({
|
||||||
|
"name": component.__name__,
|
||||||
|
"description": component.__doc__,
|
||||||
|
"params": map(lambda x: x.to_json(),
|
||||||
|
component.get_params()),
|
||||||
|
})
|
||||||
|
self.write(result)
|
||||||
|
self.flush()
|
||||||
|
self.finish()
|
||||||
|
|
||||||
|
@web.asynchronous
|
||||||
|
def post(self):
|
||||||
|
|
||||||
|
try:
|
||||||
|
body = json.loads(self.request.body)
|
||||||
|
web_service_model.banana_model(body)
|
||||||
|
type_table = self._monanas.compute_type_table(body["content"])
|
||||||
|
self.write(type_table)
|
||||||
|
except (AttributeError, voluptuous.Invalid, ValueError) as e:
|
||||||
|
logger.warn("Wrong request: {}.".
|
||||||
|
format(e))
|
||||||
|
self.set_status(400, "The request body was malformed.")
|
||||||
|
except Exception as e:
|
||||||
|
tb = traceback.format_exc()
|
||||||
|
print(tb)
|
||||||
|
logger.error("Unexpected error: {}. {}".
|
||||||
|
format(sys.exc_info()[0], e))
|
||||||
|
self.set_status(500, "Internal server error.")
|
||||||
|
|
||||||
|
self.flush()
|
||||||
|
self.finish()
|
||||||
|
|
|
@ -29,7 +29,17 @@ class WebService(web.Application):
|
||||||
params = {"monanas": self._monanas}
|
params = {"monanas": self._monanas}
|
||||||
handlers = [
|
handlers = [
|
||||||
(r"/", request_handler.MonanasHandler, params),
|
(r"/", request_handler.MonanasHandler, params),
|
||||||
(r"/banana", request_handler.BananaHandler, params),
|
(r"/banana", request_handler.BananaHandler, {
|
||||||
|
"monanas": self._monanas,
|
||||||
|
"typeck_only": False
|
||||||
|
}),
|
||||||
|
(r"/banana/typeck", request_handler.BananaHandler, {
|
||||||
|
"monanas": self._monanas,
|
||||||
|
"typeck_only": True
|
||||||
|
}),
|
||||||
|
(r"/banana/metadata", request_handler.BananaMetaDataHandler, {
|
||||||
|
"monanas": self._monanas,
|
||||||
|
})
|
||||||
]
|
]
|
||||||
|
|
||||||
settings = {}
|
settings = {}
|
||||||
|
|
Loading…
Reference in New Issue