From a2a315fb5e4b532fa64a37b5def69d402b32a02f Mon Sep 17 00:00:00 2001 From: Joan Varvenne Date: Mon, 19 Sep 2016 18:10:32 +0100 Subject: [PATCH] Improve speed in data generation while preserving semantics. This make the fake data generation much faster and configurable. For the cloud example, the phase 2 is now triggered in less than 5 seconds instead of 6 minutes. Change-Id: I9967c94d28a380fa19afe9275e769f1465f4f922 --- config/markov_source_config.banana | 2 +- config/markov_source_config.json | 1 + .../source/cloud_markov_chain.py | 4 +++ monasca_analytics/source/markov_chain/base.py | 25 +++++++++++++------ .../nested_value_in_component_params.banana | 2 +- test/source/test_markov_chain.py | 2 ++ 6 files changed, 27 insertions(+), 9 deletions(-) diff --git a/config/markov_source_config.banana b/config/markov_source_config.banana index a58f05f..2c1ec98 100644 --- a/config/markov_source_config.banana +++ b/config/markov_source_config.banana @@ -3,7 +3,7 @@ # (cloud-like data model) # -src = CloudMarkovChainSource(sleep=0.01) +src = CloudMarkovChainSource(sleep=0.01, min_event_per_burst=500) src.transitions.web_service = { "run=>slow": { diff --git a/config/markov_source_config.json b/config/markov_source_config.json index 8cdb1b1..8e64fa3 100644 --- a/config/markov_source_config.json +++ b/config/markov_source_config.json @@ -13,6 +13,7 @@ "src1": { "module": "CloudMarkovChainSource", "sleep": 0.01, + "min_event_per_burst": 500, "transitions": { "web_service": { "run=>slow": { diff --git a/monasca_analytics/source/cloud_markov_chain.py b/monasca_analytics/source/cloud_markov_chain.py index a5603d1..1f0cb4f 100644 --- a/monasca_analytics/source/cloud_markov_chain.py +++ b/monasca_analytics/source/cloud_markov_chain.py @@ -38,6 +38,7 @@ class CloudMarkovChainSource(base.MarkovChainSource): source_schema = voluptuous.Schema({ "module": voluptuous.And( basestring, vu.NoSpaceCharacter()), + "min_event_per_burst": voluptuous.Or(float, int), "sleep": voluptuous.And( float, voluptuous.Range( min=0, max=1, min_included=False, max_included=False)), @@ -90,6 +91,7 @@ class CloudMarkovChainSource(base.MarkovChainSource): return { "module": CloudMarkovChainSource.__name__, "sleep": 0.01, + "min_event_per_burst": 500, "transitions": { "web_service": { "run=>slow": { @@ -144,6 +146,8 @@ class CloudMarkovChainSource(base.MarkovChainSource): def get_params(): return [ params.ParamDescriptor('sleep', type_util.Number(), 0.01), + params.ParamDescriptor('min_event_per_burst', type_util.Number(), + 500), params.ParamDescriptor('transitions', type_util.Object({ 'web_service': type_util.Object({ 'run=>slow': type_util.Any(), diff --git a/monasca_analytics/source/markov_chain/base.py b/monasca_analytics/source/markov_chain/base.py index 74fe09f..18492fa 100644 --- a/monasca_analytics/source/markov_chain/base.py +++ b/monasca_analytics/source/markov_chain/base.py @@ -84,6 +84,11 @@ class MarkovChainSource(base.BaseSource): self._server.terminate = False self._server.system = system self._server.sleep_in_seconds = self._config["sleep"] + if "min_event_per_burst" in self._config: + self._server.min_event_per_burst = \ + int(self._config["min_event_per_burst"]) + else: + self._server.min_event_per_burst = 500 self._server_thread = threading.Thread(target=self._serve_forever) self._server_thread.start() @@ -233,9 +238,17 @@ class FMSTCPHandler(SocketServer.BaseRequestHandler): fake_date = datetime.datetime.today() hour_of_day = fake_date.hour while not self.server.terminate: - self.server.system.next_state(hour_of_day) request = RequestBuilder(self.request) - self.server.system.collect_events(hour_of_day, fake_date, request) + + # Collect some events + while request.nb_events() < self.server.min_event_per_burst: + self.server.system.next_state(hour_of_day) + self.server.system.collect_events(hour_of_day, fake_date, + request) + hour_of_day += 1 + fake_date += datetime.timedelta(hours=1) + if hour_of_day > 24: + hour_of_day = 0 try: request.finalize() @@ -245,11 +258,6 @@ class FMSTCPHandler(SocketServer.BaseRequestHandler): time.sleep(self.server.sleep_in_seconds) - hour_of_day += 1 - fake_date += datetime.timedelta(hours=1) - if hour_of_day > 24: - hour_of_day = 0 - class RequestBuilder(object): @@ -264,6 +272,9 @@ class RequestBuilder(object): """ self._collected_data.append(data) + def nb_events(self): + return len(self._collected_data) + def finalize(self): for data in self._collected_data: self._request.send("{0}\n".format(json.dumps(data, diff --git a/test/banana/typeck/should_pass/nested_value_in_component_params.banana b/test/banana/typeck/should_pass/nested_value_in_component_params.banana index 29f2d04..48331bc 100644 --- a/test/banana/typeck/should_pass/nested_value_in_component_params.banana +++ b/test/banana/typeck/should_pass/nested_value_in_component_params.banana @@ -6,4 +6,4 @@ b = a.transitions.web_service a = CloudMarkovChainSource() b = a."transitions"."web_service" -# TYPE_TABLE_EQ {Ident< a >: CloudMarkovChainSource(sleep=TypeNumber,transitions=TypeStruct < {host: TypeStruct < {off=>on: TypeNumber, on=>off: TypeNumber} >, switch: TypeStruct < {off=>on: TypeNumber, on=>off: TypeNumber} >, web_service: TypeStruct < {run=>slow: TypeAny, slow=>run: TypeAny, stop=>run: TypeAny} >} >,triggers=TypeStruct < {support: TypeStruct < {get_called: TypeAny} >} >,graph=TypeAny), Ident< b >: TypeStruct < {run=>slow: TypeAny, slow=>run: TypeAny, stop=>run: TypeAny} >} \ No newline at end of file +# TYPE_TABLE_EQ {Ident< a >: CloudMarkovChainSource(sleep=TypeNumber,min_event_per_burst=TypeNumber,transitions=TypeStruct < {host: TypeStruct < {off=>on: TypeNumber, on=>off: TypeNumber} >, switch: TypeStruct < {off=>on: TypeNumber, on=>off: TypeNumber} >, web_service: TypeStruct < {run=>slow: TypeAny, slow=>run: TypeAny, stop=>run: TypeAny} >} >,triggers=TypeStruct < {support: TypeStruct < {get_called: TypeAny} >} >,graph=TypeAny), Ident< b >: TypeStruct < {run=>slow: TypeAny, slow=>run: TypeAny, stop=>run: TypeAny} >} \ No newline at end of file diff --git a/test/source/test_markov_chain.py b/test/source/test_markov_chain.py index d877884..ac6a8d2 100644 --- a/test/source/test_markov_chain.py +++ b/test/source/test_markov_chain.py @@ -70,6 +70,7 @@ class MarkovChainSourceTest(MonanasTestCase): } }, "sleep": 0.1, + "min_event_per_burst": 500, "graph": { "h1:host": ["s1"], "h2:host": ["s1"], @@ -87,6 +88,7 @@ class MarkovChainSourceTest(MonanasTestCase): "module": 123, "transitions": dict(self.valid_config["transitions"]), "sleep": 0.1, + "min_event_per_burst": 500, "graph": {} } self.mcs = cloud.CloudMarkovChainSource("fake_id", self.valid_config)