From 3705aa1526e5969e742df3b1286e78241da7a435 Mon Sep 17 00:00:00 2001 From: Alexander Kislitsky Date: Wed, 21 Sep 2016 17:17:33 +0300 Subject: [PATCH] Fetching effective resource values optimized Now only required resource values are fetched on calculating effective values. Calculation of effective values optimised to use indexed resource values insead of going throw nested for loop. Test for fetching values and effective values for 500 items (nodes) added. Each resource value contains over than 10000 keys. Logging added to the resource values fetching. Change-Id: Icea7ece8a2bc6957bcb00c9d90d8cf59e5ca1da0 Closes-Bug: #1626147 --- tuning_box/app.py | 4 +- tuning_box/library/hierarchy_levels.py | 6 ++ tuning_box/library/resource_values.py | 97 +++++++++++++------ tuning_box/logger.py | 7 +- .../tests/library/test_resource_overrides.py | 4 +- .../tests/library/test_resource_values.py | 71 ++++++++++++++ 6 files changed, 150 insertions(+), 39 deletions(-) diff --git a/tuning_box/app.py b/tuning_box/app.py index 61dac38..1acd654 100644 --- a/tuning_box/app.py +++ b/tuning_box/app.py @@ -134,8 +134,8 @@ def build_app(configure_logging=True, with_keystone=True): handle_keys_operation_error) db.db.init_app(app) if configure_logging: - log_level = app.config.get('LOG_LEVEL', 'INFO') - logger.init_logger(log_level) + log_level = app.config.get('LOG_LEVEL', 'DEBUG') + logger.init_logger(app, log_level) if with_keystone: app.wsgi_app = keystone.KeystoneMiddleware(app) return app diff --git a/tuning_box/library/hierarchy_levels.py b/tuning_box/library/hierarchy_levels.py index dc85e3d..30e9ac8 100644 --- a/tuning_box/library/hierarchy_levels.py +++ b/tuning_box/library/hierarchy_levels.py @@ -11,6 +11,7 @@ # under the License. import flask +from flask import current_app as app import flask_restful from flask_restful import fields @@ -20,7 +21,12 @@ from tuning_box import errors def iter_environment_level_values(environment, levels): + app.logger.debug("Getting environment level values. Env: %s, " + "levels: %s", environment.id, levels) env_levels = db.EnvironmentHierarchyLevel.get_for_environment(environment) + app.logger.debug("Environment levels got. Env: %s, levels: %s", + environment.id, [l.name for l in env_levels]) + if len(env_levels) < len(levels): raise errors.TuningboxNotFound( "Levels {0} can't be matched with " diff --git a/tuning_box/library/resource_values.py b/tuning_box/library/resource_values.py index b93c807..2380e31 100644 --- a/tuning_box/library/resource_values.py +++ b/tuning_box/library/resource_values.py @@ -11,8 +11,10 @@ # under the License. import flask +from flask import current_app as app import flask_restful -import itertools +import six +from sqlalchemy import or_ from tuning_box import db from tuning_box import library @@ -39,8 +41,28 @@ class ResourceValues(flask_restful.Resource): esv.values = flask.request.json return None, 204 + def _calculate_effective_values(self, result, level_value, + resource_values_idx, show_lookup, + lookup_path): + level_value_id = getattr(level_value, 'id', None) + if level_value_id in resource_values_idx: + resource_value = resource_values_idx[level_value_id] + if show_lookup: + values = ((k, (v, lookup_path)) for k, v in + six.iteritems(resource_value.values)) + overrides = ((k, (v, lookup_path)) for k, v in + six.iteritems(resource_value.overrides)) + else: + values = resource_value.values + overrides = resource_value.overrides + result.update(values) + result.update(overrides) + @db.with_transaction def get(self, environment_id, resource_id_or_name, levels): + app.logger.debug("Getting resource value. Env: %s, " + "resource: %s, levels: %s", environment_id, + resource_id_or_name, levels) environment = db.Environment.query.get_or_404(environment_id) res_def = library.get_resource_definition( resource_id_or_name, environment_id) @@ -48,37 +70,48 @@ class ResourceValues(flask_restful.Resource): level_values = list(hierarchy_levels.iter_environment_level_values( environment, levels)) - if 'effective' in flask.request.args: - show_lookup = 'show_lookup' in flask.request.args - resource_values = db.ResourceValues.query.filter_by( - resource_definition=res_def, - environment=environment, - ).all() - result = {} - lookup_path = '' - for level_value in itertools.chain([None], level_values): - if level_value is not None: - name = level_value.level.name - value = level_value.value - lookup_path += name + '/' + value + '/' - else: - lookup_path += '/' + level_values_ids = [l.id for l in level_values] + app.logger.debug("Got level values ids: %s", level_values_ids) - for resource_value in resource_values: - if resource_value.level_value == level_value: - if show_lookup: - values = {} - for k, v in resource_value.values.items(): - values[k] = (v, lookup_path) - overrides = {} - for k, v in resource_value.overrides.items(): - overrides[k] = (v, lookup_path) - else: - values = resource_value.values - overrides = resource_value.overrides - result.update(values) - result.update(overrides) - break + if 'effective' in flask.request.args: + app.logger.debug("Getting effective resource value. Env: %s, " + "resource: %s, levels: %s", environment_id, + resource_id_or_name, levels) + show_lookup = 'show_lookup' in flask.request.args + resource_values = db.ResourceValues.query.filter( + or_( + db.ResourceValues.level_value_id.in_(level_values_ids), + db.ResourceValues.level_value_id.is_(None) + ), + db.ResourceValues.resource_definition == res_def, + db.ResourceValues.environment == environment + ).all() + app.logger.debug("Processing values for resource: %s, env: %s. " + "Loaded resource values: %s", + res_def.id, environment.id, len(resource_values)) + # Creating index of resource_values by level_value_id + resource_values_idx = {r.level_value_id: r + for r in resource_values} + app.logger.debug("Resource values index size: %s", + len(resource_values_idx)) + + result = {} + lookup_path = '/' + self._calculate_effective_values( + result, None, resource_values_idx, show_lookup, + lookup_path) + + for level_value in level_values: + name = level_value.level.name + value = level_value.value + lookup_path += name + '/' + value + '/' + + self._calculate_effective_values( + result, level_value, resource_values_idx, show_lookup, + lookup_path) + + app.logger.debug("Effective values got for resource: " + "%s, env: %s", res_def.id, environment.id) return result else: if not level_values: @@ -90,6 +123,8 @@ class ResourceValues(flask_restful.Resource): environment=environment, level_value=level_value, ).one_or_none() + app.logger.debug("Values got for resource: " + "%s, env: %s", res_def.id, environment.id) if not resource_values: return {} return resource_values.values diff --git a/tuning_box/logger.py b/tuning_box/logger.py index dd8e09c..5081b0c 100644 --- a/tuning_box/logger.py +++ b/tuning_box/logger.py @@ -20,9 +20,8 @@ def get_formatter(): return logging.Formatter(fmt=log_format, datefmt=date_format) -def init_logger(log_level): +def init_logger(app, log_level): handler = logging.StreamHandler() handler.setFormatter(get_formatter()) - logger = logging.getLogger() - logger.addHandler(handler) - logger.setLevel(log_level) + app.logger.addHandler(handler) + app.logger.setLevel(log_level) diff --git a/tuning_box/tests/library/test_resource_overrides.py b/tuning_box/tests/library/test_resource_overrides.py index 76f4010..18f6e36 100644 --- a/tuning_box/tests/library/test_resource_overrides.py +++ b/tuning_box/tests/library/test_resource_overrides.py @@ -278,8 +278,8 @@ class TestResourceOverrides(BaseTest): # Checking lookup info res = self.client.get( - '/environments/9/lvl1/1/lvl2/2/resources/5/values?' - 'effective&show_lookup', + '/environments/9/lvl1/1/lvl2/2/resources/5/values', + query_string={'effective': 1, 'show_lookup': 1} ) self.assertEqual(res.status_code, 200) expected = { diff --git a/tuning_box/tests/library/test_resource_values.py b/tuning_box/tests/library/test_resource_values.py index 6ad7d36..da64192 100644 --- a/tuning_box/tests/library/test_resource_values.py +++ b/tuning_box/tests/library/test_resource_values.py @@ -11,6 +11,9 @@ # under the License. import itertools +import uuid + +import six from tuning_box import db from tuning_box.tests.test_app import BaseTest @@ -365,3 +368,71 @@ class TestResourceValues(BaseTest): 'key3': ['root_value_3', '/'] } self.assertEqual(expected, res.json) + + def generate_values(self, size): + result = {} + for i in six.moves.range(size): + result[six.text_type(uuid.uuid4())] = i + return result + + def test_get_resource_values_effective_lot_of_data(self): + self._fixture() + env_id = 9 + res_id = 5 + keys_on_root = 10000 + keys_on_lvl1 = 15000 + keys_on_lvl2 = 20000 + values_on_level = 500 + + # Adding values on the root level + self._add_resource_values( + env_id, res_id, (), self.generate_values(keys_on_root)) + + # Adding values on the level lvl1 and lvl2 + lvl_1_values = self.generate_values(keys_on_lvl1) + lvl_2_values = self.generate_values(keys_on_lvl2) + for lvl_val in six.moves.range(values_on_level): + lvl_val = six.text_type(lvl_val) + self._add_resource_values( + env_id, res_id, (('lvl1', lvl_val),), lvl_1_values) + self._add_resource_values( + env_id, res_id, (('lvl1', lvl_val), ('lvl2', lvl_val)), + lvl_2_values) + + with self.app.app_context(): + res_vals_count = db.ResourceValues.query.count() + self.assertEqual(1 + values_on_level * 2, res_vals_count) + + # Check keys num on root level + obj_url = self.object_url.format( + env_id, self.get_levels_path(()), + res_id + ) + res = self.client.get(obj_url) + self.assertEqual(keys_on_root, len(res.json)) + + res = self.client.get(obj_url, query_string={'effective': 1}) + self.assertEqual(keys_on_root, len(res.json)) + + # Check keys num on lvl1 + obj_url = self.object_url.format( + env_id, self.get_levels_path((('lvl1', '1'),)), + res_id + ) + res = self.client.get(obj_url) + self.assertEqual(keys_on_lvl1, len(res.json)) + + res = self.client.get(obj_url, query_string={'effective': 1}) + self.assertEqual(keys_on_root + keys_on_lvl1, len(res.json)) + + # Check keys num on lvl2 + obj_url = self.object_url.format( + env_id, self.get_levels_path((('lvl1', '1'), ('lvl2', '2'))), + res_id + ) + res = self.client.get(obj_url) + self.assertEqual(keys_on_lvl2, len(res.json)) + + res = self.client.get(obj_url, query_string={'effective': 1}) + self.assertEqual(keys_on_root + keys_on_lvl1 + keys_on_lvl2, + len(res.json))