diff --git a/tuning_box/app.py b/tuning_box/app.py index 61dac38..1acd654 100644 --- a/tuning_box/app.py +++ b/tuning_box/app.py @@ -134,8 +134,8 @@ def build_app(configure_logging=True, with_keystone=True): handle_keys_operation_error) db.db.init_app(app) if configure_logging: - log_level = app.config.get('LOG_LEVEL', 'INFO') - logger.init_logger(log_level) + log_level = app.config.get('LOG_LEVEL', 'DEBUG') + logger.init_logger(app, log_level) if with_keystone: app.wsgi_app = keystone.KeystoneMiddleware(app) return app diff --git a/tuning_box/library/hierarchy_levels.py b/tuning_box/library/hierarchy_levels.py index dc85e3d..30e9ac8 100644 --- a/tuning_box/library/hierarchy_levels.py +++ b/tuning_box/library/hierarchy_levels.py @@ -11,6 +11,7 @@ # under the License. import flask +from flask import current_app as app import flask_restful from flask_restful import fields @@ -20,7 +21,12 @@ from tuning_box import errors def iter_environment_level_values(environment, levels): + app.logger.debug("Getting environment level values. Env: %s, " + "levels: %s", environment.id, levels) env_levels = db.EnvironmentHierarchyLevel.get_for_environment(environment) + app.logger.debug("Environment levels got. Env: %s, levels: %s", + environment.id, [l.name for l in env_levels]) + if len(env_levels) < len(levels): raise errors.TuningboxNotFound( "Levels {0} can't be matched with " diff --git a/tuning_box/library/resource_values.py b/tuning_box/library/resource_values.py index b93c807..2380e31 100644 --- a/tuning_box/library/resource_values.py +++ b/tuning_box/library/resource_values.py @@ -11,8 +11,10 @@ # under the License. import flask +from flask import current_app as app import flask_restful -import itertools +import six +from sqlalchemy import or_ from tuning_box import db from tuning_box import library @@ -39,8 +41,28 @@ class ResourceValues(flask_restful.Resource): esv.values = flask.request.json return None, 204 + def _calculate_effective_values(self, result, level_value, + resource_values_idx, show_lookup, + lookup_path): + level_value_id = getattr(level_value, 'id', None) + if level_value_id in resource_values_idx: + resource_value = resource_values_idx[level_value_id] + if show_lookup: + values = ((k, (v, lookup_path)) for k, v in + six.iteritems(resource_value.values)) + overrides = ((k, (v, lookup_path)) for k, v in + six.iteritems(resource_value.overrides)) + else: + values = resource_value.values + overrides = resource_value.overrides + result.update(values) + result.update(overrides) + @db.with_transaction def get(self, environment_id, resource_id_or_name, levels): + app.logger.debug("Getting resource value. Env: %s, " + "resource: %s, levels: %s", environment_id, + resource_id_or_name, levels) environment = db.Environment.query.get_or_404(environment_id) res_def = library.get_resource_definition( resource_id_or_name, environment_id) @@ -48,37 +70,48 @@ class ResourceValues(flask_restful.Resource): level_values = list(hierarchy_levels.iter_environment_level_values( environment, levels)) - if 'effective' in flask.request.args: - show_lookup = 'show_lookup' in flask.request.args - resource_values = db.ResourceValues.query.filter_by( - resource_definition=res_def, - environment=environment, - ).all() - result = {} - lookup_path = '' - for level_value in itertools.chain([None], level_values): - if level_value is not None: - name = level_value.level.name - value = level_value.value - lookup_path += name + '/' + value + '/' - else: - lookup_path += '/' + level_values_ids = [l.id for l in level_values] + app.logger.debug("Got level values ids: %s", level_values_ids) - for resource_value in resource_values: - if resource_value.level_value == level_value: - if show_lookup: - values = {} - for k, v in resource_value.values.items(): - values[k] = (v, lookup_path) - overrides = {} - for k, v in resource_value.overrides.items(): - overrides[k] = (v, lookup_path) - else: - values = resource_value.values - overrides = resource_value.overrides - result.update(values) - result.update(overrides) - break + if 'effective' in flask.request.args: + app.logger.debug("Getting effective resource value. Env: %s, " + "resource: %s, levels: %s", environment_id, + resource_id_or_name, levels) + show_lookup = 'show_lookup' in flask.request.args + resource_values = db.ResourceValues.query.filter( + or_( + db.ResourceValues.level_value_id.in_(level_values_ids), + db.ResourceValues.level_value_id.is_(None) + ), + db.ResourceValues.resource_definition == res_def, + db.ResourceValues.environment == environment + ).all() + app.logger.debug("Processing values for resource: %s, env: %s. " + "Loaded resource values: %s", + res_def.id, environment.id, len(resource_values)) + # Creating index of resource_values by level_value_id + resource_values_idx = {r.level_value_id: r + for r in resource_values} + app.logger.debug("Resource values index size: %s", + len(resource_values_idx)) + + result = {} + lookup_path = '/' + self._calculate_effective_values( + result, None, resource_values_idx, show_lookup, + lookup_path) + + for level_value in level_values: + name = level_value.level.name + value = level_value.value + lookup_path += name + '/' + value + '/' + + self._calculate_effective_values( + result, level_value, resource_values_idx, show_lookup, + lookup_path) + + app.logger.debug("Effective values got for resource: " + "%s, env: %s", res_def.id, environment.id) return result else: if not level_values: @@ -90,6 +123,8 @@ class ResourceValues(flask_restful.Resource): environment=environment, level_value=level_value, ).one_or_none() + app.logger.debug("Values got for resource: " + "%s, env: %s", res_def.id, environment.id) if not resource_values: return {} return resource_values.values diff --git a/tuning_box/logger.py b/tuning_box/logger.py index dd8e09c..5081b0c 100644 --- a/tuning_box/logger.py +++ b/tuning_box/logger.py @@ -20,9 +20,8 @@ def get_formatter(): return logging.Formatter(fmt=log_format, datefmt=date_format) -def init_logger(log_level): +def init_logger(app, log_level): handler = logging.StreamHandler() handler.setFormatter(get_formatter()) - logger = logging.getLogger() - logger.addHandler(handler) - logger.setLevel(log_level) + app.logger.addHandler(handler) + app.logger.setLevel(log_level) diff --git a/tuning_box/tests/library/test_resource_overrides.py b/tuning_box/tests/library/test_resource_overrides.py index 76f4010..18f6e36 100644 --- a/tuning_box/tests/library/test_resource_overrides.py +++ b/tuning_box/tests/library/test_resource_overrides.py @@ -278,8 +278,8 @@ class TestResourceOverrides(BaseTest): # Checking lookup info res = self.client.get( - '/environments/9/lvl1/1/lvl2/2/resources/5/values?' - 'effective&show_lookup', + '/environments/9/lvl1/1/lvl2/2/resources/5/values', + query_string={'effective': 1, 'show_lookup': 1} ) self.assertEqual(res.status_code, 200) expected = { diff --git a/tuning_box/tests/library/test_resource_values.py b/tuning_box/tests/library/test_resource_values.py index 6ad7d36..da64192 100644 --- a/tuning_box/tests/library/test_resource_values.py +++ b/tuning_box/tests/library/test_resource_values.py @@ -11,6 +11,9 @@ # under the License. import itertools +import uuid + +import six from tuning_box import db from tuning_box.tests.test_app import BaseTest @@ -365,3 +368,71 @@ class TestResourceValues(BaseTest): 'key3': ['root_value_3', '/'] } self.assertEqual(expected, res.json) + + def generate_values(self, size): + result = {} + for i in six.moves.range(size): + result[six.text_type(uuid.uuid4())] = i + return result + + def test_get_resource_values_effective_lot_of_data(self): + self._fixture() + env_id = 9 + res_id = 5 + keys_on_root = 10000 + keys_on_lvl1 = 15000 + keys_on_lvl2 = 20000 + values_on_level = 500 + + # Adding values on the root level + self._add_resource_values( + env_id, res_id, (), self.generate_values(keys_on_root)) + + # Adding values on the level lvl1 and lvl2 + lvl_1_values = self.generate_values(keys_on_lvl1) + lvl_2_values = self.generate_values(keys_on_lvl2) + for lvl_val in six.moves.range(values_on_level): + lvl_val = six.text_type(lvl_val) + self._add_resource_values( + env_id, res_id, (('lvl1', lvl_val),), lvl_1_values) + self._add_resource_values( + env_id, res_id, (('lvl1', lvl_val), ('lvl2', lvl_val)), + lvl_2_values) + + with self.app.app_context(): + res_vals_count = db.ResourceValues.query.count() + self.assertEqual(1 + values_on_level * 2, res_vals_count) + + # Check keys num on root level + obj_url = self.object_url.format( + env_id, self.get_levels_path(()), + res_id + ) + res = self.client.get(obj_url) + self.assertEqual(keys_on_root, len(res.json)) + + res = self.client.get(obj_url, query_string={'effective': 1}) + self.assertEqual(keys_on_root, len(res.json)) + + # Check keys num on lvl1 + obj_url = self.object_url.format( + env_id, self.get_levels_path((('lvl1', '1'),)), + res_id + ) + res = self.client.get(obj_url) + self.assertEqual(keys_on_lvl1, len(res.json)) + + res = self.client.get(obj_url, query_string={'effective': 1}) + self.assertEqual(keys_on_root + keys_on_lvl1, len(res.json)) + + # Check keys num on lvl2 + obj_url = self.object_url.format( + env_id, self.get_levels_path((('lvl1', '1'), ('lvl2', '2'))), + res_id + ) + res = self.client.get(obj_url) + self.assertEqual(keys_on_lvl2, len(res.json)) + + res = self.client.get(obj_url, query_string={'effective': 1}) + self.assertEqual(keys_on_root + keys_on_lvl1 + keys_on_lvl2, + len(res.json))