openstack
/
tuning-box
Code Issues Proposed changes

Nested keys handled for get operation 

Nested key can be specified with '.' separator: key1.key2.key3.
Nested keys handled in the API and in the fuel2.
Tests for keys operations moved from the test for resource values
to the test for resource key operations.

Change-Id: I73b5fd9a4a4720a96af351d7e6a7cea14d816f75
Partial-Bug: #1642330
This commit is contained in:
Alexander Kislitsky 2016-11-16 14:52:55 +03:00
parent ed1760d83e
commit 22ea50db29
8 changed files with 291 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
tuning_box/app.py
View File

@ -31,6 +31,7 @@ api_errors = {
'IntegrityError': {'status': 409}, # sqlalchemy IntegrityError
'TuningboxIntegrityError': {'status': 409},
'KeysOperationError': {'status': 409},
'RequestValidationError': {'status': 409},
'TuningboxNotFound': {'status': 404}
}
api = flask_restful.Api(errors=api_errors)
@ -99,6 +100,12 @@ api.add_resource(
)
def handle_request_validation_error(exc):
response = flask.jsonify(msg=exc.args[0])
response.status_code = 409
return response
def handle_integrity_error(exc):
response = flask.jsonify(msg=exc.args[0])
response.status_code = 409
@ -130,6 +137,8 @@ def build_app(configure_logging=True, with_keystone=True):
handle_integrity_error)
app.register_error_handler(errors.TuningboxNotFound,
handle_object_not_found)
app.register_error_handler(errors.RequestValidationError,
handle_request_validation_error)
app.register_error_handler(errors.KeysOperationError,
handle_keys_operation_error)
db.db.init_app(app)

12
tuning_box/cli/resources.py
View File

@ -62,7 +62,8 @@ class Get(show.ShowOne, ResourcesCommand):
parser.add_argument(
'-k', '--key',
type=str,
help="Name of key to get from the resource",
help="Name of key to get from the resource. For fetching nested "
"key value use '.' as delimiter. Example: k1.k2.k3",
)
parser.add_argument(
'-s', '--show-lookup',
@ -76,16 +77,17 @@ class Get(show.ShowOne, ResourcesCommand):
params = {'effective': True}
if parsed_args.show_lookup:
params['show_lookup'] = True
if parsed_args.key:
params['key'] = parsed_args.key
response = self.get_client().get(
self.get_resource_url(parsed_args),
params=params
)
key = parsed_args.key
if key is not None:
result = {key: response.get(key, {})}
if parsed_args.key:
result = {parsed_args.key: response}
else:
result = response
columns = sorted(result.keys())
columns = sorted(result)
try:
data = data_utils.get_display_data_single(columns, result)
return columns, data

4
tuning_box/errors.py
View File

@ -23,6 +23,10 @@ class TuningboxNotFound(BaseTuningboxError):
pass
class RequestValidationError(BaseTuningboxError):
pass
class KeysOperationError(BaseTuningboxError):
pass

59
tuning_box/library/resource_keys_operation.py
View File

@ -21,10 +21,11 @@ from tuning_box import library
class KeysOperationMixin(object):
OPERATION_GET = 'get'
OPERATION_SET = 'set'
OPERATION_DELETE = 'delete'
OPERATIONS = (OPERATION_SET, OPERATION_DELETE)
OPERATIONS = (OPERATION_GET, OPERATION_SET, OPERATION_DELETE)
def _check_out_of_index(self, cur_point, key, keys_path):
if isinstance(cur_point, (list, tuple)) and key >= len(cur_point):
@ -47,6 +48,60 @@ class KeysOperationMixin(object):
"in keys path {2}".format(cur_point, key, keys_path)
)
def do_get(self, storage, keys_paths):
"""Gets values from storage by keys paths.
Keys path is list of keys paths. If we have keys_paths
[['a', 'b']], then storage['a']['b'] will be get as result.
:param storage: original data
:param keys_paths: lists of keys paths to be set
:returns: value from storage specified by keys_paths
"""
# Removing show lookup information from the data
show_lookup = 'show_lookup' in flask.request.args
effective = 'effective' in flask.request.args
if effective and show_lookup:
storage_copy = copy.deepcopy(storage)
for k in storage_copy.iterkeys():
storage_copy[k] = storage[k][0]
else:
storage_copy = storage
result = []
for keys_path in keys_paths:
cur_point = storage_copy
if not keys_path:
continue
try:
for key in keys_path[:-1]:
# Keys paths are passed as part of the url, so we need
# cast list and tuple indexes to integers
if isinstance(cur_point, (list, tuple)):
key = int(key)
cur_point = cur_point[key]
key = keys_path[-1]
# Keys paths are passed as part of the url, so we need
# cast list and tuple indexes to integers
if isinstance(cur_point, (list, tuple)):
key = int(key)
self._check_path_is_reachable(cur_point, key, keys_path)
if effective and show_lookup:
result.append([cur_point[key], storage[keys_path[0]][1]])
else:
result.append(cur_point[key])
except (KeyError, IndexError):
raise errors.KeysPathNotExisted(
"Keys path doesn't exist {0}. "
"Failed on the key {1}".format(keys_path, key)
)
return result
def do_set(self, storage, keys_paths):
"""Sets values from keys paths to storage.
@ -115,6 +170,8 @@ class KeysOperationMixin(object):
return self.do_set(storage, keys_paths)
elif operation == self.OPERATION_DELETE:
return self.do_delete(storage, keys_paths)
elif operation == self.OPERATION_GET:
return self.do_get(storage, keys_paths)
else:
raise errors.UnknownKeysOperation(
"Unknown operation: {0}. "

39
tuning_box/library/resource_values.py
View File

@ -17,12 +17,16 @@ import six
from sqlalchemy import or_
from tuning_box import db
from tuning_box import errors
from tuning_box import library
from tuning_box.library import hierarchy_levels
from tuning_box.library import resource_keys_operation
from tuning_box.library.resource_keys_operation import KeysOperationMixin
class ResourceValues(flask_restful.Resource):
class ResourceValues(flask_restful.Resource, KeysOperationMixin):
KEYS_PATH_DELIMITER = '.'
@db.with_transaction
def put(self, environment_id, levels, resource_id_or_name):
@ -63,6 +67,14 @@ class ResourceValues(flask_restful.Resource):
app.logger.debug("Getting resource value. Env: %s, "
"resource: %s, levels: %s", environment_id,
resource_id_or_name, levels)
effective = 'effective' in flask.request.args
show_lookup = 'show_lookup' in flask.request.args
if show_lookup and not effective:
raise errors.RequestValidationError(
"Lookup path tracing can be done only for effective values")
environment = db.Environment.query.get_or_404(environment_id)
res_def = library.get_resource_definition(
resource_id_or_name, environment_id)
@ -73,11 +85,10 @@ class ResourceValues(flask_restful.Resource):
level_values_ids = [l.id for l in level_values]
app.logger.debug("Got level values ids: %s", level_values_ids)
if 'effective' in flask.request.args:
if effective:
app.logger.debug("Getting effective resource value. Env: %s, "
"resource: %s, levels: %s", environment_id,
resource_id_or_name, levels)
show_lookup = 'show_lookup' in flask.request.args
resource_values = db.ResourceValues.query.filter(
or_(
db.ResourceValues.level_value_id.in_(level_values_ids),
@ -112,7 +123,6 @@ class ResourceValues(flask_restful.Resource):
app.logger.debug("Effective values got for resource: "
"%s, env: %s", res_def.id, environment.id)
return result
else:
if not level_values:
level_value = None
@ -125,9 +135,24 @@ class ResourceValues(flask_restful.Resource):
).one_or_none()
app.logger.debug("Values got for resource: "
"%s, env: %s", res_def.id, environment.id)
if not resource_values:
return {}
return resource_values.values
if resource_values:
result = resource_values.values
else:
result = {}
return self._extract_keys_paths(result)
def _extract_keys_paths(self, data):
if 'key' not in flask.request.args:
return data
keys_path = flask.request.args['key'].split(self.KEYS_PATH_DELIMITER)
app.logger.debug("Extracting data by keys paths: %s", keys_path)
result = self.do_get(data, [keys_path])
# Single keys path is passed as GET request parameter, so we need
# only first result
result = result[0]
app.logger.debug("Extracted data by keys paths: %s is: %s",
keys_path, result)
return result
class ResourceValuesKeys(flask_restful.Resource,

30
tuning_box/tests/cli/test_resources.py
View File

@ -74,29 +74,29 @@ class TestGet(testscenarios.WithScenarios, _BaseCLITest):
'hello: world\n',
)),
('key,json', (
'/environments/1/resources/1/values?effective',
'get --env 1 --resource 1 --key hello --format json',
'{\n "hello": "world"\n}',
'/environments/1/resources/1/values?effective&key=k',
'get --env 1 --resource 1 --key k --format json',
'{\n "k": {\n "hello": "world"\n }\n}'
)),
('key,lookup', (
'/environments/1/resources/1/values?effective',
'get --env 1 --resource 1 --key hello --format json -s',
'{\n "hello": "world"\n}',
'/environments/1/resources/1/values?effective&key=k',
'get --env 1 --resource 1 --key k --format json -s',
'{\n "k": {\n "hello": "world"\n }\n}'
)),
('key,yaml', (
'/environments/1/resources/1/values?effective',
'get --env 1 --resource 1 --key hello --format yaml',
'hello: world\n',
'/environments/1/resources/1/values?effective&key=k',
'get --env 1 --resource 1 --key k --format yaml',
"k:\n hello: world\n"
)),
('no_key,json', (
'/environments/1/resources/1/values?effective',
'get --env 1 --resource 1 --key no --format json',
'{\n "no": {}\n}',
'/environments/1/resources/1/values?key=k&effective',
'get --env 1 --resource 1 --key k --format json',
'{\n "k": {\n "hello": "world"\n }\n}'
)),
('no_key,yaml', (
'/environments/1/resources/1/values?effective',
'get --env 1 --resource 1 --key no --format yaml',
"'no': {}\n",
'/environments/1/resources/1/values?key=no.key&effective',
'get --env 1 --resource 1 --key no.key --format yaml',
"no.key:\n hello: world\n"
))
]
]

82
tuning_box/tests/library/test_resource_key_operations.py
View File

@ -15,9 +15,11 @@ from tuning_box.library import resource_keys_operation
from tuning_box.tests.test_app import BaseTest
class TestResourceDefinitions(BaseTest):
class TestResourceKeysOperations(BaseTest):
processor = resource_keys_operation.KeysOperationMixin()
object_url = '/environments/{0}/{1}resources/{2}/values'
object_keys_url = object_url + '/keys/{3}'
def test_unknown_operation(self):
self.assertRaises(errors.UnknownKeysOperation,
@ -130,3 +132,81 @@ class TestResourceDefinitions(BaseTest):
result = self.processor.do_delete(data, keys)
self.assertEqual({'a': {'b': 'value_b'}}, data)
self.assertEqual({'a': {}}, result)
def test_put_resource_values_delete(self):
self._fixture()
environment_id = 9
res_def_id = 5
levels = (('lvl1', 'val1'), ('lvl2', 'val2'))
values = {'key_0': 'val_0', 'key_1': 'val_1'}
self._add_resource_values(environment_id, res_def_id, levels, values)
obj_url = self.object_url.format(
environment_id,
self.get_levels_path(levels),
res_def_id
)
obj_keys_url = obj_url + '/keys/delete'
data = [['key_0']]
res = self.client.put(obj_keys_url, data=data)
self.assertEqual(204, res.status_code)
res = self.client.get(obj_url)
self.assertEqual(200, res.status_code)
actual = res.json
self.assertEqual({'key_1': 'val_1'}, actual)
def test_put_resource_values_not_found(self):
self.app.config["PROPAGATE_EXCEPTIONS"] = True
self._fixture()
res = self.client.put(
'/environments/9/lvl1/val1/resources/5/values/keys/set',
data={}
)
self.assertEqual(404, res.status_code)
def test_put_resource_values_set_operation_error(self):
self.app.config["PROPAGATE_EXCEPTIONS"] = True
self._fixture()
environment_id = 9
res_def_id = 5
levels = (('lvl1', 'val1'), ('lvl2', 'val2'))
values = {'key': 'val'}
self._add_resource_values(environment_id, res_def_id, levels, values)
data = [['a', 'b', 'c', 'value']]
obj_keys_url = self.object_keys_url.format(
environment_id,
self.get_levels_path(levels),
res_def_id,
'set'
)
res = self.client.put(obj_keys_url, data=data)
self.assertEqual(409, res.status_code)
def test_put_resource_values_delete_operation_error(self):
self.app.config["PROPAGATE_EXCEPTIONS"] = True
self._fixture()
environment_id = 9
res_def_id = 5
levels = (('lvl1', 'val1'), ('lvl2', 'val2'))
values = {'key_0': 'val_0', 'key_1': 'val_1'}
self._add_resource_values(environment_id, res_def_id, levels, values)
obj_keys_url = self.object_keys_url.format(
environment_id,
self.get_levels_path(levels),
res_def_id,
'delete'
)
data = [['fake_key']]
res = self.client.put(obj_keys_url, data=data)
self.assertEqual(409, res.status_code)
data = [['key_0', 'val_0']]
res = self.client.put(obj_keys_url, data=data)
self.assertEqual(409, res.status_code)

164
tuning_box/tests/library/test_resource_values.py
View File

@ -21,7 +21,6 @@ from tuning_box.tests.test_app import BaseTest
class TestResourceValues(BaseTest):
object_url = '/environments/{0}/{1}resources/{2}/values'
object_keys_url = object_url + '/keys/{3}'
def test_put_resource_values_root(self):
self._fixture()
@ -132,37 +131,6 @@ class TestResourceValues(BaseTest):
self.assertEqual(res.status_code, 200)
self.assertEqual(expected, res.json)
def test_put_resource_values_not_found(self):
self.app.config["PROPAGATE_EXCEPTIONS"] = True
self._fixture()
res = self.client.put(
'/environments/9/lvl1/val1/resources/5/values/keys/set',
data={}
)
self.assertEqual(404, res.status_code)
def test_put_resource_values_set_operation_error(self):
self.app.config["PROPAGATE_EXCEPTIONS"] = True
self._fixture()
environment_id = 9
res_def_id = 5
levels = (('lvl1', 'val1'), ('lvl2', 'val2'))
values = {'key': 'val'}
self._add_resource_values(environment_id, res_def_id, levels, values)
data = [['a', 'b', 'c', 'value']]
obj_keys_url = self.object_keys_url.format(
environment_id,
self.get_levels_path(levels),
res_def_id,
'set'
)
res = self.client.put(obj_keys_url, data=data)
self.assertEqual(409, res.status_code)
def test_put_resource_values_set(self):
self._fixture()
environment_id = 9
@ -249,30 +217,6 @@ class TestResourceValues(BaseTest):
self.assertEqual({'key': 'key_value', 'key_x': 'key_x_value'},
actual)
def test_put_resource_values_delete(self):
self._fixture()
environment_id = 9
res_def_id = 5
levels = (('lvl1', 'val1'), ('lvl2', 'val2'))
values = {'key_0': 'val_0', 'key_1': 'val_1'}
self._add_resource_values(environment_id, res_def_id, levels, values)
obj_url = self.object_url.format(
environment_id,
self.get_levels_path(levels),
res_def_id
)
obj_keys_url = obj_url + '/keys/delete'
data = [['key_0']]
res = self.client.put(obj_keys_url, data=data)
self.assertEqual(204, res.status_code)
res = self.client.get(obj_url)
self.assertEqual(200, res.status_code)
actual = res.json
self.assertEqual({'key_1': 'val_1'}, actual)
def test_put_resource_values_delete_by_name(self):
self._fixture()
environment_id = 9
@ -304,29 +248,6 @@ class TestResourceValues(BaseTest):
actual = res.json
self.assertEqual({'key_1': 'val_1'}, actual)
def test_put_resource_values_delete_operation_error(self):
self.app.config["PROPAGATE_EXCEPTIONS"] = True
self._fixture()
environment_id = 9
res_def_id = 5
levels = (('lvl1', 'val1'), ('lvl2', 'val2'))
values = {'key_0': 'val_0', 'key_1': 'val_1'}
self._add_resource_values(environment_id, res_def_id, levels, values)
obj_keys_url = self.object_keys_url.format(
environment_id,
self.get_levels_path(levels),
res_def_id,
'delete'
)
data = [['fake_key']]
res = self.client.put(obj_keys_url, data=data)
self.assertEqual(409, res.status_code)
data = [['key_0', 'val_0']]
res = self.client.put(obj_keys_url, data=data)
self.assertEqual(409, res.status_code)
def test_get_resource_values_effective_with_lookup(self):
self._fixture()
res = self.client.put('/environments/9/resources/5/values',
@ -429,3 +350,88 @@ class TestResourceValues(BaseTest):
res = self.client.get(obj_url, query_string={'effective': 1})
self.assertEqual(keys_on_root + keys_on_lvl1 + keys_on_lvl2,
len(res.json))
def test_resource_values_get_nested_keys(self):
self.app.config["PROPAGATE_EXCEPTIONS"] = True
self._fixture()
environment_id = 9
res_def_id = 5
levels = (('lvl1', 'val1'), ('lvl2', 'val2'))
values = {'k0': 'v0', 'k1': {'k2': 'v12', 'k3': 'v13',
'k4': [{'k5': 'v1405'}, 'v141']},
'k6': [{'k7': [{'k8': 'v60708'}]}]}
self._add_resource_values(environment_id, res_def_id, levels, values)
obj_url = self.object_url.format(
environment_id,
self.get_levels_path(levels),
res_def_id
)
res = self.client.get(obj_url, query_string={'key': 'k0'})
self.assertEqual(200, res.status_code)
self.assertEqual('v0', res.json)
# Getting nested key
res = self.client.get(obj_url, query_string={'key': 'k1.k2'})
self.assertEqual(200, res.status_code)
self.assertEqual('v12', res.json)
# Getting nested key from the list
res = self.client.get(obj_url, query_string={'key': 'k1.k4.0'})
self.assertEqual(200, res.status_code)
self.assertEqual({'k5': 'v1405'}, res.json)
# Getting nested key from nested lists
res = self.client.get(obj_url, query_string={'key': 'k6.0.k7.0.k8'})
self.assertEqual(200, res.status_code)
self.assertEqual('v60708', res.json)
# Getting nested key effective value
res = self.client.get(obj_url,
query_string={'key': 'k0', 'effective': 1})
self.assertEqual(200, res.status_code)
self.assertEqual('v0', res.json)
res = self.client.get(obj_url,
query_string={'key': 'k1.k2', 'effective': 1})
self.assertEqual(200, res.status_code)
self.assertEqual('v12', res.json)
# Getting nested key value with lookup
res = self.client.get(
obj_url,
query_string={'key': 'k0', 'effective': 1, 'show_lookup': 1}
)
self.assertEqual(200, res.status_code)
self.assertEqual(['v0', '/lvl1/val1/lvl2/val2/'], res.json)
res = self.client.get(
obj_url,
query_string={'key': 'k1.k2', 'effective': 1, 'show_lookup': 1}
)
self.assertEqual(200, res.status_code)
self.assertEqual(['v12', '/lvl1/val1/lvl2/val2/'], res.json)
def test_resource_values_get_nested_keys_not_found(self):
self.app.config["PROPAGATE_EXCEPTIONS"] = True
self._fixture()
environment_id = 9
res_def_id = 5
levels = (('lvl1', 'val1'), ('lvl2', 'val2'))
values = {'k0': 'v0'}
self._add_resource_values(environment_id, res_def_id, levels, values)
obj_url = self.object_url.format(
environment_id,
self.get_levels_path(levels),
res_def_id
)
res = self.client.get(obj_url, query_string={'key': 'k0'})
self.assertEqual(200, res.status_code)
self.assertEqual('v0', res.json)
res = self.client.get(obj_url, query_string={'key': 'k1'})
self.assertEqual(409, res.status_code)
res = self.client.get(obj_url, query_string={'key': 'k1.k2'})
self.assertEqual(409, res.status_code)