From 6e5d0ce9680ce47401c9ffe78bcb78522e95626d Mon Sep 17 00:00:00 2001 From: marios Date: Tue, 24 Feb 2015 18:50:29 +0200 Subject: [PATCH] Adds the /role/role_id/extra_data path to the v2 API This uses the new TemplateExtraStore to process stored extra-data files and match them against the template for each defined role This information is included when a client requests the existing /plans/plan_uuid/templates path. To retrieve the extra-data for any particular role a new /role/role_uuid/extra_data path is added Change-Id: Ic52280248b0d169f4a3c27e10a826d3dbb2b9bf3 --- tuskar/api/controllers/v2/roles.py | 43 ++++++++++++++++++++ tuskar/common/exception.py | 6 +++ tuskar/common/utils.py | 65 ++++++++++++++++++++++++++++++ tuskar/manager/plan.py | 22 +++++++++- tuskar/manager/role.py | 45 +++++++++++++++++++++ tuskar/tests/test_utils.py | 32 ++++++++++++++- 6 files changed, 211 insertions(+), 2 deletions(-) diff --git a/tuskar/api/controllers/v2/roles.py b/tuskar/api/controllers/v2/roles.py index ad6dfef0..d9f9ab6f 100644 --- a/tuskar/api/controllers/v2/roles.py +++ b/tuskar/api/controllers/v2/roles.py @@ -17,6 +17,7 @@ from wsmeext import pecan as wsme_pecan from tuskar.api.controllers.v2 import models from tuskar.common import exception +from tuskar.common import utils from tuskar.manager.plan import PlansManager from tuskar.manager.role import RoleManager from tuskar.storage import exceptions as storage_exceptions @@ -28,6 +29,8 @@ LOG = logging.getLogger(__name__) class RolesController(rest.RestController): """REST controller for the Role class.""" + _custom_actions = {'extra_data': ['GET']} + @wsme_pecan.wsexpose([models.Role]) def get_all(self): """Returns all roles. @@ -43,6 +46,46 @@ class RolesController(rest.RestController): transfer_roles = [models.Role.from_tuskar_model(r) for r in all_roles] return transfer_roles + @wsme_pecan.wsexpose({str: str}, str) + def extra_data(self, role_uuid): + """Retrieve the extra data files associated with a given role. + + :param role_uuid: identifies the role + :type role_uuid: str + + :return: a dict where keys are filenames and values are their contents + :rtype: dict + + This method will retrieve all stored role_extra records (these are + created at the same time that the Roles are, by using --role-extra + parameter to tuskar-load-roles). + + The internal representation for a given role-extra file encodes the + file extension into the name. For instance 'hieradata/compute.yaml' + is stored as 'extra_compute_yaml'. + + The given role's template is searched for 'get_file' directives and + then matched against the stored role-extra records (based on their + name... e.g. 'extra_controller_yaml' we look for 'controller.yaml' + after a get_file directive). + + This method thus returns all the matched role-extra files for the + given role. The keys will include the relative path if one is + used in the role template: + { + "hieradata/common.yaml": "CONTENTS", + "hieradata/controller.yaml": "CONTENTS", + "hieradata/object.yaml": "CONTENTS" + } + + """ + manager = RoleManager() + db_role = manager.retrieve_db_role_by_uuid(role_uuid) + db_role_extra = manager.retrieve_db_role_extra() + role_extra_paths = utils.resolve_template_extra_data( + db_role, db_role_extra) + return manager.template_extra_data_for_output(role_extra_paths) + @wsme_pecan.wsexpose(models.Plan, str, body=models.Role, diff --git a/tuskar/common/exception.py b/tuskar/common/exception.py index 08144b2e..e3018417 100644 --- a/tuskar/common/exception.py +++ b/tuskar/common/exception.py @@ -206,3 +206,9 @@ class PlanAlreadyHasRole(DuplicateEntry): class PlanParametersNotExist(Invalid): message = _("There are no parameters named %(param_names)s" " in plan %(plan_uuid)s.") + + +class InvalidTemplateExtraStoredName(TuskarException): + # code 500 definitely internal server error. Default for TuskarException + message = _("Unexpected name for stored template extra file " + "%(name)s . Expected to start with 'extra_'") diff --git a/tuskar/common/utils.py b/tuskar/common/utils.py index 9ea10306..b60fea9d 100644 --- a/tuskar/common/utils.py +++ b/tuskar/common/utils.py @@ -19,6 +19,7 @@ """Utilities and helper functions.""" import os +import re from oslo.config import cfg @@ -108,3 +109,67 @@ def resolve_role_extra_name_from_path(role_extra_path): name_ext = os.path.basename(role_extra_path) name, extension = os.path.splitext(name_ext) return "extra_%s_%s" % (name, extension.replace('.', '')) + + +def resolve_template_file_name_from_role_extra_name(role_extra_name): + """Return the name of the included file based on the role-extra name + + The internal representation for a given role-extra file encodes the + file extension into the name. For instance 'compute.yaml' + is stored as 'extra_compute_yaml'. Here, given the stored name, + return name.extension + + Raises a InvalidTemplateExtraStoredName exception if the given + role_extra_name doesn't start with 'extra_' as a prefix. + + :param role_extra_name: the name as stored for the role-extra + :type role_extra_name: string + + :return: the name as used in the template + :rtype: string + + Returns 'compute.yaml' from 'extra_compute_yaml'. + """ + if not role_extra_name.startswith("extra_"): + raise exception.InvalidTemplateExtraStoredName(name=role_extra_name) + role_extra_name = role_extra_name[6:] + name_extension = role_extra_name.rsplit("_", 1) + if name_extension[1] == '': + return name_extension[0] + return ".".join(name_extension) + + +def resolve_template_extra_data(template, template_extra=[]): + """Match all occurences of get_file against the stored role-extra data. + + :param template: the given heat template to search for "get_file"(s) + :type template: tuskar.storage.models.StoredFile + + :param template_extra: a list of all stored role-extra data + :type template_extra: list of tuskar.storage.models.StoredFile + + :return: a dict of 'name'=>'path' for each matched role-extra + :rtype: dict + + Using regex, compile a list of all occurences of 'get_file:' in the + template. Match each of the stored role-extra data based on their name. + + For each match capture the full path as it appears in the template + and couple it to the name of the role-extra we have on record. For + example: + + [{'extra_common_yaml': 'hieradata/common.yaml'}, + {'extra_object_yaml': 'hieradata/object.yaml'}] + + """ + included_files = [] + all_get_files = re.findall("get_file:.*\n", template.contents) + # looks like: ["get_file: hieradata/common.yaml}", ... ] + for te in template_extra: + token = resolve_template_file_name_from_role_extra_name(te.name) + for get_file in all_get_files: + if re.match("get_file:.*%s[}]*\n" % token, get_file): + path = get_file.replace("get_file:", "").lstrip().replace( + "}", "").rstrip() + included_files.append({te.name: path}) + return included_files diff --git a/tuskar/manager/plan.py b/tuskar/manager/plan.py index fef1d954..4f131a9a 100644 --- a/tuskar/manager/plan.py +++ b/tuskar/manager/plan.py @@ -13,8 +13,10 @@ import logging from tuskar.common import exception +from tuskar.common import utils from tuskar.manager import models from tuskar.manager import name_utils +from tuskar.manager.role import RoleManager from tuskar.storage.exceptions import UnknownName from tuskar.storage.load_roles import RESOURCE_REGISTRY_NAME from tuskar.storage.load_roles import role_name_from_path @@ -24,6 +26,7 @@ from tuskar.storage.stores import MasterSeedStore from tuskar.storage.stores import MasterTemplateStore from tuskar.storage.stores import ResourceRegistryMappingStore from tuskar.storage.stores import ResourceRegistryStore +from tuskar.storage.stores import TemplateExtraStore from tuskar.storage.stores import TemplateStore from tuskar.templates import composer from tuskar.templates.heat import RegistryEntry @@ -46,6 +49,7 @@ class PlansManager(object): self.registry_store = ResourceRegistryStore() self.registry_mapping_store = ResourceRegistryMappingStore() self.template_store = TemplateStore() + self.template_extra_store = TemplateExtraStore() self.master_template_store = MasterTemplateStore() self.environment_store = EnvironmentFileStore() @@ -367,18 +371,34 @@ class PlansManager(object): } plan_roles = self._find_roles(environment) - + manager = RoleManager() for role in plan_roles: contents = composer.compose_template(role.template) filename = name_utils.role_template_filename(role.name, role.version) files_dict[filename] = contents + def _add_template_extra_data_for(templates, template_store): + template_extra_data = manager.retrieve_db_role_extra() + for template in templates: + db_template = template_store.retrieve_by_name(template.name) + template_extra_paths = utils.resolve_template_extra_data( + db_template, template_extra_data) + extra_data_output = manager.template_extra_data_for_output( + template_extra_paths) + files_dict.update(extra_data_output) + + # also grab any extradata files for the role + _add_template_extra_data_for(plan_roles, self.template_store) + # in addition to provider roles above, return non-role template files reg_mapping = self.registry_mapping_store.list() for entry in reg_mapping: files_dict[entry.name] = entry.contents + # similarly, also grab extradata files for the non role templates + _add_template_extra_data_for(reg_mapping, self.registry_mapping_store) + return files_dict def _find_roles(self, environment): diff --git a/tuskar/manager/role.py b/tuskar/manager/role.py index 619606c4..ebc5b7dc 100644 --- a/tuskar/manager/role.py +++ b/tuskar/manager/role.py @@ -11,6 +11,7 @@ # under the License. from tuskar.manager import models +from tuskar.storage.stores import TemplateExtraStore from tuskar.storage.stores import TemplateStore from tuskar.templates import parser @@ -20,6 +21,7 @@ class RoleManager(object): def __init__(self): super(RoleManager, self).__init__() self.template_store = TemplateStore() + self.template_extra_store = TemplateExtraStore() def list_roles(self, only_latest=False): """Returns a list of all roles known to Tuskar. @@ -46,6 +48,49 @@ class RoleManager(object): role = self._role_to_tuskar_object(db_role) return role + def retrieve_db_role_by_uuid(self, role_uuid): + return self.template_store.retrieve(role_uuid) + + def retrieve_db_role_extra(self): + return self.template_extra_store.list(only_latest=False) + + def template_extra_data_for_output(self, template_extra_paths): + """Compile and return role-extra data for output as a string + + :param template_extra_paths: a list of {k,v} (name=>path) + :type template_extra_paths: list of dict + + :return: a dict of path=>contents + :rtype: dict + + The keys in template_extra_paths correspond to the names of stored + role-extra data and the values are the paths at which the + corresponding files ares expected to be. This list is returned by + common.utils.resolve_template_extra_data for example: + + [{'extra_common_yaml': 'hieradata/common.yaml'}, + {'extra_object_yaml': 'hieradata/object.yaml'}] + + Using this create a new dict that maps the path (values above) as + key to the contents of the corresponding stored role-extra object + (using the name above to retrieve it). For the example input + above, the output would be like: + + { + "hieradata/common.yaml": "CONTENTS", + "hieradata/object.yaml": "CONTENTS" + } + + """ + res = {} + for path in template_extra_paths: + role_extra_name = path.keys()[0] + role_extra_path = path[role_extra_name] + db_role_extra = self.template_extra_store.retrieve_by_name( + role_extra_name) + res[role_extra_path] = db_role_extra.contents + return res + @staticmethod def _role_to_tuskar_object(db_role): parsed = parser.parse_template(db_role.contents) diff --git a/tuskar/tests/test_utils.py b/tuskar/tests/test_utils.py index 5d50ace5..3d6fce32 100644 --- a/tuskar/tests/test_utils.py +++ b/tuskar/tests/test_utils.py @@ -14,6 +14,7 @@ # under the License. from tuskar.common import utils +from tuskar.storage import models from tuskar.tests import base @@ -24,13 +25,42 @@ class CommonUtilsTestCase(base.TestCase): {"/hieradata/config.yaml": "extra_config_yaml"}, {"./name.has.dots": "extra_name.has_dots"}, {"/path/name.": "extra_name_"}, - {"/path/cdefile.c": "extra_cdefile_c"}, ] + {"/path/cdefile.c": "extra_cdefile_c"}, + {"./name_underscore_no_extension": + "extra_name_underscore_no_extension_"}, + {"/path/name_underscore.ext": + "extra_name_underscore_ext"}, ] for params in expected: path = params.keys()[0] res = utils.resolve_role_extra_name_from_path(path) self.assertEqual(params[path], res) + def test_resolve_template_file_name_from_role_extra_name(self): + expected = [{"extra_FOO_": "FOO"}, + {"extra_config_yaml": "config.yaml"}, + {"extra_name.has_dots": "name.has.dots"}, + {"extra_name_": "name"}, + {"extra_cdefile_c": "cdefile.c"}, + {"extra_name_underscore_no_extension_": + "name_underscore_no_extension"}, + {"extra_name_underscore_ext": "name_underscore.ext"}, ] + for params in expected: + name = params.keys()[0] + res = utils.resolve_template_file_name_from_role_extra_name(name) + self.assertEqual(params[name], res) + + def test_resolve_template_extra_data(self): + template_contents = """ Foo Bar Baz + get_file: foo/bar.baz + """ + template_extra = models.StoredFile( + uuid="1234", contents="boo!", store=None, name="extra_bar_baz") + template = models.StoredFile( + uuid="1234", contents=template_contents, store=None) + res = utils.resolve_template_extra_data(template, [template_extra]) + self.assertEqual(res, [{"extra_bar_baz": "foo/bar.baz"}]) + class IntLikeTestCase(base.TestCase):