x
/
tuskar
Code Issues Proposed changes

Adds the /role/role_id/extra_data path to the v2 API 

This uses the new TemplateExtraStore to process stored extra-data
files and match them against the template for each defined role

This information is included when a client requests the existing
/plans/plan_uuid/templates path. To retrieve the extra-data for
any particular role a new /role/role_uuid/extra_data path is added

Change-Id: Ic52280248b0d169f4a3c27e10a826d3dbb2b9bf3
This commit is contained in:
marios 2015-02-24 18:50:29 +02:00
parent 472d69fd96
commit 6e5d0ce968
6 changed files with 211 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
tuskar/api/controllers/v2/roles.py
View File

@ -17,6 +17,7 @@ from wsmeext import pecan as wsme_pecan
from tuskar.api.controllers.v2 import models
from tuskar.common import exception
from tuskar.common import utils
from tuskar.manager.plan import PlansManager
from tuskar.manager.role import RoleManager
from tuskar.storage import exceptions as storage_exceptions
@ -28,6 +29,8 @@ LOG = logging.getLogger(__name__)
class RolesController(rest.RestController):
"""REST controller for the Role class."""
_custom_actions = {'extra_data': ['GET']}
@wsme_pecan.wsexpose([models.Role])
def get_all(self):
"""Returns all roles.
@ -43,6 +46,46 @@ class RolesController(rest.RestController):
transfer_roles = [models.Role.from_tuskar_model(r) for r in all_roles]
return transfer_roles
@wsme_pecan.wsexpose({str: str}, str)
def extra_data(self, role_uuid):
"""Retrieve the extra data files associated with a given role.
:param role_uuid: identifies the role
:type role_uuid: str
:return: a dict where keys are filenames and values are their contents
:rtype: dict
This method will retrieve all stored role_extra records (these are
created at the same time that the Roles are, by using --role-extra
parameter to tuskar-load-roles).
The internal representation for a given role-extra file encodes the
file extension into the name. For instance 'hieradata/compute.yaml'
is stored as 'extra_compute_yaml'.
The given role's template is searched for 'get_file' directives and
then matched against the stored role-extra records (based on their
name... e.g. 'extra_controller_yaml' we look for 'controller.yaml'
after a get_file directive).
This method thus returns all the matched role-extra files for the
given role. The keys will include the relative path if one is
used in the role template:
{
"hieradata/common.yaml": "CONTENTS",
"hieradata/controller.yaml": "CONTENTS",
"hieradata/object.yaml": "CONTENTS"
}
"""
manager = RoleManager()
db_role = manager.retrieve_db_role_by_uuid(role_uuid)
db_role_extra = manager.retrieve_db_role_extra()
role_extra_paths = utils.resolve_template_extra_data(
db_role, db_role_extra)
return manager.template_extra_data_for_output(role_extra_paths)
@wsme_pecan.wsexpose(models.Plan,
str,
body=models.Role,

6
tuskar/common/exception.py
View File

@ -206,3 +206,9 @@ class PlanAlreadyHasRole(DuplicateEntry):
class PlanParametersNotExist(Invalid):
message = _("There are no parameters named %(param_names)s"
" in plan %(plan_uuid)s.")
class InvalidTemplateExtraStoredName(TuskarException):
# code 500 definitely internal server error. Default for TuskarException
message = _("Unexpected name for stored template extra file "
"%(name)s . Expected to start with 'extra_'")

65
tuskar/common/utils.py
View File

@ -19,6 +19,7 @@
"""Utilities and helper functions."""
import os
import re
from oslo.config import cfg
@ -108,3 +109,67 @@ def resolve_role_extra_name_from_path(role_extra_path):
name_ext = os.path.basename(role_extra_path)
name, extension = os.path.splitext(name_ext)
return "extra_%s_%s" % (name, extension.replace('.', ''))
def resolve_template_file_name_from_role_extra_name(role_extra_name):
"""Return the name of the included file based on the role-extra name
The internal representation for a given role-extra file encodes the
file extension into the name. For instance 'compute.yaml'
is stored as 'extra_compute_yaml'. Here, given the stored name,
return name.extension
Raises a InvalidTemplateExtraStoredName exception if the given
role_extra_name doesn't start with 'extra_' as a prefix.
:param role_extra_name: the name as stored for the role-extra
:type role_extra_name: string
:return: the name as used in the template
:rtype: string
Returns 'compute.yaml' from 'extra_compute_yaml'.
"""
if not role_extra_name.startswith("extra_"):
raise exception.InvalidTemplateExtraStoredName(name=role_extra_name)
role_extra_name = role_extra_name[6:]
name_extension = role_extra_name.rsplit("_", 1)
if name_extension[1] == '':
return name_extension[0]
return ".".join(name_extension)
def resolve_template_extra_data(template, template_extra=[]):
"""Match all occurences of get_file against the stored role-extra data.
:param template: the given heat template to search for "get_file"(s)
:type template: tuskar.storage.models.StoredFile
:param template_extra: a list of all stored role-extra data
:type template_extra: list of tuskar.storage.models.StoredFile
:return: a dict of 'name'=>'path' for each matched role-extra
:rtype: dict
Using regex, compile a list of all occurences of 'get_file:' in the
template. Match each of the stored role-extra data based on their name.
For each match capture the full path as it appears in the template
and couple it to the name of the role-extra we have on record. For
example:
[{'extra_common_yaml': 'hieradata/common.yaml'},
{'extra_object_yaml': 'hieradata/object.yaml'}]
"""
included_files = []
all_get_files = re.findall("get_file:.*\n", template.contents)
# looks like: ["get_file: hieradata/common.yaml}", ... ]
for te in template_extra:
token = resolve_template_file_name_from_role_extra_name(te.name)
for get_file in all_get_files:
if re.match("get_file:.*%s[}]*\n" % token, get_file):
path = get_file.replace("get_file:", "").lstrip().replace(
"}", "").rstrip()
included_files.append({te.name: path})
return included_files

22
tuskar/manager/plan.py
View File

@ -13,8 +13,10 @@
import logging
from tuskar.common import exception
from tuskar.common import utils
from tuskar.manager import models
from tuskar.manager import name_utils
from tuskar.manager.role import RoleManager
from tuskar.storage.exceptions import UnknownName
from tuskar.storage.load_roles import RESOURCE_REGISTRY_NAME
from tuskar.storage.load_roles import role_name_from_path
@ -24,6 +26,7 @@ from tuskar.storage.stores import MasterSeedStore
from tuskar.storage.stores import MasterTemplateStore
from tuskar.storage.stores import ResourceRegistryMappingStore
from tuskar.storage.stores import ResourceRegistryStore
from tuskar.storage.stores import TemplateExtraStore
from tuskar.storage.stores import TemplateStore
from tuskar.templates import composer
from tuskar.templates.heat import RegistryEntry
@ -46,6 +49,7 @@ class PlansManager(object):
self.registry_store = ResourceRegistryStore()
self.registry_mapping_store = ResourceRegistryMappingStore()
self.template_store = TemplateStore()
self.template_extra_store = TemplateExtraStore()
self.master_template_store = MasterTemplateStore()
self.environment_store = EnvironmentFileStore()
@ -367,18 +371,34 @@ class PlansManager(object):
}
plan_roles = self._find_roles(environment)
manager = RoleManager()
for role in plan_roles:
contents = composer.compose_template(role.template)
filename = name_utils.role_template_filename(role.name,
role.version)
files_dict[filename] = contents
def _add_template_extra_data_for(templates, template_store):
template_extra_data = manager.retrieve_db_role_extra()
for template in templates:
db_template = template_store.retrieve_by_name(template.name)
template_extra_paths = utils.resolve_template_extra_data(
db_template, template_extra_data)
extra_data_output = manager.template_extra_data_for_output(
template_extra_paths)
files_dict.update(extra_data_output)
# also grab any extradata files for the role
_add_template_extra_data_for(plan_roles, self.template_store)
# in addition to provider roles above, return non-role template files
reg_mapping = self.registry_mapping_store.list()
for entry in reg_mapping:
files_dict[entry.name] = entry.contents
# similarly, also grab extradata files for the non role templates
_add_template_extra_data_for(reg_mapping, self.registry_mapping_store)
return files_dict
def _find_roles(self, environment):

45
tuskar/manager/role.py
View File

@ -11,6 +11,7 @@
# under the License.
from tuskar.manager import models
from tuskar.storage.stores import TemplateExtraStore
from tuskar.storage.stores import TemplateStore
from tuskar.templates import parser
@ -20,6 +21,7 @@ class RoleManager(object):
def __init__(self):
super(RoleManager, self).__init__()
self.template_store = TemplateStore()
self.template_extra_store = TemplateExtraStore()
def list_roles(self, only_latest=False):
"""Returns a list of all roles known to Tuskar.
@ -46,6 +48,49 @@ class RoleManager(object):
role = self._role_to_tuskar_object(db_role)
return role
def retrieve_db_role_by_uuid(self, role_uuid):
return self.template_store.retrieve(role_uuid)
def retrieve_db_role_extra(self):
return self.template_extra_store.list(only_latest=False)
def template_extra_data_for_output(self, template_extra_paths):
"""Compile and return role-extra data for output as a string
:param template_extra_paths: a list of {k,v} (name=>path)
:type template_extra_paths: list of dict
:return: a dict of path=>contents
:rtype: dict
The keys in template_extra_paths correspond to the names of stored
role-extra data and the values are the paths at which the
corresponding files ares expected to be. This list is returned by
common.utils.resolve_template_extra_data for example:
[{'extra_common_yaml': 'hieradata/common.yaml'},
{'extra_object_yaml': 'hieradata/object.yaml'}]
Using this create a new dict that maps the path (values above) as
key to the contents of the corresponding stored role-extra object
(using the name above to retrieve it). For the example input
above, the output would be like:
{
"hieradata/common.yaml": "CONTENTS",
"hieradata/object.yaml": "CONTENTS"
}
"""
res = {}
for path in template_extra_paths:
role_extra_name = path.keys()[0]
role_extra_path = path[role_extra_name]
db_role_extra = self.template_extra_store.retrieve_by_name(
role_extra_name)
res[role_extra_path] = db_role_extra.contents
return res
@staticmethod
def _role_to_tuskar_object(db_role):
parsed = parser.parse_template(db_role.contents)

32
tuskar/tests/test_utils.py
View File

@ -14,6 +14,7 @@
# under the License.
from tuskar.common import utils
from tuskar.storage import models
from tuskar.tests import base
@ -24,13 +25,42 @@ class CommonUtilsTestCase(base.TestCase):
{"/hieradata/config.yaml": "extra_config_yaml"},
{"./name.has.dots": "extra_name.has_dots"},
{"/path/name.": "extra_name_"},
{"/path/cdefile.c": "extra_cdefile_c"}, ]
{"/path/cdefile.c": "extra_cdefile_c"},
{"./name_underscore_no_extension":
"extra_name_underscore_no_extension_"},
{"/path/name_underscore.ext":
"extra_name_underscore_ext"}, ]
for params in expected:
path = params.keys()[0]
res = utils.resolve_role_extra_name_from_path(path)
self.assertEqual(params[path], res)
def test_resolve_template_file_name_from_role_extra_name(self):
expected = [{"extra_FOO_": "FOO"},
{"extra_config_yaml": "config.yaml"},
{"extra_name.has_dots": "name.has.dots"},
{"extra_name_": "name"},
{"extra_cdefile_c": "cdefile.c"},
{"extra_name_underscore_no_extension_":
"name_underscore_no_extension"},
{"extra_name_underscore_ext": "name_underscore.ext"}, ]
for params in expected:
name = params.keys()[0]
res = utils.resolve_template_file_name_from_role_extra_name(name)
self.assertEqual(params[name], res)
def test_resolve_template_extra_data(self):
template_contents = """ Foo Bar Baz
get_file: foo/bar.baz
"""
template_extra = models.StoredFile(
uuid="1234", contents="boo!", store=None, name="extra_bar_baz")
template = models.StoredFile(
uuid="1234", contents=template_contents, store=None)
res = utils.resolve_template_extra_data(template, [template_extra])
self.assertEqual(res, [{"extra_bar_baz": "foo/bar.baz"}])
class IntLikeTestCase(base.TestCase):