openstack
/
murano
Code Issues Proposed changes

Package versioning 

With this change MuranoPackage becomes first-class DSL citizen.
Packages have version, runtime_version (that is specified
in Format attribute of the manifest file) and a list of classes.
Previously engine used to have package loader which had most
of "load" functionality and class loader that mostly acted as an
adapter from package loader to interface that DSL used to
get classes. Now class loader is gone and is replaced with
package loader at the DSL level. Package loader is responsible
for loading packages by either package or class name (as it was
before) plus semantic_version spec (for example ">=1.2,<2.0").
Package loader can now keep track of several versions of the same
package.

Also packages now have requirements with version specs.
All class names that are encountered in application code are
looked up within requirements only. As a consequence
packages that use other packages without referencing
them explicitly will become broken. An exception from this rule
is core library which is referenced automatically.

Partially implements: blueprint murano-versioning

Change-Id: I8789ba45b6210e71bf4977a766f82b66d2a2d270
This commit is contained in:
Stan Lagun 2015-08-26 05:28:39 +03:00 committed by Victor Ryzhenkin
parent 74c30daae9
commit 068831ccd8
43 changed files with 1075 additions and 703 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
murano/cmd/test_runner.py
View File

@ -27,15 +27,14 @@ from oslo_utils import importutils
from murano import version
from murano.common.i18n import _, _LE
from murano.common import config
from murano.common import engine
from murano.dsl import constants
from murano.dsl import exceptions
from murano.dsl import executor
from murano.dsl import helpers
from murano.engine import client_manager
from murano.engine import environment
from murano.engine import package_class_loader
from murano.engine import package_loader
from murano.engine.system import system_objects
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
@ -55,7 +54,13 @@ else:
def _load_package(pkg_loader, name):
try:
package = pkg_loader.get_package(name)
parts = name.rsplit('/')
if len(parts) == 2:
name, pkg_version = parts
version_spec = helpers.parse_version_spec(pkg_version)
else:
version_spec = helpers.parse_version_spec('*')
package = pkg_loader.load_package(name, version_spec)
except exceptions.NoPackageFound:
if not CONF.engine.load_packages_from:
msg = _('Local package is not found since "load-packages-from" '
@ -108,7 +113,7 @@ def _get_methods_to_run(package, tests_to_run, class_to_methods):
return methods_to_run
def _get_all_test_methods(exc, package, class_loader):
def _get_all_test_methods(exc, package):
"""Initiate objects of package classes and get test methods.
Check, if test class and test case name are valid.
@ -120,10 +125,10 @@ def _get_all_test_methods(exc, package, class_loader):
child_context[constants.CTX_ALLOW_PROPERTY_WRITES] = True
for pkg_class_name in package.classes:
class_obj = class_loader.get_class(pkg_class_name)
class_obj = package.find_class(pkg_class_name, False)
obj = class_obj.new(None, exc.object_store, child_context)()
if BASE_CLASS not in [p.name for p in class_obj.parents]:
if not helpers.is_instance_of(obj, BASE_CLASS, '*'):
LOG.debug('Class {0} is not inherited from {1}. '
'Skipping it.'.format(pkg_class_name, BASE_CLASS))
continue
@ -203,17 +208,13 @@ def run_tests(args):
# Replace location of loading packages with provided from command line.
if load_packages_from:
cfg.CONF.engine.load_packages_from = load_packages_from
with package_loader.CombinedPackageLoader(murano_client_factory,
client.tenant_id) as pkg_loader:
class_loader = package_class_loader.PackageClassLoader(pkg_loader)
engine.get_plugin_loader().register_in_loader(class_loader)
system_objects.register(class_loader, pkg_loader)
exc = executor.MuranoDslExecutor(class_loader, test_env)
with package_loader.CombinedPackageLoader(
murano_client_factory, client.tenant_id) as pkg_loader:
# engine.get_plugin_loader().register_in_loader(class_loader)
exc = executor.MuranoDslExecutor(pkg_loader, test_env)
package = _load_package(pkg_loader, provided_pkg_name)
class_to_methods, class_to_obj = _get_all_test_methods(exc,
package,
class_loader)
class_to_methods, class_to_obj = _get_all_test_methods(exc, package)
run_set = _get_methods_to_run(package, tests_to_run, class_to_methods)
if run_set:

20
murano/common/engine.py
View File

@ -29,13 +29,11 @@ from murano.common.helpers import token_sanitizer
from murano.common import plugin_loader
from murano.common import rpc
from murano.dsl import dsl_exception
from murano.dsl import executor as dsl_executor
from murano.dsl import serializer
from murano.engine import environment
from murano.engine import package_class_loader
from murano.engine import executor as engine_executor
from murano.engine import package_loader
from murano.engine.system import status_reporter
import murano.engine.system.system_objects as system_objects
from murano.common.i18n import _LI, _LE, _LW
from murano.policy import model_policy_enforcer as enforcer
@ -149,12 +147,10 @@ class TaskExecutor(object):
return result
def _execute(self, pkg_loader):
class_loader = package_class_loader.PackageClassLoader(pkg_loader)
system_objects.register(class_loader, pkg_loader)
get_plugin_loader().register_in_loader(class_loader)
# get_plugin_loader().register_in_loader(class_loader)
executor = dsl_executor.MuranoDslExecutor(
class_loader, self.environment)
executor = engine_executor.Executor(
pkg_loader, self.environment)
try:
obj = executor.load(self.model)
except Exception as e:
@ -162,7 +158,7 @@ class TaskExecutor(object):
if obj is not None:
try:
self._validate_model(obj.object, self.action, class_loader)
self._validate_model(obj.object, self.action, pkg_loader)
except Exception as e:
return self.exception_result(e, obj, '<validate>')
@ -229,11 +225,11 @@ class TaskExecutor(object):
}
}
def _validate_model(self, obj, action, class_loader):
def _validate_model(self, obj, action, package_loader):
if CONF.engine.enable_model_policy_enforcer:
if action is not None and action['method'] == 'deploy':
self._model_policy_enforcer.validate(obj.to_dictionary(),
class_loader)
self._model_policy_enforcer.validate(
obj.to_dictionary(), package_loader)
def _invoke(self, mpl_executor):
obj = mpl_executor.object_store.get(self.action['object_id'])

127
murano/dsl/class_loader.py
View File

@ -1,127 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import inspect
import types
from murano.dsl import exceptions
from murano.dsl import murano_class
from murano.dsl import murano_object
from murano.dsl import namespace_resolver
from murano.dsl import principal_objects
from murano.dsl import typespec
from murano.dsl import yaql_integration
class MuranoClassLoader(object):
def __init__(self):
self._loaded_types = {}
self._packages_cache = {}
self._imported_types = {object, murano_object.MuranoObject}
principal_objects.register(self)
def _get_package_for_class(self, class_name):
package_name = self.find_package_name(class_name)
if package_name is None:
raise exceptions.NoPackageForClassFound(class_name)
if package_name not in self._packages_cache:
package = self.load_package(package_name)
self._packages_cache[package_name] = package
return self._packages_cache[package_name]
def get_class(self, name, create_missing=False):
if name in self._loaded_types:
return self._loaded_types[name]
try:
data = self.load_definition(name)
package = self._get_package_for_class(name)
except (exceptions.NoPackageForClassFound, exceptions.NoClassFound):
if create_missing:
data = {'Name': name}
package = None
else:
raise
namespaces = data.get('Namespaces') or {}
ns_resolver = namespace_resolver.NamespaceResolver(namespaces)
parent_class_names = data.get('Extends')
parent_classes = []
if parent_class_names:
if not isinstance(parent_class_names, types.ListType):
parent_class_names = [parent_class_names]
for parent_name in parent_class_names:
full_name = ns_resolver.resolve_name(parent_name)
parent_classes.append(self.get_class(full_name))
type_obj = murano_class.MuranoClass(self, ns_resolver, name,
package, parent_classes)
properties = data.get('Properties') or {}
for property_name, property_spec in properties.iteritems():
spec = typespec.PropertySpec(property_spec, type_obj)
type_obj.add_property(property_name, spec)
methods = data.get('Methods') or data.get('Workflow') or {}
method_mappings = {
'initialize': '.init',
'destroy': '.destroy'
}
for method_name, payload in methods.iteritems():
type_obj.add_method(
method_mappings.get(method_name, method_name), payload)
self._loaded_types[name] = type_obj
return type_obj
def load_definition(self, name):
raise NotImplementedError()
def find_package_name(self, class_name):
raise NotImplementedError()
def load_package(self, class_name):
raise NotImplementedError()
def create_root_context(self):
return yaql_integration.create_context()
def get_class_config(self, name):
return {}
def create_local_context(self, parent_context, murano_class):
return parent_context.create_child_context()
def import_class(self, cls, name=None):
if cls in self._imported_types:
return
name = name or getattr(cls, '__murano_name', None) or cls.__name__
m_class = self.get_class(name, create_missing=True)
m_class.extend_with_class(cls)
for method_name in dir(cls):
if method_name.startswith('_'):
continue
method = getattr(cls, method_name)
if not inspect.ismethod(method):
continue
m_class.add_method(
yaql_integration.CONVENTION.convert_function_name(
method_name),
method)
self._imported_types.add(cls)

7
murano/dsl/constants.py
View File

@ -13,20 +13,20 @@
# under the License.
EXPRESSION_MEMORY_QUOTA = 512 * 1024
ITERATORS_LIMIT = 200
ITERATORS_LIMIT = 2000
CTX_ACTIONS_ONLY = '?actionsOnly'
CTX_ALLOW_PROPERTY_WRITES = '$?allowPropertyWrites'
CTX_ARGUMENT_OWNER = '$?argumentOwner'
CTX_ATTRIBUTE_STORE = '$?attributeStore'
CTX_CALLER_CONTEXT = '$?callerContext'
CTX_CLASS_LOADER = '$?classLoader'
CTX_CURRENT_INSTRUCTION = '$?currentInstruction'
CTX_CURRENT_EXCEPTION = '$?currentException'
CTX_CURRENT_METHOD = '$?currentMethod'
CTX_ENVIRONMENT = '$?environment'
CTX_EXECUTOR = '$?executor'
CTX_OBJECT_STORE = '$?objectStore'
CTX_PACKAGE_LOADER = '$?packageLoader'
CTX_SKIP_FRAME = '$?skipFrame'
CTX_THIS = '$?this'
CTX_TYPE = '$?type'
@ -36,3 +36,6 @@ DM_OBJECTS_COPY = 'ObjectsCopy'
DM_ATTRIBUTES = 'Attributes'
META_NO_TRACE = '?noTrace'
CORE_LIBRARY = 'io.murano'
CORE_LIBRARY_OBJECT = 'io.murano.Object'

52
murano/dsl/dsl.py
View File

@ -35,14 +35,15 @@ def name(dsl_name):
return wrapper
class MuranoObjectType(yaqltypes.PythonType):
def __init__(self, murano_class, nullable=False):
class MuranoType(yaqltypes.PythonType):
def __init__(self, murano_class, nullable=False, version_spec=None):
self.murano_class = murano_class
super(MuranoObjectType, self).__init__(
self.version_spec = version_spec
super(MuranoType, self).__init__(
(dsl_types.MuranoObject, MuranoObjectInterface), nullable)
def check(self, value, context, *args, **kwargs):
if not super(MuranoObjectType, self).check(
if not super(MuranoType, self).check(
value, context, *args, **kwargs):
return False
if isinstance(value, MuranoObjectInterface):
@ -51,13 +52,16 @@ class MuranoObjectType(yaqltypes.PythonType):
return True
murano_class = self.murano_class
if isinstance(murano_class, types.StringTypes):
class_loader = helpers.get_class_loader(context)
murano_class = class_loader.get_class(self.murano_class)
return murano_class.is_compatible(value)
murano_class_name = murano_class
else:
murano_class_name = murano_class.name
return helpers.is_instance_of(
value, murano_class_name,
self.version_spec or helpers.get_type(context))
def convert(self, value, sender, context, function_spec, engine,
*args, **kwargs):
result = super(MuranoObjectType, self).convert(
result = super(MuranoType, self).convert(
value, sender, context, function_spec, engine, *args, **kwargs)
if isinstance(result, dsl_types.MuranoObject):
return MuranoObjectInterface(result, engine)
@ -100,11 +104,11 @@ class MuranoTypeName(yaqltypes.LazyParameterType, yaqltypes.PythonType):
value = super(MuranoTypeName, self).convert(
value, sender, context, function_spec, engine)
if isinstance(value, types.StringTypes):
class_loader = helpers.get_class_loader(context)
murano_type = helpers.get_type(context)
value = dsl_types.MuranoClassReference(
class_loader.get_class(
murano_type.namespace_resolver.resolve_name(value)))
helpers.get_class(
murano_type.namespace_resolver.resolve_name(
value), context))
return value
@ -168,6 +172,10 @@ class MuranoObjectInterface(dsl_types.MuranoObjectInterface):
def type(self):
return self.__object.type
@property
def package(self):
return self.type.package
def data(self):
return MuranoObjectInterface.DataInterface(self)
@ -175,9 +183,20 @@ class MuranoObjectInterface(dsl_types.MuranoObjectInterface):
def extension(self):
return self.__object.extension
def cast(self, murano_class):
def cast(self, murano_class, version_spec=None):
return MuranoObjectInterface(
self.__object.cast(murano_class), self.__engine, self.__executor)
helpers.cast(
self.__object, murano_class,
version_spec or helpers.get_type()),
self.__engine, self.__executor)
def is_instance_of(self, murano_class, version_spec=None):
return helpers.is_instance_of(
self.__object, murano_class,
version_spec or helpers.get_type())
def ancestors(self):
return self.type.ancestors()
def __getitem__(self, item):
context = helpers.get_context()
@ -282,11 +301,12 @@ class Interfaces(object):
@property
def class_config(self):
return self.class_loader.get_class_config(self.__object.type.name)
return self.__object.type.package.get_class_config(
self.__object.type.name)
@property
def class_loader(self):
return helpers.get_class_loader()
def package_loader(self):
return helpers.get_package_loader()
class NativeInstruction(object):

6
murano/dsl/exceptions.py
View File

@ -72,6 +72,12 @@ class AmbiguousMethodName(Exception):
'Found more that one method "%s"' % name)
class AmbiguousClassName(Exception):
def __init__(self, name):
super(AmbiguousClassName, self).__init__(
'Found more that one version of class "%s"' % name)
class DslContractSyntaxError(Exception):
pass

27
murano/dsl/executor.py
View File

@ -38,20 +38,19 @@ LOG = logging.getLogger(__name__)
class MuranoDslExecutor(object):
def __init__(self, class_loader, environment=None):
self._class_loader = class_loader
def __init__(self, package_loader, environment=None):
self._package_loader = package_loader
self._attribute_store = attribute_store.AttributeStore()
self._root_context = \
class_loader.create_root_context().create_child_context()
self.create_root_context().create_child_context()
self._root_context[constants.CTX_EXECUTOR] = weakref.proxy(self)
self._root_context[
constants.CTX_CLASS_LOADER] = weakref.proxy(self._class_loader)
constants.CTX_PACKAGE_LOADER] = weakref.proxy(self._package_loader)
self._root_context[constants.CTX_ENVIRONMENT] = environment
self._root_context[constants.CTX_ATTRIBUTE_STORE] = weakref.proxy(
self._attribute_store)
self._object_store = object_store.ObjectStore(self._root_context)
self._locks = {}
yaql_functions.register(self._root_context)
@property
def object_store(self):
@ -62,8 +61,8 @@ class MuranoDslExecutor(object):
return self._attribute_store
@property
def class_loader(self):
return self._class_loader
def package_loader(self):
return self._package_loader
def invoke_method(self, method, this, context, args, kwargs,
skip_stub=False):
@ -180,7 +179,7 @@ class MuranoDslExecutor(object):
def _create_method_context(self, this, method, context=None,
actions_only=False, skip_frame=False):
new_context = self._class_loader.create_local_context(
new_context = self.create_local_context(
parent_context=this.context,
murano_class=this.type)
caller = context
@ -220,7 +219,7 @@ class MuranoDslExecutor(object):
objects_to_clean.append(obj)
if objects_to_clean:
for obj in objects_to_clean:
methods = obj.type.find_all_methods('.destroy')
methods = obj.type.find_methods(lambda m: m.name == '.destroy')
for method in methods:
try:
method.invoke(self, obj, (), {}, None)
@ -244,3 +243,13 @@ class MuranoDslExecutor(object):
for val in data:
for res in self._list_potential_object_ids(val):
yield res
# noinspection PyMethodMayBeStatic
def create_local_context(self, parent_context, murano_class):
return parent_context.create_child_context()
# noinspection PyMethodMayBeStatic
def create_root_context(self):
context = yaql_integration.create_context()
yaql_functions.register(context)
return context

102
murano/dsl/helpers.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2014 #Mirantis, Inc.
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -12,14 +12,17 @@
# License for the specific language governing permissions and limitations
# under the License.
import collections
import contextlib
import re
import string
import sys
import types
import uuid
import eventlet.greenpool
import eventlet.greenthread
import semantic_version
import yaql.language.exceptions
import yaql.language.expressions
from yaql.language import utils as yaqlutils
@ -28,6 +31,7 @@ from yaql.language import utils as yaqlutils
from murano.common import utils
from murano.dsl import constants
from murano.dsl import dsl_types
from murano.dsl import exceptions
KEYWORD_REGEX = re.compile(r'(?!__)\b[^\W\d]\w*\b')
@ -134,11 +138,6 @@ def get_executor(context=None):
return context[constants.CTX_EXECUTOR]
def get_class_loader(context=None):
context = context or get_context()
return context[constants.CTX_CLASS_LOADER]
def get_type(context=None):
context = context or get_context()
return context[constants.CTX_TYPE]
@ -154,6 +153,11 @@ def get_object_store(context=None):
return context[constants.CTX_OBJECT_STORE]
def get_package_loader(context=None):
context = context or get_context()
return context[constants.CTX_PACKAGE_LOADER]
def get_this(context=None):
context = context or get_context()
return context[constants.CTX_THIS]
@ -189,6 +193,12 @@ def are_property_modifications_allowed(context=None):
return context[constants.CTX_ALLOW_PROPERTY_WRITES] or False
def get_class(name, context=None):
context = context or get_context()
murano_class = get_type(context)
return murano_class.package.find_class(name)
def is_keyword(text):
return KEYWORD_REGEX.match(text) is not None
@ -218,3 +228,83 @@ def contextual(ctx):
setattr(current_thread, '__murano_context', current_context)
elif hasattr(current_thread, '__murano_context'):
delattr(current_thread, '__murano_context')
def parse_version_spec(version_spec):
if isinstance(version_spec, semantic_version.Spec):
return version_spec
if isinstance(version_spec, semantic_version.Version):
return semantic_version.Spec('==' + str(version_spec))
if not version_spec:
version_spec = '0'
version_spec = str(version_spec).translate(None, string.whitespace)
if version_spec[0].isdigit():
version_spec = '==' + str(version_spec)
version_spec = semantic_version.Spec(version_spec)
return version_spec
def traverse(seed, producer=None, track_visited=True):
if not yaqlutils.is_iterable(seed):
seed = [seed]
visited = None if not track_visited else set()
queue = collections.deque(seed)
while queue:
item = queue.popleft()
if track_visited:
if item in visited:
continue
visited.add(item)
produced = (yield item)
if produced is None and producer:
produced = producer(item)
if produced:
queue.extend(produced)
def cast(obj, murano_class, pov_or_version_spec=None):
if isinstance(obj, dsl_types.MuranoObjectInterface):
obj = obj.object
if isinstance(pov_or_version_spec, dsl_types.MuranoClass):
pov_or_version_spec = pov_or_version_spec.package
elif isinstance(pov_or_version_spec, types.StringTypes):
pov_or_version_spec = parse_version_spec(pov_or_version_spec)
if isinstance(murano_class, dsl_types.MuranoClass):
if pov_or_version_spec is None:
pov_or_version_spec = parse_version_spec(murano_class.version)
murano_class = murano_class.name
candidates = []
for cls in obj.type.ancestors():
if cls.name != murano_class:
continue
elif isinstance(pov_or_version_spec, semantic_version.Version):
if cls.version != pov_or_version_spec:
continue
elif isinstance(pov_or_version_spec, semantic_version.Spec):
if cls.version not in pov_or_version_spec:
continue
elif isinstance(pov_or_version_spec, dsl_types.MuranoPackage):
requirement = pov_or_version_spec.requirements.get(
cls.package.name)
if requirement is None:
raise exceptions.NoClassFound(murano_class)
if cls.version not in requirement:
continue
elif pov_or_version_spec is not None:
raise ValueError('pov_or_version_spec of unsupported '
'type {0}'.format(type(pov_or_version_spec)))
candidates.append(cls)
if not candidates:
raise exceptions.NoClassFound(murano_class)
elif len(candidates) > 1:
raise exceptions.AmbiguousClassName(murano_class)
return obj.cast(candidates[0])
def is_instance_of(obj, class_name, pov_or_version_spec=None):
try:
cast(obj, class_name, pov_or_version_spec)
return True
except (exceptions.NoClassFound, exceptions.AmbiguousClassName):
return False

209
murano/dsl/murano_class.py
View File

@ -13,37 +13,37 @@
# under the License.
import collections
import weakref
import semantic_version
from murano.dsl import constants
from murano.dsl import dsl
from murano.dsl import dsl_types
from murano.dsl import exceptions
from murano.dsl import helpers
from murano.dsl import murano_method
from murano.dsl import murano_object
from murano.dsl import typespec
from murano.dsl import yaql_integration
class GeneratedNativeTypeMetaClass(type):
def __str__(cls):
return cls.__name__
class MuranoClass(dsl_types.MuranoClass):
def __init__(self, class_loader, namespace_resolver, name, package,
def __init__(self, namespace_resolver, name, package,
parents=None):
self._package = package
self._class_loader = class_loader
self._package = weakref.ref(package)
self._methods = {}
self._namespace_resolver = namespace_resolver
self._name = namespace_resolver.resolve_name(name)
self._name = name
self._properties = {}
self._config = {}
if self._name == 'io.murano.Object':
if self._name == constants.CORE_LIBRARY_OBJECT:
self._parents = []
else:
self._parents = parents or [
class_loader.get_class('io.murano.Object')]
package.find_class(constants.CORE_LIBRARY_OBJECT)]
self._unique_methods = None
self._parent_mappings = self._build_parent_remappings()
@property
def name(self):
@ -51,20 +51,24 @@ class MuranoClass(dsl_types.MuranoClass):
@property
def package(self):
return self._package
return self._package()
@property
def namespace_resolver(self):
return self._namespace_resolver
@property
def parents(self):
def declared_parents(self):
return self._parents
@property
def methods(self):
return self._methods
@property
def parent_mappings(self):
return self._parent_mappings
def extend_with_class(self, cls):
ctor = yaql_integration.get_class_factory_definition(cls)
self.add_method('__init__', ctor)
@ -102,29 +106,24 @@ class MuranoClass(dsl_types.MuranoClass):
def get_property(self, name):
return self._properties[name]
def _find_method_chains(self, name):
initial = [self.methods[name]] if name in self.methods else []
yielded = False
for parent in self.parents:
for seq in parent._find_method_chains(name):
yield initial + list(seq)
yielded = True
if initial and not yielded:
yield initial
def find_method(self, name):
if name in self._methods:
return [(self, name)]
if not self._parents:
return []
return list(set(reduce(
lambda x, y: x + y,
[p.find_method(name) for p in self._parents])))
def _find_method_chains(self, name, origin):
queue = collections.deque([(self, ())])
while queue:
cls, path = queue.popleft()
segment = (cls.methods[name],) if name in cls.methods else ()
leaf = True
for p in cls.parents(origin):
leaf = False
queue.append((p, path + segment))
if leaf:
path = path + segment
if path:
yield path
def find_single_method(self, name):
chains = sorted(self._find_method_chains(name), key=lambda t: len(t))
chains = sorted(self._find_method_chains(name, self),
key=lambda t: len(t))
result = []
for i in range(len(chains)):
if chains[i][0] in result:
continue
@ -149,38 +148,45 @@ class MuranoClass(dsl_types.MuranoClass):
raise exceptions.AmbiguousMethodName(name)
return result[0]
def find_all_methods(self, name):
def find_methods(self, predicate):
result = []
queue = collections.deque([self])
while queue:
c = queue.popleft()
if name in c.methods:
method = c.methods[name]
if method not in result:
for c in self.ancestors():
for method in c.methods.itervalues():
if predicate(method) and method not in result:
result.append(method)
queue.extend(c.parents)
return result
def _iterate_unique_methods(self):
names = set()
queue = collections.deque([self])
while queue:
c = queue.popleft()
for c in self.ancestors():
names.update(c.methods.keys())
queue.extend(c.parents)
for name in names:
yield self.find_single_method(name)
def find_property(self, name):
result = []
types = collections.deque([self])
while len(types) > 0:
mc = types.popleft()
for mc in self.ancestors():
if name in mc.properties and mc not in result:
result.append(mc)
types.extend(mc.parents)
return result
def find_single_property(self, name):
result = None
parents = None
gen = helpers.traverse(self)
while True:
try:
mc = gen.send(parents)
if name in mc.properties:
if result and result != mc:
raise exceptions.AmbiguousPropertyNameError(name)
result = mc
parents = []
else:
parents = mc.parents(self)
except StopIteration:
return result
def invoke(self, name, executor, this, args, kwargs, context=None):
method = self.find_single_method(name)
return method.invoke(executor, this, args, kwargs, context)
@ -188,13 +194,8 @@ class MuranoClass(dsl_types.MuranoClass):
def is_compatible(self, obj):
if isinstance(obj, (murano_object.MuranoObject,
dsl.MuranoObjectInterface)):
return self.is_compatible(obj.type)
if obj is self:
return True
for parent in obj.parents:
if self.is_compatible(parent):
return True
return False
obj = obj.type
return any(cls is self for cls in obj.ancestors())
def new(self, owner, object_store, context=None, **kwargs):
if context is None:
@ -204,12 +205,102 @@ class MuranoClass(dsl_types.MuranoClass):
def initializer(**params):
init_context = context.create_child_context()
init_context['?allowPropertyWrites'] = True
init_context[constants.CTX_ALLOW_PROPERTY_WRITES] = True
obj.initialize(init_context, object_store, params)
return obj
initializer.object = obj
return initializer
def __str__(self):
return 'MuranoClass({0})'.format(self.name)
def __repr__(self):
return 'MuranoClass({0}/{1})'.format(self.name, self.version)
@property
def version(self):
return self.package.version
def _build_parent_remappings(self):
"""Remaps class parents.
In case of multiple inheritance class may indirectly get several
versions of the same class. It is reasonable to try to replace them
with single version to avoid conflicts. We can do that when within
versions that satisfy our class package requirements.
But in order to merge several classes that are not our parents but
grand parents we will need to modify classes that may be used
somewhere else (with another set of requirements). We cannot do this.
So instead we build translation table that will tell which ancestor
class need to be replaced with which so that we minimize number of
versions used for single class (or technically packages since version
is a package attribute). For translation table to work there should
be a method that returns all class virtual ancestors so that everybody
will see them instead of accessing class parents directly and getting
declared ancestors.
"""
result = {}
aggregation = {
self.package.name: {(
self.package,
semantic_version.Spec('==' + str(self.package.version))
)}
}
for cls, parent in helpers.traverse(
((self, parent) for parent in self._parents),
lambda (c, p): ((p, anc) for anc in p.declared_parents)):
if cls.package != parent.package:
requirement = cls.package.requirements[parent.package.name]
aggregation.setdefault(parent.package.name, set()).add(
(parent.package, requirement))
package_bindings = {}
for versions in aggregation.itervalues():
mappings = self._remap_package(versions)
package_bindings.update(mappings)
for cls in helpers.traverse(
self.declared_parents, lambda c: c.declared_parents):
if cls.package in package_bindings:
package2 = package_bindings[cls.package]
cls2 = package2.classes[cls.name]
result[cls] = cls2
return result
@staticmethod
def _remap_package(versions):
result = {}
reverse_mappings = {}
versions_list = sorted(versions, key=lambda x: x[0].version)
i = 0
while i < len(versions_list):
package1, requirement1 = versions_list[i]
dst_package = None
for j, (package2, requirement2) in enumerate(versions_list):
if i == j:
continue
if package2.version in requirement1 and (
dst_package is None or
dst_package.version < package2.version):
dst_package = package2
if dst_package:
result[package1] = dst_package
reverse_mappings.setdefault(dst_package, []).append(package1)
for package in reverse_mappings.get(package1, []):
result[package] = dst_package
del versions_list[i]
else:
i += 1
return result
def parents(self, origin):
mappings = origin.parent_mappings
yielded = set()
for p in self._parents:
parent = mappings.get(p, p)
if parent not in yielded:
yielded.add(parent)
yield parent
def ancestors(self):
for c in helpers.traverse(self, lambda t: t.parents(self)):
yield c

34
murano/dsl/murano_object.py
View File

@ -37,12 +37,12 @@ class MuranoObject(dsl_types.MuranoObject):
self.__extension = None
self.__context = self.__setup_context(context)
object_store = helpers.get_object_store(context)
self.__config = object_store.class_loader.get_class_config(
self.__config = murano_class.package.get_class_config(
murano_class.name)
if not isinstance(self.__config, dict):
self.__config = {}
known_classes[murano_class.name] = self
for parent_class in murano_class.parents:
for parent_class in murano_class.parents(self.real_this.type):
name = parent_class.name
if name not in known_classes:
obj = parent_class.new(
@ -182,11 +182,9 @@ class MuranoObject(dsl_types.MuranoObject):
if name in start_type.properties:
return self.cast(start_type)._get_property_value(name)
else:
declared_properties = start_type.find_property(name)
if len(declared_properties) == 1:
return self.cast(declared_properties[0]).__properties[name]
elif len(declared_properties) > 1:
raise exceptions.AmbiguousPropertyNameError(name)
declared_properties = start_type.find_single_property(name)
if declared_properties:
return self.cast(declared_properties).__properties[name]
elif derived:
return self.cast(caller_class)._get_property_value(name)
else:
@ -232,19 +230,15 @@ class MuranoObject(dsl_types.MuranoObject):
else:
raise exceptions.PropertyWriteError(name, start_type)
def cast(self, type):
if self.type is type:
return self
for parent in self.__parents.values():
try:
return parent.cast(type)
except TypeError:
continue
raise TypeError('Cannot cast {0} to {1}'.format(self, type))
def cast(self, cls):
for p in helpers.traverse(self, lambda t: t.__parents.values()):
if p.type is cls:
return p
raise TypeError('Cannot cast {0} to {1}'.format(self.type, cls))
def __repr__(self):
return '<{0} {1} ({2})>'.format(
self.type.name, self.object_id, id(self))
return '<{0}/{1} {2} ({3})>'.format(
self.type.name, self.type.version, self.object_id, id(self))
def to_dictionary(self, include_hidden=False):
result = {}
@ -253,7 +247,9 @@ class MuranoObject(dsl_types.MuranoObject):
result.update({'?': {
'type': self.type.name,
'id': self.object_id,
'name': self.name
'name': self.name,
'classVersion': str(self.type.version),
'package': self.type.package.name
}})
if include_hidden:
result.update(self.__properties)

173
murano/dsl/murano_package.py
View File

@ -12,18 +12,181 @@
# License for the specific language governing permissions and limitations
# under the License.
import inspect
import weakref
import semantic_version
from yaql.language import utils
from murano.dsl import constants
from murano.dsl import dsl_types
from murano.dsl import exceptions
from murano.dsl import helpers
from murano.dsl import murano_class
from murano.dsl import murano_object
from murano.dsl import namespace_resolver
from murano.dsl import principal_objects
from murano.dsl import typespec
from murano.dsl import yaql_integration
class MuranoPackage(dsl_types.MuranoPackage):
def __init__(self):
def __init__(self, package_loader, name, version=None,
runtime_version=None, requirements=None):
super(MuranoPackage, self).__init__()
self._name = None
self._package_loader = weakref.proxy(package_loader)
self._name = name
self._version = self._parse_version(version)
self._runtime_version = self._parse_version(runtime_version)
self._requirements = {
name: semantic_version.Spec('==' + str(self._version.major))
}
if name != constants.CORE_LIBRARY:
self._requirements[constants.CORE_LIBRARY] = \
semantic_version.Spec('==0')
self._classes = {}
self._imported_types = {object, murano_object.MuranoObject}
for key, value in (requirements or {}).iteritems():
self._requirements[key] = helpers.parse_version_spec(value)
self._load_queue = {}
self._native_load_queue = {}
if self.name == constants.CORE_LIBRARY:
principal_objects.register(self)
@property
def package_loader(self):
return self._package_loader
@property
def name(self):
return self._name
@name.setter
def name(self, value):
self._name = value
@property
def version(self):
return self._version
@property
def runtime_version(self):
return self._runtime_version
@property
def requirements(self):
return self._requirements
@property
def classes(self):
return set(self._classes.keys() +
self._load_queue.keys() +
self._native_load_queue.keys())
def get_resource(self, name):
raise NotImplementedError('resource API is not implemented')
# noinspection PyMethodMayBeStatic
def get_class_config(self, name):
return {}
def _register_mpl_class(self, data, name=None):
if name in self._classes:
return self._classes[name]
namespaces = data.get('Namespaces') or {}
ns_resolver = namespace_resolver.NamespaceResolver(namespaces)
parent_class_names = data.get('Extends')
parent_classes = []
if parent_class_names:
if not utils.is_sequence(parent_class_names):
parent_class_names = [parent_class_names]
for parent_name in parent_class_names:
full_name = ns_resolver.resolve_name(parent_name)
parent_classes.append(self.find_class(full_name))
type_obj = murano_class.MuranoClass(
ns_resolver, name, self, parent_classes)
properties = data.get('Properties') or {}
for property_name, property_spec in properties.iteritems():
spec = typespec.PropertySpec(property_spec, type_obj)
type_obj.add_property(property_name, spec)
methods = data.get('Methods') or data.get('Workflow') or {}
method_mappings = {
'initialize': '.init',
'destroy': '.destroy'
}
for method_name, payload in methods.iteritems():
type_obj.add_method(
method_mappings.get(method_name, method_name), payload)
self._classes[name] = type_obj
return type_obj
def _register_native_class(self, cls, name):
if cls in self._imported_types:
return self._classes[name]
try:
m_class = self.find_class(name, False)
except exceptions.NoClassFound:
m_class = self._register_mpl_class({'Name': name}, name)
m_class.extend_with_class(cls)
for method_name in dir(cls):
if method_name.startswith('_'):
continue
method = getattr(cls, method_name)
if not inspect.ismethod(method):
continue
m_class.add_method(
yaql_integration.CONVENTION.convert_function_name(
method_name),
method)
self._imported_types.add(cls)
return m_class
def register_class(self, cls, name=None):
if inspect.isclass(cls):
name = name or getattr(cls, '__murano_name', None) or cls.__name__
self._native_load_queue[name] = cls
else:
self._load_queue[name] = cls
def find_class(self, name, search_requirements=True):
payload = self._native_load_queue.pop(name, None)
if payload is not None:
return self._register_native_class(payload, name)
payload = self._load_queue.pop(name, None)
if payload is not None:
if callable(payload):
payload = payload()
return self._register_mpl_class(payload, name)
result = self._classes.get(name)
if result:
return result
if search_requirements:
for package_name, version_spec in self._requirements.iteritems():
if package_name == self.name:
continue
referenced_package = self._package_loader.load_package(
package_name, version_spec)
try:
return referenced_package.find_class(name, False)
except exceptions.NoClassFound:
continue
raise exceptions.NoClassFound(name)
@staticmethod
def _parse_version(version):
if isinstance(version, semantic_version.Version):
return version
if not version:
version = '0'
return semantic_version.Version.coerce(str(version))

19
murano/dsl/object_store.py
View File

@ -20,7 +20,7 @@ from murano.dsl import helpers
class ObjectStore(object):
def __init__(self, context, parent_store=None):
self._context = context.create_child_context()
self._class_loader = helpers.get_class_loader(context)
self._package_loader = helpers.get_package_loader(context)
self._context[constants.CTX_OBJECT_STORE] = self
self._parent_store = parent_store
self._store = {}
@ -31,10 +31,6 @@ class ObjectStore(object):
def initializing(self):
return self._initializing
@property
def class_loader(self):
return self._class_loader
@property
def context(self):
return self._context
@ -63,9 +59,16 @@ class ObjectStore(object):
system_key = value['?']
object_id = system_key['id']
obj_type = system_key['type']
class_obj = self._class_loader.get_class(obj_type)
if not class_obj:
raise ValueError()
version_spec = helpers.parse_version_spec(
system_key.get('classVersion'))
if 'package' not in system_key:
package = self._package_loader.load_class_package(
obj_type, version_spec)
else:
package = self._package_loader.load_package(
system_key['package'], version_spec)
class_obj = package.find_class(obj_type, False)
try:
if owner is None:

32
murano/dsl/package_loader.py Normal file
View File

@ -0,0 +1,32 @@
# Copyright (c) 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import abc
import six
@six.add_metaclass(abc.ABCMeta)
class MuranoPackageLoader(object):
@abc.abstractmethod
def load_package(self, package_name, version_spec):
pass
@abc.abstractmethod
def load_class_package(self, class_name, version_spec):
pass
@abc.abstractmethod
def register_package(self, package):
pass

19
murano/dsl/principal_objects/__init__.py
View File

@ -12,17 +12,12 @@
# License for the specific language governing permissions and limitations
# under the License.
import murano.dsl.principal_objects.exception
import murano.dsl.principal_objects.stack_trace
import murano.dsl.principal_objects.sys_object
from murano.dsl.principal_objects import exception
from murano.dsl.principal_objects import stack_trace
from murano.dsl.principal_objects import sys_object
def register(class_loader):
sys_object = murano.dsl.principal_objects.sys_object
class_loader.import_class(sys_object.SysObject)
stack_trace = murano.dsl.principal_objects.stack_trace
class_loader.import_class(stack_trace.StackTrace)
exception = murano.dsl.principal_objects.exception
class_loader.import_class(exception.DslException)
def register(package):
package.register_class(sys_object.SysObject)
package.register_class(stack_trace.StackTrace)
package.register_class(exception.DslException)

37
murano/dsl/serializer.py
View File

@ -18,7 +18,6 @@ from yaql import utils
from murano.dsl import dsl
from murano.dsl import dsl_types
from murano.dsl import helpers
from murano.dsl import murano_method
@ -71,28 +70,15 @@ def serialize_model(root_object, executor, allow_refs=False):
}
def _serialize_available_action(obj):
def _serialize(obj_type):
actions = {}
for name, method in obj_type.methods.iteritems():
if method.usage == murano_method.MethodUsages.Action:
action_id = '{0}_{1}'.format(obj.object_id, name)
actions[action_id] = {
'name': name,
'enabled': True
}
for parent in obj_type.parents:
parent_actions = _serialize(parent)
actions = helpers.merge_dicts(parent_actions, actions)
return actions
return _serialize(obj.type)
def _merge_actions(dict1, dict2):
result = helpers.merge_dicts(dict1, dict2)
for action_id in dict1:
if action_id not in dict2:
del result[action_id]
def _serialize_available_action(obj, current_actions):
result = {}
actions = obj.type.find_methods(
lambda m: m.usage == murano_method.MethodUsages.Action)
for action in actions:
action_id = '{0}_{1}'.format(obj.object_id, action.name)
entry = current_actions.get(action_id, {'enabled': True})
entry['name'] = action.name
result[action_id] = entry
return result
@ -117,9 +103,8 @@ def _pass12_serialize(value, parent, serialized_objects,
if designer_attributes_getter is not None:
result['?'].update(designer_attributes_getter(value.object_id))
# deserialize and merge list of actions
actions = _serialize_available_action(value)
result['?']['_actions'] = _merge_actions(
result['?'].get('_actions', {}), actions)
result['?']['_actions'] = _serialize_available_action(
value, result['?'].get('_actions', {}))
serialized_objects.add(value.object_id)
return _pass12_serialize(
result, value, serialized_objects, designer_attributes_getter)

13
murano/dsl/type_scheme.py
View File

@ -148,15 +148,13 @@ class TypeScheme(object):
@specs.parameter('default_name', dsl.MuranoTypeName(
True, root_context))
@specs.parameter('value', nullable=True)
@specs.parameter('version_spec', yaqltypes.String(True))
@specs.method
def class_(value, name, default_name=None):
def class_(value, name, default_name=None, version_spec=None):
object_store = helpers.get_object_store(root_context)
if not default_name:
default_name = name
murano_class = name.murano_class
if not murano_class:
raise exceptions.NoClassFound(
'Class {0} cannot be found'.format(name))
if value is None:
return None
if isinstance(value, dsl_types.MuranoObject):
@ -167,7 +165,8 @@ class TypeScheme(object):
if '?' not in value:
new_value = {'?': {
'id': uuid.uuid4().hex,
'type': default_name.murano_class.name
'type': default_name.murano_class.name,
'classVersion': str(default_name.murano_class.version)
}}
new_value.update(value)
value = new_value
@ -184,7 +183,9 @@ class TypeScheme(object):
raise exceptions.ContractViolationException(
'Value {0} cannot be represented as class {1}'.format(
value, name))
if not murano_class.is_compatible(obj):
if not helpers.is_instance_of(
obj, murano_class.name,
version_spec or helpers.get_type(root_context)):
raise exceptions.ContractViolationException(
'Object of type {0} is not compatible with '
'requested type {1}'.format(obj.type.name, name))

23
murano/dsl/yaql_functions.py
View File

@ -32,10 +32,13 @@ def id_(value):
@specs.parameter('value', dsl_types.MuranoObject)
@specs.parameter('type', dsl.MuranoTypeName())
@specs.parameter('type__', dsl.MuranoTypeName())
@specs.parameter('version_spec', yaqltypes.String(True))
@specs.extension_method
def cast(value, type):
return value.cast(type.murano_class)
def cast(context, value, type__, version_spec=None):
return helpers.cast(
value, type__.murano_class.name,
version_spec or helpers.get_type(context))
@specs.parameter('__type_name', dsl.MuranoTypeName())
@ -65,14 +68,15 @@ def new_from_dict(type_name, context, parameters,
**yaql_integration.filter_parameters_dict(parameters))
@specs.parameter('value', dsl_types.MuranoObject)
@specs.parameter('sender', dsl_types.MuranoObject)
@specs.parameter('func', yaqltypes.Lambda())
@specs.extension_method
def super_(context, value, func=None):
def super_(context, sender, func=None):
cast_type = helpers.get_type(context)
if func is None:
return [value.cast(type) for type in cast_type.parents]
return itertools.imap(func, super_(context, value))
return [sender.cast(type) for type in cast_type.parents(
sender.real_this.type)]
return itertools.imap(func, super_(context, sender))
@specs.parameter('value', dsl_types.MuranoObject)
@ -141,11 +145,10 @@ def op_dot(context, sender, expr, operator):
@specs.name('#operator_:')
def ns_resolve(context, prefix, name):
murano_type = helpers.get_type(context)
class_loader = helpers.get_class_loader(context)
return dsl_types.MuranoClassReference(
class_loader.get_class(
helpers.get_class(
murano_type.namespace_resolver.resolve_name(
prefix + ':' + name)))
prefix + ':' + name), context))
@specs.parameter('obj1', dsl_types.MuranoObject, nullable=True)

20
murano/dsl/yaql_integration.py
View File

@ -145,31 +145,23 @@ def build_wrapper_function_definition(murano_method):
def _build_native_wrapper_function_definition(murano_method):
@specs.method
@specs.name(murano_method.name)
def payload(__context, __sender, *args, **kwargs):
executor = helpers.get_executor(__context)
args = tuple(to_mutable(arg) for arg in args)
kwargs = to_mutable(kwargs)
return murano_method.invoke(
executor, __sender, args[1:], kwargs, __context, True)
executor, __sender, args, kwargs, __context, True)
fd = murano_method.body.strip_hidden_parameters()
fd.payload = payload
for v in fd.parameters.itervalues():
if v.position == 0:
v.value_type = yaqltypes.PythonType(dsl_types.MuranoObject, False)
break
fd.insert_parameter(specs.ParameterDefinition(
'?1', yaqltypes.Context(), 0))
fd.insert_parameter(specs.ParameterDefinition(
'?2', yaqltypes.Sender(), 1))
return fd
return specs.get_function_definition(payload)
def _build_mpl_wrapper_function_definition(murano_method):
def payload(__context, __sender, *args, **kwargs):
executor = helpers.get_executor(__context)
return murano_method.invoke(
executor, __sender.object, args, kwargs, __context, True)
executor, __sender, args, kwargs, __context, True)
fd = specs.FunctionDefinition(
murano_method.name, payload, is_function=False, is_method=True)
@ -185,7 +177,7 @@ def _build_mpl_wrapper_function_definition(murano_method):
'__context', yaqltypes.Context(), 0))
fd.set_parameter(specs.ParameterDefinition(
'__sender', dsl.MuranoObjectType(murano_method.murano_class), 1))
'__sender', yaqltypes.PythonType(dsl_types.MuranoObject, False), 1))
return fd

27
murano/engine/executor.py Normal file
View File

@ -0,0 +1,27 @@
# Copyright (c) 2015 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from murano.dsl import executor
from murano.engine.system import yaql_functions
class Executor(executor.MuranoDslExecutor):
def __init__(self, package_loader, env):
super(Executor, self).__init__(package_loader, env)
def create_root_context(self):
context = super(Executor, self).create_root_context()
yaql_functions.register(context)
return context

52
murano/engine/murano_package.py Normal file
View File

@ -0,0 +1,52 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
from oslo_config import cfg
from oslo_log import log as logging
import yaml
from murano.dsl import murano_package
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class MuranoPackage(murano_package.MuranoPackage):
def __init__(self, package_loader, application_package):
self.application_package = application_package
super(MuranoPackage, self).__init__(
package_loader,
application_package.full_name,
application_package.version,
application_package.runtime_version,
application_package.requirements
)
def get_class_config(self, name):
json_config = os.path.join(CONF.engine.class_configs, name + '.json')
if os.path.exists(json_config):
with open(json_config) as f:
return json.load(f)
yaml_config = os.path.join(CONF.engine.class_configs, name + '.yaml')
if os.path.exists(yaml_config):
with open(yaml_config) as f:
return yaml.safe_load(f)
return {}
def get_resource(self, name):
return self.application_package.get_resource(name)

86
murano/engine/package_class_loader.py
View File

@ -1,86 +0,0 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os.path
import sys
from oslo_config import cfg
from oslo_log import log as logging
import yaml
from murano.dsl import class_loader
from murano.dsl import exceptions
from murano.dsl import murano_package
from murano.engine.system import yaql_functions
from murano.packages import exceptions as pkg_exceptions
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class PackageClassLoader(class_loader.MuranoClassLoader):
def __init__(self, package_loader):
self.package_loader = package_loader
self._class_packages = {}
super(PackageClassLoader, self).__init__()
def _get_package_for(self, class_name):
package = self._class_packages.get(class_name, None)
if package is None:
package = self.package_loader.get_package_by_class(class_name)
if package is not None:
for cn in package.classes:
self._class_packages[cn] = package
return package
def load_definition(self, name):
try:
package = self._get_package_for(name)
if package is None:
raise exceptions.NoPackageForClassFound(name)
return package.get_class(name)
# (sjmc7) This is used as a control condition for system classes;
# do not delete (although I think it needs a better solution)
except exceptions.NoPackageForClassFound:
raise
except Exception as e:
msg = "Error loading {0}: {1}".format(name, str(e))
raise pkg_exceptions.PackageLoadError(msg), None, sys.exc_info()[2]
def load_package(self, name):
package = murano_package.MuranoPackage()
package.name = name
return package
def find_package_name(self, class_name):
app_pkg = self._get_package_for(class_name)
return None if app_pkg is None else app_pkg.full_name
def create_root_context(self):
context = super(PackageClassLoader, self).create_root_context()
yaql_functions.register(context)
return context
def get_class_config(self, name):
json_config = os.path.join(CONF.engine.class_configs, name + '.json')
if os.path.exists(json_config):
with open(json_config) as f:
return json.load(f)
yaml_config = os.path.join(CONF.engine.class_configs, name + '.yaml')
if os.path.exists(yaml_config):
with open(yaml_config) as f:
return yaml.safe_load(f)
return {}

230
murano/engine/package_loader.py
View File

@ -13,8 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
import os
import os.path
import shutil
import sys
import tempfile
@ -23,10 +23,13 @@ import uuid
from muranoclient.common import exceptions as muranoclient_exc
from oslo_config import cfg
from oslo_log import log as logging
import six
from murano.common.i18n import _LE, _LI
from murano.dsl import constants
from murano.dsl import exceptions
from murano.dsl import package_loader
from murano.engine import murano_package
from murano.engine.system import system_objects
from murano.engine import yaql_yaml_loader
from murano.packages import exceptions as pkg_exc
from murano.packages import load_utils
@ -35,39 +38,53 @@ CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class PackageLoader(six.with_metaclass(abc.ABCMeta)):
@abc.abstractmethod
def get_package(self, name):
pass
@abc.abstractmethod
def get_package_by_class(self, name):
pass
class ApiPackageLoader(PackageLoader):
def __init__(self, murano_client_factory, tenant_id):
class ApiPackageLoader(package_loader.MuranoPackageLoader):
def __init__(self, murano_client_factory, tenant_id, root_loader=None):
self._cache_directory = self._get_cache_directory()
self._murano_client_factory = murano_client_factory
self.tenant_id = tenant_id
self._class_cache = {}
self._package_cache = {}
self._root_loader = root_loader or self
def get_package_by_class(self, name):
filter_opts = {'class_name': name}
def load_class_package(self, class_name, version_spec):
packages = self._class_cache.get(class_name)
if packages:
version = version_spec.select(packages.iterkeys())
if version:
return packages[version]
filter_opts = {'class_name': class_name}
try:
package_definition = self._get_definition(filter_opts)
except LookupError:
exc_info = sys.exc_info()
raise exceptions.NoPackageForClassFound(name), None, exc_info[2]
return self._get_package_by_definition(package_definition)
raise (exceptions.NoPackageForClassFound(class_name),
None, exc_info[2])
return self._to_dsl_package(
self._get_package_by_definition(package_definition))
def get_package(self, name):
filter_opts = {'fqn': name}
def load_package(self, package_name, version_spec):
packages = self._package_cache.get(package_name)
if packages:
version = version_spec.select(packages.iterkeys())
if version:
return packages[version]
filter_opts = {'fqn': package_name}
try:
package_definition = self._get_definition(filter_opts)
except LookupError:
exc_info = sys.exc_info()
raise exceptions.NoPackageFound(name), None, exc_info[2]
return self._get_package_by_definition(package_definition)
raise exceptions.NoPackageFound(package_name), None, exc_info[2]
return self._to_dsl_package(
self._get_package_by_definition(package_definition))
def register_package(self, package):
for name in package.classes:
self._class_cache.setdefault(name, {})[package.version] = package
self._package_cache.setdefault(package.name, {})[
package.version] = package
@staticmethod
def _get_cache_directory():
@ -76,7 +93,7 @@ class ApiPackageLoader(PackageLoader):
os.path.join(tempfile.gettempdir(), 'murano-packages-cache')
)
directory = os.path.abspath(os.path.join(base_directory,
str(uuid.uuid4())))
uuid.uuid4().hex))
os.makedirs(directory)
LOG.debug('Cache for package loader is located at: %s' % directory)
@ -103,10 +120,21 @@ class ApiPackageLoader(PackageLoader):
LOG.debug('Failed to get package definition from repository')
raise LookupError()
def _to_dsl_package(self, app_package):
dsl_package = murano_package.MuranoPackage(
self._root_loader, app_package)
for name in app_package.classes:
dsl_package.register_class(
(lambda cls: lambda: app_package.get_class(cls))(name),
name)
if app_package.full_name == constants.CORE_LIBRARY:
system_objects.register(dsl_package)
self.register_package(dsl_package)
return dsl_package
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_name = package_def.fully_qualified_name
package_directory = os.path.join(self._cache_directory, package_name)
package_directory = os.path.join(self._cache_directory, package_id)
if os.path.exists(package_directory):
try:
@ -135,7 +163,8 @@ class ApiPackageLoader(PackageLoader):
package_file.name,
target_dir=package_directory,
drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader
loader=yaql_yaml_loader.YaqlYamlLoader,
preload=False
)
except IOError:
msg = 'Unable to extract package data for %s' % package_id
@ -174,75 +203,128 @@ class ApiPackageLoader(PackageLoader):
return False
class DirectoryPackageLoader(PackageLoader):
def __init__(self, base_path):
class DirectoryPackageLoader(package_loader.MuranoPackageLoader):
def __init__(self, base_path, root_loader=None):
self._base_path = base_path
self._processed_entries = set()
self._packages_by_class = {}
self._packages_by_name = {}
self._loaded_packages = set()
self._root_loader = root_loader or self
self._build_index()
def get_package(self, name):
return self._packages_by_name.get(name)
def get_package_by_class(self, name):
return self._packages_by_class.get(name)
def _build_index(self):
for entry in os.listdir(self._base_path):
folder = os.path.join(self._base_path, entry)
if not os.path.isdir(folder) or entry in self._processed_entries:
continue
for folder in self.search_package_folders(self._base_path):
try:
package = load_utils.load_from_dir(
folder, preload=True,
folder, preload=False,
loader=yaql_yaml_loader.YaqlYamlLoader)
dsl_package = murano_package.MuranoPackage(
self._root_loader, package)
for class_name in package.classes:
dsl_package.register_class(
(lambda pkg, cls:
lambda: pkg.get_class(cls))(package, class_name),
class_name
)
if dsl_package.name == constants.CORE_LIBRARY:
system_objects.register(dsl_package)
self.register_package(dsl_package)
except pkg_exc.PackageLoadError:
LOG.info(_LI('Unable to load package from path: {0}').format(
os.path.join(self._base_path, entry)))
folder))
continue
LOG.info(_LI('Loaded package from path {0}').format(
os.path.join(self._base_path, entry)))
for c in package.classes:
self._packages_by_class[c] = package
self._packages_by_name[package.full_name] = package
LOG.info(_LI('Loaded package from path {0}').format(folder))
self._processed_entries.add(entry)
def load_class_package(self, class_name, version_spec):
packages = self._packages_by_class.get(class_name)
if not packages:
raise exceptions.NoPackageForClassFound(class_name)
version = version_spec.select(packages.iterkeys())
if not version:
raise exceptions.NoPackageForClassFound(class_name)
return packages[version]
def load_package(self, package_name, version_spec):
packages = self._packages_by_name.get(package_name)
if not packages:
raise exceptions.NoPackageFound(package_name)
version = version_spec.select(packages.iterkeys())
if not version:
raise exceptions.NoPackageFound(package_name)
return packages[version]
def register_package(self, package):
for c in package.classes:
self._packages_by_class.setdefault(c, {})[
package.version] = package
self._packages_by_name.setdefault(package.name, {})[
package.version] = package
@property
def packages(self):
for package_versions in self._packages_by_name.itervalues():
for package in package_versions.itervalues():
yield package
@staticmethod
def split_path(path):
tail = True
while tail:
path, tail = os.path.split(path)
if tail:
yield path
@classmethod
def search_package_folders(cls, path):
packages = set()
for folder, _, files in os.walk(path):
if 'manifest.yaml' in files:
found = False
for part in cls.split_path(folder):
if part in packages:
found = True
break
if not found:
packages.add(folder)
yield folder
class CombinedPackageLoader(PackageLoader):
def __init__(self, murano_client_factory, tenant_id):
self.murano_client_factory = murano_client_factory
self.tenant_id = tenant_id
self.loader_from_api = ApiPackageLoader(self.murano_client_factory,
self.tenant_id)
self.loaders_from_dir = []
class CombinedPackageLoader(package_loader.MuranoPackageLoader):
def __init__(self, murano_client_factory, tenant_id, root_loader=None):
root_loader = root_loader or self
self.api_loader = ApiPackageLoader(
murano_client_factory, tenant_id, root_loader)
self.directory_loaders = []
for directory in CONF.engine.load_packages_from:
if os.path.exists(directory):
self.loaders_from_dir.append(DirectoryPackageLoader(directory))
for folder in CONF.engine.load_packages_from:
if os.path.exists(folder):
self.directory_loaders.append(DirectoryPackageLoader(
folder, root_loader))
def get_package_by_class(self, name):
for loader in self.loaders_from_dir:
pkg = loader.get_package_by_class(name)
if pkg:
return pkg
return self.loader_from_api.get_package_by_class(name)
def load_package(self, package_name, version_spec):
for loader in self.directory_loaders:
try:
return loader.load_package(package_name, version_spec)
except exceptions.NoPackageFound:
continue
return self.api_loader.load_package(
package_name, version_spec)
def get_package(self, name):
# Try to load from local directory first
for loader in self.loaders_from_dir:
pkg = loader.get_package(name)
if pkg:
return pkg
# If no package found, load package by API
return self.loader_from_api.get_package(name)
def load_class_package(self, class_name, version_spec):
for loader in self.directory_loaders:
try:
return loader.load_class_package(class_name, version_spec)
except exceptions.NoPackageForClassFound:
continue
return self.api_loader.load_class_package(
class_name, version_spec)
def register_package(self, package):
self.api_loader.register_package(package)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.loader_from_api.cleanup()
self.api_loader.cleanup()
return False

4
murano/engine/system/agent.py
View File

@ -121,7 +121,7 @@ class Agent(object):
return None
@specs.parameter(
'resources', dsl.MuranoObjectType('io.murano.system.Resources'))
'resources', dsl.MuranoType('io.murano.system.Resources'))
def call(self, template, resources, timeout=None):
if timeout is None:
timeout = CONF.engine.agent_timeout
@ -130,7 +130,7 @@ class Agent(object):
return self._send(plan, True, timeout)
@specs.parameter(
'resources', dsl.MuranoObjectType('io.murano.system.Resources'))
'resources', dsl.MuranoType('io.murano.system.Resources'))
def send(self, template, resources):
self._check_enabled()
plan = self.build_execution_plan(template, resources())

8
murano/engine/system/resource_manager.py
View File

@ -17,7 +17,8 @@ import json as jsonlib
import yaml as yamllib
import murano.dsl.helpers as helpers
from murano.dsl import dsl
from murano.dsl import helpers
if hasattr(yamllib, 'CSafeLoader'):
yaml_loader = yamllib.CSafeLoader
@ -39,10 +40,11 @@ yaml_loader.add_constructor(u'tag:yaml.org,2002:timestamp',
_construct_yaml_str)
@dsl.name('io.murano.system.Resources')
class ResourceManager(object):
def __init__(self, package_loader, context):
def __init__(self, context):
murano_class = helpers.get_type(helpers.get_caller_context(context))
self._package = package_loader.get_package(murano_class.package.name)
self._package = murano_class.package
def string(self, name):
path = self._package.get_resource(name)

28
murano/engine/system/system_objects.py
View File

@ -12,7 +12,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from murano.dsl import dsl
from murano.engine.system import agent
from murano.engine.system import agent_listener
from murano.engine.system import heat_stack
@ -24,19 +24,13 @@ from murano.engine.system import resource_manager
from murano.engine.system import status_reporter
def register(class_loader, package_loader):
@dsl.name('io.murano.system.Resources')
class ResourceManagerWrapper(resource_manager.ResourceManager):
def __init__(self, context):
super(ResourceManagerWrapper, self).__init__(
package_loader, context)
class_loader.import_class(agent.Agent)
class_loader.import_class(agent_listener.AgentListener)
class_loader.import_class(heat_stack.HeatStack)
class_loader.import_class(mistralclient.MistralClient)
class_loader.import_class(ResourceManagerWrapper)
class_loader.import_class(instance_reporter.InstanceReportNotifier)
class_loader.import_class(status_reporter.StatusReporter)
class_loader.import_class(net_explorer.NetworkExplorer)
class_loader.import_class(logger.Logger)
def register(package):
package.register_class(agent.Agent)
package.register_class(agent_listener.AgentListener)
package.register_class(heat_stack.HeatStack)
package.register_class(mistralclient.MistralClient)
package.register_class(resource_manager.ResourceManager)
package.register_class(instance_reporter.InstanceReportNotifier)
package.register_class(status_reporter.StatusReporter)
package.register_class(net_explorer.NetworkExplorer)
package.register_class(logger.Logger)

12
murano/packages/application_package.py
View File

@ -47,6 +47,7 @@ class ApplicationPackage(object):
self._blob_cache = None
self._version = None
self._runtime_version = None
self._requirements = {}
@property
def full_name(self):
@ -56,10 +57,18 @@ class ApplicationPackage(object):
def version(self):
return self._version
@property
def classes(self):
return tuple()
@property
def runtime_version(self):
return self._runtime_version
@property
def requirements(self):
return self._requirements
@property
def package_type(self):
return self._package_type
@ -102,6 +111,9 @@ class ApplicationPackage(object):
self._blob_cache = _pack_dir(self._source_directory)
return self._blob_cache
def get_class(self, name):
raise exceptions.PackageClassLoadError(name)
def get_resource(self, name):
resources_dir = os.path.join(self._source_directory, 'Resources')
if not os.path.exists(resources_dir):

4
murano/packages/hot_package.py
View File

@ -62,8 +62,8 @@ class HotPackage(application_package.ApplicationPackage):
self._supplier = manifest.get('Supplier') or {}
self._logo = manifest.get('Logo')
self._tags = manifest.get('Tags')
self._version = semantic_version.Version(manifest.get(
'Version', '0.0.0'))
self._version = semantic_version.Version.coerce(str(manifest.get(
'Version', '0.0.0')))
self._runtime_version = runtime_version
@property

4
murano/packages/load_utils.py
View File

@ -30,7 +30,7 @@ import murano.packages.mpl_package
def load_from_file(archive_path, target_dir=None, drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader):
loader=yaql_yaml_loader.YaqlYamlLoader, preload=True):
if not os.path.isfile(archive_path):
raise e.PackageLoadError('Unable to find package file')
created = False
@ -50,7 +50,7 @@ def load_from_file(archive_path, target_dir=None, drop_dir=False,
"zip archive".format(archive_path))
package = zipfile.ZipFile(archive_path)
package.extractall(path=target_dir)
return load_from_dir(target_dir, preload=True, loader=loader)
return load_from_dir(target_dir, preload=preload, loader=loader)
except ValueError as err:
raise e.PackageLoadError("Couldn't load package from file: "
"{0}".format(err))

7
murano/packages/mpl_package.py
View File

@ -48,9 +48,10 @@ class MuranoPlPackage(application_package.ApplicationPackage):
self._ui = manifest.get('UI', 'ui.yaml')
self._logo = manifest.get('Logo')
self._tags = manifest.get('Tags')
self._version = semantic_version.Version(manifest.get(
'Version', '0.0.0'))
self._version = semantic_version.Version.coerce(str(manifest.get(
'Version', '0.0.0')))
self._runtime_version = runtime_version
self._requirements = manifest.get('Require') or {}
@property
def classes(self):
@ -97,7 +98,7 @@ class MuranoPlPackage(application_package.ApplicationPackage):
def _load_class(self, name):
if name not in self._classes:
raise exceptions.PackageClassLoadError(
name, 'Class not defined in this package')
name, 'Class not defined in package ' + self.full_name)
def_file = self._classes[name]
full_path = os.path.join(self._source_directory, 'Classes', def_file)
if not os.path.isfile(full_path):

38
murano/policy/congress_rules.py
View File

@ -13,6 +13,10 @@
# License for the specific language governing permissions and limitations
# under the License.
import semantic_version
from murano.dsl import helpers
class CongressRulesManager(object):
"""Converts murano model to list of congress rules:
@ -25,11 +29,11 @@ class CongressRulesManager(object):
_rules = []
_env_id = ''
_class_loader = None
_package_loader = None
def convert(self, model, class_loader=None, tenant_id=None):
def convert(self, model, package_loader=None, tenant_id=None):
self._rules = []
self._class_loader = class_loader
self._package_loader = package_loader
if model is None:
return self._rules
@ -115,7 +119,13 @@ class CongressRulesManager(object):
self._create_propety_rules(obj_rule.obj_id, obj))
cls = obj['?']['type']
types = self._get_parent_types(cls, self._class_loader)
if 'classVersion' in obj['?']:
version_spec = helpers.parse_version_spec(
semantic_version.Version(obj['?']['classVersion']))
else:
version_spec = semantic_version.Spec('*')
types = self._get_parent_types(
cls, self._package_loader, version_spec)
self._rules.extend(self._create_parent_type_rules(obj['?']['id'],
types))
# current object will be the owner for its subtree
@ -176,17 +186,15 @@ class CongressRulesManager(object):
else:
return rule
def _get_parent_types(self, type_name, class_loader):
types = set()
types.add(type_name)
if class_loader is not None:
cls = class_loader.get_class(type_name)
if cls is not None:
for parent in cls.parents:
types.add(parent.name)
types = types.union(
self._get_parent_types(parent.name, class_loader))
return types
@staticmethod
def _get_parent_types(type_name, package_loader, version_spec):
result = {type_name}
if package_loader:
pkg = package_loader.load_class_package(type_name, version_spec)
cls = pkg.find_class(type_name, False)
if cls:
result.update(t.name for t in cls.ancestors())
return result
@staticmethod
def _create_parent_type_rules(app_id, types):

8
murano/policy/model_policy_enforcer.py
View File

@ -43,14 +43,14 @@ class ModelPolicyEnforcer(object):
self._environment = environment
self._client_manager = environment.clients
def validate(self, model, class_loader=None):
def validate(self, model, package_loader=None):
"""Validate model using Congress rule engine.
@type model: dict
@param model: Dictionary representation of model starting on
environment level (['Objects'])
@type class_loader: murano.dsl.class_loader.MuranoClassLoader
@param class_loader: Optional. Used for evaluating parent class types
@type package_loader: murano.dsl.package_loader.MuranoPackageLoader
@param package_loader: Optional. Used for evaluating parent class types
@raises ValidationError in case validation was not successful
"""
@ -65,7 +65,7 @@ class ModelPolicyEnforcer(object):
LOG.debug(model)
rules = congress_rules.CongressRulesManager().convert(
model, class_loader, self._environment.tenant_id)
model, package_loader, self._environment.tenant_id)
rules_str = map(str, rules)
env_id = model['?']['id']

20
murano/tests/unit/dsl/foundation/runner.py
View File

@ -22,9 +22,23 @@ from murano.dsl import murano_object
from murano.dsl import serializer
from murano.dsl import yaql_integration
from murano.engine import environment
from murano.engine.system import yaql_functions
from murano.tests.unit.dsl.foundation import object_model
class TestExecutor(executor.MuranoDslExecutor):
def __init__(self, package_loader, env, functions):
self.__functions = functions
super(TestExecutor, self).__init__(package_loader, env)
def create_root_context(self):
context = super(TestExecutor, self).create_root_context()
yaql_functions.register(context)
for name, func in self.__functions.iteritems():
context.register_function(func, name)
return context
class Runner(object):
class DslObjectWrapper(object):
def __init__(self, obj, runner):
@ -48,15 +62,15 @@ class Runner(object):
if item.startswith('test'):
return call
def __init__(self, model, class_loader):
def __init__(self, model, package_loader, functions):
if isinstance(model, types.StringTypes):
model = object_model.Object(model)
model = object_model.build_model(model)
if 'Objects' not in model:
model = {'Objects': model}
self.executor = executor.MuranoDslExecutor(
class_loader, environment.Environment())
self.executor = TestExecutor(
package_loader, environment.Environment(), functions)
self._root = self.executor.load(model).object
def _execute(self, name, object_id, *args, **kwargs):

23
murano/tests/unit/dsl/foundation/test_case.py
View File

@ -20,7 +20,7 @@ import eventlet.debug
from murano.tests.unit import base
from murano.tests.unit.dsl.foundation import runner
from murano.tests.unit.dsl.foundation import test_class_loader
from murano.tests.unit.dsl.foundation import test_package_loader
class DslTestCase(base.MuranoTestCase):
@ -30,19 +30,19 @@ class DslTestCase(base.MuranoTestCase):
inspect.getfile(self.__class__)), 'meta')
root_meta_directory = os.path.join(
os.path.dirname(__file__), '../../../../../meta')
sys_class_loader = test_class_loader.TestClassLoader(
sys_package_loader = test_package_loader.TestPackageLoader(
os.path.join(root_meta_directory, 'io.murano/Classes'),
'murano.io')
self._class_loader = test_class_loader.TestClassLoader(
directory, 'tests', sys_class_loader)
'io.murano')
self._package_loader = test_package_loader.TestPackageLoader(
directory, 'tests', sys_package_loader)
self._functions = {}
self.register_function(
lambda data: self._traces.append(data), 'trace')
self._traces = []
test_class_loader.TestClassLoader.clear_configs()
eventlet.debug.hub_exceptions(False)
def new_runner(self, model):
return runner.Runner(model, self.class_loader)
return runner.Runner(model, self.package_loader, self._functions)
@property
def traces(self):
@ -53,13 +53,14 @@ class DslTestCase(base.MuranoTestCase):
self._traces = []
@property
def class_loader(self):
return self._class_loader
def package_loader(self):
return self._package_loader
def register_function(self, func, name):
self.class_loader.register_function(func, name)
self._functions[name] = func
def find_attribute(self, model, obj_id, obj_type, name):
@staticmethod
def find_attribute(model, obj_id, obj_type, name):
for entry in model['Attributes']:
if tuple(entry[:3]) == (obj_id, obj_type, name):
return entry[3]

88
murano/tests/unit/dsl/foundation/test_class_loader.py → murano/tests/unit/dsl/foundation/test_package_loader.py
View File

@ -17,49 +17,59 @@ import os.path
import yaml
from murano.dsl import class_loader
from murano.dsl import exceptions
from murano.dsl import murano_package
from murano.dsl import namespace_resolver
from murano.engine.system import yaql_functions
from murano.dsl import package_loader
from murano.engine import yaql_yaml_loader
from murano.tests.unit.dsl.foundation import object_model
class TestClassLoader(class_loader.MuranoClassLoader):
class TestPackage(murano_package.MuranoPackage):
def __init__(self, package_loader, name, version,
runtime_version, requirements, configs):
self.__configs = configs
super(TestPackage, self).__init__(
package_loader, name, version,
runtime_version, requirements)
def get_class_config(self, name):
return self.__configs.get(name, {})
class TestPackageLoader(package_loader.MuranoPackageLoader):
_classes_cache = {}
_configs = {}
def __init__(self, directory, package_name, parent_loader=None):
self._package = murano_package.MuranoPackage()
self._package.name = package_name
self._parent = parent_loader
if directory in TestClassLoader._classes_cache:
self._classes = TestClassLoader._classes_cache[directory]
self._package_name = package_name
if directory in TestPackageLoader._classes_cache:
self._classes = TestPackageLoader._classes_cache[directory]
else:
self._classes = {}
self._build_index(directory)
TestClassLoader._classes_cache[directory] = self._classes
self._functions = {}
super(TestClassLoader, self).__init__()
TestPackageLoader._classes_cache[directory] = self._classes
self._parent = parent_loader
self._configs = {}
self._package = TestPackage(
self, package_name, None, None, None, self._configs)
for name, payload in self._classes.iteritems():
self._package.register_class(payload, name)
super(TestPackageLoader, self).__init__()
def find_package_name(self, class_name):
def load_package(self, package_name, version_spec):
if package_name == self._package_name:
return self._package
elif self._parent:
return self._parent.load_package(package_name, version_spec)
else:
raise KeyError(package_name)
def load_class_package(self, class_name, version_spec):
if class_name in self._classes:
return self._package.name
if self._parent:
return self._parent.find_package_name(class_name)
return None
def load_package(self, class_name):
return self._package
def load_definition(self, name):
try:
return self._classes[name]
except KeyError:
if self._parent:
return self._parent.load_definition(name)
raise exceptions.NoClassFound(name)
return self._package
elif self._parent:
return self._parent.load_class_package(class_name, version_spec)
else:
raise KeyError(class_name)
def _build_index(self, directory):
yamls = [os.path.join(dirpath, f)
@ -84,25 +94,11 @@ class TestClassLoader(class_loader.MuranoClassLoader):
class_name = ns.resolve_name(data['Name'])
self._classes[class_name] = data
def create_root_context(self):
context = super(TestClassLoader, self).create_root_context()
yaql_functions.register(context)
for name, func in self._functions.iteritems():
context.register_function(func, name)
return context
def register_function(self, func, name):
self._functions[name] = func
def get_class_config(self, name):
return TestClassLoader._configs.get(name, {})
def set_config_value(self, class_name, property_name, value):
if isinstance(class_name, object_model.Object):
class_name = class_name.type_name
TestClassLoader._configs.setdefault(class_name, {})[
self._configs.setdefault(class_name, {})[
property_name] = value
@staticmethod
def clear_configs():
TestClassLoader._configs = {}
def register_package(self, package):
super(TestPackageLoader, self).register_package(package)

6
murano/tests/unit/dsl/test_agent.py
View File

@ -31,7 +31,8 @@ class TestAgentListener(test_case.DslTestCase):
super(TestAgentListener, self).setUp()
# Register Agent class
self.class_loader.import_class(agent_listener.AgentListener)
self.package_loader.load_package('io.murano', None).register_class(
agent_listener.AgentListener)
model = om.Object(
'AgentListenerTests')
self.runner = self.new_runner(model)
@ -62,7 +63,8 @@ class TestAgent(test_case.DslTestCase):
super(TestAgent, self).setUp()
# Register Agent class
self.class_loader.import_class(agent.Agent)
self.package_loader.load_package('io.murano', None).register_class(
agent.Agent)
model = om.Object(
'AgentTests')
self.runner = self.new_runner(model)

6
murano/tests/unit/dsl/test_config_properties.py
View File

@ -19,7 +19,7 @@ from murano.tests.unit.dsl.foundation import test_case
class TestConfigProperties(test_case.DslTestCase):
def test_config_property(self):
obj = om.Object('ConfigProperties')
self.class_loader.set_config_value(obj, 'cfgProperty', '987')
self.package_loader.set_config_value(obj, 'cfgProperty', '987')
runner = self.new_runner(obj)
runner.testPropertyValues()
self.assertEqual(
@ -38,7 +38,7 @@ class TestConfigProperties(test_case.DslTestCase):
def test_config_affects_default(self):
obj = om.Object('ConfigProperties')
self.class_loader.set_config_value(obj, 'normalProperty', 'custom')
self.package_loader.set_config_value(obj, 'normalProperty', 'custom')
runner = self.new_runner(obj)
runner.testPropertyValues()
self.assertEqual(
@ -48,7 +48,7 @@ class TestConfigProperties(test_case.DslTestCase):
def test_config_not_affects_in_properties(self):
obj = om.Object('ConfigProperties', normalProperty='qq')
self.class_loader.set_config_value(obj, 'normalProperty', 'custom')
self.package_loader.set_config_value(obj, 'normalProperty', 'custom')
runner = self.new_runner(obj)
runner.testPropertyValues()
self.assertEqual(

7
murano/tests/unit/dsl/test_logger.py
View File

@ -16,6 +16,7 @@ from mock import ANY
from mock import MagicMock
from mock.mock import call
from murano.dsl import helpers
from murano.engine.system import logger
from murano.tests.unit.dsl.foundation import object_model as om
from murano.tests.unit.dsl.foundation import test_case
@ -35,13 +36,13 @@ class TestLogger(test_case.DslTestCase):
def setUp(self):
super(TestLogger, self).setUp()
self._runner = self.new_runner(om.Object('TestLogger'))
self.class_loader.import_class(logger.Logger)
self.package_loader.load_package('io.murano', None).register_class(
logger.Logger)
def test_create(self):
cls = self.class_loader.get_class('io.murano.system.Logger')
logger_instance = self._runner.testCreate()
self.assertTrue(
cls.is_compatible(logger_instance),
helpers.is_instance_of(logger_instance, 'io.murano.system.Logger'),
'Function should return io.murano.system.Logger instance')
def _create_logger_mock(self):

2
murano/tests/unit/engine/system/test_agent.py
View File

@ -35,7 +35,7 @@ class TestExecutionPlan(base.MuranoTestCase):
self.mock_murano_class = mock.Mock(spec=murano_class.MuranoClass)
self.mock_murano_class.name = 'io.murano.system.Agent'
self.mock_murano_class.parents = []
self.mock_murano_class.declared_parents = []
self.mock_object_store = mock.Mock(spec=object_store.ObjectStore)
object_interface = mock.Mock(spec=murano_object.MuranoObject)

33
murano/tests/unit/engine/test_package_loader.py
View File

@ -14,16 +14,16 @@ import os
import mock
from oslo_config import cfg
import semantic_version
from murano.dsl import murano_package as dsl_package
from murano.engine import package_loader
from murano.packages import mpl_package
from murano.tests.unit import base
CONF = cfg.CONF
class TestCombinedPackageLoader(base.MuranoTestCase):
@classmethod
def setUpClass(cls):
super(TestCombinedPackageLoader, cls).setUpClass()
@ -34,34 +34,39 @@ class TestCombinedPackageLoader(base.MuranoTestCase):
cls.loader = package_loader.CombinedPackageLoader(
cls.murano_client_factory, 'test_tenant_id')
cls.api_loader = mock.MagicMock()
cls.loader.loader_from_api = cls.api_loader
cls.loader.api_loader = cls.api_loader
cls.local_pkg_name = 'io.murano.test.MyTest'
cls.api_pkg_name = 'test.mpl.v1.app.Thing'
def test_loaders_initialized(self):
self.assertEqual(1, len(self.loader.loaders_from_dir),
self.assertEqual(1, len(self.loader.directory_loaders),
'One directory class loader should be initialized'
' since there is one valid murano pl package in the'
' provided directory in config.')
self.assertIsInstance(self.loader.loaders_from_dir[0],
self.assertIsInstance(self.loader.directory_loaders[0],
package_loader.DirectoryPackageLoader)
def test_get_package_by_class_directory_loader(self):
result = self.loader.get_package_by_class(self.local_pkg_name)
self.assertIsInstance(result, mpl_package.MuranoPlPackage)
spec = semantic_version.Spec('*')
result = self.loader.load_class_package(self.local_pkg_name, spec)
self.assertIsInstance(result, dsl_package.MuranoPackage)
def test_get_package_by_name_directory_loader(self):
result = self.loader.get_package(self.local_pkg_name)
self.assertIsInstance(result, mpl_package.MuranoPlPackage)
spec = semantic_version.Spec('*')
result = self.loader.load_package(self.local_pkg_name, spec)
self.assertIsInstance(result, dsl_package.MuranoPackage)
def test_get_package_by_class_api_loader(self):
self.loader.get_package(self.api_pkg_name)
spec = semantic_version.Spec('*')
self.loader.load_package(self.api_pkg_name, spec)
self.api_loader.get_package.assert_called_with(self.api_pkg_name)
self.api_loader.load_package.assert_called_with(
self.api_pkg_name, spec)
def test_get_package_api_loader(self):
self.loader.get_package_by_class(self.api_pkg_name)
spec = semantic_version.Spec('*')
self.loader.load_class_package(self.api_pkg_name, spec)
self.api_loader.get_package_by_class.assert_called_with(
self.api_pkg_name)
self.api_loader.load_class_package.assert_called_with(
self.api_pkg_name, spec)

71
murano/tests/unit/policy/test_congress_rules.py
View File

@ -20,36 +20,65 @@ import unittest2 as unittest
import yaml
from murano.common import uuidutils
from murano.dsl import helpers
from murano.dsl import package_loader
import murano.policy.congress_rules as congress
TENANT_ID = 'de305d5475b4431badb2eb6b9e546013'
class MockClassLoader(object):
class MockPackageLoader(package_loader.MuranoPackageLoader):
def __init__(self, rules):
"""Create rules like this: ['child->parent', 'child->parent2']."""
self._rules_dict = {}
self._classes = {}
rules_dict = {}
for rule in rules:
split = rule.split('->')
if split[0] in self._rules_dict:
self._rules_dict[split[0]].append(split[1])
else:
self._rules_dict[split[0]] = [split[1]]
rules_dict.setdefault(split[0], []).append(split[1])
classes = (self.get_class(cls, rules_dict) for cls in rules_dict)
self._package = MockPackage(classes)
def get_class(self, name):
if name not in self._rules_dict:
return None
parents = []
for parent_name in self._rules_dict[name]:
parents.append(MockClass({'name': parent_name}))
return MockClass({'parents': parents})
def get_class(self, name, rules_dict):
if name in self._classes:
return self._classes[name]
parents = [self.get_class(parent, rules_dict)
for parent in rules_dict.get(name, [])]
result = MockClass({'name': name, 'declared_parents': parents})
self._classes[name] = result
return result
def register_package(self, package):
pass
def load_class_package(self, class_name, version_spec):
return self._package
def load_package(self, package_name, version_spec):
return self._package
class MockPackage(object):
def __init__(self, classes):
self._classes = {}
for cls in classes:
self._classes[cls.name] = cls
@property
def classes(self):
return self._classes.keys()
def find_class(self, name, *args, **kwargs):
return self._classes.get(name)
class MockClass(object):
def __init__(self, entries):
self.__dict__.update(entries)
def ancestors(self):
return helpers.traverse(self, lambda t: t.declared_parents)
class TestCongressRules(unittest.TestCase):
@ -60,11 +89,11 @@ class TestCongressRules(unittest.TestCase):
with open(model_file) as stream:
return yaml.load(stream)
def _create_rules_str(self, model_file, class_loader=None):
def _create_rules_str(self, model_file, package_loader=None):
model = self._load_file(model_file)
congress_rules = congress.CongressRulesManager()
rules = congress_rules.convert(model, class_loader,
rules = congress_rules.convert(model, package_loader,
tenant_id=TENANT_ID)
rules_str = ", \n".join(map(str, rules))
print rules_str
@ -143,14 +172,14 @@ class TestCongressRules(unittest.TestCase):
# \ /
# io.murano.apps.linux.Git
class_loader = MockClassLoader([
package_loader = MockPackageLoader([
'io.murano.apps.linux.Git->parent1',
'io.murano.apps.linux.Git->parent2',
'parent1->grand-parent',
'parent2->grand-parent'
])
rules_str = self._create_rules_str('model.yaml', class_loader)
rules_str = self._create_rules_str('model.yaml', package_loader)
self.assertTrue(
'murano:parent_types+("0c810278-7282-4e4a-9d69-7b4c36b6ce6f",'
@ -211,7 +240,7 @@ class TestCongressRules(unittest.TestCase):
'"tenant1", "io.murano.Environment")' in rules_str)
def test_wordpress(self):
class_loader = MockClassLoader([
package_loader = MockPackageLoader([
'io.murano.Environment->io.murano.Object',
'io.murano.resources.NeutronNetwork->io.murano.resources.Network',
'io.murano.resources.Network->io.murano.Object',
@ -230,11 +259,11 @@ class TestCongressRules(unittest.TestCase):
'io.murano.resources.LinuxInstance'
])
self._create_and_check_rules_str('wordpress', class_loader)
self._create_and_check_rules_str('wordpress', package_loader)
def _create_and_check_rules_str(self, model_name, class_loader=None):
def _create_and_check_rules_str(self, model_name, package_loader=None):
rules_str = self._create_rules_str(
'{0}.yaml'.format(model_name), class_loader)
'{0}.yaml'.format(model_name), package_loader)
self._check_expected_rules(rules_str,
'expected_rules_{0}.txt'.format(model_name))
return rules_str

10
murano/tests/unit/policy/test_model_policy_enforcer.py
View File

@ -27,7 +27,7 @@ CONF = cfg.CONF
class TestModelPolicyEnforcer(base.MuranoTestCase):
obj = mock.Mock()
class_loader = mock.Mock()
package_loader = mock.Mock()
model_dict = mock.Mock()
obj.to_dictionary = mock.Mock(return_value=model_dict)
@ -59,7 +59,7 @@ class TestModelPolicyEnforcer(base.MuranoTestCase):
CONF.engine.enable_model_policy_enforcer = False
executor._validate_model(self.obj, self.task['action'],
self.class_loader)
self.package_loader)
self.assertFalse(executor._model_policy_enforcer.validate.called)
@ -69,11 +69,11 @@ class TestModelPolicyEnforcer(base.MuranoTestCase):
CONF.engine.enable_model_policy_enforcer = True
executor._validate_model(self.obj, self.task['action'],
self.class_loader)
self.package_loader)
executor._model_policy_enforcer \
.validate.assert_called_once_with(self.model_dict,
self.class_loader)
self.package_loader)
def test_validation_pass(self):
self.congress_client_mock.execute_policy_action.return_value = \
@ -114,6 +114,6 @@ class TestModelPolicyEnforcer(base.MuranoTestCase):
CONF.engine.enable_model_policy_enforcer = True
executor._validate_model(self.obj, {'method': 'not_deploy'},
self.class_loader)
self.package_loader)
self.assertFalse(executor._model_policy_enforcer.validate.called)

46
murano/tests/unit/test_actions.py
View File

@ -24,59 +24,35 @@ class TestActionsSerializer(base.MuranoTestCase):
def setUp(self):
super(TestActionsSerializer, self).setUp()
def test_old_actions_deletion(self):
old = {
'action1': {'name': 'name1', 'enabled': True},
'action2': {'name': 'name2', 'enabled': True},
'action3': {'name': 'name3', 'enabled': True},
}
new = {
'action2': {'name': 'name2', 'enabled': False},
'action3': {'name': 'name3', 'enabled': True},
}
result = serializer._merge_actions(old, new)
self.assertEqual(2, len(result))
self.assertNotIn('action1', result)
def test_actions_state_update(self):
old = {
'action1': {'name': 'name1', 'enabled': True},
'action2': {'name': 'name2', 'enabled': True},
}
new = {
'action1': {'name': 'name2', 'enabled': False},
'action2': {'name': 'name3', 'enabled': True},
}
result = serializer._merge_actions(old, new)
self.assertFalse(result['action1']['enabled'])
def _get_mocked_obj(self):
method1 = mock.Mock()
method1.usage = murano_method.MethodUsages.Action
method1.name = 'method1'
method2 = mock.Mock()
method2.usage = murano_method.MethodUsages.Runtime
method2.name = 'method2'
method3 = mock.Mock()
method3.usage = murano_method.MethodUsages.Action
method3.name = 'method3'
obj2_type = mock.Mock()
obj2_type.parents = []
obj2_type.declared_parents = []
obj2_type.methods = {'method3': method3}
obj2_type.type.find_methods = lambda p: filter(p, [method3])
obj = mock.Mock()
obj.object_id = 'id1'
obj.type.parents = [obj2_type]
obj.type.declared_parents = [obj2_type]
obj.type.methods = {'method1': method1, 'method2': method2}
obj.type.find_methods = lambda p: filter(
p, [method1, method2, method3])
return obj
def test_object_actions_serialization(self):
obj = self._get_mocked_obj()
obj_actions = serializer._serialize_available_action(obj)
obj_actions = serializer._serialize_available_action(obj, {})
expected_result = {'name': 'method1', 'enabled': True}
self.assertIn('id1_method1', obj_actions)
@ -84,13 +60,13 @@ class TestActionsSerializer(base.MuranoTestCase):
def test_that_only_actions_are_serialized(self):
obj = self._get_mocked_obj()
obj_actions = serializer._serialize_available_action(obj)
obj_actions = serializer._serialize_available_action(obj, {})
self.assertNotIn('id1_method2', obj_actions)
def test_parent_actions_are_serialized(self):
obj = self._get_mocked_obj()
obj_actions = serializer._serialize_available_action(obj)
obj_actions = serializer._serialize_available_action(obj, {})
expected_result = {'name': 'method3', 'enabled': True}
self.assertIn('id1_method3', obj_actions)

5
murano/tests/unit/test_heat_stack.py
View File

@ -16,7 +16,6 @@
from heatclient.v1 import stacks
import mock
from murano.dsl import class_loader
from murano.dsl import constants
from murano.dsl import helpers
from murano.dsl import murano_class
@ -34,12 +33,10 @@ class TestHeatStack(base.MuranoTestCase):
super(TestHeatStack, self).setUp()
self.mock_murano_class = mock.Mock(spec=murano_class.MuranoClass)
self.mock_murano_class.name = 'io.murano.system.HeatStack'
self.mock_murano_class.parents = []
self.mock_murano_class.declared_parents = []
self.heat_client_mock = mock.MagicMock()
self.heat_client_mock.stacks = mock.MagicMock(spec=stacks.StackManager)
self.mock_object_store = mock.Mock(spec=object_store.ObjectStore)
self.mock_object_store.class_loader = mock.Mock(
spec=class_loader.MuranoClassLoader)
self.environment_mock = mock.Mock(
spec=environment.Environment)
client_manager_mock = mock.Mock(spec=client_manager.ClientManager)