openstack
/
murano
Code Issues Proposed changes

Merge "Major refactoring of how OS clients are created and managed"

This commit is contained in:
Jenkins 2016-02-22 15:13:25 +00:00 committed by Gerrit Code Review
parent 16b5aa7af6 fc76b3b1b4
commit 9eab33151d
27 changed files with 608 additions and 591 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
contrib/plugins/murano_exampleplugin/murano_exampleplugin/__init__.py
View File

@ -21,7 +21,7 @@ from oslo_config import cfg as config
from oslo_log import log as logging
import murano.dsl.helpers as helpers
from murano.common import auth_utils
CONF = config.CONF
@ -30,9 +30,7 @@ LOG = logging.getLogger(__name__)
class GlanceClient(object):
def __init__(self, context):
client_manager = helpers.get_environment(context).clients
self.client = client_manager.get_client("glance", True,
self.create_glance_client)
self.client = self.create_glance_client()
def list(self):
images = self.client.images.list()
@ -67,14 +65,11 @@ class GlanceClient(object):
def init_plugin(cls):
cls.CONF = cfg.init_config(CONF)
def create_glance_client(self, keystone_client, auth_token):
def create_glance_client(self):
LOG.debug("Creating a glance client")
glance_endpoint = keystone_client.service_catalog.url_for(
service_type='image', endpoint_type=self.CONF.endpoint_type)
client = glanceclient.Client(self.CONF.api_version,
endpoint=glance_endpoint,
token=auth_token)
return client
params = auth_utils.get_session_client_parameters(
service_type='image', conf=self.CONF)
return glanceclient.Client(self.CONF.api_version, **params)
class AmbiguousNameException(Exception):

22
murano/cmd/test_runner.py
View File

@ -33,8 +33,7 @@ from murano.common import engine
from murano.dsl import exceptions
from murano.dsl import executor
from murano.dsl import helpers
from murano.engine import client_manager
from murano.engine import environment
from murano.engine import execution_session
from murano.engine import mock_context_manager
from murano.engine import package_loader
@ -202,19 +201,14 @@ class MuranoTestRunner(object):
ks_opts = self._validate_keystone_opts(self.args)
client = ks_client.Client(**ks_opts)
test_env = environment.Environment()
test_env.token = client.auth_token
test_env.tenant_id = client.auth_tenant_id
test_env.clients = client_manager.ClientManager(test_env)
murano_client_factory = lambda: \
test_env.clients.get_murano_client(test_env)
test_session = execution_session.ExecutionSession()
test_session.token = client.auth_token
test_session.project_id = client.project_id
# Replace location of loading packages with provided from command line.
if load_packages_from:
cfg.CONF.packages_opts.load_packages_from = load_packages_from
with package_loader.CombinedPackageLoader(
murano_client_factory, client.tenant_id) as pkg_loader:
with package_loader.CombinedPackageLoader(test_session) as pkg_loader:
engine.get_plugin_loader().register_in_loader(pkg_loader)
package = self._load_package(pkg_loader, provided_pkg_name)
@ -236,13 +230,13 @@ class MuranoTestRunner(object):
dsl_executor = executor.MuranoDslExecutor(
pkg_loader,
mock_context_manager.MockContextManager(),
test_env)
test_session)
obj = package.find_class(pkg_class, False).new(
None, dsl_executor.object_store, dsl_executor)(None)
self._call_service_method('setUp', dsl_executor, obj)
obj.type.methods[m].usage = 'Action'
test_env.start()
test_session.start()
try:
obj.type.invoke(m, dsl_executor, obj, (), {})
LOG.debug('\n.....{0}.{1}.....OK'.format(obj.type.name,
@ -254,7 +248,7 @@ class MuranoTestRunner(object):
''.format(obj.type.name, m))
exit_code = 1
finally:
test_env.finish()
test_session.finish()
return exit_code
def get_parser(self):

217
murano/common/auth_utils.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2014 Mirantis, Inc.
# Copyright (c) 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -13,92 +13,151 @@
# under the License.
from keystoneclient.auth import identity
from keystoneclient import session as ks_session
from keystoneclient.v3 import client as ks_client
from oslo_config import cfg
from oslo_utils import importutils
def get_client(token, project_id):
settings = _get_keystone_settings()
kwargs = {
'token': token,
'project_id': project_id,
'auth_url': settings['auth_url']
}
kwargs.update(settings['ssl'])
kwargs['region_name'] = settings['region_name']
keystone = ks_client.Client(**kwargs)
keystone.management_url = settings['auth_url']
return keystone
from murano.dsl import helpers
def get_client_for_admin(project_name):
return _admin_client(project_name=project_name)
def _admin_client(trust_id=None, project_name=None):
settings = _get_keystone_settings()
kwargs = {
'project_name': project_name,
'trust_id': trust_id
}
for key in ('username', 'password', 'auth_url'):
kwargs[key] = settings[key]
kwargs.update(settings['ssl'])
kwargs['region_name'] = settings['region_name']
client = ks_client.Client(**kwargs)
# without resetting this attributes keystone client cannot re-authenticate
client.project_id = None
client.project_name = None
client.management_url = settings['auth_url']
return client
def get_client_for_trusts(trust_id):
return _admin_client(trust_id)
def create_trust(token, project_id):
client = get_client(token, project_id)
settings = _get_keystone_settings()
trustee_id = get_client_for_admin(
settings['project_name']).user_id
roles = [t['name'] for t in client.auth_ref['roles']]
trust = client.trusts.create(trustor_user=client.user_id,
trustee_user=trustee_id,
impersonation=True,
role_names=roles,
project=project_id)
return trust.id
def delete_trust(trust_id):
keystone_client = get_client_for_trusts(trust_id)
keystone_client.trusts.delete(trust_id)
def _get_keystone_settings():
@helpers.memoize
def _get_keystone_admin_parameters(scoped):
importutils.import_module('keystonemiddleware.auth_token')
return {
settings = {
'auth_url': cfg.CONF.keystone_authtoken.auth_uri.replace('v2.0', 'v3'),
'username': cfg.CONF.keystone_authtoken.admin_user,
'password': cfg.CONF.keystone_authtoken.admin_password,
'project_name': cfg.CONF.keystone_authtoken.admin_tenant_name,
'ssl': {
'cacert': cfg.CONF.keystone.ca_file,
'insecure': cfg.CONF.keystone.insecure,
'cert': cfg.CONF.keystone.cert_file,
'key': cfg.CONF.keystone.key_file,
},
'region_name': cfg.CONF.murano.region_name_for_services
'user_domain_name': 'default'
}
if scoped:
settings.update({
'project_name': cfg.CONF.keystone_authtoken.admin_tenant_name,
'project_domain_name': 'default'
})
return settings
@helpers.memoize
def create_keystone_admin_client(scoped):
kwargs = _get_keystone_admin_parameters(scoped)
password_auth = identity.Password(**kwargs)
session = ks_session.Session(auth=password_auth)
_set_ssl_parameters(cfg.CONF.keystone_authtoken, session)
return ks_client.Client(session=session)
def get_client_session(execution_session=None, conf=None):
if not execution_session:
execution_session = helpers.get_execution_session()
trust_id = execution_session.trust_id
if trust_id is None:
return get_token_client_session(
token=execution_session.token,
project_id=execution_session.project_id)
kwargs = _get_keystone_admin_parameters(False)
kwargs['trust_id'] = trust_id
password_auth = identity.Password(**kwargs)
session = ks_session.Session(auth=password_auth)
_set_ssl_parameters(conf, session)
return session
def get_token_client_session(token=None, project_id=None, conf=None):
auth_url = _get_keystone_admin_parameters(False)['auth_url']
if token is None or project_id is None:
execution_session = helpers.get_execution_session()
token = execution_session.token
project_id = execution_session.project_id
token_auth = identity.Token(auth_url, token=token, project_id=project_id)
session = ks_session.Session(auth=token_auth)
_set_ssl_parameters(conf, session)
return session
def create_keystone_client(token=None, project_id=None, conf=None):
return ks_client.Client(session=get_token_client_session(
token=token, project_id=project_id, conf=conf))
def create_trust(trustee_token=None, trustee_project_id=None):
admin_client = create_keystone_admin_client(True)
user_client = create_keystone_client(
token=trustee_token, project_id=trustee_project_id)
trustee_user = admin_client.session.auth.get_user_id(admin_client.session)
auth_ref = user_client.session.auth.get_access(user_client.session)
trustor_user = auth_ref.user_id
project = auth_ref.project_id
roles = auth_ref.role_names
trust = user_client.trusts.create(
trustor_user=trustor_user,
trustee_user=trustee_user,
impersonation=True,
role_names=roles,
project=project)
return trust.id
def delete_trust(trust):
user_client = create_keystone_admin_client(True)
user_client.trusts.delete(trust)
def _get_config_option(conf_section, option_names, default=None):
if not isinstance(option_names, (list, tuple)):
option_names = (option_names,)
for name in option_names:
if hasattr(conf_section, name):
return getattr(conf_section, name)
return default
def _set_ssl_parameters(conf_section, session):
if not conf_section:
return
insecure = _get_config_option(conf_section, 'insecure', False)
if insecure:
session.verify = False
else:
session.verify = _get_config_option(
conf_section, ('ca_file', 'cafile', 'cacert')) or True
cert_file = _get_config_option(conf_section, ('cert_file', 'certfile'))
key_file = _get_config_option(conf_section, ('key_file', 'keyfile'))
if cert_file and key_file:
session.cert = (cert_file, key_file)
elif cert_file:
session.cert = cert_file
else:
session.cert = None
def get_session_client_parameters(service_type=None,
region='',
interface=None,
service_name=None,
conf=None,
session=None,
execution_session=None):
if region == '':
region = cfg.CONF.home_region
result = {
'session': session or get_client_session(
execution_session=execution_session, conf=conf)
}
url = _get_config_option(conf, 'url')
if url:
result['endpoint_override'] = url
else:
if not interface:
interface = _get_config_option(conf, 'endpoint_type')
result.update({
'service_type': service_type,
'service_name': service_name,
'interface': interface,
'region_name': region
})
return result

47
murano/common/config.py
View File

@ -58,6 +58,8 @@ rabbit_opts = [
]
heat_opts = [
cfg.StrOpt('url', help='Optional heat endpoint override'),
cfg.BoolOpt('insecure', default=False,
help='This option explicitly allows Murano to perform '
'"insecure" SSL connections and transfers with Heat API.'),
@ -82,13 +84,26 @@ heat_opts = [
]
mistral_opts = [
cfg.StrOpt('url', help='Optional mistral endpoint override'),
cfg.StrOpt('endpoint_type', default='publicURL',
help='Mistral endpoint type.'),
cfg.StrOpt('service_type', default='workflowv2',
help='Mistral service type.')
help='Mistral service type.'),
cfg.BoolOpt('insecure', default=False,
help='This option explicitly allows Murano to perform '
'"insecure" SSL connections and transfers with Mistral.'),
cfg.StrOpt('ca_cert',
help='(SSL) Tells Murano to use the specified client '
'certificate file when communicating with Mistral.')
]
neutron_opts = [
cfg.StrOpt('url', help='Optional neutron endpoint override'),
cfg.BoolOpt('insecure', default=False,
help='This option explicitly allows Murano to perform '
'"insecure" SSL connections and transfers with Neutron API.'),
@ -101,24 +116,6 @@ neutron_opts = [
help='Neutron endpoint type.')
]
keystone_opts = [
cfg.BoolOpt('insecure', default=False,
help='This option explicitly allows Murano to perform '
'"insecure" SSL connections and transfers with '
'Keystone API running Kyestone API.'),
cfg.StrOpt('ca_file',
help='(SSL) Tells Murano to use the specified certificate file '
'to verify the peer when communicating with Keystone.'),
cfg.StrOpt('cert_file',
help='(SSL) Tells Murano to use the specified client '
'certificate file when communicating with Keystone.'),
cfg.StrOpt('key_file', help='(SSL/SSH) Private key file name to '
'communicate with Keystone API')
]
murano_opts = [
cfg.StrOpt('url', help='Optional murano url in format '
'like http://0.0.0.0:8082 used by Murano engine'),
@ -148,10 +145,7 @@ murano_opts = [
cfg.ListOpt('enabled_plugins',
help="List of enabled Extension Plugins. "
"Remove or leave commented to enable all installed "
"plugins."),
cfg.StrOpt('region_name_for_services',
help="Default region name used to get services endpoints.")
"plugins.")
]
networking_opts = [
@ -271,6 +265,11 @@ file_server = [
help='Set a file server.')
]
home_region = cfg.StrOpt(
'home_region', default=None,
help="Default region name used to get services endpoints.")
CONF = cfg.CONF
CONF.register_opts(paste_deploy_opts, group='paste_deploy')
CONF.register_cli_opts(bind_opts)
@ -278,10 +277,10 @@ CONF.register_opts(rabbit_opts, group='rabbitmq')
CONF.register_opts(heat_opts, group='heat')
CONF.register_opts(mistral_opts, group='mistral')
CONF.register_opts(neutron_opts, group='neutron')
CONF.register_opts(keystone_opts, group='keystone')
CONF.register_opts(murano_opts, group='murano')
CONF.register_opts(engine_opts, group='engine')
CONF.register_opts(file_server)
CONF.register_opt(home_region)
CONF.register_cli_opts(metadata_dir)
CONF.register_opts(packages_opts, group='packages_opts')
CONF.register_opts(stats_opts, group='stats')

52
murano/common/engine.py
View File

@ -34,7 +34,7 @@ from murano.dsl import dsl_exception
from murano.dsl import executor as dsl_executor
from murano.dsl import helpers
from murano.dsl import serializer
from murano.engine import environment
from murano.engine import execution_session
from murano.engine import package_loader
from murano.engine.system import status_reporter
from murano.engine.system import yaql_functions
@ -125,8 +125,8 @@ class TaskExecutor(object):
return self._action
@property
def environment(self):
return self._environment
def session(self):
return self._session
@property
def model(self):
@ -137,14 +137,14 @@ class TaskExecutor(object):
reporter = status_reporter.StatusReporter(task['id'])
self._action = task.get('action')
self._model = task['model']
self._environment = environment.Environment()
self._environment.token = task['token']
self._environment.tenant_id = task['tenant_id']
self._environment.system_attributes = self._model.get('SystemData', {})
self._session = execution_session.ExecutionSession()
self._session.token = task['token']
self._session.project_id = task['tenant_id']
self._session.system_attributes = self._model.get('SystemData', {})
self._reporter = reporter
self._model_policy_enforcer = enforcer.ModelPolicyEnforcer(
self._environment)
self._session)
def execute(self):
try:
@ -152,13 +152,9 @@ class TaskExecutor(object):
except Exception as e:
return self.exception_result(e, None, '<system>')
murano_client_factory = \
lambda: self._environment.clients.get_murano_client()
with package_loader.CombinedPackageLoader(
murano_client_factory,
self._environment.tenant_id) as pkg_loader:
with package_loader.CombinedPackageLoader(self._session) as pkg_loader:
result = self._execute(pkg_loader)
self._model['SystemData'] = self._environment.system_attributes
self._model['SystemData'] = self._session.system_attributes
result['model'] = self._model
if (not self._model.get('Objects') and
@ -174,7 +170,7 @@ class TaskExecutor(object):
get_plugin_loader().register_in_loader(pkg_loader)
executor = dsl_executor.MuranoDslExecutor(
pkg_loader, ContextManager(), self.environment)
pkg_loader, ContextManager(), self.session)
try:
obj = executor.load(self.model)
except Exception as e:
@ -188,20 +184,20 @@ class TaskExecutor(object):
try:
LOG.debug('Invoking pre-cleanup hooks')
self.environment.start()
self.session.start()
executor.cleanup(self._model)
except Exception as e:
return self.exception_result(e, obj, '<GC>')
finally:
LOG.debug('Invoking post-cleanup hooks')
self.environment.finish()
self.session.finish()
self._model['ObjectsCopy'] = copy.deepcopy(self._model.get('Objects'))
action_result = None
if self.action:
try:
LOG.debug('Invoking pre-execution hooks')
self.environment.start()
self.session.start()
try:
action_result = self._invoke(executor)
finally:
@ -213,7 +209,7 @@ class TaskExecutor(object):
return self.exception_result(e, obj, self.action['method'])
finally:
LOG.debug('Invoking post-execution hooks')
self.environment.finish()
self.session.finish()
try:
action_result = serializer.serialize(action_result)
@ -266,16 +262,16 @@ class TaskExecutor(object):
def _create_trust(self):
if not CONF.engine.use_trusts:
return
trust_id = self._environment.system_attributes.get('TrustId')
trust_id = self._session.system_attributes.get('TrustId')
if not trust_id:
trust_id = auth_utils.create_trust(self._environment.token,
self._environment.tenant_id)
self._environment.system_attributes['TrustId'] = trust_id
self._environment.trust_id = trust_id
trust_id = auth_utils.create_trust(
self._session.token, self._session.project_id)
self._session.system_attributes['TrustId'] = trust_id
self._session.trust_id = trust_id
def _delete_trust(self):
trust_id = self._environment.trust_id
trust_id = self._session.trust_id
if trust_id:
auth_utils.delete_trust(self._environment.trust_id)
self._environment.system_attributes['TrustId'] = None
self._environment.trust_id = None
auth_utils.delete_trust(self._session.trust_id)
self._session.system_attributes['TrustId'] = None
self._session.trust_id = None

5
murano/db/services/environments.py
View File

@ -263,9 +263,10 @@ class EnvironmentServices(object):
@staticmethod
def get_network_driver(context):
ks = auth_utils.get_client(context.auth_token, context.tenant)
session = auth_utils.get_token_client_session(
context.auth_token, context.tenant)
try:
ks.service_catalog.url_for(service_type='network')
session.get_endpoint(service_type='network')
except ks_exceptions.EndpointNotFound:
LOG.debug("Will use NovaNetwork as a network driver")
return "nova"

8
murano/dsl/constants.py
View File

@ -25,15 +25,15 @@ CTX_CALLER_CONTEXT = '$?callerContext'
CTX_CURRENT_INSTRUCTION = '$?currentInstruction'
CTX_CURRENT_EXCEPTION = '$?currentException'
CTX_CURRENT_METHOD = '$?currentMethod'
CTX_ENVIRONMENT = '$?environment'
CTX_EXECUTOR = '$?executor'
CTX_EXECUTION_SESSION = '$?executionSession'
CTX_ORIGINAL_CONTEXT = '$?originalContext'
CTX_PACKAGE_LOADER = '$?packageLoader'
CTX_SKIP_FRAME = '$?skipFrame'
CTX_THIS = '$?this'
CTX_TYPE = '$?type'
CTX_VARIABLE_SCOPE = '$?variableScope'
CTX_YAQL_ENGINE = '$?yaqlEngine'
CTX_ORIGINAL_CONTEXT = '$?originalContext'
DM_OBJECTS = 'Objects'
DM_OBJECTS_COPY = 'ObjectsCopy'
@ -45,6 +45,10 @@ META_NO_TRACE = '?noTrace'
CORE_LIBRARY = 'io.murano'
CORE_LIBRARY_OBJECT = 'io.murano.Object'
TL_CONTEXT = '__murano_context'
TL_ID = '__thread_id'
TL_SESSION = '__murano_execution_session'
RUNTIME_VERSION_1_0 = semantic_version.Version('1.0.0')
RUNTIME_VERSION_1_1 = semantic_version.Version('1.1.0')
RUNTIME_VERSION_1_2 = semantic_version.Version('1.2.0')

4
murano/dsl/dsl.py
View File

@ -247,8 +247,8 @@ class Interfaces(object):
return MuranoObjectInterface(mpl_object)
@property
def environment(self):
return helpers.get_environment()
def execution_session(self):
return helpers.get_execution_session()
@property
def caller(self):

20
murano/dsl/executor.py
View File

@ -39,10 +39,10 @@ LOG = logging.getLogger(__name__)
class MuranoDslExecutor(object):
def __init__(self, package_loader, context_manager, environment=None):
def __init__(self, package_loader, context_manager, session=None):
self._package_loader = package_loader
self._context_manager = context_manager
self._environment = environment
self._session = session
self._attribute_store = attribute_store.AttributeStore()
self._object_store = object_store.ObjectStore(self)
self._locks = {}
@ -66,6 +66,12 @@ class MuranoDslExecutor(object):
def invoke_method(self, method, this, context, args, kwargs,
skip_stub=False):
with helpers.execution_session(self._session):
return self._invoke_method(
method, this, context, args, kwargs, skip_stub=skip_stub)
def _invoke_method(self, method, this, context, args, kwargs,
skip_stub=False):
if isinstance(this, dsl.MuranoObjectInterface):
this = this.object
kwargs = utils.filter_parameters_dict(kwargs)
@ -190,6 +196,10 @@ class MuranoDslExecutor(object):
return tuple(), parameter_values
def load(self, data):
with helpers.execution_session(self._session):
return self._load(data)
def _load(self, data):
if not isinstance(data, dict):
raise TypeError()
self._attribute_store.load(data.get(constants.DM_ATTRIBUTES) or [])
@ -199,6 +209,10 @@ class MuranoDslExecutor(object):
return dsl.MuranoObjectInterface(result, executor=self)
def cleanup(self, data):
with helpers.execution_session(self._session):
return self._cleanup(data)
def _cleanup(self, data):
objects_copy = data.get(constants.DM_OBJECTS_COPY)
if not objects_copy:
return
@ -244,7 +258,7 @@ class MuranoDslExecutor(object):
context[constants.CTX_EXECUTOR] = weakref.ref(self)
context[constants.CTX_PACKAGE_LOADER] = weakref.ref(
self._package_loader)
context[constants.CTX_ENVIRONMENT] = self._environment
context[constants.CTX_EXECUTION_SESSION] = self._session
context[constants.CTX_ATTRIBUTE_STORE] = weakref.ref(
self._attribute_store)
self._root_context_cache[runtime_version] = context

53
murano/dsl/helpers.py
View File

@ -109,10 +109,14 @@ def generate_id():
def parallel_select(collection, func, limit=1000):
# workaround for eventlet issue 232
# https://github.com/eventlet/eventlet/issues/232
context = get_context()
session = get_execution_session()
def wrapper(element):
try:
with contextual(get_context()):
return func(element), False, None
with contextual(context):
with execution_session(session):
return func(element), False, None
except Exception as e:
return e, True, sys.exc_info()[2]
@ -132,7 +136,7 @@ def enum(**enums):
def get_context():
current_thread = eventlet.greenthread.getcurrent()
return getattr(current_thread, '__murano_context', None)
return getattr(current_thread, constants.TL_CONTEXT, None)
def get_executor(context=None):
@ -146,9 +150,15 @@ def get_type(context=None):
return context[constants.CTX_TYPE]
def get_environment(context=None):
def get_execution_session(context=None):
context = context or get_context()
return context[constants.CTX_ENVIRONMENT]
session = None
if context is not None:
session = context[constants.CTX_EXECUTION_SESSION]
if session is None:
current_thread = eventlet.greenthread.getcurrent()
session = getattr(current_thread, constants.TL_SESSION, None)
return session
def get_object_store(context=None):
@ -215,27 +225,37 @@ def get_current_thread_id():
global _threads_sequencer
current_thread = eventlet.greenthread.getcurrent()
thread_id = getattr(current_thread, '__thread_id', None)
thread_id = getattr(current_thread, constants.TL_ID, None)
if thread_id is None:
thread_id = 'T' + str(_threads_sequencer)
_threads_sequencer += 1
setattr(current_thread, '__thread_id', thread_id)
setattr(current_thread, constants.TL_ID, thread_id)
return thread_id
@contextlib.contextmanager
def contextual(ctx):
def thread_local_attribute(name, value):
current_thread = eventlet.greenthread.getcurrent()
current_context = getattr(current_thread, '__murano_context', None)
if ctx:
setattr(current_thread, '__murano_context', ctx)
old_value = getattr(current_thread, name, None)
if value is not None:
setattr(current_thread, name, value)
elif hasattr(current_thread, name):
delattr(current_thread, name)
try:
yield
finally:
if current_context:
setattr(current_thread, '__murano_context', current_context)
elif hasattr(current_thread, '__murano_context'):
delattr(current_thread, '__murano_context')
if old_value is not None:
setattr(current_thread, name, old_value)
elif hasattr(current_thread, name):
delattr(current_thread, name)
def contextual(ctx):
return thread_local_attribute(constants.TL_CONTEXT, ctx)
def execution_session(session):
return thread_local_attribute(constants.TL_SESSION, session)
def parse_version_spec(version_spec):
@ -332,7 +352,10 @@ def is_instance_of(obj, class_name, pov_or_version_spec=None):
def memoize(func):
cache = {}
return get_memoize_func(func, cache)
def get_memoize_func(func, cache):
@functools.wraps(func)
def wrap(*args):
if args not in cache:

97
murano/dsl/session_local_storage.py Normal file
View File

@ -0,0 +1,97 @@
# Copyright (c) 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This code is almost a complete copy of eventlet.corolocal with only
# the concept of current thread replaced with current session
import weakref
import six
from murano.dsl import helpers
# the entire purpose of this class is to store off the constructor
# arguments in a local variable without calling __init__ directly
class _localbase(object):
__slots__ = '_local__args', '_local__sessions'
def __new__(cls, *args, **kw):
self = object.__new__(cls)
object.__setattr__(self, '_local__args', (args, kw))
object.__setattr__(
self, '_local__sessions', weakref.WeakKeyDictionary())
if (args or kw) and (cls.__init__ is object.__init__):
raise TypeError('Initialization arguments are not supported')
return self
def _patch(session_local):
sessions_dict = object.__getattribute__(session_local, '_local__sessions')
session = helpers.get_execution_session()
localdict = sessions_dict.get(session)
if localdict is None:
# must be the first time we've seen this session, call __init__
localdict = {}
sessions_dict[session] = localdict
cls = type(session_local)
if cls.__init__ is not object.__init__:
args, kw = object.__getattribute__(session_local, '_local__args')
session_local.__init__(*args, **kw)
object.__setattr__(session_local, '__dict__', localdict)
class _local(_localbase):
def __getattribute__(self, attr):
_patch(self)
return object.__getattribute__(self, attr)
def __setattr__(self, attr, value):
_patch(self)
return object.__setattr__(self, attr, value)
def __delattr__(self, attr):
_patch(self)
return object.__delattr__(self, attr)
def session_local(cls):
return type(cls.__name__, (cls, _local), {})
class SessionLocalDict(six.moves.UserDict, object):
def __init__(self, **kwargs):
self.__session_data = weakref.WeakKeyDictionary()
self.__default = {}
super(SessionLocalDict, self).__init__(**kwargs)
@property
def data(self):
session = helpers.get_execution_session()
if session is None:
return self.__default
return self.__session_data.setdefault(session, {})
@data.setter
def data(self, value):
session = helpers.get_execution_session()
if session is None:
self.__default = value
else:
self.__session_data[session] = value
def execution_session_memoize(func):
cache = SessionLocalDict()
return helpers.get_memoize_func(func, cache)

241
murano/engine/client_manager.py
View File

@ -1,241 +0,0 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import weakref
from eventlet import semaphore
import heatclient.client as hclient
import keystoneclient
import keystoneclient.auth.identity.access as access
import muranoclient.v1.client as muranoclient
import neutronclient.v2_0.client as nclient
from oslo_config import cfg
from murano.common import auth_utils
from muranoclient.glance import client as art_client
try:
# integration with congress is optional
import congressclient.v1.client as congress_client
except ImportError as congress_client_import_error:
congress_client = None
try:
import mistralclient.api.client as mistralclient
except ImportError as mistral_import_error:
mistralclient = None
CONF = cfg.CONF
class ClientManager(object):
def __init__(self, environment):
self._trusts_keystone_client = None
self._token_keystone_client = None
self._cache = {}
self._semaphore = semaphore.BoundedSemaphore()
self._environment = weakref.proxy(environment)
def get_client(self, name, use_trusts, client_factory):
if not CONF.engine.use_trusts:
use_trusts = False
keystone_client = None if name == 'keystone' else \
self.get_keystone_client(use_trusts)
self._semaphore.acquire()
try:
client, used_token = self._cache.get(
(name, use_trusts), (None, None))
fresh_token = None if keystone_client is None \
else keystone_client.auth_token
if use_trusts and used_token != fresh_token:
client = None
if not client:
token = fresh_token
if not use_trusts:
token = self._environment.token
client = client_factory(keystone_client, token)
self._cache[(name, use_trusts)] = (client, token)
return client
finally:
self._semaphore.release()
def get_keystone_client(self, use_trusts=True):
if not CONF.engine.use_trusts:
use_trusts = False
factory = lambda _1, _2: \
auth_utils.get_client_for_trusts(self._environment.trust_id) \
if use_trusts else auth_utils.get_client(
self._environment.token, self._environment.tenant_id)
return self.get_client('keystone', use_trusts, factory)
def get_congress_client(self, use_trusts=True):
"""Client for congress services
:return: initialized congress client
:raise ImportError: in case that python-congressclient
is not present on python path
"""
if not congress_client:
# congress client was not imported
raise congress_client_import_error
if not CONF.engine.use_trusts:
use_trusts = False
def factory(keystone_client, auth_token):
auth = access.AccessInfoPlugin(keystone_client.auth_ref)
session = keystoneclient.session.Session(auth=auth)
return congress_client.Client(session=session,
service_type='policy')
return self.get_client('congress', use_trusts, factory)
def get_heat_client(self, use_trusts=True):
if not CONF.engine.use_trusts:
use_trusts = False
def factory(keystone_client, auth_token):
heat_settings = CONF.heat
heat_url = keystone_client.service_catalog.url_for(
service_type='orchestration',
endpoint_type=heat_settings.endpoint_type)
kwargs = {
'token': auth_token,
'ca_file': heat_settings.ca_file or None,
'cert_file': heat_settings.cert_file or None,
'key_file': heat_settings.key_file or None,
'insecure': heat_settings.insecure
}
if not CONF.engine.use_trusts:
kwargs.update({
'username': 'badusername',
'password': 'badpassword'
})
return hclient.Client('1', heat_url, **kwargs)
return self.get_client('heat', use_trusts, factory)
def get_neutron_client(self, use_trusts=True):
if not CONF.engine.use_trusts:
use_trusts = False
def factory(keystone_client, auth_token):
neutron_settings = CONF.neutron
neutron_url = keystone_client.service_catalog.url_for(
service_type='network',
endpoint_type=neutron_settings.endpoint_type)
return nclient.Client(
endpoint_url=neutron_url,
token=auth_token,
ca_cert=neutron_settings.ca_cert or None,
insecure=neutron_settings.insecure)
return self.get_client('neutron', use_trusts, factory)
def get_murano_client(self, use_trusts=True):
if not CONF.engine.use_trusts:
use_trusts = False
def factory(keystone_client, auth_token):
murano_settings = CONF.murano
murano_url = \
murano_settings.url or keystone_client.service_catalog.url_for(
service_type='application-catalog',
endpoint_type=murano_settings.endpoint_type)
if CONF.packages_opts.packages_service == 'glance':
glance_settings = CONF.glance
glance_url = (glance_settings.url or
keystone_client.service_catalog.url_for(
service_type='image',
endpoint_type=glance_settings.endpoint_type))
arts = art_client.Client(
endpoint=glance_url, token=auth_token,
insecure=glance_settings.insecure,
key_file=glance_settings.key_file or None,
ca_file=glance_settings.ca_file or None,
cert_file=glance_settings.cert_file or None,
type_name='murano',
type_version=1)
else:
arts = None
return muranoclient.Client(
endpoint=murano_url,
key_file=murano_settings.key_file or None,
ca_file=murano_settings.cacert or None,
cert_file=murano_settings.cert_file or None,
insecure=murano_settings.insecure,
auth_url=keystone_client.auth_url,
token=auth_token,
artifacts_client=arts)
return self.get_client('murano', use_trusts, factory)
def get_mistral_client(self, use_trusts=True):
if not mistralclient:
raise mistral_import_error
if not CONF.engine.use_trusts:
use_trusts = False
def factory(keystone_client, auth_token):
mistral_settings = CONF.mistral
endpoint_type = mistral_settings.endpoint_type
service_type = mistral_settings.service_type
mistral_url = keystone_client.service_catalog.url_for(
service_type=service_type,
endpoint_type=endpoint_type)
return mistralclient.client(mistral_url=mistral_url,
project_id=keystone_client.tenant_id,
endpoint_type=endpoint_type,
service_type=service_type,
auth_token=auth_token,
user_id=keystone_client.user_id)
return self.get_client('mistral', use_trusts, factory)
def get_artifacts_client(self, use_trusts=True):
if not CONF.engine.use_trusts:
use_trusts = False
def factory(keystone_client, auth_token):
glance_settings = CONF.glance
glance_url = (glance_settings.url or
keystone_client.service_catalog.url_for(
service_type='image',
endpoint_type=glance_settings.endpoint_type))
return art_client.Client(endpoint=glance_url, token=auth_token,
insecure=glance_settings.insecure,
key_file=glance_settings.key_file or None,
cacert=glance_settings.cacert or None,
cert_file=(glance_settings.cert_file or
None),
type_name='murano',
type_version=1)
return self.get_client('artifacts', use_trusts, factory)

6
murano/engine/environment.py → murano/engine/execution_session.py
View File

@ -16,18 +16,16 @@
from oslo_log import log as logging
from murano.common.i18n import _LE
from murano.engine import client_manager
LOG = logging.getLogger(__name__)
class Environment(object):
class ExecutionSession(object):
def __init__(self):
self.token = None
self.tenant_id = None
self.project_id = None
self.trust_id = None
self.system_attributes = {}
self.clients = client_manager.ClientManager(self)
self._set_up_list = []
self._tear_down_list = []

78
murano/engine/package_loader.py
View File

@ -24,10 +24,13 @@ import uuid
import eventlet
from muranoclient.common import exceptions as muranoclient_exc
from muranoclient.glance import client as glare_client
import muranoclient.v1.client as muranoclient
from oslo_config import cfg
from oslo_log import log as logging
import six
from murano.common import auth_utils
from murano.common.i18n import _LE, _LI, _LW
from murano.dsl import constants
from murano.dsl import exceptions
@ -48,18 +51,67 @@ usage_mem_locks = collections.defaultdict(m_utils.ReaderWriterLock)
class ApiPackageLoader(package_loader.MuranoPackageLoader):
def __init__(self, murano_client_factory, tenant_id, root_loader=None):
def __init__(self, execution_session, root_loader=None):
self._cache_directory = self._get_cache_directory()
self._murano_client_factory = murano_client_factory
self.tenant_id = tenant_id
self._class_cache = {}
self._package_cache = {}
self._root_loader = root_loader or self
self._execution_session = execution_session
self._last_glare_token = None
self._glare_client = None
self._murano_client = None
self._murano_client_session = None
self._mem_locks = []
self._ipc_locks = []
self._downloaded = []
def _get_glare_client(self):
glance_settings = CONF.glance
session = auth_utils.get_client_session(self._execution_session)
token = session.auth.get_token(session)
if self._last_glare_token != token:
self._last_glare_token = token
self._glare_client = None
if self._glare_client is None:
url = glance_settings.url
if not url:
url = session.get_endpoint(
service_type='image',
interface=glance_settings.endpoint_type,
region_name=CONF.home_region)
self._glare_client = glare_client.Client(
endpoint=url, token=token,
insecure=glance_settings.insecure,
key_file=glance_settings.key_file or None,
ca_file=glance_settings.ca_file or None,
cert_file=glance_settings.cert_file or None,
type_name='murano',
type_version=1)
return self._glare_client
@property
def client(self):
murano_settings = CONF.murano
last_glare_client = self._glare_client
if CONF.packages_opts.packages_service == 'glance':
artifacts_client = self._get_glare_client()
else:
artifacts_client = None
if artifacts_client != last_glare_client:
self._murano_client = None
if not self._murano_client:
parameters = auth_utils.get_session_client_parameters(
service_type='application-catalog',
execution_session=self._execution_session,
conf=murano_settings
)
self._murano_client = muranoclient.Client(
artifacts_client=artifacts_client, **parameters)
return self._murano_client
def load_class_package(self, class_name, version_spec):
packages = self._class_cache.get(class_name)
if packages:
@ -97,8 +149,9 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
exc_info = sys.exc_info()
six.reraise(exceptions.NoPackageFound(package_name),
None, exc_info[2])
return self._to_dsl_package(
self._get_package_by_definition(package_definition))
else:
return self._to_dsl_package(
self._get_package_by_definition(package_definition))
def register_package(self, package):
for name in package.classes:
@ -129,7 +182,7 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
def _get_definition(self, filter_opts):
filter_opts['catalog'] = True
try:
packages = list(self._murano_client_factory().packages.filter(
packages = list(self.client.packages.filter(
**filter_opts))
if len(packages) > 1:
LOG.debug('Ambiguous package resolution: more then 1 package '
@ -180,8 +233,7 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
download_ipc_lock = m_utils.ExclusiveInterProcessLock(
path=download_lock_path, sleep_func=eventlet.sleep)
with download_mem_locks[package_id].write_lock(),\
download_ipc_lock:
with download_mem_locks[package_id].write_lock(), download_ipc_lock:
# NOTE(kzaitsev):
# in case there were 2 concurrent threads/processes one might have
@ -198,8 +250,7 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
try:
LOG.debug("Attempting to download package {} {}".format(
package_def.fully_qualified_name, package_id))
package_data = self._murano_client_factory().packages.download(
package_id)
package_data = self.client.packages.download(package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e)
@ -304,7 +355,7 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
public = None
other = []
for package in packages:
if package.owner_id == self.tenant_id:
if package.owner_id == self._execution_session.project_id:
return package
elif package.is_public:
public = package
@ -451,10 +502,9 @@ class DirectoryPackageLoader(package_loader.MuranoPackageLoader):
class CombinedPackageLoader(package_loader.MuranoPackageLoader):
def __init__(self, murano_client_factory, tenant_id, root_loader=None):
def __init__(self, execution_session, root_loader=None):
root_loader = root_loader or self
self.api_loader = ApiPackageLoader(
murano_client_factory, tenant_id, root_loader)
self.api_loader = ApiPackageLoader(execution_session, root_loader)
self.directory_loaders = []
for folder in CONF.packages_opts.load_packages_from:

2
murano/engine/system/agent_listener.py
View File

@ -66,7 +66,7 @@ class AgentListener(object):
return
if self._receive_thread is None:
helpers.get_environment().on_session_finish(
helpers.get_execution_session().on_session_finish(
lambda: self.stop())
self._receive_thread = eventlet.spawn(self._receive)

42
murano/engine/system/heat_stack.py
View File

@ -16,15 +16,18 @@
import copy
import eventlet
import heatclient.client as hclient
import heatclient.exc as heat_exc
from oslo_config import cfg
from oslo_log import log as logging
import six
from murano.common import auth_utils
from murano.common.i18n import _LW
from murano.common import utils
from murano.dsl import dsl
from murano.dsl import helpers
from murano.dsl import session_local_storage
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
@ -46,17 +49,35 @@ class HeatStack(object):
self._hot_environment = ''
self._applied = True
self._description = description
self._clients = helpers.get_environment().clients
self._last_stack_timestamps = (None, None)
self._tags = ''
self._client = self._get_client(CONF.home_region)
pass
@staticmethod
def _create_client(session, region_name):
parameters = auth_utils.get_session_client_parameters(
service_type='orchestration', region=region_name,
conf=CONF.heat, session=session)
return hclient.Client('1', **parameters)
@staticmethod
@session_local_storage.execution_session_memoize
def _get_client(region_name):
session = auth_utils.get_client_session(conf=CONF.heat)
return HeatStack._create_client(session, region_name)
@classmethod
def _get_token_client(cls):
ks_session = auth_utils.get_token_client_session(conf=CONF.heat)
return cls._create_client(ks_session, CONF.home_region)
def current(self):
client = self._clients.get_heat_client()
if self._template is not None:
return self._template
try:
stack_info = client.stacks.get(stack_id=self._name)
template = client.stacks.template(
stack_info = self._client.stacks.get(stack_id=self._name)
template = self._client.stacks.template(
stack_id='{0}/{1}'.format(
stack_info.stack_name,
stack_info.id))
@ -126,11 +147,11 @@ class HeatStack(object):
def _wait_state(self, status_func, wait_progress=False):
tries = 4
delay = 1
while tries > 0:
while True:
client = self._clients.get_heat_client()
try:
stack_info = client.stacks.get(
stack_info = self._client.stacks.get(
stack_id=self._name)
status = stack_info.stack_status
tries = 4
@ -194,7 +215,7 @@ class HeatStack(object):
resources = template.get('Resources') or template.get('resources')
if current_status == 'NOT_FOUND':
if resources is not None:
token_client = self._clients.get_heat_client(use_trusts=False)
token_client = self._get_token_client()
token_client.stacks.create(
stack_name=self._name,
parameters=self._parameters,
@ -207,9 +228,7 @@ class HeatStack(object):
self._wait_state(lambda status: status == 'CREATE_COMPLETE')
else:
if resources is not None:
trust_client = self._clients.get_heat_client()
trust_client.stacks.update(
self._client.stacks.update(
stack_id=self._name,
parameters=self._parameters,
files=self._files,
@ -225,11 +244,10 @@ class HeatStack(object):
self._applied = not utils.is_different(self._template, template)
def delete(self):
client = self._clients.get_heat_client()
try:
if not self.current():
return
client.stacks.delete(stack_id=self._name)
self._client.stacks.delete(stack_id=self._name)
self._wait_state(
lambda status: status in ('DELETE_COMPLETE', 'NOT_FOUND'),
wait_progress=True)

54
murano/engine/system/mistralclient.py
View File

@ -17,9 +17,17 @@
import json
import eventlet
try:
import mistralclient.api.client as mistralclient
except ImportError as mistral_import_error:
mistralclient = None
from oslo_config import cfg
from murano.common import auth_utils
from murano.dsl import dsl
from murano.dsl import helpers
from murano.dsl import session_local_storage
CONF = cfg.CONF
class MistralError(Exception):
@ -28,18 +36,44 @@ class MistralError(Exception):
@dsl.name('io.murano.system.MistralClient')
class MistralClient(object):
def __init__(self, context):
self._clients = helpers.get_environment(context).clients
def __init__(self):
self._client = self._create_client(CONF.home_region)
@staticmethod
@session_local_storage.execution_session_memoize
def _create_client(region):
if not mistralclient:
raise mistral_import_error
mistral_settings = CONF.mistral
endpoint_type = mistral_settings.endpoint_type
service_type = mistral_settings.service_type
session = auth_utils.get_client_session()
mistral_url = mistral_settings.url or session.get_endpoint(
service_type=service_type,
endpoint_type=endpoint_type,
region_name=region)
auth_ref = session.auth.get_access(session)
return mistralclient.client(
mistral_url=mistral_url,
project_id=auth_ref.project_id,
endpoint_type=endpoint_type,
service_type=service_type,
auth_token=auth_ref.auth_token,
user_id=auth_ref.user_id,
insecure=mistral_settings.insecure,
cacert=mistral_settings.ca_cert
)
def upload(self, definition):
mistral_client = self._clients.get_mistral_client()
mistral_client.workflows.create(definition)
self._client.workflows.create(definition)
def run(self, name, timeout=600, inputs=None, params=None):
mistral_client = self._clients.get_mistral_client()
execution = mistral_client.executions.create(workflow_name=name,
workflow_input=inputs,
params=params)
execution = self._client.executions.create(
workflow_name=name, workflow_input=inputs, params=params)
# For the fire and forget functionality - when we do not want to wait
# for the result of the run.
if timeout == 0:
@ -51,7 +85,7 @@ class MistralClient(object):
with eventlet.timeout.Timeout(timeout):
while state not in ('ERROR', 'SUCCESS'):
eventlet.sleep(2)
execution = mistral_client.executions.get(execution.id)
execution = self._client.executions.get(execution.id)
state = execution.state
except eventlet.timeout.Timeout:
error_message = (

49
murano/engine/system/net_explorer.py
View File

@ -16,15 +16,18 @@ import math
import netaddr
from netaddr.strategy import ipv4
import neutronclient.v2_0.client as nclient
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import uuidutils
import retrying
from murano.common import auth_utils
from murano.common import exceptions as exc
from murano.common.i18n import _LI
from murano.dsl import dsl
from murano.dsl import helpers
from murano.dsl import session_local_storage
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
@ -33,11 +36,19 @@ LOG = logging.getLogger(__name__)
@dsl.name('io.murano.system.NetworkExplorer')
class NetworkExplorer(object):
def __init__(self):
environment = helpers.get_environment()
self._clients = environment.clients
self._tenant_id = environment.tenant_id
session = helpers.get_execution_session()
self._project_id = session.project_id
self._settings = CONF.networking
self._available_cidrs = self._generate_possible_cidrs()
self._client = self._get_client(CONF.home_region)
@staticmethod
@session_local_storage.execution_session_memoize
def _get_client(region_name):
neutron_settings = CONF.neutron
return nclient.Client(**auth_utils.get_session_client_parameters(
service_type='network', region=region_name, conf=neutron_settings
))
# NOTE(starodubcevna): to avoid simultaneous router requests we use retry
# decorator with random delay 1-10 seconds between attempts and maximum
@ -47,11 +58,10 @@ class NetworkExplorer(object):
wait_random_min=1000, wait_random_max=10000,
stop_max_delay=30000)
def get_default_router(self):
client = self._clients.get_neutron_client()
router_name = self._settings.router_name
routers = client.list_routers(
tenant_id=self._tenant_id, name=router_name).get('routers')
routers = self._client.list_routers(
tenant_id=self._project_id, name=router_name).get('routers')
if len(routers) == 0:
LOG.debug('Router {name} not found'.format(name=router_name))
if self._settings.create_router:
@ -61,7 +71,7 @@ class NetworkExplorer(object):
kwargs = {'id': external_network} \
if uuidutils.is_uuid_like(external_network) \
else {'name': external_network}
networks = client.list_networks(**kwargs).get('networks')
networks = self._client.list_networks(**kwargs).get('networks')
ext_nets = filter(lambda n: n['router:external'], networks)
if len(ext_nets) == 0:
raise KeyError('Router %s could not be created, '
@ -77,7 +87,8 @@ class NetworkExplorer(object):
'admin_state_up': True,
}
}
router = client.create_router(body=body_data).get('router')
router = self._client.create_router(
body=body_data).get('router')
LOG.info(_LI('Created router: {id}').format(id=router['id']))
return router['id']
else:
@ -112,20 +123,18 @@ class NetworkExplorer(object):
return self._settings.default_dns
def get_external_network_id_for_router(self, router_id):
client = self._clients.get_neutron_client()
router = client.show_router(router_id).get('router')
router = self._client.show_router(router_id).get('router')
if not router or 'external_gateway_info' not in router:
return None
return router['external_gateway_info'].get('network_id')
def get_external_network_id_for_network(self, network_id):
client = self._clients.get_neutron_client()
network = client.show_network(network_id).get('network')
network = self._client.show_network(network_id).get('network')
if network.get('router:external', False):
return network_id
# Get router interfaces of the network
router_ports = client.list_ports(
router_ports = self._client.list_ports(
**{'device_owner': 'network:router_interface',
'network_id': network_id}).get('ports')
@ -141,14 +150,13 @@ class NetworkExplorer(object):
def _get_cidrs_taken_by_router(self, router_id):
if not router_id:
return []
client = self._clients.get_neutron_client()
ports = client.list_ports(device_id=router_id)['ports']
ports = self._client.list_ports(device_id=router_id)['ports']
subnet_ids = []
for port in ports:
for fixed_ip in port['fixed_ips']:
subnet_ids.append(fixed_ip['subnet_id'])
all_subnets = client.list_subnets()['subnets']
all_subnets = self._client.list_subnets()['subnets']
filtered_cidrs = [netaddr.IPNetwork(subnet['cidr']) for subnet in
all_subnets if subnet['id'] in subnet_ids]
@ -169,13 +177,10 @@ class NetworkExplorer(object):
return list(net.subnet(width - bits_for_hosts))
def list_networks(self):
client = self._clients.get_neutron_client()
return client.list_networks()['networks']
return self._client.list_networks()['networks']
def list_subnetworks(self):
client = self._clients.get_neutron_client()
return client.list_subnets()['subnets']
return self._client.list_subnets()['subnets']
def list_ports(self):
client = self._clients.get_neutron_client()
return client.list_ports()['ports']
return self._client.list_ports()['ports']

8
murano/engine/system/test_fixture.py
View File

@ -34,12 +34,12 @@ class TestFixture(object):
return exc.load(model)
def finish_env(self):
env = helpers.get_environment()
env.finish()
session = helpers.get_execution_session()
session.finish()
def start_env(self):
env = helpers.get_environment()
env.start()
session = helpers.get_execution_session()
session.start()
def assert_equal(self, expected, observed, message=None):
self._test_case.assertEqual(expected, observed, message)

3
murano/opts.py
View File

@ -34,8 +34,9 @@ _opt_lists = [
('rabbitmq', murano.common.config.rabbit_opts),
('heat', murano.common.config.heat_opts),
('neutron', murano.common.config.neutron_opts),
('keystone', murano.common.config.keystone_opts),
('murano', murano.common.config.murano_opts),
('glance', murano.common.config.glance_opts),
('mistral', murano.common.config.mistral_opts),
('networking', murano.common.config.networking_opts),
('stats', murano.common.config.stats_opts),
('packages_opts', murano.common.config.packages_opts),

41
murano/policy/model_policy_enforcer.py
View File

@ -15,14 +15,22 @@
import re
try:
# integration with congress is optional
import congressclient.v1.client as congress_client
except ImportError as congress_client_import_error:
congress_client = None
from oslo_config import cfg
from oslo_log import log as logging
from murano.common import auth_utils
from murano.common.i18n import _, _LI
from murano.policy import congress_rules
from murano.policy.modify.actions import action_manager as am
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class ValidationError(Exception):
@ -40,10 +48,25 @@ class ModelPolicyEnforcer(object):
table along with congress data rules to return validation results.
"""
def __init__(self, environment, action_manager=None):
self._environment = environment
self._client_manager = environment.clients
def __init__(self, execution_session, action_manager=None):
self._execution_session = execution_session
self._action_manager = action_manager or am.ModifyActionManager()
self._client = None
def _create_client(self):
if not congress_client:
# congress client was not imported
raise congress_client_import_error
return congress_client.Client(
**auth_utils.get_session_client_parameters(
service_type='policy',
execution_session=self._execution_session))
@property
def client(self):
if self._client is None:
self._client = self._create_client()
return self._client
def modify(self, obj, package_loader=None):
"""Modifies model using Congress rule engine.
@ -110,7 +133,7 @@ class ModelPolicyEnforcer(object):
def _execute_simulation(self, package_loader, env_id, model, query):
rules = congress_rules.CongressRulesManager().convert(
model, package_loader, self._environment.tenant_id)
model, package_loader, self._execution_session.project_id)
rules_str = list(map(str, rules))
# cleanup of data populated by murano driver
rules_str.insert(0, 'deleteEnv("{0}")'.format(env_id))
@ -118,9 +141,7 @@ class ModelPolicyEnforcer(object):
LOG.debug('Congress rules: \n {rules} '
.format(rules='\n '.join(rules_str)))
client = self._check_client()
validation_result = client.execute_policy_action(
validation_result = self.client.execute_policy_action(
"murano_system",
"simulate",
False,
@ -130,12 +151,6 @@ class ModelPolicyEnforcer(object):
'sequence': rules_line})
return validation_result
def _check_client(self):
client = self._client_manager.get_congress_client(self._environment)
if not client:
raise ValueError(_('Congress client is not configured!'))
return client
@staticmethod
def _parse_simulation_result(query, env_id, results):
"""Transforms list of strings in format

4
murano/tests/unit/dsl/foundation/runner.py
View File

@ -25,7 +25,7 @@ from murano.dsl import helpers
from murano.dsl import murano_object
from murano.dsl import serializer
from murano.dsl import yaql_integration
from murano.engine import environment
from murano.engine import execution_session
from murano.engine.system import yaql_functions
from murano.tests.unit.dsl.foundation import object_model
@ -77,7 +77,7 @@ class Runner(object):
self.executor = executor.MuranoDslExecutor(
package_loader, TestContextManager(functions),
environment.Environment())
execution_session.ExecutionSession())
self._root = self.executor.load(model).object
def _execute(self, name, object_id, *args, **kwargs):

5
murano/tests/unit/dsl/test_agent.py
View File

@ -19,7 +19,7 @@ from murano.common import exceptions as exc
from murano.dsl import constants
from murano.dsl import helpers
from murano.dsl import yaql_integration
from murano.engine import environment
from murano.engine import execution_session
from murano.engine.system import agent
from murano.engine.system import agent_listener
from murano.tests.unit.dsl.foundation import object_model as om
@ -37,7 +37,8 @@ class TestAgentListener(test_case.DslTestCase):
'AgentListenerTests')
self.runner = self.new_runner(model)
self.context = yaql_integration.create_empty_context()
self.context[constants.CTX_ENVIRONMENT] = environment.Environment()
self.context[constants.CTX_EXECUTION_SESSION] = \
execution_session.ExecutionSession()
def test_listener_enabled(self):
self.override_config('disable_murano_agent', False, 'engine')

4
murano/tests/unit/engine/test_mock_context_manager.py
View File

@ -18,7 +18,7 @@ from yaql import specs
from murano.dsl import constants
from murano.dsl import executor
from murano.dsl import murano_class
from murano.engine import environment
from murano.engine import execution_session
from murano.engine import mock_context_manager
from murano.engine.system import test_fixture
from murano.tests.unit import base
@ -58,7 +58,7 @@ class MockRunner(runner.Runner):
model = {'Objects': model}
self.executor = executor.MuranoDslExecutor(
package_loader, TestMockContextManager(functions),
environment.Environment())
execution_session.ExecutionSession())
self._root = self.executor.load(model).object

14
murano/tests/unit/engine/test_package_loader.py
View File

@ -17,6 +17,7 @@ import tempfile
import mock
from oslo_config import cfg
import semantic_version
import testtools
from murano.dsl import murano_package as dsl_package
from murano.engine import package_loader
@ -37,23 +38,22 @@ class TestPackageCache(base.MuranoTestCase):
CONF.set_override('packages_cache', self.location, 'packages_opts')
self.murano_client = mock.MagicMock()
self.murano_client_factory = mock.MagicMock(
return_value=self.murano_client)
self.loader = package_loader.ApiPackageLoader(
self.murano_client_factory, 'test_tenant_id')
package_loader.ApiPackageLoader.client = self.murano_client
self.loader = package_loader.ApiPackageLoader(None)
def tearDown(self):
CONF.set_override('packages_cache', self.old_location, 'packages_opts')
shutil.rmtree(self.location, ignore_errors=True)
super(TestPackageCache, self).tearDown()
@testtools.skipIf(os.name == 'nt', "Doesn't work on Windows")
def test_load_package(self):
fqn = 'io.murano.apps.test'
path, name = utils.compose_package(
'test',
os.path.join(self.location, 'manifest.yaml'),
self.location, archive_dir=self.location)
with open(path) as f:
with open(path, 'rb') as f:
package_data = f.read()
spec = semantic_version.Spec('*')
@ -154,9 +154,9 @@ class TestCombinedPackageLoader(base.MuranoTestCase):
location = os.path.dirname(__file__)
CONF.set_override('load_packages_from', [location], 'packages_opts',
enforce_type=True)
cls.murano_client_factory = mock.MagicMock()
cls.execution_session = mock.MagicMock()
cls.loader = package_loader.CombinedPackageLoader(
cls.murano_client_factory, 'test_tenant_id')
cls.execution_session)
cls.api_loader = mock.MagicMock()
cls.loader.api_loader = cls.api_loader

25
murano/tests/unit/policy/test_model_policy_enforcer.py
View File

@ -18,7 +18,6 @@ import mock
from oslo_config import cfg
from murano.common import engine
from murano.engine import client_manager
from murano.policy import model_policy_enforcer
from murano.tests.unit import base
@ -45,14 +44,8 @@ class TestModelPolicyEnforcer(base.MuranoTestCase):
self.congress_client_mock = \
mock.Mock(spec=congressclient.v1.client.Client)
self.client_manager_mock = mock.Mock(spec=client_manager.ClientManager)
self.client_manager_mock.get_congress_client.return_value = \
self.congress_client_mock
self.environment = mock.Mock()
self.environment.clients = self.client_manager_mock
model_policy_enforcer.ModelPolicyEnforcer._create_client = mock.Mock(
return_value=self.congress_client_mock)
def test_enforcer_disabled(self):
executor = engine.TaskExecutor(self.task)
@ -74,17 +67,11 @@ class TestModelPolicyEnforcer(base.MuranoTestCase):
.validate.assert_called_once_with(self.model_dict,
self.package_loader)
def test_enforcer_no_client(self):
self.client_manager_mock.get_congress_client.return_value = None
enforcer = model_policy_enforcer.ModelPolicyEnforcer(self.environment)
model = {'?': {'id': '123', 'type': 'class'}}
self.assertRaises(ValueError, enforcer.validate, model)
def test_validation_pass(self):
self.congress_client_mock.execute_policy_action.return_value = \
{"result": []}
model = {'?': {'id': '123', 'type': 'class'}}
enforcer = model_policy_enforcer.ModelPolicyEnforcer(self.environment)
enforcer = model_policy_enforcer.ModelPolicyEnforcer(mock.Mock())
enforcer.validate(model)
def test_validation_failure(self):
@ -92,7 +79,7 @@ class TestModelPolicyEnforcer(base.MuranoTestCase):
{"result": ['predeploy_errors("123","instance1","failure")']}
model = {'?': {'id': '123', 'type': 'class'}}
enforcer = model_policy_enforcer.ModelPolicyEnforcer(self.environment)
enforcer = model_policy_enforcer.ModelPolicyEnforcer(mock.Mock())
self.assertRaises(model_policy_enforcer.ValidationError,
enforcer.validate, model)
@ -107,7 +94,7 @@ class TestModelPolicyEnforcer(base.MuranoTestCase):
action_manager = mock.MagicMock()
enforcer = model_policy_enforcer.ModelPolicyEnforcer(
self.environment, action_manager)
mock.Mock(), action_manager)
enforcer.modify(obj)
self.assertTrue(action_manager.apply_action.called)
@ -120,7 +107,7 @@ class TestModelPolicyEnforcer(base.MuranoTestCase):
'predeploy_errors("env2","instance1","Instance 3 has problem")'
]
enforcer = model_policy_enforcer.ModelPolicyEnforcer(self.environment)
enforcer = model_policy_enforcer.ModelPolicyEnforcer(None)
result = enforcer._parse_simulation_result(
'predeploy_errors', 'env1', congress_response)

81
murano/tests/unit/test_heat_stack.py
View File

@ -17,50 +17,34 @@ from heatclient.v1 import stacks
import mock
from oslo_config import cfg
from murano.dsl import constants
from murano.dsl import helpers
from murano.dsl import murano_class
from murano.dsl import object_store
from murano.engine import client_manager
from murano.engine import environment
from murano.engine.system import heat_stack
from murano.tests.unit import base
MOD_NAME = 'murano.engine.system.heat_stack'
CLS_NAME = 'murano.engine.system.heat_stack.HeatStack'
CONF = cfg.CONF
class TestHeatStack(base.MuranoTestCase):
def setUp(self):
super(TestHeatStack, self).setUp()
self.mock_murano_class = mock.Mock(spec=murano_class.MuranoClass)
self.mock_murano_class.name = 'io.murano.system.HeatStack'
self.mock_murano_class.declared_parents = []
self.heat_client_mock = mock.MagicMock()
self.heat_client_mock = mock.Mock()
self.heat_client_mock.stacks = mock.MagicMock(spec=stacks.StackManager)
self.mock_object_store = mock.Mock(spec=object_store.ObjectStore)
self.environment_mock = mock.Mock(
spec=environment.Environment)
client_manager_mock = mock.Mock(spec=client_manager.ClientManager)
client_manager_mock.get_heat_client.return_value = \
self.heat_client_mock
self.environment_mock.clients = client_manager_mock
CONF.set_override('stack_tags', ['test-murano'], 'heat',
enforce_type=True)
self.mock_tag = ','.join(CONF.heat.stack_tags)
heat_stack.HeatStack._get_token_client = mock.Mock(
return_value=self.heat_client_mock)
heat_stack.HeatStack._get_client = mock.Mock(
return_value=self.heat_client_mock)
@mock.patch(MOD_NAME + '.HeatStack._wait_state')
@mock.patch(MOD_NAME + '.HeatStack._get_status')
@mock.patch(CLS_NAME + '._wait_state')
@mock.patch(CLS_NAME + '._get_status')
def test_push_adds_version(self, status_get, wait_st):
"""Assert that if heat_template_version is omitted, it's added."""
status_get.return_value = 'NOT_FOUND'
wait_st.return_value = {}
context = {constants.CTX_ENVIRONMENT: self.environment_mock}
with helpers.contextual(context):
hs = heat_stack.HeatStack(
'test-stack', 'Generated by TestHeatStack')
hs = heat_stack.HeatStack('test-stack', 'Generated by TestHeatStack')
hs._template = {'resources': {'test': 1}}
hs._files = {}
hs._hot_environment = ''
@ -68,9 +52,7 @@ class TestHeatStack(base.MuranoTestCase):
hs._applied = False
hs.push()
with helpers.contextual(context):
hs = heat_stack.HeatStack(
'test-stack', 'Generated by TestHeatStack')
hs = heat_stack.HeatStack('test-stack', 'Generated by TestHeatStack')
hs._template = {'resources': {'test': 1}}
hs._files = {}
hs._parameters = {}
@ -93,17 +75,14 @@ class TestHeatStack(base.MuranoTestCase):
)
self.assertTrue(hs._applied)
@mock.patch(MOD_NAME + '.HeatStack._wait_state')
@mock.patch(MOD_NAME + '.HeatStack._get_status')
@mock.patch(CLS_NAME + '._wait_state')
@mock.patch(CLS_NAME + '._get_status')
def test_description_is_optional(self, status_get, wait_st):
"""Assert that if heat_template_version is omitted, it's added."""
status_get.return_value = 'NOT_FOUND'
wait_st.return_value = {}
context = {constants.CTX_ENVIRONMENT: self.environment_mock}
with helpers.contextual(context):
hs = heat_stack.HeatStack('test-stack', None)
hs = heat_stack.HeatStack('test-stack', None)
hs._template = {'resources': {'test': 1}}
hs._files = {}
hs._hot_environment = ''
@ -126,17 +105,14 @@ class TestHeatStack(base.MuranoTestCase):
)
self.assertTrue(hs._applied)
@mock.patch(MOD_NAME + '.HeatStack._wait_state')
@mock.patch(MOD_NAME + '.HeatStack._get_status')
@mock.patch(CLS_NAME + '._wait_state')
@mock.patch(CLS_NAME + '._get_status')
def test_heat_files_are_sent(self, status_get, wait_st):
"""Assert that if heat_template_version is omitted, it's added."""
status_get.return_value = 'NOT_FOUND'
wait_st.return_value = {}
context = {constants.CTX_ENVIRONMENT: self.environment_mock}
with helpers.contextual(context):
hs = heat_stack.HeatStack('test-stack', None)
hs = heat_stack.HeatStack('test-stack', None)
hs._description = None
hs._template = {'resources': {'test': 1}}
hs._files = {"heatFile": "file"}
@ -160,17 +136,14 @@ class TestHeatStack(base.MuranoTestCase):
)
self.assertTrue(hs._applied)
@mock.patch(MOD_NAME + '.HeatStack._wait_state')
@mock.patch(MOD_NAME + '.HeatStack._get_status')
@mock.patch(CLS_NAME + '._wait_state')
@mock.patch(CLS_NAME + '._get_status')
def test_heat_environments_are_sent(self, status_get, wait_st):
"""Assert that if heat_template_version is omitted, it's added."""
status_get.return_value = 'NOT_FOUND'
wait_st.return_value = {}
context = {constants.CTX_ENVIRONMENT: self.environment_mock}
with helpers.contextual(context):
hs = heat_stack.HeatStack('test-stack', None)
hs = heat_stack.HeatStack('test-stack', None)
hs._description = None
hs._template = {'resources': {'test': 1}}
hs._files = {"heatFile": "file"}
@ -194,14 +167,11 @@ class TestHeatStack(base.MuranoTestCase):
)
self.assertTrue(hs._applied)
@mock.patch(MOD_NAME + '.HeatStack.current')
@mock.patch(CLS_NAME + '.current')
def test_update_wrong_template_version(self, current):
"""Template version other than expected should cause error."""
context = {constants.CTX_ENVIRONMENT: self.environment_mock}
with helpers.contextual(context):
hs = heat_stack.HeatStack(
'test-stack', 'Generated by TestHeatStack')
hs = heat_stack.HeatStack('test-stack', 'Generated by TestHeatStack')
hs._template = {'resources': {'test': 1}}
invalid_template = {
@ -227,8 +197,8 @@ class TestHeatStack(base.MuranoTestCase):
expected['heat_template_version'] = '2013-05-23'
self.assertEqual(expected, hs._template)
@mock.patch(MOD_NAME + '.HeatStack._wait_state')
@mock.patch(MOD_NAME + '.HeatStack._get_status')
@mock.patch(CLS_NAME + '._wait_state')
@mock.patch(CLS_NAME + '._get_status')
def test_heat_stack_tags_are_sent(self, status_get, wait_st):
"""Assert that heat_stack `tags` parameter get push & with
value from config parameter `stack_tags`.
@ -238,10 +208,7 @@ class TestHeatStack(base.MuranoTestCase):
wait_st.return_value = {}
CONF.set_override('stack_tags', ['test-murano', 'murano-tag'], 'heat',
enforce_type=True)
context = {constants.CTX_ENVIRONMENT: self.environment_mock}
with helpers.contextual(context):
hs = heat_stack.HeatStack('test-stack', None)
hs = heat_stack.HeatStack('test-stack', None)
hs._description = None
hs._template = {'resources': {'test': 1}}
hs._files = {}