diff --git a/murano/api/v1/catalog.py b/murano/api/v1/catalog.py index be7e65eee..b10667f61 100644 --- a/murano/api/v1/catalog.py +++ b/murano/api/v1/catalog.py @@ -29,8 +29,8 @@ from murano.openstack.common import exception from murano.openstack.common.gettextutils import _ # noqa from murano.openstack.common import log as logging from murano.openstack.common import wsgi -from murano.packages import application_package as app_pkg from murano.packages import exceptions as pkg_exc +from murano.packages import load_utils LOG = logging.getLogger(__name__) CONF = cfg.CONF @@ -201,14 +201,13 @@ class Controller(object): tempf.write(content) package_meta['archive'] = content try: - LOG.debug("Deleting package archive temporary file") - pkg_to_upload = app_pkg.load_from_file(tempf.name, - target_dir=None, - drop_dir=True) + pkg_to_upload = load_utils.load_from_file( + tempf.name, target_dir=None, drop_dir=True) except pkg_exc.PackageLoadError as e: LOG.exception(e) raise exc.HTTPBadRequest(e) finally: + LOG.debug("Deleting package archive temporary file") os.remove(tempf.name) # extend dictionary for update db diff --git a/murano/cmd/manage.py b/murano/cmd/manage.py index 3399dd114..918f1c8b9 100644 --- a/murano/cmd/manage.py +++ b/murano/cmd/manage.py @@ -27,7 +27,7 @@ from murano.db.catalog import api as db_catalog_api from murano.db import session as db_session from murano.openstack.common.db import exception as db_exception from murano.openstack.common import log as logging -from murano.packages import application_package +from murano.packages import load_utils CONF = cfg.CONF @@ -50,7 +50,7 @@ class AdminContext(object): def _do_import_package(_dir, categories, update=False): LOG.info("Going to import Murano package from {0}".format(_dir)) - pkg = application_package.load_from_dir(_dir) + pkg = load_utils.load_from_dir(_dir) LOG.info("Checking for existing") existing = db_catalog_api.package_search( diff --git a/murano/common/engine.py b/murano/common/engine.py index 05fa646ab..cc405a254 100644 --- a/murano/common/engine.py +++ b/murano/common/engine.py @@ -73,10 +73,11 @@ class TaskProcessingEndpoint(object): # TODO(slagun) code below needs complete rewrite and redesign LOG.exception("Error during task execution for tenant %s", env.tenant_id) - msg_env = Environment(task['model']['Objects']['?']['id']) - reporter = status_reporter.StatusReporter() - reporter.initialize(msg_env) - reporter.report_error(msg_env, '{0}'.format(e)) + if task['model']['Objects']: + msg_env = Environment(task['model']['Objects']['?']['id']) + reporter = status_reporter.StatusReporter() + reporter.initialize(msg_env) + reporter.report_error(msg_env, '{0}'.format(e)) rpc.api().process_result(task['model']) diff --git a/murano/common/server.py b/murano/common/server.py index 5b9798fd9..c2eac8f1d 100644 --- a/murano/common/server.py +++ b/murano/common/server.py @@ -43,13 +43,12 @@ class ResultEndpoint(object): LOG.debug('Got result from orchestration ' 'engine:\n{0}'.format(secure_result)) - result_id = result['Objects']['?']['id'] - - if 'deleted' in result: - LOG.debug('Result for environment {0} is dropped. Environment ' - 'is deleted'.format(result_id)) + if not result['Objects']: + LOG.debug('Ignoring result for deleted environment') return + result_id = result['Objects']['?']['id'] + unit = session.get_session() environment = unit.query(models.Environment).get(result_id) diff --git a/murano/dsl/executor.py b/murano/dsl/executor.py index 27575abcc..a667cc221 100644 --- a/murano/dsl/executor.py +++ b/murano/dsl/executor.py @@ -233,7 +233,7 @@ class MuranoDslExecutor(object): if not isinstance(data, types.DictionaryType): raise TypeError() self._attribute_store.load(data.get('Attributes') or []) - result = self._object_store.load(data.get('Objects') or {}, + result = self._object_store.load(data.get('Objects'), None, self._root_context) self.cleanup(data) return result diff --git a/murano/dsl/yaql_functions.py b/murano/dsl/yaql_functions.py index 538e693ef..07b686878 100644 --- a/murano/dsl/yaql_functions.py +++ b/murano/dsl/yaql_functions.py @@ -129,6 +129,11 @@ def _sleep(seconds): eventlet.sleep(seconds) +@yaql.context.EvalArg('value', murano_object.MuranoObject) +def _type(value): + return value.type.name + + def register(context): context.register_function(_resolve, '#resolve') context.register_function(_cast, 'cast') @@ -140,3 +145,4 @@ def register(context): context.register_function(_require, 'require') context.register_function(_get_container, 'find') context.register_function(_sleep, 'sleep') + context.register_function(_type, 'type') diff --git a/murano/engine/package_loader.py b/murano/engine/package_loader.py index 52ea059de..ec685504f 100644 --- a/murano/engine/package_loader.py +++ b/murano/engine/package_loader.py @@ -23,31 +23,17 @@ from keystoneclient.v2_0 import client as keystoneclient from muranoclient.common import exceptions as muranoclient_exc from muranoclient.v1 import client as muranoclient import six -import yaml from murano.common import config from murano.dsl import exceptions -from murano.dsl import yaql_expression +from murano.engine import yaql_yaml_loader from murano.openstack.common import log as logging -from murano.packages import application_package as app_pkg from murano.packages import exceptions as pkg_exc +from murano.packages import load_utils LOG = logging.getLogger(__name__) -class YaqlYamlLoader(yaml.Loader): - pass - - -def yaql_constructor(loader, node): - value = loader.construct_scalar(node) - return yaql_expression.YaqlExpression(value) - -yaml.add_constructor(u'!yaql', yaql_constructor, YaqlYamlLoader) -yaml.add_implicit_resolver(u'!yaql', yaql_expression.YaqlExpression, - Loader=YaqlYamlLoader) - - class PackageLoader(six.with_metaclass(abc.ABCMeta)): @abc.abstractmethod def get_package(self, name): @@ -149,34 +135,36 @@ class ApiPackageLoader(PackageLoader): if os.path.exists(package_directory): try: - return app_pkg.load_from_dir(package_directory, preload=True, - loader=YaqlYamlLoader) + return load_utils.load_from_dir( + package_directory, preload=True, + loader=yaql_yaml_loader.YaqlYamlLoader) except pkg_exc.PackageLoadError: LOG.exception('Unable to load package from cache. Clean-up...') shutil.rmtree(package_directory, ignore_errors=True) - try: package_data = self._client.packages.download(package_id) except muranoclient_exc.HTTPException: LOG.exception('Unable to download ' 'package with id {0}'.format(package_id)) raise pkg_exc.PackageLoadError() + package_file = None try: with tempfile.NamedTemporaryFile(delete=False) as package_file: package_file.write(package_data) - return app_pkg.load_from_file( + return load_utils.load_from_file( package_file.name, target_dir=package_directory, drop_dir=False, - loader=YaqlYamlLoader + loader=yaql_yaml_loader.YaqlYamlLoader ) except IOError: LOG.exception('Unable to write package file') raise pkg_exc.PackageLoadError() finally: try: - os.remove(package_file.name) + if package_file: + os.remove(package_file.name) except OSError: pass @@ -213,8 +201,9 @@ class DirectoryPackageLoader(PackageLoader): continue try: - package = app_pkg.load_from_dir(folder, preload=True, - loader=YaqlYamlLoader) + package = load_utils.load_from_dir( + folder, preload=True, + loader=yaql_yaml_loader.YaqlYamlLoader) except pkg_exc.PackageLoadError: LOG.exception('Unable to load package from path: ' '{0}'.format(entry)) diff --git a/murano/engine/system/heat_stack.py b/murano/engine/system/heat_stack.py index e0519c769..fdee339a1 100644 --- a/murano/engine/system/heat_stack.py +++ b/murano/engine/system/heat_stack.py @@ -26,7 +26,7 @@ import murano.dsl.murano_class as murano_class import murano.dsl.murano_object as murano_object import murano.openstack.common.log as logging -log = logging.getLogger(__name__) +LOG = logging.getLogger(__name__) @murano_class.classname('io.murano.system.HeatStack') @@ -103,6 +103,10 @@ class HeatStack(murano_object.MuranoObject): self._parameters.clear() self._applied = False + def setParameters(self, parameters): + self._parameters = parameters + self._applied = False + def updateTemplate(self, template): self.current() self._template = helpers.merge_dicts(self._template, template) @@ -161,13 +165,13 @@ class HeatStack(murano_object.MuranoObject): if self._applied or self._template is None: return - log.info('Pushing: {0}'.format(self._template)) + LOG.info('Pushing: {0}'.format(self._template)) current_status = self._get_status() + resources = self._template.get('Resources') or \ + self._template.get('resources') if current_status == 'NOT_FOUND': - # For now, allow older CFN style templates as well, but this - # should be removed to avoid mixing them - if 'resources' in self._template or 'Resources' in self._template: + if resources: self._heat_client.stacks.create( stack_name=self._name, parameters=self._parameters, @@ -177,9 +181,7 @@ class HeatStack(murano_object.MuranoObject): self._wait_state( lambda status: status == 'CREATE_COMPLETE') else: - # For now, allow older CFN style templates as well, but this - # should be removed to avoid mixing them - if 'resources' in self._template or 'Resources' in self._template: + if resources: self._heat_client.stacks.update( stack_id=self._name, parameters=self._parameters, diff --git a/murano/engine/system/resource_manager.py b/murano/engine/system/resource_manager.py index 8a1da9a3c..47f04609d 100644 --- a/murano/engine/system/resource_manager.py +++ b/murano/engine/system/resource_manager.py @@ -18,6 +18,30 @@ import yaml as yamllib import murano.dsl.murano_object as murano_object +if hasattr(yamllib, 'CSafeLoader'): + yaml_loader = yamllib.CSafeLoader +else: + yaml_loader = yamllib.SafeLoader + +if hasattr(yamllib, 'CSafeDumper'): + yaml_dumper = yamllib.CSafeDumper +else: + yaml_dumper = yamllib.SafeDumper + + +def _construct_yaml_str(self, node): + # Override the default string handling function + # to always return unicode objects + return self.construct_scalar(node) + +yaml_loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str) +# Unquoted dates like 2013-05-23 in yaml files get loaded as objects of type +# datetime.data which causes problems in API layer when being processed by +# openstack.common.jsonutils. Therefore, make unicode string out of timestamps +# until jsonutils can handle dates. +yaml_loader.add_constructor(u'tag:yaml.org,2002:timestamp', + _construct_yaml_str) + class ResourceManager(murano_object.MuranoObject): def initialize(self, package_loader, _context, _class): diff --git a/murano/engine/system/yaql_functions.py b/murano/engine/system/yaql_functions.py index 1624fe25f..6da0dc1c6 100644 --- a/murano/engine/system/yaql_functions.py +++ b/murano/engine/system/yaql_functions.py @@ -15,7 +15,10 @@ import base64 import collections +import random import re +import string +import time import types import jsonpatch @@ -27,6 +30,9 @@ import murano.common.config as cfg import murano.dsl.helpers as helpers +_random_string_counter = None + + def _transform_json(json, mappings): if isinstance(json, types.ListType): return [_transform_json(t, mappings) for t in json] @@ -204,6 +210,55 @@ def _patch(obj, patch): return obj +def _int2base(x, base): + """Converts decimal integers into another number base + from base-2 to base-36. + + :param x: decimal integer + :param base: number base, max value is 36 + :return: integer converted to the specified base + """ + digs = string.digits + string.lowercase + if x < 0: + sign = -1 + elif x == 0: + return '0' + else: + sign = 1 + x *= sign + digits = [] + while x: + digits.append(digs[x % base]) + x /= base + if sign < 0: + digits.append('-') + digits.reverse() + return ''.join(digits) + + +def _random_name(): + """Replace '#' char in pattern with supplied number, if no pattern is + supplied generate short and unique name for the host. + + :param pattern: hostname pattern + :param number: number to replace with in pattern + :return: hostname + """ + global _random_string_counter + + counter = _random_string_counter or 1 + # generate first 5 random chars + prefix = ''.join(random.choice(string.lowercase) for _ in range(5)) + # convert timestamp to higher base to shorten hostname string + # (up to 8 chars) + timestamp = _int2base(int(time.time() * 1000), 36)[:8] + # third part of random name up to 2 chars + # (1295 is last 2-digit number in base-36, 1296 is first 3-digit number) + suffix = _int2base(counter, 36) + _random_string_counter = (counter + 1) % 1296 + return prefix + timestamp + suffix + + @yaql.context.EvalArg('self', dict) def _values(self): return self.values() @@ -250,6 +305,7 @@ def register(context): context.register_function(_str, 'str') context.register_function(_int, 'int') context.register_function(_patch, 'patch') + context.register_function(_random_name, 'randomName') # Temporary workaround as YAQL does not provide "where" function for # dictionaries, and there is no easy way to implement it there. context.register_function(yaql_builtin.dict_attribution, 'get') diff --git a/murano/engine/yaql_yaml_loader.py b/murano/engine/yaql_yaml_loader.py new file mode 100644 index 000000000..05972e05b --- /dev/null +++ b/murano/engine/yaql_yaml_loader.py @@ -0,0 +1,37 @@ +# Copyright (c) 2014 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml + +from murano.dsl import yaql_expression + + +class YaqlYamlLoader(yaml.Loader): + pass + +# workaround for PyYAML bug: http://pyyaml.org/ticket/221 +resolvers = {} +for k, v in yaml.Loader.yaml_implicit_resolvers.items(): + resolvers[k] = v[:] +YaqlYamlLoader.yaml_implicit_resolvers = resolvers + + +def yaql_constructor(loader, node): + value = loader.construct_scalar(node) + return yaql_expression.YaqlExpression(value) + +yaml.add_constructor(u'!yaql', yaql_constructor, YaqlYamlLoader) +yaml.add_implicit_resolver(u'!yaql', yaql_expression.YaqlExpression, + Loader=YaqlYamlLoader) diff --git a/murano/packages/application_package.py b/murano/packages/application_package.py index bc158ca45..ec3d7163f 100644 --- a/murano/packages/application_package.py +++ b/murano/packages/application_package.py @@ -16,26 +16,10 @@ import imghdr import io import os -import shutil import sys -import tempfile -import yaml import zipfile - import murano.packages.exceptions as e -import murano.packages.versions.v1 - - -class DummyLoader(yaml.Loader): - pass - - -def yaql_constructor(loader, node): - value = loader.construct_scalar(node) - return value - -yaml.add_constructor(u'!yaql', yaql_constructor, DummyLoader) class PackageTypes(object): @@ -45,7 +29,7 @@ class PackageTypes(object): class ApplicationPackage(object): - def __init__(self, source_directory, manifest, loader=DummyLoader): + def __init__(self, source_directory, manifest, loader): self.yaml_loader = loader self._source_directory = source_directory self._full_name = None @@ -54,14 +38,9 @@ class ApplicationPackage(object): self._description = None self._author = None self._tags = None - self._classes = None - self._ui = None self._logo = None self._format = manifest.get('Format') - self._ui_cache = None - self._raw_ui_cache = None self._logo_cache = None - self._classes_cache = {} self._blob_cache = None @property @@ -88,22 +67,6 @@ class ApplicationPackage(object): def tags(self): return tuple(self._tags) - @property - def classes(self): - return tuple(self._classes.keys()) - - @property - def ui(self): - if not self._ui_cache: - self._load_ui(True) - return self._ui_cache - - @property - def raw_ui(self): - if not self._raw_ui_cache: - self._load_ui(False) - return self._raw_ui_cache - @property def logo(self): if not self._logo_cache: @@ -116,42 +79,15 @@ class ApplicationPackage(object): self._blob_cache = _pack_dir(self._source_directory) return self._blob_cache - def get_class(self, name): - if name not in self._classes_cache: - self._load_class(name) - return self._classes_cache[name] - def get_resource(self, name): - return os.path.join(self._source_directory, 'Resources', name) + resources_dir = os.path.join(self._source_directory, 'Resources') + if not os.path.exists(resources_dir): + os.makedirs(resources_dir) + return os.path.join(resources_dir, name) def validate(self): - self._classes_cache.clear() - for class_name in self._classes: - self.get_class(class_name) - self._load_ui(True) self._load_logo(True) - # Private methods - def _load_ui(self, load_yaml=False): - if self._raw_ui_cache and load_yaml: - self._ui_cache = yaml.load(self._raw_ui_cache, self.yaml_loader) - else: - ui_file = self._ui - full_path = os.path.join(self._source_directory, 'UI', ui_file) - if not os.path.isfile(full_path): - self._raw_ui_cache = None - self._ui_cache = None - return - try: - with open(full_path) as stream: - self._raw_ui_cache = stream.read() - if load_yaml: - self._ui_cache = yaml.load(self._raw_ui_cache, - self.yaml_loader) - except Exception as ex: - trace = sys.exc_info()[2] - raise e.PackageUILoadError(str(ex)), None, trace - def _load_logo(self, validate=False): logo_file = self._logo or 'logo.png' full_path = os.path.join(self._source_directory, logo_file) @@ -169,54 +105,6 @@ class ApplicationPackage(object): raise e.PackageLoadError( "Unable to load logo: " + str(ex)), None, trace - def _load_class(self, name): - if name not in self._classes: - raise e.PackageClassLoadError(name, 'Class not defined ' - 'in this package') - def_file = self._classes[name] - full_path = os.path.join(self._source_directory, 'Classes', def_file) - if not os.path.isfile(full_path): - raise e.PackageClassLoadError(name, 'File with class ' - 'definition not found') - try: - with open(full_path) as stream: - self._classes_cache[name] = yaml.load(stream, self.yaml_loader) - except Exception as ex: - trace = sys.exc_info()[2] - msg = 'Unable to load class definition due to "{0}"'.format( - str(ex)) - raise e.PackageClassLoadError(name, msg), None, trace - - -def load_from_dir(source_directory, filename='manifest.yaml', preload=False, - loader=DummyLoader): - formats = {'1.0': murano.packages.versions.v1} - - if not os.path.isdir(source_directory) or not os.path.exists( - source_directory): - raise e.PackageLoadError('Invalid package directory') - full_path = os.path.join(source_directory, filename) - if not os.path.isfile(full_path): - raise e.PackageLoadError('Unable to find package manifest') - - try: - with open(full_path) as stream: - content = yaml.load(stream, DummyLoader) - except Exception as ex: - trace = sys.exc_info()[2] - raise e.PackageLoadError( - "Unable to load due to '{0}'".format(str(ex))), None, trace - if content: - p_format = str(content.get('Format')) - if not p_format or p_format not in formats: - raise e.PackageFormatError( - 'Unknown or missing format version') - package = ApplicationPackage(source_directory, content, loader) - formats[p_format].load(package, content) - if preload: - package.validate() - return package - def _zipdir(path, zipf): for root, dirs, files in os.walk(path): @@ -233,34 +121,3 @@ def _pack_dir(source_directory): zipf.close() return blob.getvalue() - - -def load_from_file(archive_path, target_dir=None, drop_dir=False, - loader=DummyLoader): - if not os.path.isfile(archive_path): - raise e.PackageLoadError('Unable to find package file') - created = False - if not target_dir: - target_dir = tempfile.mkdtemp() - created = True - elif not os.path.exists(target_dir): - os.mkdir(target_dir) - created = True - else: - if os.listdir(target_dir): - raise e.PackageLoadError('Target directory is not empty') - - try: - if not zipfile.is_zipfile(archive_path): - raise e.PackageFormatError("Uploading file should be a " - "zip' archive") - package = zipfile.ZipFile(archive_path) - package.extractall(path=target_dir) - return load_from_dir(target_dir, preload=True, loader=loader) - finally: - if drop_dir: - if created: - shutil.rmtree(target_dir) - else: - for f in os.listdir(target_dir): - os.unlink(os.path.join(target_dir, f)) diff --git a/murano/packages/hot_package.py b/murano/packages/hot_package.py new file mode 100644 index 000000000..5cfa767ee --- /dev/null +++ b/murano/packages/hot_package.py @@ -0,0 +1,389 @@ +# Copyright (c) 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import shutil +import sys +import types +import yaml + +from murano.dsl import yaql_expression +import murano.packages.application_package +from murano.packages import exceptions + + +YAQL = yaql_expression.YaqlExpression + + +class Dumper(yaml.Dumper): + pass + + +def yaql_representer(dumper, data): + return dumper.represent_scalar(u'!yaql', str(data)) + +Dumper.add_representer(YAQL, yaql_representer) + + +class HotPackage(murano.packages.application_package.ApplicationPackage): + def __init__(self, source_directory, manifest, loader): + super(HotPackage, self).__init__(source_directory, manifest, loader) + self._translated_class = None + self._source_directory = source_directory + self._translated_ui = None + + @property + def classes(self): + return self.full_name, + + @property + def ui(self): + if not self._translated_ui: + self._translated_ui = self._translate_ui() + return self._translated_ui + + @property + def raw_ui(self): + ui_obj = self.ui + result = yaml.dump(ui_obj, Dumper=Dumper, default_style='"') + return result + + def get_class(self, name): + if name != self.full_name: + raise exceptions.PackageClassLoadError( + name, 'Class not defined in this package') + if not self._translated_class: + self._translate_class() + return self._translated_class + + def validate(self): + self.get_class(self.full_name) + if not self._translated_ui: + self._translated_ui = self._translate_ui() + super(HotPackage, self).validate() + + def _translate_class(self): + template_file = os.path.join(self._source_directory, 'template.yaml') + shutil.copy(template_file, self.get_resource(self.full_name)) + + if not os.path.isfile(template_file): + raise exceptions.PackageClassLoadError( + self.full_name, 'File with class definition not found') + with open(template_file) as stream: + hot = yaml.safe_load(stream) + if 'resources' not in hot: + raise exceptions.PackageFormatError('Not a HOT template') + translated = { + 'Name': self.full_name, + 'Extends': 'io.murano.Application' + } + + parameters = HotPackage._translate_parameters(hot) + parameters.update(HotPackage._translate_outputs(hot)) + translated['Properties'] = parameters + + translated.update(HotPackage._generate_workflow(hot)) + self._translated_class = translated + + @staticmethod + def _translate_parameters(hot): + result = { + 'generatedHeatStackName': { + 'Contract': YAQL('$.string()'), + 'Usage': 'Out' + } + } + for key, value in (hot.get('parameters') or {}).items(): + result[key] = HotPackage._translate_parameter(value) + result['name'] = {'Usage': 'In', + 'Contract': YAQL('$.string().notNull()')} + return result + + @staticmethod + def _translate_parameter(value): + contract = '$' + + parameter_type = value['type'] + if parameter_type == 'string': + contract += '.string()' + elif parameter_type == 'number': + contract += '.int()' + elif parameter_type == 'json': + contract += '.object()' + else: + raise ValueError('Unsupported parameter type ' + parameter_type) + + constraints = value.get('constraints') or [] + for constraint in constraints: + translated = HotPackage._translate_constraint(constraint) + if translated: + contract += translated + + result = { + 'Contract': YAQL(contract), + "Usage": "In" + } + if 'default' in value: + result['Default'] = value['default'] + return result + + @staticmethod + def _translate_outputs(hot): + result = {} + for key, value in (hot.get('outputs') or {}).items(): + result[key] = { + "Contract": YAQL("$.string()"), + "Usage": "Out" + } + return result + + @staticmethod + def _translate_constraint(constraint): + if 'allowed_values' in constraint: + return HotPackage._translate_allowed_values_constraint( + constraint['allowed_values']) + elif 'length' in constraint: + return HotPackage._translate_length_constraint( + constraint['length']) + elif 'range' in constraint: + return HotPackage._translate_range_constraint( + constraint['range']) + elif 'allowed_pattern' in constraint: + return HotPackage._translate_allowed_pattern_constraint( + constraint['allowed_pattern']) + + @staticmethod + def _translate_allowed_pattern_constraint(value): + return ".check(matches($, '{0}'))".format(value) + + @staticmethod + def _translate_allowed_values_constraint(values): + return '.check($ in list({0}))'.format( + ', '.join([HotPackage._format_value(v) for v in values])) + + @staticmethod + def _translate_length_constraint(value): + if 'min' in value and 'max' in value: + return '.check(len($) >= {0} and len($) <= {1})'.format( + int(value['min']), int(value['max'])) + elif 'min' in value: + return '.check(len($) >= {0})'.format(int(value['min'])) + elif 'max' in value: + return '.check(len($) <= {0})'.format(int(value['max'])) + + @staticmethod + def _translate_range_constraint(value): + if 'min' in value and 'max' in value: + return '.check($ >= {0} and $ <= {1})'.format( + int(value['min']), int(value['max'])) + elif 'min' in value: + return '.check($ >= {0})'.format(int(value['min'])) + elif 'max' in value: + return '.check($ <= {0})'.format(int(value['max'])) + + @staticmethod + def _format_value(value): + if isinstance(value, types.StringTypes): + return str("'" + value + "'") + return str(value) + + @staticmethod + def _generate_workflow(hot): + template_parameters = {} + for key, value in (hot.get('parameters') or {}).items(): + template_parameters[key] = YAQL("$." + key) + + deploy = [ + { + 'If': YAQL('$.generatedHeatStackName = null'), + 'Then': [ + {YAQL('$.generatedHeatStackName'): YAQL('randomName()')} + ] + }, + + {YAQL('$stack'): YAQL( + "new('io.murano.system.HeatStack', " + "name => $.generatedHeatStackName)")}, + {YAQL('$resources'): YAQL("new('io.murano.system.Resources')")}, + {YAQL('$template'): YAQL("$resources.yaml(type($this))")}, + {YAQL('$parameters'): template_parameters}, + YAQL('$stack.setTemplate($template)'), + YAQL('$stack.setParameters($parameters)'), + YAQL('$stack.push()'), + {YAQL('$outputs'): YAQL('$stack.output()')} + ] + for key, value in (hot.get('outputs') or {}).items(): + deploy.append({YAQL('$.' + key): YAQL( + '$outputs.' + key)}) + + destroy = [ + {YAQL('$stack'): YAQL( + "new('io.murano.system.HeatStack', " + "name => $.generatedHeatStackName)")}, + YAQL('$stack.delete()') + ] + + return { + 'Workflow': { + 'deploy': { + 'Body': deploy + }, + 'destroy': { + 'Body': destroy + } + } + } + + @staticmethod + def _translate_ui_parameters(hot, title): + result = [ + { + 'name': 'title', + 'type': 'string', + 'required': False, + 'hidden': True, + 'description': title + }, + { + 'name': 'name', + 'type': 'string', + 'label': 'Application Name', + 'required': True, + 'description': + 'Enter a desired name for the application.' + ' Just A-Z, a-z, 0-9, and dash are allowed' + } + ] + for key, value in (hot.get('parameters') or {}).items(): + result.append(HotPackage._translate_ui_parameter(key, value)) + return result + + @staticmethod + def _translate_ui_parameter(name, parameter_spec): + translated = { + 'name': name, + 'label': name.title().replace('_', ' ') + } + parameter_type = parameter_spec['type'] + if parameter_type == 'string': + translated['type'] = 'string' + elif parameter_type == 'number': + translated['type'] = 'integer' + + if 'description' in parameter_spec: + translated['description'] = parameter_spec['description'] + + if 'default' in parameter_spec: + translated['initial'] = parameter_spec['default'] + translated['required'] = False + else: + translated['required'] = True + + constraints = parameter_spec.get('constraints') or [] + translated_constraints = [] + + for constraint in constraints: + if 'length' in constraint: + spec = constraint['length'] + if 'min' in spec: + translated['minLength'] = max( + translated.get('minLength', -sys.maxint - 1), + int(spec['min'])) + if 'max' in spec: + translated['maxLength'] = min( + translated.get('maxLength', sys.maxint), + int(spec['max'])) + + elif 'range' in constraint: + spec = constraint['range'] + if 'min' in spec and 'max' in spec: + ui_constraint = { + 'expr': YAQL('$ >= {0} and $ <= {1}'.format( + spec['min'], spec['max'])) + } + elif 'min' in spec: + ui_constraint = { + 'expr': YAQL('$ >= {0}'.format(spec['min'])) + } + else: + ui_constraint = { + 'expr': YAQL('$ <= {0}'.format(spec['max'])) + } + if 'description' in constraint: + ui_constraint['message'] = constraint['description'] + translated_constraints.append(ui_constraint) + + elif 'allowed_values' in constraint: + values = constraint['allowed_values'] + ui_constraint = { + 'expr': YAQL('$ in list({0})'.format(', '.join( + [HotPackage._format_value(v) for v in values]))) + } + if 'description' in constraint: + ui_constraint['message'] = constraint['description'] + translated_constraints.append(ui_constraint) + + elif 'allowed_pattern' in constraint: + pattern = constraint['allowed_pattern'] + ui_constraint = { + 'expr': { + 'regexpValidator': pattern + } + } + if 'description' in constraint: + ui_constraint['message'] = constraint['description'] + translated_constraints.append(ui_constraint) + + if translated_constraints: + translated['validators'] = translated_constraints + + return translated + + @staticmethod + def _generate_application_ui(hot, type_name): + app = { + '?': { + 'type': type_name + } + } + for key in (hot.get('parameters') or {}).keys(): + app[key] = YAQL('$.appConfiguration.' + key) + app['name'] = YAQL('$.appConfiguration.name') + + return app + + def _translate_ui(self): + template_file = os.path.join(self._source_directory, 'template.yaml') + + if not os.path.isfile(template_file): + raise exceptions.PackageClassLoadError( + self.full_name, 'File with class definition not found') + with open(template_file) as stream: + hot = yaml.safe_load(stream) + + translated = { + 'Version': 2, + 'Application': HotPackage._generate_application_ui( + hot, self.full_name), + 'Forms': [ + { + 'appConfiguration': { + 'fields': HotPackage._translate_ui_parameters( + hot, self.description) + } + } + ] + } + + return translated diff --git a/murano/packages/load_utils.py b/murano/packages/load_utils.py new file mode 100644 index 000000000..5ce2b75a9 --- /dev/null +++ b/murano/packages/load_utils.py @@ -0,0 +1,91 @@ +# Copyright (c) 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import shutil +import sys +import tempfile +import yaml +import zipfile + +from murano.engine import yaql_yaml_loader +import murano.packages.application_package +import murano.packages.exceptions as e +import murano.packages.versions.hot_v1 +import murano.packages.versions.mpl_v1 + + +def load_from_file(archive_path, target_dir=None, drop_dir=False, + loader=yaql_yaml_loader.YaqlYamlLoader): + if not os.path.isfile(archive_path): + raise e.PackageLoadError('Unable to find package file') + created = False + if not target_dir: + target_dir = tempfile.mkdtemp() + created = True + elif not os.path.exists(target_dir): + os.mkdir(target_dir) + created = True + else: + if os.listdir(target_dir): + raise e.PackageLoadError('Target directory is not empty') + + try: + if not zipfile.is_zipfile(archive_path): + raise e.PackageFormatError("Uploading file should be a " + "zip' archive") + package = zipfile.ZipFile(archive_path) + package.extractall(path=target_dir) + return load_from_dir(target_dir, preload=True, loader=loader) + finally: + if drop_dir: + if created: + shutil.rmtree(target_dir) + else: + for f in os.listdir(target_dir): + os.unlink(os.path.join(target_dir, f)) + + +def load_from_dir(source_directory, filename='manifest.yaml', preload=False, + loader=yaql_yaml_loader.YaqlYamlLoader): + formats = { + '1.0': murano.packages.versions.mpl_v1, + 'MuranoPL/1.0': murano.packages.versions.mpl_v1, + 'Heat.HOT/1.0': murano.packages.versions.hot_v1 + } + + if not os.path.isdir(source_directory) or not os.path.exists( + source_directory): + raise e.PackageLoadError('Invalid package directory') + full_path = os.path.join(source_directory, filename) + if not os.path.isfile(full_path): + raise e.PackageLoadError('Unable to find package manifest') + + try: + with open(full_path) as stream: + content = yaml.safe_load(stream) + except Exception as ex: + trace = sys.exc_info()[2] + raise e.PackageLoadError( + "Unable to load due to '{0}'".format(str(ex))), None, trace + if content: + p_format = str(content.get('Format')) + if not p_format or p_format not in formats: + raise e.PackageFormatError( + 'Unknown or missing format version') + package = formats[p_format].create(source_directory, content, loader) + formats[p_format].load(package, content) + if preload: + package.validate() + return package diff --git a/murano/packages/mpl_package.py b/murano/packages/mpl_package.py new file mode 100644 index 000000000..4258503d2 --- /dev/null +++ b/murano/packages/mpl_package.py @@ -0,0 +1,99 @@ +# Copyright (c) 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys +import yaml + +import murano.packages.application_package +from murano.packages import exceptions + + +class MuranoPlPackage(murano.packages.application_package.ApplicationPackage): + def __init__(self, source_directory, manifest, loader): + super(MuranoPlPackage, self).__init__( + source_directory, manifest, loader) + + self._classes = None + self._ui = None + self._ui_cache = None + self._raw_ui_cache = None + self._classes_cache = {} + + @property + def classes(self): + return tuple(self._classes.keys()) + + @property + def ui(self): + if not self._ui_cache: + self._load_ui(True) + return self._ui_cache + + @property + def raw_ui(self): + if not self._raw_ui_cache: + self._load_ui(False) + return self._raw_ui_cache + + def get_class(self, name): + if name not in self._classes_cache: + self._load_class(name) + return self._classes_cache[name] + + # Private methods + def _load_ui(self, load_yaml=False): + if self._raw_ui_cache and load_yaml: + self._ui_cache = yaml.load(self._raw_ui_cache, self.yaml_loader) + else: + ui_file = self._ui + full_path = os.path.join(self._source_directory, 'UI', ui_file) + if not os.path.isfile(full_path): + self._raw_ui_cache = None + self._ui_cache = None + return + try: + with open(full_path) as stream: + self._raw_ui_cache = stream.read() + if load_yaml: + self._ui_cache = yaml.load(self._raw_ui_cache, + self.yaml_loader) + except Exception as ex: + trace = sys.exc_info()[2] + raise exceptions.PackageUILoadError(str(ex)), None, trace + + def _load_class(self, name): + if name not in self._classes: + raise exceptions.PackageClassLoadError( + name, 'Class not defined in this package') + def_file = self._classes[name] + full_path = os.path.join(self._source_directory, 'Classes', def_file) + if not os.path.isfile(full_path): + raise exceptions.PackageClassLoadError( + name, 'File with class definition not found') + try: + with open(full_path) as stream: + self._classes_cache[name] = yaml.load(stream, self.yaml_loader) + except Exception as ex: + trace = sys.exc_info()[2] + msg = 'Unable to load class definition due to "{0}"'.format( + str(ex)) + raise exceptions.PackageClassLoadError(name, msg), None, trace + + def validate(self): + self._classes_cache.clear() + for class_name in self._classes: + self.get_class(class_name) + self._load_ui(True) + super(MuranoPlPackage, self).validate() diff --git a/murano/packages/versions/hot_v1.py b/murano/packages/versions/hot_v1.py new file mode 100644 index 000000000..86befc923 --- /dev/null +++ b/murano/packages/versions/hot_v1.py @@ -0,0 +1,57 @@ +# Copyright (c) 2014 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import murano.packages.application_package +import murano.packages.exceptions as e +import murano.packages.hot_package + + +# noinspection PyProtectedMember + + +def load(package, yaml_content): + package._full_name = yaml_content.get('FullName') + if not package._full_name: + raise murano.packages.exceptions.PackageFormatError( + 'FullName not specified') + _check_full_name(package._full_name) + package._package_type = yaml_content.get('Type') + if not package._package_type or package._package_type not in \ + murano.packages.application_package.PackageTypes.ALL: + raise e.PackageFormatError('Invalid Package Type') + package._display_name = yaml_content.get('Name', package._full_name) + package._description = yaml_content.get('Description') + package._author = yaml_content.get('Author') + package._logo = yaml_content.get('Logo') + package._tags = yaml_content.get('Tags') + + +def create(source_directory, content, loader): + return murano.packages.hot_package.HotPackage( + source_directory, content, loader) + + +def _check_full_name(full_name): + error = murano.packages.exceptions.PackageFormatError( + 'Invalid FullName') + if re.match(r'^[\w\.]+$', full_name): + if full_name.startswith('.') or full_name.endswith('.'): + raise error + if '..' in full_name: + raise error + else: + raise error diff --git a/murano/packages/versions/v1.py b/murano/packages/versions/mpl_v1.py similarity index 91% rename from murano/packages/versions/v1.py rename to murano/packages/versions/mpl_v1.py index 295d7b4fb..1d204d2b6 100644 --- a/murano/packages/versions/v1.py +++ b/murano/packages/versions/mpl_v1.py @@ -17,9 +17,12 @@ import re import murano.packages.application_package import murano.packages.exceptions as e +import murano.packages.mpl_package # noinspection PyProtectedMember + + def load(package, yaml_content): package._full_name = yaml_content.get('FullName') if not package._full_name: @@ -39,6 +42,11 @@ def load(package, yaml_content): package._tags = yaml_content.get('Tags') +def create(source_directory, content, loader): + return murano.packages.mpl_package.MuranoPlPackage( + source_directory, content, loader) + + def _check_full_name(full_name): error = murano.packages.exceptions.PackageFormatError( 'Invalid FullName')