From df95ad397e3930737d92ddac7e580fedcfc0b6ce Mon Sep 17 00:00:00 2001 From: Alexander Tivelkov Date: Wed, 2 Sep 2015 22:18:42 +0300 Subject: [PATCH] Added the support of Glance Artifact Repository Murano's application packages can now be stored either in database (using murano-api) or in Glance Artifact repository (so-called Glance V3 api). This patch adds an experimental support of the latter approach to murano engine. As all the difference between these two storages is incapsulated in the murano client, engine just needs to configure it properly by passing an instance of glance v3 client on creation. This is controlled by a 'packages_service' parameter of 'packages_opts' configuration group. It is set to 'murano' by default and indicates the usage of old, database-backed storage. If set to 'glance', the murano client will encapsulate glance v3 connector and thus the packages will be accessed from Glance Artifact Repository. The settings of Glance client are also added to the configuration, as well as a client factory to generate the client. As these settings may now conflict with the settings "demo plugin", the appropriate configuration section is renamed in the latter. This patch also contains a couple of utility functions to transform partial semver version specs into non-partial ones and - further - to a set of Glance query parameters needed to filter the artifacts based on that spec. Change-Id: I690467e43b6b63850ebecef756635241e623554c Implements-blueprint: artifact-repository-support --- .../murano_exampleplugin/cfg.py | 2 +- murano/common/config.py | 31 +++++++- murano/dsl/helpers.py | 74 ++++++++++++++++++- murano/engine/client_manager.py | 47 +++++++++++- murano/engine/package_loader.py | 9 ++- 5 files changed, 153 insertions(+), 10 deletions(-) diff --git a/contrib/plugins/murano_exampleplugin/murano_exampleplugin/cfg.py b/contrib/plugins/murano_exampleplugin/murano_exampleplugin/cfg.py index 85b8b6099..eed15e6b3 100644 --- a/contrib/plugins/murano_exampleplugin/murano_exampleplugin/cfg.py +++ b/contrib/plugins/murano_exampleplugin/murano_exampleplugin/cfg.py @@ -20,5 +20,5 @@ def init_config(conf): cfg.IntOpt('api_version', default=2), cfg.StrOpt('endpoint_type', default='publicURL') ] - conf.register_opts(opts, group="glance") + conf.register_opts(opts, group="images") return conf.glance diff --git a/murano/common/config.py b/murano/common/config.py index f9be1ae28..3539f7a03 100644 --- a/murano/common/config.py +++ b/murano/common/config.py @@ -229,7 +229,35 @@ packages_opts = [ cfg.IntOpt('api_limit_max', default=100, help='Maximum number of packages to be returned in a single ' - 'pagination request') + 'pagination request'), + + cfg.StrOpt('packages_service', default='murano', + help=_('The service to store murano packages: murano (stands ' + 'for legacy behavior using murano-api) or glance ' + '(stands for Glance V3 artifact repository)')) +] + +glance_opts = [ + cfg.StrOpt('url', help='Optional murano url in format ' + 'like http://0.0.0.0:9292 used by Glance API'), + + cfg.BoolOpt('insecure', default=False, + help='This option explicitly allows Murano to perform ' + '"insecure" SSL connections and transfers with Glance API.'), + + cfg.StrOpt('ca_file', + help='(SSL) Tells Murano to use the specified certificate file ' + 'to verify the peer running Glance API.'), + + cfg.StrOpt('cert_file', + help='(SSL) Tells Murano to use the specified client ' + 'certificate file when communicating with Glance.'), + + cfg.StrOpt('key_file', help='(SSL/SSH) Private key file name to ' + 'communicate with Glance API.'), + + cfg.StrOpt('endpoint_type', default='publicURL', + help='Glance endpoint type.') ] file_server = [ @@ -251,6 +279,7 @@ CONF.register_cli_opts(metadata_dir) CONF.register_opts(packages_opts, group='packages_opts') CONF.register_opts(stats_opts, group='stats') CONF.register_opts(networking_opts, group='networking') +CONF.register_opts(glance_opts, group='glance') def parse_args(args=None, usage=None, default_config_files=None): diff --git a/murano/dsl/helpers.py b/murano/dsl/helpers.py index 2cf2d0e77..395cdf403 100644 --- a/murano/dsl/helpers.py +++ b/murano/dsl/helpers.py @@ -241,16 +241,17 @@ def contextual(ctx): def parse_version_spec(version_spec): if isinstance(version_spec, semantic_version.Spec): - return version_spec + return normalize_version_spec(version_spec) if isinstance(version_spec, semantic_version.Version): - return semantic_version.Spec('==' + str(version_spec)) + return normalize_version_spec( + semantic_version.Spec('==' + str(version_spec))) if not version_spec: version_spec = '0' version_spec = str(version_spec).translate(None, string.whitespace) if version_spec[0].isdigit(): version_spec = '==' + str(version_spec) version_spec = semantic_version.Spec(version_spec) - return version_spec + return normalize_version_spec(version_spec) def parse_version(version): @@ -347,3 +348,70 @@ def memoize(func): else: return cache[args] return wrap + + +def normalize_version_spec(version_spec): + def coerce(v): + return semantic_version.Version('{0}.{1}.{2}'.format( + v.major, v.minor or 0, v.patch or 0 + )) + + def increment(v): + # NOTE(ativelkov): replace these implementations with next_minor() and + # next_major() calls when the semantic_version is updated in global + # requirements. + if v.minor is None: + return semantic_version.Version( + '.'.join(str(x) for x in [v.major + 1, 0, 0])) + else: + return semantic_version.Version( + '.'.join(str(x) for x in [v.major, v.minor + 1, 0])) + + def extend(v): + return semantic_version.Version(str(v) + '-0') + + transformations = { + '>': [('>=', (increment, extend))], + '>=': [('>=', (coerce,))], + '<': [('<', (coerce, extend))], + '<=': [('<', (increment, extend))], + '!=': [('>=', (increment, extend))], + '==': [('>=', (coerce,)), ('<', (increment, coerce, extend))] + } + + new_parts = [] + for item in version_spec.specs: + if item.kind == '*': + continue + elif item.spec.patch is not None: + new_parts.append(str(item)) + else: + for op, funcs in transformations[item.kind]: + new_parts.append('{0}{1}'.format( + op, + reduce(lambda v, f: f(v), funcs, item.spec) + )) + if not new_parts: + return semantic_version.Spec('*') + return semantic_version.Spec(*new_parts) + + +semver_to_api_map = { + '>': 'gt', + '>=': 'ge', + '<': 'lt', + '<=': 'le', + '!=': 'ne', + '==': 'eq' +} + + +def breakdown_spec_to_query(normalized_spec): + res = [] + for item in normalized_spec.specs: + if item.kind == '*': + continue + else: + res.append("%s:%s" % (semver_to_api_map[item.kind], + item.spec)) + return res diff --git a/murano/engine/client_manager.py b/murano/engine/client_manager.py index fbd19a678..f921029e8 100644 --- a/murano/engine/client_manager.py +++ b/murano/engine/client_manager.py @@ -23,7 +23,7 @@ import neutronclient.v2_0.client as nclient from oslo_config import cfg from murano.common import auth_utils - +from muranoclient.glance import client as art_client try: # integration with congress is optional @@ -162,14 +162,33 @@ class ClientManager(object): service_type='application_catalog', endpoint_type=murano_settings.endpoint_type) + if CONF.packages_opts.packages_service == 'glance': + glance_settings = CONF.glance + glance_url = (glance_settings.url or + keystone_client.service_catalog.url_for( + service_type='image', + endpoint_type=glance_settings.endpoint_type)) + + arts = art_client.Client( + endpoint=glance_url, token=auth_token, + insecure=glance_settings.insecure, + key_file=glance_settings.key_file or None, + ca_file=glance_settings.ca_file or None, + cert_file=glance_settings.cert_file or None, + type_name='murano', + type_version=1) + else: + arts = None + return muranoclient.Client( endpoint=murano_url, key_file=murano_settings.key_file or None, - cacert=murano_settings.cacert or None, + ca_file=murano_settings.cacert or None, cert_file=murano_settings.cert_file or None, insecure=murano_settings.insecure, auth_url=keystone_client.auth_url, - token=auth_token) + token=auth_token, + artifacts_client=arts) return self.get_client('murano', use_trusts, factory) @@ -198,3 +217,25 @@ class ClientManager(object): user_id=keystone_client.user_id) return self.get_client('mistral', use_trusts, factory) + + def get_artifacts_client(self, use_trusts=True): + if not CONF.engine.use_trusts: + use_trusts = False + + def factory(keystone_client, auth_token): + glance_settings = CONF.glance + + glance_url = (glance_settings.url or + keystone_client.service_catalog.url_for( + service_type='image', + endpoint_type=glance_settings.endpoint_type)) + + return art_client.Client(endpoint=glance_url, token=auth_token, + insecure=glance_settings.insecure, + key_file=glance_settings.key_file or None, + cacert=glance_settings.cacert or None, + cert_file=(glance_settings.cert_file or + None), + type_name='murano', + type_version=1) + return self.get_client('artifacts', use_trusts, factory) diff --git a/murano/engine/package_loader.py b/murano/engine/package_loader.py index 369722ebe..d1300ce31 100644 --- a/murano/engine/package_loader.py +++ b/murano/engine/package_loader.py @@ -27,6 +27,7 @@ from oslo_log import log as logging from murano.common.i18n import _LE, _LI from murano.dsl import constants from murano.dsl import exceptions +from murano.dsl import helpers from murano.dsl import package_loader from murano.engine import murano_package from murano.engine.system import system_objects @@ -54,7 +55,9 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader): if version: return packages[version] - filter_opts = {'class_name': class_name} + filter_opts = {'class_name': class_name, + 'version': helpers.breakdown_spec_to_query( + version_spec)} try: package_definition = self._get_definition(filter_opts) except LookupError: @@ -71,7 +74,9 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader): if version: return packages[version] - filter_opts = {'fqn': package_name} + filter_opts = {'fqn': package_name, + 'version': helpers.breakdown_spec_to_query( + version_spec)} try: package_definition = self._get_definition(filter_opts) except LookupError: