Initial implementation of Plugable Classes

Adds a PluginLoader which loads classes defined as stevedore plugins at
io.murano.extension namespace and registers them as MuranoPL classes in
class loader.

Modifies the ClientManager class to make the _get_client method public,
so other code may use it to add custom clients. This is useful for
plugins which may define their own clients.

Modifies the configuration settings adding 'enabled_plugins' parameter to
control which of the installed plugins are active.

Adds an example plugin which encapsulates Glance interaction logic to:
	* List all available glance images
	* Get Image by ID
	* Get Image by Name
	* Output image info with murano-related metadata

Adds a demo application which demonstrates the usage of plugin. The app
consist of the following components:
	* An 'ImageValidatorMixin' class which inherits generic instance
	  class (io.murano.resources.Instance) and adds a method capable to
	  validate Instance's image for having appropriate murano metadata
	  type. This class may be used as a mixin when added to inheritance
	  hierarchy of concrete instance classes.
	* A concrete class called DemoInstance which inherits from
	  io.murano.resources.LinuxMuranoInstance and ImageValidatorMixin
	  to add the image validation logic to standard Murano-enabled
	  Linux-based instance.
	* An application which deploys a single VM using the DemoInstance
	  class if the tag on user-supplied image matches the user-supplied
	  constant.

The ImageValidatorMixin demonstrates the instantiation of
plugin-provided class and its usage, as well as handling of exception
which may be thrown if the plugin is not installed in the environment.

Change-Id: I978339d87033bbe38dad4c2102612d8f3a1eb3c3
Implements-blueprint: plugable-classes

contrib/plugins/murano_exampleplugin/example-app/io.murano.apps.demo.DemoApp/Classes/DemoApp.yaml

@@ -0,0 +1,40 @@
+Namespaces:
+  =: io.murano.apps.example.plugin
+  std: io.murano
+  res: io.murano.resources
+  sys: io.murano.system
+
+
+Name: DemoApp
+
+Extends: std:Application
+
+Properties:
+  name:
+    Contract: $.string().notNull()
+
+  instance:
+    Contract: $.class(res:Instance).notNull()
+
+Workflow:
+  initialize:
+    Body:
+      - $.environment: $.find(std:Environment).require()
+
+  deploy:
+    Body:
+      - If: !yaql "not bool($.getAttr(deployed))"
+        Then:
+          - $this.find(std:Environment).reporter.report($this, 'Creating VM ')
+          - $securityGroupIngress:
+              - ToPort: 22
+                FromPort: 22
+                IpProtocol: tcp
+                External: True
+          - $.environment.securityGroupManager.addGroupIngress($securityGroupIngress)
+          - $.instance.deploy()
+          - $resources: new(sys:Resources)
+          - $this.find(std:Environment).reporter.report($this, 'Test VM is installed')
+          - $.host: $.instance.ipAddresses[0]
+          - $.user: 'root'
+          - $.setAttr(deployed, True)

contrib/plugins/murano_exampleplugin/example-app/io.murano.apps.demo.DemoApp/Classes/DemoInstance.yaml

@@ -0,0 +1,15 @@
+Namespaces:
+  =: io.murano.apps.example.plugin
+  res: io.murano.resources
+
+Name: DemoInstance
+
+Extends:
+  - res:LinuxMuranoInstance
+  - ImageValidatorMixin
+
+Workflow:
+  deploy:
+    Body:
+      - $.validateImage()
+      - $.super($.deploy())

contrib/plugins/murano_exampleplugin/example-app/io.murano.apps.demo.DemoApp/Classes/ImageValidatorMixin.yaml

@@ -0,0 +1,36 @@
+Namespaces:
+  =: io.murano.apps.example.plugin
+  res: io.murano.resources
+
+Name: ImageValidatorMixin
+
+Extends:
+  - res:Instance
+
+Properties:
+  requiredType:
+    Contract: $.string().notNull()
+
+Workflow:
+  validateImage:
+    Body:
+      - Try:
+        - $glance: new('io.murano.extensions.mirantis.example.Glance')
+        Catch:
+          With: 'murano.dsl.exceptions.NoPackageForClassFound'
+          Do:
+            Throw: PluginNotFoundException
+            Message: 'Plugin for interaction with Glance is not installed'
+      - $glanceImage: $glance.getById($.image)
+      - If: $glanceImage = null
+        Then:
+          Throw: ImageNotFoundException
+          Message: 'Image with specified Id was not found'
+      - If: $glanceImage.meta = null
+        Then:
+          Throw: InvalidImageException
+          Message: 'Image does not contain Murano metadata tag'
+      - If: $glanceImage.meta.type != $.requiredType
+        Then:
+          Throw: InvalidImageException
+          Message: 'Image has unappropriate Murano type'

contrib/plugins/murano_exampleplugin/example-app/io.murano.apps.demo.DemoApp/UI/ui.yaml

@@ -0,0 +1,81 @@
+Version: 2
+
+Application:
+  ?:
+    type: io.murano.apps.example.plugin.DemoApp
+  name: $.appConfiguration.name
+  instance:
+    ?:
+      type: io.murano.apps.example.plugin.DemoInstance
+    name: generateHostname($.instanceConfiguration.unitNamingPattern, 1)
+    flavor: $.instanceConfiguration.flavor
+    image: $.instanceConfiguration.osImage
+    requiredType: $.appConfiguration.requiredType
+    assignFloatingIp: $.appConfiguration.assignFloatingIP
+    keyname: $.instanceConfiguration.keyPair
+
+Forms:
+  - appConfiguration:
+      fields:
+        - name: name
+          type: string
+          label: Application Name
+          initial: Demo
+          description: >-
+            Enter a desired name for the application. Just A-Z, a-z, 0-9, dash and
+            underline are allowed
+        - name: requiredType
+          type: string
+          label: Required MuranoImage Type
+          initial: linux
+          description: >-
+            Enter a value to be matched against 'type' field of MuranoImage metadata
+        - name: assignFloatingIP
+          type: boolean
+          label: Assign Floating IP
+          description: >-
+             Select to true to assign floating IP automatically
+          initial: false
+          required: false
+          widgetMedia:
+            css: {all: ['muranodashboard/css/checkbox.css']}
+  - instanceConfiguration:
+      fields:
+        - name: title
+          type: string
+          required: false
+          hidden: true
+          description: Specify some instance parameters on which the application would be created
+        - name: flavor
+          type: flavor
+          label: Instance flavor
+          description: >-
+            Select registered in Openstack flavor. Consider that application performance
+            depends on this parameter.
+          required: false
+        - name: osImage
+          type: image
+          imageType: linux
+          label: Instance image
+          description: >-
+            Select a valid image for the application. Image should already be prepared and
+            registered in glance.
+        - name: keyPair
+          type: keypair
+          label: Key Pair
+          description: >-
+            Select a Key Pair to control access to instances. You can login to
+            instances using this KeyPair after the deployment of application.
+          required: false
+        - name: availabilityZone
+          type: azone
+          label: Availability zone
+          description: Select availability zone where the application would be installed.
+          required: false
+        - name: unitNamingPattern
+          label: Hostname
+          type: string
+          required: false
+          widgetMedia:
+            js: ['muranodashboard/js/support_placeholder.js']
+            css: {all: ['muranodashboard/css/support_placeholder.css']}

contrib/plugins/murano_exampleplugin/example-app/io.murano.apps.demo.DemoApp/manifest.yaml

@@ -0,0 +1,12 @@
+Format: 1.0
+Type: Application
+FullName: io.murano.apps.example.plugin.DemoApp
+Name: Plugin Demo App
+Description: |
+ Demo App to validate Glance Images
+Author: 'Mirantis, Inc'
+Tags: [Demo, Images]
+Classes:
+ io.murano.apps.example.plugin.ImageValidatorMixin: ImageValidatorMixin.yaml
+ io.murano.apps.example.plugin.DemoInstance: DemoInstance.yaml
+ io.murano.apps.example.plugin.DemoApp: DemoApp.yaml

contrib/plugins/murano_exampleplugin/murano_exampleplugin/__init__.py

@@ -0,0 +1,82 @@
+# Copyright (c) 2015 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import json
+
+import glanceclient
+from murano.common import config
+import murano.dsl.helpers as helpers
+from murano.openstack.common import log as logging
+
+import cfg
+
+
+LOG = logging.getLogger(__name__)
+
+
+# noinspection PyPep8Naming
+class GlanceClient(object):
+    def initialize(self, _context):
+        client_manager = helpers.get_environment(_context).clients
+        self.client = client_manager.get_client(_context, "glance", True,
+                                                self.create_glance_client)
+
+    def list(self):
+        images = self.client.images.list()
+        while True:
+            try:
+                image = images.next()
+                yield GlanceClient._format(image)
+            except StopIteration:
+                break
+
+    def getByName(self, name):
+        images = list(self.client.images.list(filters={"name": name}))
+        if len(images) > 1:
+            raise AmbiguousNameException(name)
+        elif len(images) == 0:
+            return None
+        else:
+            return GlanceClient._format(images[0])
+
+    def getById(self, imageId):
+        image = self.client.images.get(imageId)
+        return GlanceClient._format(image)
+
+    @staticmethod
+    def _format(image):
+        res = {"id": image.id, "name": image.name}
+        if hasattr(image, "murano_image_info"):
+            res["meta"] = json.loads(image.murano_image_info)
+        return res
+
+    @classmethod
+    def init_plugin(cls):
+        cls.CONF = cfg.init_config(config.CONF)
+
+    def create_glance_client(self, keystone_client, auth_token):
+        LOG.debug("Creating a glance client")
+        glance_endpoint = keystone_client.service_catalog.url_for(
+            service_type='image', endpoint_type=self.CONF.endpoint_type)
+        client = glanceclient.Client(self.CONF.api_version,
+                                     endpoint=glance_endpoint,
+                                     token=auth_token)
+        return client
+
+
+class AmbiguousNameException(Exception):
+    def __init__(self, name):
+        super(AmbiguousNameException, self).__init__("Image name '%s'"
+                                                     " is ambiguous" % name)

contrib/plugins/murano_exampleplugin/murano_exampleplugin/cfg.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2015 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo.config import cfg
+
+
+def init_config(conf):
+    opts = [
+        cfg.IntOpt('api_version', default=2),
+        cfg.StrOpt('endpoint_type', default='publicURL')
+    ]
+    conf.register_opts(opts, group="glance")
+    return conf.glance

contrib/plugins/murano_exampleplugin/requiremenets.txt

@@ -0,0 +1 @@
+python-glanceclient>=0.15.0

contrib/plugins/murano_exampleplugin/setup.cfg

@@ -0,0 +1,18 @@
+[metadata]
+version = 1.0
+name = murano.plugins.example
+description = Example Plugin to extend collection of MuranoPL system classes
+summary = An example Murano Plugin demonstrating extensibility of MuranoPL
+          classes with code written in Python. This particular plugin uses
+          python-glanceclient to call OpenStack Images API to list available
+          images and return their ids to caller. Anther available method allows
+          to get murano-related metadata from image with a given id.
+author = Alexander Tivelkov
+author-email = ativelkov@mirantis.com
+
+[files]
+packages = murano_exampleplugin
+
+[entry_points]
+io.murano.extensions =
+    mirantis.example.Glance = murano_exampleplugin:GlanceClient

contrib/plugins/murano_exampleplugin/setup.py

@@ -0,0 +1,20 @@
+# Copyright 2011-2012 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import setuptools
+
+# all other params will be taken from setup.cfg
+setuptools.setup(packages=setuptools.find_packages(),
+                 setup_requires=['pbr'], pbr=True)

murano/common/config.py

@@ -146,7 +146,12 @@ murano_opts = [
                                 'Murano engine.'),
 
     cfg.StrOpt('endpoint_type', default='publicURL',
-               help='Murno endpoint type used by Murano engine.')
+               help='Murano endpoint type used by Murano engine.'),
+
+    cfg.ListOpt('enabled_plugins', default=None,
+                help="List of enabled Extension Plugins. "
+                     "Remove or leave commented to enable all installed "
+                     "plugins.")
 ]
 
 networking_opts = [

murano/common/engine.py

@@ -23,6 +23,7 @@ from oslo.serialization import jsonutils
 
 from murano.common import config
 from murano.common.helpers import token_sanitizer
+from murano.common import plugin_loader
 from murano.common import rpc
 from murano.dsl import dsl_exception
 from murano.dsl import executor
@@ -38,7 +39,9 @@ from murano.common.i18n import _LI, _LE
 from murano.openstack.common import log as logging
 from murano.policy import model_policy_enforcer as enforcer
 
+
 RPC_SERVICE = None
+PLUGIN_LOADER = None
 
 LOG = logging.getLogger(__name__)
 
@@ -84,6 +87,14 @@ def get_rpc_service():
     return RPC_SERVICE
 
 
+def get_plugin_loader():
+    global PLUGIN_LOADER
+
+    if PLUGIN_LOADER is None:
+        PLUGIN_LOADER = plugin_loader.PluginLoader()
+    return PLUGIN_LOADER
+
+
 class Environment(object):
     def __init__(self, object_id):
         self.object_id = object_id
@@ -137,6 +148,7 @@ class TaskExecutor(object):
     def _execute(self, pkg_loader):
         class_loader = package_class_loader.PackageClassLoader(pkg_loader)
         system_objects.register(class_loader, pkg_loader)
+        get_plugin_loader().register_in_loader(class_loader)
 
         exc = executor.MuranoDslExecutor(class_loader, self.environment)
         obj = exc.load(self.model)

murano/common/plugin_loader.py

@@ -0,0 +1,128 @@
+# Copyright (c) 2015 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import inspect
+import re
+
+import six
+from stevedore import dispatch
+
+from murano.common import config
+from murano.common.i18n import _LE, _LI, _LW
+from murano.openstack.common import log as logging
+
+
+CONF = config.CONF
+LOG = logging.getLogger(__name__)
+
+# regexp validator to ensure that the entry-point name is a valid MuranoPL
+# class name with an optional namespace name
+NAME_RE = re.compile(r'^[a-zA-Z]\w*(\.[a-zA-Z]\w*)*$')
+
+
+class PluginLoader(object):
+    def __init__(self, namespace="io.murano.extensions"):
+        LOG.debug('Loading plugins')
+        self.namespace = namespace
+        extension_manager = dispatch.EnabledExtensionManager(
+            self.namespace,
+            PluginLoader.is_plugin_enabled,
+            on_load_failure_callback=PluginLoader._on_load_failure)
+        self.packages = {}
+        name_map = {}
+        for ext in extension_manager.extensions:
+            self.load_extension(ext, name_map)
+        self.cleanup_duplicates(name_map)
+
+    def load_extension(self, extension, name_map):
+        dist_name = str(extension.entry_point.dist)
+        name = extension.entry_point.name
+        if not NAME_RE.match(name):
+            LOG.warning(_LW("Entry-point 'name' %s is invalid") % name)
+            return
+        name = "%s.%s" % (self.namespace, name)
+        name_map.setdefault(name, []).append(dist_name)
+        if dist_name in self.packages:
+            package = self.packages[dist_name]
+        else:
+            package = PackageDefinition(extension.entry_point.dist)
+            self.packages[dist_name] = package
+
+        plugin = extension.plugin
+        try:
+            package.classes[name] = initialize_plugin(plugin)
+        except Exception:
+            LOG.exception(_LE("Unable to initialize plugin for %s") % name)
+            return
+        LOG.info(_LI("Loaded class '%(class_name)s' from '%(dist)s'")
+                 % dict(class_name=name, dist=dist_name))
+
+    def cleanup_duplicates(self, name_map):
+        for class_name, package_names in six.iteritems(name_map):
+            if len(package_names) >= 2:
+                LOG.warning(_LW("Class is defined in multiple packages!"))
+                for package_name in package_names:
+                    LOG.warning(_LW("Disabling class '%(class_name)s' in "
+                                    "'%(dist)s' due to conflict") %
+                                dict(class_name=class_name, dist=package_name))
+                    self.packages[package_name].plugins.pop(class_name)
+
+    @staticmethod
+    def is_plugin_enabled(extension):
+        if CONF.murano.enabled_plugins is None:
+            return True
+        else:
+            return (extension.entry_point.dist.project_name in
+                    CONF.murano.enabled_plugins)
+
+    @staticmethod
+    def _on_load_failure(manager, ep, exc):
+        LOG.warning(_LW("Error loading entry-point '%(ep)s' "
+                        "from package '%(dist)s': %(err)s")
+                    % dict(ep=ep.name, dist=ep.dist, err=exc))
+
+    def register_in_loader(self, class_loader):
+        for package in six.itervalues(self.packages):
+            for class_name, clazz in six.iteritems(package.classes):
+                if hasattr(clazz, "_murano_class_name"):
+                    LOG.warning(_LW("Class '%(class_name)s' has a MuranoPL "
+                                    "name '%(name)s' defined which will be "
+                                    "ignored") %
+                                dict(class_name=class_name,
+                                name=getattr(clazz, "_murano_class_name")))
+                LOG.debug("Registering '%s' from '%s' in class loader"
+                          % (class_name, package.name))
+                class_loader.import_class(clazz, name=class_name)
+
+
+def initialize_plugin(plugin):
+    if hasattr(plugin, "init_plugin"):
+        initializer = getattr(plugin, "init_plugin")
+        if inspect.ismethod(initializer) and initializer.__self__ is plugin:
+            LOG.debug("Initializing plugin class %s" % plugin)
+            initializer()
+    return plugin
+
+
+class PackageDefinition(object):
+    def __init__(self, distribution):
+        self.name = distribution.project_name
+        self.version = distribution.version
+        if distribution.has_metadata(distribution.PKG_INFO):
+            # This has all the package metadata, including Author,
+            # description, License etc
+            self.info = distribution.get_metadata(distribution.PKG_INFO)
+        else:
+            self.info = None
+        self.classes = {}

murano/engine/client_manager.py

@@ -47,7 +47,7 @@ class ClientManager(object):
             return context
         return helpers.get_environment(context)
 
-    def _get_client(self, context, name, use_trusts, client_factory):
+    def get_client(self, context, name, use_trusts, client_factory):
         if not config.CONF.engine.use_trusts:
             use_trusts = False
 
@@ -80,7 +80,7 @@ class ClientManager(object):
         factory = lambda _1, _2: auth_utils.get_client_for_trusts(env) \
             if use_trusts else auth_utils.get_client(env)
 
-        return self._get_client(context, 'keystone', use_trusts, factory)
+        return self.get_client(context, 'keystone', use_trusts, factory)
 
     def get_congress_client(self, context, use_trusts=True):
         """Client for congress services
@@ -105,7 +105,7 @@ class ClientManager(object):
             return congress_client.Client(session=session,
                                           service_type='policy')
 
-        return self._get_client(context, 'congress', use_trusts, factory)
+        return self.get_client(context, 'congress', use_trusts, factory)
 
     def get_heat_client(self, context, use_trusts=True):
         if not config.CONF.engine.use_trusts:
@@ -133,7 +133,7 @@ class ClientManager(object):
                 })
             return hclient.Client('1', heat_url, **kwargs)
 
-        return self._get_client(context, 'heat', use_trusts, factory)
+        return self.get_client(context, 'heat', use_trusts, factory)
 
     def get_neutron_client(self, context, use_trusts=True):
         if not config.CONF.engine.use_trusts:
@@ -152,7 +152,7 @@ class ClientManager(object):
                 ca_cert=neutron_settings.ca_cert or None,
                 insecure=neutron_settings.insecure)
 
-        return self._get_client(context, 'neutron', use_trusts, factory)
+        return self.get_client(context, 'neutron', use_trusts, factory)
 
     def get_murano_client(self, context, use_trusts=True):
         if not config.CONF.engine.use_trusts:
@@ -175,7 +175,7 @@ class ClientManager(object):
                 auth_url=keystone_client.auth_url,
                 token=auth_token)
 
-        return self._get_client(context, 'murano', use_trusts, factory)
+        return self.get_client(context, 'murano', use_trusts, factory)
 
     def get_mistral_client(self, context, use_trusts=True):
         if not mistralclient:
@@ -202,4 +202,4 @@ class ClientManager(object):
                                         auth_token=auth_token,
                                         user_id=keystone_client.user_id)
 
-        return self._get_client(context, 'mistral', use_trusts, factory)
+        return self.get_client(context, 'mistral', use_trusts, factory)