Taking a first stab at the plugin system

Jose and I taking a first stab at a plugin system for opencafe.
* Making more changes
* Adding simple docstring
* Cleaning up a plugin install a tiny bit
* Adding license to __init__.py files
* Adding Plugin List
* Adding note to the end of setup

Change-Id: Id0a61549f37ada7f9a254f8cc944957c4d2c4694

cafe/configurator/cli.py

@@ -16,7 +16,7 @@ limitations under the License.
 
 import argparse
 from cafe.configurator.managers import (
-    EngineDirectoryManager, EngineConfigManager)
+    EngineDirectoryManager, EngineConfigManager, EnginePluginManager)
 
 
 class EngineActions(object):
@@ -30,6 +30,29 @@ class EngineActions(object):
             print "================================="
 
 
+class PluginActions(object):
+    class AddPluginCache(argparse.Action):
+        def __call__(self, parser, namespace, values, option_string=None):
+            print "================================="
+            print "* Adding Plugin Cache"
+            EnginePluginManager.populate_plugin_cache(values)
+            print "================================="
+
+    class InstallPlugin(argparse.Action):
+        def __call__(self, parser, namespace, values, option_string=None):
+            print "================================="
+            print "* Installing Plugins"
+            EnginePluginManager.install_plugins(values)
+            print "================================="
+
+    class ListPlugins(argparse.Action):
+        def __call__(self, parser, namespace, values, option_string=None):
+            print "================================="
+            print "* Available Plugins"
+            EnginePluginManager.list_plugins()
+            print "================================="
+
+
 class ConfiguratorCLI(object):
     """CLI for future engine management and configuration options."""
 
@@ -43,6 +66,24 @@ class ConfiguratorCLI(object):
         subparser_engine_config.add_argument(
             '--init-install', action=EngineActions.InitInstall, nargs=0)
 
+        # Plugin argument subparser
+        subparser_plugins = subparsers.add_parser('plugins')
+        plugin_args = subparser_plugins.add_subparsers(dest='plugin_args')
+
+        plugins_add_parser = plugin_args.add_parser('add')
+        plugins_add_parser.add_argument(
+            'plugin_dir', action=PluginActions.AddPluginCache,
+            type=str)
+
+        plugins_add_parser = plugin_args.add_parser('list')
+        plugins_add_parser.add_argument(
+            'list_plugins', action=PluginActions.ListPlugins, nargs=0)
+
+        plugins_install_parser = plugin_args.add_parser('install')
+        plugins_install_parser.add_argument(
+            'plugin-name', action=PluginActions.InstallPlugin,
+            type=str, nargs='*')
+
         return parser.parse_args()
 
 

cafe/configurator/managers.py

@@ -22,6 +22,8 @@ import sys
 import textwrap
 import getpass
 import shutil
+from subprocess import Popen, PIPE
+
 from ConfigParser import SafeConfigParser
 from cafe.engine.config import EngineConfig
 
@@ -267,7 +269,8 @@ class EngineDirectoryManager(object):
         LOG_DIR=os.path.join(OPENCAFE_ROOT_DIR, 'logs'),
         DATA_DIR=os.path.join(OPENCAFE_ROOT_DIR, 'data'),
         TEMP_DIR=os.path.join(OPENCAFE_ROOT_DIR, 'temp'),
-        CONFIG_DIR=os.path.join(OPENCAFE_ROOT_DIR, 'configs'))
+        CONFIG_DIR=os.path.join(OPENCAFE_ROOT_DIR, 'configs'),
+        PLUGIN_CACHE=os.path.join(OPENCAFE_ROOT_DIR, 'plugin_cache'))
 
     @classmethod
     def update_deprecated_engine_directories(cls):
@@ -573,3 +576,82 @@ class EngineConfigManager(object):
                         _printed.append(destination_dir)
 
                 PlatformManager.safe_chown(destination_file)
+
+
+class EnginePluginManager(object):
+
+    @classmethod
+    def copy_plugin_to_cache(
+            cls, plugins_src_dir, plugins_dest_dir, plugin_name):
+        """ Copies an individual plugin to the .opencafe plugin cache """
+        src_plugin_path = os.path.join(plugins_src_dir, plugin_name)
+        dest_plugin_path = os.path.join(plugins_dest_dir, plugin_name)
+
+        if os.path.exists(dest_plugin_path):
+            shutil.rmtree(dest_plugin_path)
+
+        shutil.copytree(src_plugin_path, dest_plugin_path)
+
+    @classmethod
+    def populate_plugin_cache(cls, plugins_src_dir):
+        """ Handles moving all plugin src data from package into the user's
+        .opencafe folder for installation by the cafe-config tool.
+        """
+        default_dest = EngineDirectoryManager.OPENCAFE_SUB_DIRS.PLUGIN_CACHE
+        plugins = os.walk(plugins_src_dir).next()[1]
+
+        for plugin_name in plugins:
+            cls.copy_plugin_to_cache(
+                plugins_src_dir, default_dest, plugin_name)
+
+    @classmethod
+    def list_plugins(cls):
+        """ Lists all plugins currently available in user's .opencafe cache"""
+        plugin_cache = EngineDirectoryManager.OPENCAFE_SUB_DIRS.PLUGIN_CACHE
+        plugin_folders = os.walk(plugin_cache).next()[1]
+        wrap = textwrap.TextWrapper(initial_indent="  ",
+                                    subsequent_indent="  ",
+                                    break_long_words=False).fill
+
+        for plugin_folder in plugin_folders:
+            print wrap('... {name}'.format(name=plugin_folder))
+
+    @classmethod
+    def install_plugins(cls, plugin_names):
+        """ Installs a list of plugins into the current environment"""
+        for plugin_name in plugin_names:
+            cls.install_plugin(plugin_name)
+
+    @classmethod
+    def install_plugin(cls, plugin_name):
+        """ Install a single plugin by name into the current environment"""
+        plugin_cache = EngineDirectoryManager.OPENCAFE_SUB_DIRS.PLUGIN_CACHE
+        plugin_dir = os.path.join(plugin_cache, plugin_name)
+        wrap = textwrap.TextWrapper(initial_indent="  ",
+                                    subsequent_indent="  ",
+                                    break_long_words=False).fill
+
+        # Pretty output of plugin name
+        print wrap('... {name}'.format(name=plugin_name))
+
+        # Verify that the plugin exists
+        if not os.path.exists(plugin_dir):
+            print wrap('* Failed to install plugin: {0}'.format(plugin_name))
+            return
+
+        # Install Plugin
+        process, standard_out, standard_error = None, None, None
+        cmd = 'pip install {name} --upgrade'.format(name=plugin_dir)
+
+        try:
+            process = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
+            standard_out, standard_error = process.communicate()
+        except Exception as e:
+            msg = '* Plugin install failed {0}\n{1}\n'.format(cmd, e)
+            print wrap(msg)
+
+        # Print failure if we receive an error code
+        if process and process.returncode != 0:
+            print wrap(standard_out)
+            print wrap(standard_error)
+            print wrap('* Failed to install plugin: {0}'.format(plugin_name))

cafe/engine/models/data_interfaces.py

@@ -20,7 +20,14 @@ import os
 import ConfigParser
 
 from cafe.common.reporting import cclogging
-from cafe.engine.mongo.client import BaseMongoClient
+try:
+    from cafe.engine.mongo.client import BaseMongoClient
+except:
+    """ We are temporarily ignoring errors due to plugin refactor.
+    The mongo data-source is currently not being used. and needs to be
+    abstracted out into a data-source plugin.
+    """
+    pass
 
 
 class ConfigDataException(Exception):

pip-requires

@@ -1,7 +1,5 @@
 decorator
 paramiko
-requests==1.2.0
-pymongo==2.6
 argparse==1.2.1
 unittest2==0.5.1
 launchpadlib==1.10.2

plugins/http/cafe/engine/__init__.py

@@ -12,4 +12,4 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-"""
+"""

plugins/http/cafe/engine/clients/__init__.py

@@ -0,0 +1,15 @@
+"""
+Copyright 2013 Rackspace
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""

plugins/http/cafe/engine/http/__init__.py

@@ -0,0 +1,15 @@
+"""
+Copyright 2013 Rackspace
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""

plugins/http/setup.py

@@ -0,0 +1,28 @@
+"""
+Copyright 2013 Rackspace
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from setuptools import setup, find_packages
+
+setup(
+    name='cafe_http_plugin',
+    version='0.0.1',
+    description='The Common Automation Framework Engine',
+    author='Rackspace Cloud QE',
+    author_email='cloud-cafe@lists.rackspace.com',
+    url='http://rackspace.com',
+    packages=find_packages(),
+    package_dir={'cafe': 'cafe'},
+    install_requires=['requests'],
+    zip_safe=False)

plugins/mongo/cafe/__init__.py

@@ -0,0 +1,15 @@
+"""
+Copyright 2013 Rackspace
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""

plugins/mongo/cafe/engine/__init__.py

@@ -0,0 +1,15 @@
+"""
+Copyright 2013 Rackspace
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""

plugins/mongo/cafe/engine/clients/__init__.py

@@ -0,0 +1,15 @@
+"""
+Copyright 2013 Rackspace
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""

plugins/mongo/cafe/engine/clients/mongo.py

@@ -19,4 +19,4 @@ warn("cafe.engine.clients.mongo has been moved to cafe.engine.mongo.client",
      DeprecationWarning)
 
 from cafe.engine.mongo.client import (
-    pymongo, MongoClient, cclogging, BaseClient, BaseMongoClient)
+    pymongo, MongoClient, cclogging, BaseClient, BaseMongoClient)

plugins/mongo/cafe/engine/mongo/__init__.py

@@ -0,0 +1,15 @@
+"""
+Copyright 2013 Rackspace
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""

plugins/mongo/setup.py

@@ -0,0 +1,28 @@
+"""
+Copyright 2013 Rackspace
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from setuptools import setup, find_packages
+
+setup(
+    name='cafe_mongo_plugin',
+    version='0.0.1',
+    description='The Common Automation Framework Engine',
+    author='Rackspace Cloud QE',
+    author_email='cloud-cafe@lists.rackspace.com',
+    url='http://rackspace.com',
+    packages=find_packages(),
+    package_dir={'cafe': 'cafe'},
+    install_requires=['pymongo'],
+    zip_safe=False)

setup.py

@@ -34,6 +34,8 @@ if sys.argv[-1] == 'publish':
 #Post-install engine configuration
 def _post_install(dir):
     call(['cafe-config', 'engine', '--init-install'])
+    call(['cafe-config', 'plugins', 'add', 'plugins'])
+    call(['cafe-config', 'plugins', 'install', 'http'])
     print(
         """
          ( (
@@ -45,7 +47,15 @@ def _post_install(dir):
         |      |___|
         |______|
     === OpenCAFE ===
-        """)
+
+    -----------------------------------------------------------------
+    If you wish to install additional plugins, you can do so through
+    the cafe-config tool.
+
+    Example:
+    $ cafe-config plugins install mongo
+    -----------------------------------------------------------------
+    """)
 
 
 #cmdclass hook allows setup to make post install call