Merge "Dynamically loading plugins"

.zuul.yaml

@@ -110,7 +110,7 @@
     name: sahara-buildimages-cloudera
     parent: sahara-buildimages-base
     vars:
-      sahara_plugin: cloudera
+      sahara_plugin: cdh
 
 - job:
     name: sahara-buildimages-mapr

devstack/plugin.sh

@@ -291,7 +291,7 @@ function start_sahara {
 function configure_tempest_for_sahara {
     if is_service_enabled tempest; then
         iniset $TEMPEST_CONFIG service_available sahara True
-        iniset $TEMPEST_CONFIG data-processing-feature-enabled plugins $SAHARA_ENABLED_PLUGINS
+        iniset $TEMPEST_CONFIG data-processing-feature-enabled plugins $SAHARA_INSTALLED_PLUGINS
     fi
 }
 
@@ -320,8 +320,8 @@ function is_sahara_enabled {
     fi
 }
 
-function is_plugin_enabled {
-    if [ "${SAHARA_ENABLED_PLUGINS/$1}" = "$SAHARA_ENABLED_PLUGINS" ] ; then
+function is_plugin_required {
+    if [ "${SAHARA_INSTALLED_PLUGINS/$1}" = "$SAHARA_INSTALLED_PLUGINS" ] ; then
         return 1
     else
         return 0
@@ -333,22 +333,22 @@ if is_service_enabled sahara; then
     if [[ "$1" == "stack" && "$2" == "install" ]]; then
         echo_summary "Installing sahara"
         install_sahara
-        if is_plugin_enabled ambari; then
+        if is_plugin_required ambari; then
             install_ambari
         fi
-        if is_plugin_enabled cdh; then
+        if is_plugin_required cdh; then
             install_cdh
         fi
-        if is_plugin_enabled mapr; then
+        if is_plugin_required mapr; then
             install_mapr
         fi
-        if is_plugin_enabled spark; then
+        if is_plugin_required spark; then
             install_spark
         fi
-        if is_plugin_enabled storm; then
+        if is_plugin_required storm; then
             install_storm
         fi
-        if is_plugin_enabled vanilla; then
+        if is_plugin_required vanilla; then
             install_vanilla
         fi
         install_python_saharaclient

devstack/settings

@@ -16,17 +16,17 @@ VANILLA_PLUGIN_DIR=$DEST/sahara-plugin-vanilla
 SAHARACLIENT_REPO=${SAHARACLIENT_REPO:-\
 ${GIT_BASE}/openstack/python-saharaclient.git}
 SAHARACLIENT_BRANCH=${SAHARACLIENT_BRANCH:-master}
-AMBARI_PLUGIN_REPO=${AMBARI_PLUGIN_REPO:-http://git.openstack.org/cgit/openstack/sahara-plugin-ambari/}
+AMBARI_PLUGIN_REPO=${AMBARI_PLUGIN_REPO:-https://git.openstack.org/openstack/sahara-plugin-ambari/}
 AMBARI_PLUGIN_BRANCH=${AMBARI_PLUGIN_BRANCH:-master}
-CDH_PLUGIN_REPO=${CDH_PLUGIN_REPO:-http://git.openstack.org/cgit/openstack/sahara-plugin-cdh/}
+CDH_PLUGIN_REPO=${CDH_PLUGIN_REPO:-https://git.openstack.org/openstack/sahara-plugin-cdh/}
 CDH_PLUGIN_BRANCH=${CDH_PLUGIN_BRANCH:-master}
-MAPR_PLUGIN_REPO=${MAPR_PLUGIN_REPO:-http://git.openstack.org/cgit/openstack/sahara-plugin-mapr/}
+MAPR_PLUGIN_REPO=${MAPR_PLUGIN_REPO:-https://git.openstack.org/openstack/sahara-plugin-mapr/}
 MAPR_PLUGIN_BRANCH=${MAPR_PLUGIN_BRANCH:-master}
-SPARK_PLUGIN_REPO=${SPARK_PLUGIN_REPO:-http://git.openstack.org/cgit/openstack/sahara-plugin-spark/}
+SPARK_PLUGIN_REPO=${SPARK_PLUGIN_REPO:-https://git.openstack.org/openstack/sahara-plugin-spark/}
 SPARK_PLUGIN_BRANCH=${SPARK_PLUGIN_BRANCH:-master}
-STORM_PLUGIN_REPO=${STORM_PLUGIN_REPO:-http://git.openstack.org/cgit/openstack/sahara-plugin-storm/}
+STORM_PLUGIN_REPO=${STORM_PLUGIN_REPO:-https://git.openstack.org/openstack/sahara-plugin-storm/}
 STORM_PLUGIN_BRANCH=${STORM_PLUGIN_BRANCH:-master}
-VANILLA_PLUGIN_REPO=${VANILLA_PLUGIN_REPO:-http://git.openstack.org/cgit/openstack/sahara-plugin-vanilla/}
+VANILLA_PLUGIN_REPO=${VANILLA_PLUGIN_REPO:-https://git.openstack.org/openstack/sahara-plugin-vanilla/}
 VANILLA_PLUGIN_BRANCH=${VANILLA_PLUGIN_BRANCH:-master}
 
 
@@ -48,6 +48,8 @@ SAHARA_AUTH_CACHE_DIR=${SAHARA_AUTH_CACHE_DIR:-/var/cache/sahara}
 
 SAHARA_ENABLED_PLUGINS=${SAHARA_ENABLED_PLUGINS:-\
 vanilla,ambari,cdh,mapr,spark,storm,fake}
+SAHARA_INSTALLED_PLUGINS=${SAHARA_INSTALLED_PLUGINS:-\
+vanilla,ambari,cdh,mapr,spark,storm,fake}
 SAHARA_BIN_DIR=$(get_python_exec_prefix)
 
 SAHARA_ENABLE_DISTRIBUTED_PERIODICS=${SAHARA_ENABLE_DISTRIBUTED_PERIODICS:-\

roles/build-sahara-images-cli/tasks/main.yaml

@@ -1,3 +1,8 @@
+- name: Installs needed plugins to build its images
+  command: tox -e images -- pip install {{ ansible_user_dir }}/src/git.openstack.org/openstack/sahara-plugin-{{ sahara_plugin }}
+  args:
+    chdir: "{{ sahara_src_dir }}"
+
 - name: Build all the images for the selected plugin
   command: ./tools/gate/build-images "{{ sahara_plugin }}"
   args:

sahara/cli/image_pack/api.py

@@ -111,6 +111,10 @@ def setup_plugins():
     plugins_base.setup_plugins()
 
 
+def get_loaded_plugins():
+    return plugins_base.PLUGINS.plugins
+
+
 def get_plugin_arguments(plugin_name):
     """Gets plugin arguments, as a dict of version to argument list."""
     plugin = plugins_base.PLUGINS.get_plugin(plugin_name)

sahara/cli/image_pack/cli.py

@@ -62,7 +62,7 @@ for extra_opt in ["log-exchange", "host", "port"]:
 
 def add_plugin_parsers(subparsers):
     api.setup_plugins()
-    for plugin in CONF.plugins:
+    for plugin in api.get_loaded_plugins():
         args_by_version = api.get_plugin_arguments(plugin)
         if all(args is NotImplemented for version, args
                in six.iteritems(args_by_version)):

sahara/plugins/base.py

@@ -103,16 +103,16 @@ class PluginManager(object):
                      entry_point=ext.entry_point_target))
 
         if len(self.plugins) < len(config_plugins):
-            loaded_plugins = set(six.iterkeys(self.plugins))
+            self.loaded_plugins = set(six.iterkeys(self.plugins))
             requested_plugins = set(config_plugins)
-            raise ex.ConfigurationError(
-                _("Plugins couldn't be loaded: %s") %
-                ", ".join(requested_plugins - loaded_plugins))
+            LOG.warning("Plugins couldn't be loaded: %s",
+                        ", ".join(requested_plugins - self.loaded_plugins))
 
     def get_plugins(self, serialized=False):
         if serialized:
-            return [self.serialize_plugin(name) for name in CONF.plugins]
-        return [self.get_plugin(name) for name in CONF.plugins]
+            return [self.serialize_plugin(name)
+                    for name in PLUGINS.plugins]
+        return [self.get_plugin(name) for name in PLUGINS.plugins]
 
     def get_plugin(self, plugin_name):
         return self.plugins.get(plugin_name)

sahara/tests/unit/plugins/test_base_plugins_support.py

@@ -23,7 +23,7 @@ class BasePluginsSupportTest(testtools.TestCase):
 
     def setUp(self):
         super(BasePluginsSupportTest, self).setUp()
-        main.CONF.set_override('plugins', ['fake'])
+        main.CONF.set_override('plugins', ['fake', 'cdh', 'spark'])
         pb.setup_plugins()
 
     def test_plugins_loaded(self):

tools/gate/build-images

@@ -64,7 +64,7 @@ setup_build_env
 # build_images "<plugin_name>" "<plugin_version>" "<distribution> <distribution>"
 
 case "$PLUGIN" in
-    "cloudera")
+    "cdh")
         build_images "cdh" "5.9.0" "centos7"
         build_images "cdh" "5.11.0" "centos7"
         build_images "cdh" "5.13.0" "centos7"