Dynamically loading plugins

Since plugins are not part of Sahara core anymore, loading plugins
like we currently do fails when plugins are not installed.
Currently we check the list of loaded plugins by checking the CONF
file plugins session. To fix that, we are changing to check the list
of loaded plugins on the PluginManager variable loaded_plugins, this
way we can instead of failing to start sahara if a desired plugin
is not installed, we can warn that the plugin was not loaded and
continue the application.

Change-Id: Iaae3a75cd75b06e94bc4492e5dc654a6281fdff4
This commit is contained in:
Telles Nobrega 2019-01-17 16:21:13 -03:00
parent 92686f5a10
commit 55bd5da57f
9 changed files with 36 additions and 25 deletions

View File

@ -110,7 +110,7 @@
name: sahara-buildimages-cloudera
parent: sahara-buildimages-base
vars:
sahara_plugin: cloudera
sahara_plugin: cdh
- job:
name: sahara-buildimages-mapr

View File

@ -291,7 +291,7 @@ function start_sahara {
function configure_tempest_for_sahara {
if is_service_enabled tempest; then
iniset $TEMPEST_CONFIG service_available sahara True
iniset $TEMPEST_CONFIG data-processing-feature-enabled plugins $SAHARA_ENABLED_PLUGINS
iniset $TEMPEST_CONFIG data-processing-feature-enabled plugins $SAHARA_INSTALLED_PLUGINS
fi
}
@ -320,8 +320,8 @@ function is_sahara_enabled {
fi
}
function is_plugin_enabled {
if [ "${SAHARA_ENABLED_PLUGINS/$1}" = "$SAHARA_ENABLED_PLUGINS" ] ; then
function is_plugin_required {
if [ "${SAHARA_INSTALLED_PLUGINS/$1}" = "$SAHARA_INSTALLED_PLUGINS" ] ; then
return 1
else
return 0
@ -333,22 +333,22 @@ if is_service_enabled sahara; then
if [[ "$1" == "stack" && "$2" == "install" ]]; then
echo_summary "Installing sahara"
install_sahara
if is_plugin_enabled ambari; then
if is_plugin_required ambari; then
install_ambari
fi
if is_plugin_enabled cdh; then
if is_plugin_required cdh; then
install_cdh
fi
if is_plugin_enabled mapr; then
if is_plugin_required mapr; then
install_mapr
fi
if is_plugin_enabled spark; then
if is_plugin_required spark; then
install_spark
fi
if is_plugin_enabled storm; then
if is_plugin_required storm; then
install_storm
fi
if is_plugin_enabled vanilla; then
if is_plugin_required vanilla; then
install_vanilla
fi
install_python_saharaclient

View File

@ -16,17 +16,17 @@ VANILLA_PLUGIN_DIR=$DEST/sahara-plugin-vanilla
SAHARACLIENT_REPO=${SAHARACLIENT_REPO:-\
${GIT_BASE}/openstack/python-saharaclient.git}
SAHARACLIENT_BRANCH=${SAHARACLIENT_BRANCH:-master}
AMBARI_PLUGIN_REPO=${AMBARI_PLUGIN_REPO:-http://git.openstack.org/cgit/openstack/sahara-plugin-ambari/}
AMBARI_PLUGIN_REPO=${AMBARI_PLUGIN_REPO:-https://git.openstack.org/openstack/sahara-plugin-ambari/}
AMBARI_PLUGIN_BRANCH=${AMBARI_PLUGIN_BRANCH:-master}
CDH_PLUGIN_REPO=${CDH_PLUGIN_REPO:-http://git.openstack.org/cgit/openstack/sahara-plugin-cdh/}
CDH_PLUGIN_REPO=${CDH_PLUGIN_REPO:-https://git.openstack.org/openstack/sahara-plugin-cdh/}
CDH_PLUGIN_BRANCH=${CDH_PLUGIN_BRANCH:-master}
MAPR_PLUGIN_REPO=${MAPR_PLUGIN_REPO:-http://git.openstack.org/cgit/openstack/sahara-plugin-mapr/}
MAPR_PLUGIN_REPO=${MAPR_PLUGIN_REPO:-https://git.openstack.org/openstack/sahara-plugin-mapr/}
MAPR_PLUGIN_BRANCH=${MAPR_PLUGIN_BRANCH:-master}
SPARK_PLUGIN_REPO=${SPARK_PLUGIN_REPO:-http://git.openstack.org/cgit/openstack/sahara-plugin-spark/}
SPARK_PLUGIN_REPO=${SPARK_PLUGIN_REPO:-https://git.openstack.org/openstack/sahara-plugin-spark/}
SPARK_PLUGIN_BRANCH=${SPARK_PLUGIN_BRANCH:-master}
STORM_PLUGIN_REPO=${STORM_PLUGIN_REPO:-http://git.openstack.org/cgit/openstack/sahara-plugin-storm/}
STORM_PLUGIN_REPO=${STORM_PLUGIN_REPO:-https://git.openstack.org/openstack/sahara-plugin-storm/}
STORM_PLUGIN_BRANCH=${STORM_PLUGIN_BRANCH:-master}
VANILLA_PLUGIN_REPO=${VANILLA_PLUGIN_REPO:-http://git.openstack.org/cgit/openstack/sahara-plugin-vanilla/}
VANILLA_PLUGIN_REPO=${VANILLA_PLUGIN_REPO:-https://git.openstack.org/openstack/sahara-plugin-vanilla/}
VANILLA_PLUGIN_BRANCH=${VANILLA_PLUGIN_BRANCH:-master}
@ -48,6 +48,8 @@ SAHARA_AUTH_CACHE_DIR=${SAHARA_AUTH_CACHE_DIR:-/var/cache/sahara}
SAHARA_ENABLED_PLUGINS=${SAHARA_ENABLED_PLUGINS:-\
vanilla,ambari,cdh,mapr,spark,storm,fake}
SAHARA_INSTALLED_PLUGINS=${SAHARA_INSTALLED_PLUGINS:-\
vanilla,ambari,cdh,mapr,spark,storm,fake}
SAHARA_BIN_DIR=$(get_python_exec_prefix)
SAHARA_ENABLE_DISTRIBUTED_PERIODICS=${SAHARA_ENABLE_DISTRIBUTED_PERIODICS:-\

View File

@ -1,3 +1,8 @@
- name: Installs needed plugins to build its images
command: tox -e images -- pip install {{ ansible_user_dir }}/src/git.openstack.org/openstack/sahara-plugin-{{ sahara_plugin }}
args:
chdir: "{{ sahara_src_dir }}"
- name: Build all the images for the selected plugin
command: ./tools/gate/build-images "{{ sahara_plugin }}"
args:

View File

@ -111,6 +111,10 @@ def setup_plugins():
plugins_base.setup_plugins()
def get_loaded_plugins():
return plugins_base.PLUGINS.plugins
def get_plugin_arguments(plugin_name):
"""Gets plugin arguments, as a dict of version to argument list."""
plugin = plugins_base.PLUGINS.get_plugin(plugin_name)

View File

@ -62,7 +62,7 @@ for extra_opt in ["log-exchange", "host", "port"]:
def add_plugin_parsers(subparsers):
api.setup_plugins()
for plugin in CONF.plugins:
for plugin in api.get_loaded_plugins():
args_by_version = api.get_plugin_arguments(plugin)
if all(args is NotImplemented for version, args
in six.iteritems(args_by_version)):

View File

@ -103,16 +103,16 @@ class PluginManager(object):
entry_point=ext.entry_point_target))
if len(self.plugins) < len(config_plugins):
loaded_plugins = set(six.iterkeys(self.plugins))
self.loaded_plugins = set(six.iterkeys(self.plugins))
requested_plugins = set(config_plugins)
raise ex.ConfigurationError(
_("Plugins couldn't be loaded: %s") %
", ".join(requested_plugins - loaded_plugins))
LOG.warning("Plugins couldn't be loaded: %s",
", ".join(requested_plugins - self.loaded_plugins))
def get_plugins(self, serialized=False):
if serialized:
return [self.serialize_plugin(name) for name in CONF.plugins]
return [self.get_plugin(name) for name in CONF.plugins]
return [self.serialize_plugin(name)
for name in PLUGINS.plugins]
return [self.get_plugin(name) for name in PLUGINS.plugins]
def get_plugin(self, plugin_name):
return self.plugins.get(plugin_name)

View File

@ -23,7 +23,7 @@ class BasePluginsSupportTest(testtools.TestCase):
def setUp(self):
super(BasePluginsSupportTest, self).setUp()
main.CONF.set_override('plugins', ['fake'])
main.CONF.set_override('plugins', ['fake', 'cdh', 'spark'])
pb.setup_plugins()
def test_plugins_loaded(self):

View File

@ -64,7 +64,7 @@ setup_build_env
# build_images "<plugin_name>" "<plugin_version>" "<distribution> <distribution>"
case "$PLUGIN" in
"cloudera")
"cdh")
build_images "cdh" "5.9.0" "centos7"
build_images "cdh" "5.11.0" "centos7"
build_images "cdh" "5.13.0" "centos7"