diff --git a/devstack/files/monasca-transform/monasca-transform.conf b/devstack/files/monasca-transform/monasca-transform.conf index 2421554..be44d3e 100644 --- a/devstack/files/monasca-transform/monasca-transform.conf +++ b/devstack/files/monasca-transform/monasca-transform.conf @@ -50,6 +50,9 @@ spark_jars_list = /opt/spark/current/lib/spark-streaming-kafka_2.10-1.6.1.jar,/o # A list of where the Spark master(s) should run spark_master_list = spark://localhost:7077 +# spark_home for the environment +spark_home = /opt/spark/current + # Python files for Spark to use spark_python_files = /opt/monasca/transform/lib/monasca-transform.zip diff --git a/devstack/plugin.sh b/devstack/plugin.sh index c5c7b7a..6b47a56 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -87,7 +87,7 @@ function install_java_libs { function link_spark_streaming_lib { pushd /opt/spark/current/lib - ln -sf spark-streaming-kafka.jar spark-streaming-kafka_2.10-1.6.0.jar + ln -sf spark-streaming-kafka.jar spark-streaming-kafka_2.10-1.6.1.jar popd } diff --git a/etc/monasca-transform.conf b/etc/monasca-transform.conf index 05ec9ab..63ed73a 100644 --- a/etc/monasca-transform.conf +++ b/etc/monasca-transform.conf @@ -56,6 +56,9 @@ spark_jars_list = /opt/spark/current/lib/spark-streaming-kafka.jar,/opt/spark/cu # A list of where the Spark master(s) should run spark_master_list = spark://192.168.10.4:7077,192.168.10.5:7077 +# spark_home for the environment +spark_home = /opt/spark/current + # Python files for Spark to use spark_python_files = /opt/stack/monasca-transform/dist/monasca_transform-0.0.1.egg diff --git a/monasca_transform/config/config_initializer.py b/monasca_transform/config/config_initializer.py index ff5913e..e4b5b32 100644 --- a/monasca_transform/config/config_initializer.py +++ b/monasca_transform/config/config_initializer.py @@ -98,7 +98,8 @@ class ConfigInitializer(object): cfg.StrOpt('spark_master_list'), cfg.StrOpt('spark_python_files'), cfg.IntOpt('stream_interval'), - cfg.StrOpt('work_dir') + cfg.StrOpt('work_dir'), + cfg.StrOpt('spark_home') ] service_group = cfg.OptGroup(name='service', title='service') cfg.CONF.register_group(service_group) diff --git a/monasca_transform/service/transform_service.py b/monasca_transform/service/transform_service.py index 4126521..cd51141 100644 --- a/monasca_transform/service/transform_service.py +++ b/monasca_transform/service/transform_service.py @@ -95,7 +95,8 @@ class TransformService(threading.Thread): else: pyfiles = '' # Build the command to start the Spark driver - spark_cmd = ("export SPARK_HOME=/opt/spark/current && " + spark_cmd = ("export SPARK_HOME=" + + CONF.service.spark_home + " && " "spark-submit --supervise --master " + CONF.service.spark_master_list + " --conf spark.eventLog.enabled=" +