Removed spark-events from config
Allow spark to configure the location of spark-events. Add spark events log config to spark-defaults in devstack plugin. Move spark events logging to /var/log/spark/events for devstack plugin. Set group permissions to ensure spark events log directory is group, but not world writable. Change-Id: I26aef23a9a801a02a20e14899e1c89b10556e4d4
This commit is contained in:
parent
72f240b926
commit
02b23741a5
|
@ -38,9 +38,6 @@ spark_driver = /opt/monasca/transform/lib/driver.py
|
||||||
# the location for the transform-service log
|
# the location for the transform-service log
|
||||||
service_log_path=/var/log/monasca/transform/
|
service_log_path=/var/log/monasca/transform/
|
||||||
|
|
||||||
# The location where Spark event logs should be written
|
|
||||||
spark_event_logging_dest = /var/log/spark-events
|
|
||||||
|
|
||||||
# Whether Spark event logging should be enabled (true/false)
|
# Whether Spark event logging should be enabled (true/false)
|
||||||
spark_event_logging_enabled = true
|
spark_event_logging_enabled = true
|
||||||
|
|
||||||
|
|
|
@ -8,3 +8,4 @@ spark.executor.port 7115
|
||||||
spark.fileserver.port 7120
|
spark.fileserver.port 7120
|
||||||
spark.speculation true
|
spark.speculation true
|
||||||
spark.speculation.interval 200
|
spark.speculation.interval 200
|
||||||
|
spark.eventLog.dir /var/log/spark/events
|
||||||
|
|
|
@ -13,6 +13,6 @@ export SPARK_WORKER_DIR=/var/run/spark/work
|
||||||
export SPARK_WORKER_MEMORY=2g
|
export SPARK_WORKER_MEMORY=2g
|
||||||
export SPARK_WORKER_CORES=2
|
export SPARK_WORKER_CORES=2
|
||||||
|
|
||||||
export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS -Dspark.history.fs.logDirectory=file://var/log/spark/spark-events -Dspark.history.ui.port=18082"
|
export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS -Dspark.history.fs.logDirectory=file://var/log/spark/events -Dspark.history.ui.port=18082"
|
||||||
export SPARK_LOG_DIR=/var/log/spark
|
export SPARK_LOG_DIR=/var/log/spark
|
||||||
export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=127.0.0.1:2181 -Dspark.deploy.zookeeper.dir=/var/run/spark"
|
export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=127.0.0.1:2181 -Dspark.deploy.zookeeper.dir=/var/run/spark"
|
||||||
|
|
|
@ -13,6 +13,6 @@ export SPARK_WORKER_DIR=/var/run/spark/work
|
||||||
export SPARK_WORKER_MEMORY=2g
|
export SPARK_WORKER_MEMORY=2g
|
||||||
export SPARK_WORKER_CORES=2
|
export SPARK_WORKER_CORES=2
|
||||||
|
|
||||||
export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS -Dspark.history.fs.logDirectory=file://var/log/spark/spark-events -Dspark.history.ui.port=18082"
|
export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS -Dspark.history.fs.logDirectory=file://var/log/spark/events -Dspark.history.ui.port=18082"
|
||||||
export SPARK_LOG_DIR=/var/log/spark
|
export SPARK_LOG_DIR=/var/log/spark
|
||||||
export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=127.0.0.1:2181 -Dspark.deploy.zookeeper.dir=/var/run/spark"
|
export SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=127.0.0.1:2181 -Dspark.deploy.zookeeper.dir=/var/run/spark"
|
||||||
|
|
|
@ -168,8 +168,9 @@ function create_spark_directories {
|
||||||
sudo chmod 755 ${SPARK_DIRECTORY}
|
sudo chmod 755 ${SPARK_DIRECTORY}
|
||||||
done
|
done
|
||||||
|
|
||||||
sudo mkdir -p /var/log/spark-events
|
sudo mkdir -p /var/log/spark/events
|
||||||
sudo chmod "a+rw" /var/log/spark-events
|
sudo chown spark:spark /var/log/spark/events
|
||||||
|
sudo chmod 775 /var/log/spark/events
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -180,7 +181,7 @@ function delete_spark_directories {
|
||||||
sudo rm -rf ${SPARK_DIRECTORY} || true
|
sudo rm -rf ${SPARK_DIRECTORY} || true
|
||||||
done
|
done
|
||||||
|
|
||||||
sudo rm -rf /var/log/spark-events || true
|
sudo rm -rf /var/log/spark/events || true
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -264,6 +265,7 @@ function install_monasca_transform {
|
||||||
sudo groupadd --system monasca-transform || true
|
sudo groupadd --system monasca-transform || true
|
||||||
|
|
||||||
sudo useradd --system -g monasca-transform monasca-transform || true
|
sudo useradd --system -g monasca-transform monasca-transform || true
|
||||||
|
sudo usermod -a -G spark monasca-transform
|
||||||
|
|
||||||
create_monasca_transform_directories
|
create_monasca_transform_directories
|
||||||
copy_monasca_transform_files
|
copy_monasca_transform_files
|
||||||
|
|
|
@ -44,9 +44,6 @@ spark_driver = /opt/stack/monasca-transform/monasca_transform/driver/mon_metrics
|
||||||
# the location for the transform-service log
|
# the location for the transform-service log
|
||||||
service_log_path=/opt/stack/monasca-transform
|
service_log_path=/opt/stack/monasca-transform
|
||||||
|
|
||||||
# The location where Spark event logs should be written
|
|
||||||
spark_event_logging_dest = /var/log/spark-events
|
|
||||||
|
|
||||||
# Whether Spark event logging should be enabled (true/false)
|
# Whether Spark event logging should be enabled (true/false)
|
||||||
spark_event_logging_enabled = true
|
spark_event_logging_enabled = true
|
||||||
|
|
||||||
|
|
|
@ -94,6 +94,14 @@ class TransformService(threading.Thread):
|
||||||
pyfiles = " --py-files %s" % CONF.service.spark_python_files
|
pyfiles = " --py-files %s" % CONF.service.spark_python_files
|
||||||
else:
|
else:
|
||||||
pyfiles = ''
|
pyfiles = ''
|
||||||
|
|
||||||
|
if (CONF.service.spark_event_logging_enabled and
|
||||||
|
CONF.service.spark_event_logging_dest):
|
||||||
|
event_logging_dest = (" --conf spark.eventLog.dir=file://%s" %
|
||||||
|
CONF.service.spark_event_logging_dest)
|
||||||
|
else:
|
||||||
|
event_logging_dest = ''
|
||||||
|
|
||||||
# Build the command to start the Spark driver
|
# Build the command to start the Spark driver
|
||||||
spark_cmd = ("export SPARK_HOME=" +
|
spark_cmd = ("export SPARK_HOME=" +
|
||||||
CONF.service.spark_home + " && "
|
CONF.service.spark_home + " && "
|
||||||
|
@ -101,8 +109,7 @@ class TransformService(threading.Thread):
|
||||||
CONF.service.spark_master_list +
|
CONF.service.spark_master_list +
|
||||||
" --conf spark.eventLog.enabled=" +
|
" --conf spark.eventLog.enabled=" +
|
||||||
CONF.service.spark_event_logging_enabled +
|
CONF.service.spark_event_logging_enabled +
|
||||||
" --conf spark.eventLog.dir=file://" +
|
event_logging_dest +
|
||||||
CONF.service.spark_event_logging_dest +
|
|
||||||
" --jars " + CONF.service.spark_jars_list +
|
" --jars " + CONF.service.spark_jars_list +
|
||||||
pyfiles +
|
pyfiles +
|
||||||
" " + CONF.service.spark_driver)
|
" " + CONF.service.spark_driver)
|
||||||
|
|
|
@ -12,7 +12,7 @@ export SPARK_HOME=/opt/spark/current/
|
||||||
# control-c is pressed.
|
# control-c is pressed.
|
||||||
COUNTER=0
|
COUNTER=0
|
||||||
while [ $COUNTER -lt 2 ]; do
|
while [ $COUNTER -lt 2 ]; do
|
||||||
spark-submit --supervise --master spark://192.168.10.4:7077,192.168.10.5:7077 --conf spark.eventLog.enabled=true --conf spark.eventLog.dir=file:///var/log/spark-events --jars $JARS_PATH --py-files dist/$new_filename /opt/monasca/transform/lib/driver.py || break
|
spark-submit --supervise --master spark://192.168.10.4:7077,192.168.10.5:7077 --conf spark.eventLog.enabled=true --jars $JARS_PATH --py-files dist/$new_filename /opt/monasca/transform/lib/driver.py || break
|
||||||
let COUNTER=COUNTER+1
|
let COUNTER=COUNTER+1
|
||||||
done
|
done
|
||||||
popd
|
popd
|
||||||
|
|
Loading…
Reference in New Issue