Add scripts for multinode job with spark
Change-Id: I0e359c2aa5f456e9d943befe966bdb82ea2b2615
This commit is contained in:
parent
5d62413bc1
commit
6430b8b210
|
@ -13,3 +13,31 @@ edp_jobs_flow:
|
||||||
additional_libs:
|
additional_libs:
|
||||||
- type: swift
|
- type: swift
|
||||||
source: edp-examples/edp-pig/cleanup-string/edp-pig-udf-stringcleaner.jar
|
source: edp-examples/edp-pig/cleanup-string/edp-pig-udf-stringcleaner.jar
|
||||||
|
spark_pi:
|
||||||
|
- type: Spark
|
||||||
|
main_lib:
|
||||||
|
type: database
|
||||||
|
source: edp-examples/edp-spark/spark-pi.py
|
||||||
|
configs:
|
||||||
|
edp.java.main_class: main
|
||||||
|
args:
|
||||||
|
- 2
|
||||||
|
spark_wordcount:
|
||||||
|
- type: Spark
|
||||||
|
input_datasource:
|
||||||
|
type: swift
|
||||||
|
source: edp-examples/edp-spark/sample_input.txt
|
||||||
|
output_datasource:
|
||||||
|
type: swift
|
||||||
|
destination: edp-output
|
||||||
|
main_lib:
|
||||||
|
type: database
|
||||||
|
source: edp-examples/edp-spark/spark-wordcount.jar
|
||||||
|
configs:
|
||||||
|
edp.java.main_class: sahara.edp.spark.SparkWordCount
|
||||||
|
edp.spark.adapt_for_swift: true
|
||||||
|
fs.swift.service.sahara.username: ${os_username}
|
||||||
|
fs.swift.service.sahara.password: ${os_password}
|
||||||
|
args:
|
||||||
|
- '{input_datasource}'
|
||||||
|
- '{output_datasource}'
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
clusters:
|
clusters:
|
||||||
- plugin_name: fake
|
- plugin_name: fake
|
||||||
plugin_version: "0.1"
|
plugin_version: "0.1"
|
||||||
image: ${fake_plugin_image}
|
image: ${plugin_image}
|
||||||
node_group_templates:
|
node_group_templates:
|
||||||
- name: aio
|
- name: aio
|
||||||
flavor: ${ci_flavor_id}
|
flavor: ${ci_flavor_id}
|
||||||
|
|
|
@ -0,0 +1,39 @@
|
||||||
|
<%page args="is_proxy_gateway='true', use_auto_security_group='true', cluster_name='ct', ci_flavor_id='m1.small'"/>
|
||||||
|
|
||||||
|
clusters:
|
||||||
|
- plugin_name: spark
|
||||||
|
plugin_version: 1.6.0
|
||||||
|
image: ${plugin_image}
|
||||||
|
node_group_templates:
|
||||||
|
- name: master
|
||||||
|
flavor: ${ci_flavor_id}
|
||||||
|
node_processes:
|
||||||
|
- master
|
||||||
|
- namenode
|
||||||
|
- datanode
|
||||||
|
- slave
|
||||||
|
auto_security_group: ${use_auto_security_group}
|
||||||
|
is_proxy_gateway: ${is_proxy_gateway}
|
||||||
|
- name: worker
|
||||||
|
flavor: ${ci_flavor_id}
|
||||||
|
node_processes:
|
||||||
|
- datanode
|
||||||
|
- slave
|
||||||
|
auto_security_group: ${use_auto_security_group}
|
||||||
|
cluster_template:
|
||||||
|
name: spark160
|
||||||
|
node_group_templates:
|
||||||
|
master: 1
|
||||||
|
cluster_configs:
|
||||||
|
HDFS:
|
||||||
|
dfs.replication: 1
|
||||||
|
scaling:
|
||||||
|
- operation: add
|
||||||
|
node_group: worker
|
||||||
|
size: 1
|
||||||
|
scenario:
|
||||||
|
- scale
|
||||||
|
edp_jobs_flow:
|
||||||
|
- spark_pi
|
||||||
|
cluster:
|
||||||
|
name: ${cluster_name}
|
|
@ -16,6 +16,7 @@
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
export NETWORK=${1:-neutron}
|
export NETWORK=${1:-neutron}
|
||||||
|
export PLUGIN=${2:-fake}
|
||||||
export IS_TRANSIENT=true
|
export IS_TRANSIENT=true
|
||||||
export AUTO_SECURITY_GROUP=true
|
export AUTO_SECURITY_GROUP=true
|
||||||
# Normalize network name
|
# Normalize network name
|
||||||
|
@ -35,10 +36,11 @@ export SAHARA_TESTS_DIR=${SAHARA_TESTS_DIR:-$DEST/sahara-tests}
|
||||||
|
|
||||||
export LOCALCONF_PATH=$DEVSTACK_DIR/local.conf
|
export LOCALCONF_PATH=$DEVSTACK_DIR/local.conf
|
||||||
|
|
||||||
function sahara_register_fake_plugin_image {
|
function sahara_register_image {
|
||||||
openstack dataprocessing image register --username $SAHARA_FAKE_PLUGIN_IMAGE_USERNAME \
|
openstack dataprocessing image register --username $SAHARA_IMAGE_USERNAME \
|
||||||
$SAHARA_FAKE_PLUGIN_IMAGE_NAME
|
$SAHARA_IMAGE_NAME
|
||||||
openstack dataprocessing image tags add $SAHARA_FAKE_PLUGIN_IMAGE_NAME --tags 0.1 fake
|
openstack dataprocessing image tags add $SAHARA_IMAGE_NAME --tags \
|
||||||
|
$SAHARA_PLUGIN_VERSION $SAHARA_PLUGIN_NAME
|
||||||
}
|
}
|
||||||
|
|
||||||
function sahara_register_flavor {
|
function sahara_register_flavor {
|
||||||
|
|
|
@ -27,7 +27,7 @@ source $DEVSTACK_DIR/openrc admin admin
|
||||||
set -x
|
set -x
|
||||||
|
|
||||||
# Make public and register in Sahara as admin
|
# Make public and register in Sahara as admin
|
||||||
sahara_register_fake_plugin_image
|
sahara_register_image
|
||||||
|
|
||||||
# Register sahara specific flavor for gate
|
# Register sahara specific flavor for gate
|
||||||
sahara_register_flavor
|
sahara_register_flavor
|
||||||
|
@ -41,7 +41,7 @@ sudo -E -u jenkins tee template_vars.ini <<EOF
|
||||||
network_type: ${NETWORK}
|
network_type: ${NETWORK}
|
||||||
network_private_name: ${PRIVATE_NETWORK_NAME}
|
network_private_name: ${PRIVATE_NETWORK_NAME}
|
||||||
network_public_name: ${PUBLIC_NETWORK_NAME}
|
network_public_name: ${PUBLIC_NETWORK_NAME}
|
||||||
fake_plugin_image: ${SAHARA_FAKE_PLUGIN_IMAGE_NAME}
|
plugin_image: ${SAHARA_IMAGE_NAME}
|
||||||
ci_flavor_id: '${SAHARA_FLAVOR_ID}'
|
ci_flavor_id: '${SAHARA_FLAVOR_ID}'
|
||||||
cluster_name: fake-cluster
|
cluster_name: fake-cluster
|
||||||
is_transient: ${IS_TRANSIENT}
|
is_transient: ${IS_TRANSIENT}
|
||||||
|
@ -53,7 +53,7 @@ echo "Running scenario tests"
|
||||||
sudo -u jenkins tox -e venv -- sahara-scenario --verbose -V template_vars.ini \
|
sudo -u jenkins tox -e venv -- sahara-scenario --verbose -V template_vars.ini \
|
||||||
etc/scenario/gate/credentials.yaml.mako \
|
etc/scenario/gate/credentials.yaml.mako \
|
||||||
etc/scenario/gate/edp.yaml.mako \
|
etc/scenario/gate/edp.yaml.mako \
|
||||||
etc/scenario/gate/fake.yaml.mako \
|
etc/scenario/gate/$SAHARA_SCENARIO_TEMPLATE \
|
||||||
--os-cloud devstack \
|
--os-cloud devstack \
|
||||||
| tee scenario.log
|
| tee scenario.log
|
||||||
|
|
||||||
|
|
|
@ -22,7 +22,7 @@ set -ex
|
||||||
source commons $@
|
source commons $@
|
||||||
|
|
||||||
echo "[[local|localrc]]" >> $LOCALCONF_PATH
|
echo "[[local|localrc]]" >> $LOCALCONF_PATH
|
||||||
echo "IMAGE_URLS=$SAHARA_FAKE_PLUGIN_IMAGE" >> $LOCALCONF_PATH
|
echo "IMAGE_URLS=$SAHARA_IMAGE" >> $LOCALCONF_PATH
|
||||||
|
|
||||||
# Here we can set some configurations for local.conf
|
# Here we can set some configurations for local.conf
|
||||||
# for example, to pass some config options directly to sahara.conf file
|
# for example, to pass some config options directly to sahara.conf file
|
||||||
|
|
|
@ -16,11 +16,28 @@
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
#TODO(slukjanov): replace with special image for fake plugin (cloud ubuntu?)
|
#TODO(slukjanov): replace with special image for fake plugin (cloud ubuntu?)
|
||||||
export SAHARA_FAKE_PLUGIN_IMAGE=https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img
|
if [ $PLUGIN == fake ] ; then
|
||||||
export SAHARA_FAKE_PLUGIN_IMAGE_NAME=$(basename $SAHARA_FAKE_PLUGIN_IMAGE .img)
|
export SAHARA_IMAGE=https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img
|
||||||
export SAHARA_FAKE_PLUGIN_IMAGE_USERNAME=ubuntu
|
export SAHARA_IMAGE_NAME=$(basename $SAHARA_IMAGE .img)
|
||||||
export SAHARA_FLAVOR_NAME=sahara-flavor
|
export SAHARA_IMAGE_USERNAME=ubuntu
|
||||||
export SAHARA_FLAVOR_ID=20
|
export SAHARA_PLUGIN_NAME=fake
|
||||||
export SAHARA_FLAVOR_RAM=512
|
export SAHARA_PLUGIN_VERSION=0.1
|
||||||
export SAHARA_FLAVOR_DISK=10
|
export SAHARA_SCENARIO_TEMPLATE=fake.yaml.mako
|
||||||
export SAHARA_FLAVOR_VCPUS=1
|
export SAHARA_FLAVOR_NAME=sahara-flavor
|
||||||
|
export SAHARA_FLAVOR_ID=20
|
||||||
|
export SAHARA_FLAVOR_RAM=512
|
||||||
|
export SAHARA_FLAVOR_DISK=10
|
||||||
|
export SAHARA_FLAVOR_VCPUS=1
|
||||||
|
else
|
||||||
|
export SAHARA_IMAGE=http://sahara-files.mirantis.com/images/upstream/newton/sahara-newton-spark-1.6.0-ubuntu.qcow2
|
||||||
|
export SAHARA_IMAGE_NAME=$(basename $SAHARA_IMAGE .qcow2)
|
||||||
|
export SAHARA_IMAGE_USERNAME=ubuntu
|
||||||
|
export SAHARA_PLUGIN_NAME=spark
|
||||||
|
export SAHARA_PLUGIN_VERSION=1.6.0
|
||||||
|
export SAHARA_SCENARIO_TEMPLATE=spark-1.6.0.yaml.mako
|
||||||
|
export SAHARA_FLAVOR_NAME=sahara-flavor
|
||||||
|
export SAHARA_FLAVOR_ID=20
|
||||||
|
export SAHARA_FLAVOR_RAM=1536
|
||||||
|
export SAHARA_FLAVOR_DISK=20
|
||||||
|
export SAHARA_FLAVOR_VCPUS=1
|
||||||
|
fi
|
||||||
|
|
Loading…
Reference in New Issue