openstack
/
sahara-tests
Code Issues Proposed changes

Add scripts for multinode job with spark 

Change-Id: I0e359c2aa5f456e9d943befe966bdb82ea2b2615
This commit is contained in:
Vitaly Gridnev 2017-03-16 13:54:35 +04:00
parent 5d62413bc1
commit 6430b8b210
7 changed files with 103 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
etc/scenario/gate/edp.yaml.mako
View File

@ -13,3 +13,31 @@ edp_jobs_flow:
additional_libs:
- type: swift
source: edp-examples/edp-pig/cleanup-string/edp-pig-udf-stringcleaner.jar
spark_pi:
- type: Spark
main_lib:
type: database
source: edp-examples/edp-spark/spark-pi.py
configs:
edp.java.main_class: main
args:
- 2
spark_wordcount:
- type: Spark
input_datasource:
type: swift
source: edp-examples/edp-spark/sample_input.txt
output_datasource:
type: swift
destination: edp-output
main_lib:
type: database
source: edp-examples/edp-spark/spark-wordcount.jar
configs:
edp.java.main_class: sahara.edp.spark.SparkWordCount
edp.spark.adapt_for_swift: true
fs.swift.service.sahara.username: ${os_username}
fs.swift.service.sahara.password: ${os_password}
args:
- '{input_datasource}'
- '{output_datasource}'

2
etc/scenario/gate/fake.yaml.mako
View File

@ -3,7 +3,7 @@
clusters:
- plugin_name: fake
plugin_version: "0.1"
image: ${fake_plugin_image}
image: ${plugin_image}
node_group_templates:
- name: aio
flavor: ${ci_flavor_id}

39
etc/scenario/gate/spark-1.6.0.yaml.mako Normal file
View File

@ -0,0 +1,39 @@
<%page args="is_proxy_gateway='true', use_auto_security_group='true', cluster_name='ct', ci_flavor_id='m1.small'"/>
clusters:
- plugin_name: spark
plugin_version: 1.6.0
image: ${plugin_image}
node_group_templates:
- name: master
flavor: ${ci_flavor_id}
node_processes:
- master
- namenode
- datanode
- slave
auto_security_group: ${use_auto_security_group}
is_proxy_gateway: ${is_proxy_gateway}
- name: worker
flavor: ${ci_flavor_id}
node_processes:
- datanode
- slave
auto_security_group: ${use_auto_security_group}
cluster_template:
name: spark160
node_group_templates:
master: 1
cluster_configs:
HDFS:
dfs.replication: 1
scaling:
- operation: add
node_group: worker
size: 1
scenario:
- scale
edp_jobs_flow:
- spark_pi
cluster:
name: ${cluster_name}

10
tools/gate/scenario/commons
View File

@ -16,6 +16,7 @@
# limitations under the License.
export NETWORK=${1:-neutron}
export PLUGIN=${2:-fake}
export IS_TRANSIENT=true
export AUTO_SECURITY_GROUP=true
# Normalize network name
@ -35,10 +36,11 @@ export SAHARA_TESTS_DIR=${SAHARA_TESTS_DIR:-$DEST/sahara-tests}
export LOCALCONF_PATH=$DEVSTACK_DIR/local.conf
function sahara_register_fake_plugin_image {
openstack dataprocessing image register --username $SAHARA_FAKE_PLUGIN_IMAGE_USERNAME \
$SAHARA_FAKE_PLUGIN_IMAGE_NAME
openstack dataprocessing image tags add $SAHARA_FAKE_PLUGIN_IMAGE_NAME --tags 0.1 fake
function sahara_register_image {
openstack dataprocessing image register --username $SAHARA_IMAGE_USERNAME \
$SAHARA_IMAGE_NAME
openstack dataprocessing image tags add $SAHARA_IMAGE_NAME --tags \
$SAHARA_PLUGIN_VERSION $SAHARA_PLUGIN_NAME
}
function sahara_register_flavor {

6
tools/gate/scenario/post_test_hook.sh
View File

@ -27,7 +27,7 @@ source $DEVSTACK_DIR/openrc admin admin
set -x
# Make public and register in Sahara as admin
sahara_register_fake_plugin_image
sahara_register_image
# Register sahara specific flavor for gate
sahara_register_flavor
@ -41,7 +41,7 @@ sudo -E -u jenkins tee template_vars.ini <<EOF
network_type: ${NETWORK}
network_private_name: ${PRIVATE_NETWORK_NAME}
network_public_name: ${PUBLIC_NETWORK_NAME}
fake_plugin_image: ${SAHARA_FAKE_PLUGIN_IMAGE_NAME}
plugin_image: ${SAHARA_IMAGE_NAME}
ci_flavor_id: '${SAHARA_FLAVOR_ID}'
cluster_name: fake-cluster
is_transient: ${IS_TRANSIENT}
@ -53,7 +53,7 @@ echo "Running scenario tests"
sudo -u jenkins tox -e venv -- sahara-scenario --verbose -V template_vars.ini \
etc/scenario/gate/credentials.yaml.mako \
etc/scenario/gate/edp.yaml.mako \
etc/scenario/gate/fake.yaml.mako \
etc/scenario/gate/$SAHARA_SCENARIO_TEMPLATE \
--os-cloud devstack \
| tee scenario.log

2
tools/gate/scenario/pre_test_hook.sh
View File

@ -22,7 +22,7 @@ set -ex
source commons $@
echo "[[local|localrc]]" >> $LOCALCONF_PATH
echo "IMAGE_URLS=$SAHARA_FAKE_PLUGIN_IMAGE" >> $LOCALCONF_PATH
echo "IMAGE_URLS=$SAHARA_IMAGE" >> $LOCALCONF_PATH
# Here we can set some configurations for local.conf
# for example, to pass some config options directly to sahara.conf file

33
tools/gate/scenario/settings
View File

@ -16,11 +16,28 @@
# limitations under the License.
#TODO(slukjanov): replace with special image for fake plugin (cloud ubuntu?)
export SAHARA_FAKE_PLUGIN_IMAGE=https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img
export SAHARA_FAKE_PLUGIN_IMAGE_NAME=$(basename $SAHARA_FAKE_PLUGIN_IMAGE .img)
export SAHARA_FAKE_PLUGIN_IMAGE_USERNAME=ubuntu
export SAHARA_FLAVOR_NAME=sahara-flavor
export SAHARA_FLAVOR_ID=20
export SAHARA_FLAVOR_RAM=512
export SAHARA_FLAVOR_DISK=10
export SAHARA_FLAVOR_VCPUS=1
if [ $PLUGIN == fake ] ; then
export SAHARA_IMAGE=https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img
export SAHARA_IMAGE_NAME=$(basename $SAHARA_IMAGE .img)
export SAHARA_IMAGE_USERNAME=ubuntu
export SAHARA_PLUGIN_NAME=fake
export SAHARA_PLUGIN_VERSION=0.1
export SAHARA_SCENARIO_TEMPLATE=fake.yaml.mako
export SAHARA_FLAVOR_NAME=sahara-flavor
export SAHARA_FLAVOR_ID=20
export SAHARA_FLAVOR_RAM=512
export SAHARA_FLAVOR_DISK=10
export SAHARA_FLAVOR_VCPUS=1
else
export SAHARA_IMAGE=http://sahara-files.mirantis.com/images/upstream/newton/sahara-newton-spark-1.6.0-ubuntu.qcow2
export SAHARA_IMAGE_NAME=$(basename $SAHARA_IMAGE .qcow2)
export SAHARA_IMAGE_USERNAME=ubuntu
export SAHARA_PLUGIN_NAME=spark
export SAHARA_PLUGIN_VERSION=1.6.0
export SAHARA_SCENARIO_TEMPLATE=spark-1.6.0.yaml.mako
export SAHARA_FLAVOR_NAME=sahara-flavor
export SAHARA_FLAVOR_ID=20
export SAHARA_FLAVOR_RAM=1536
export SAHARA_FLAVOR_DISK=20
export SAHARA_FLAVOR_VCPUS=1
fi