diff --git a/releasenotes/notes/test-templates-rocky-c271d2acb0d3c42a.yaml b/releasenotes/notes/test-templates-rocky-c271d2acb0d3c42a.yaml new file mode 100644 index 00000000..e0a1dad3 --- /dev/null +++ b/releasenotes/notes/test-templates-rocky-c271d2acb0d3c42a.yaml @@ -0,0 +1,16 @@ +--- +prelude: > + Rocky test templates are now available. +features: + - A folder with scenario templates for Rocky was added. + It is a subset of the templates in the main directory, + and includes all non-deprecated plugin/versions. + - The available default test templates now supports also + Spark 2.3, Storm 1.2, Vanilla 2.7.5, and the Ambari-based + HDP 2.6 and 2.5. +fixes: + - The CDH 5.11 test template, previously only available for queens, + was added to main (unversioned) templates directory. +upgrades: + - All the templates for deprecated version/plugins were removed + from the main (unversioned) templates directory. diff --git a/sahara_tests/scenario/defaults/ambari-2.5.yaml.mako b/sahara_tests/scenario/defaults/ambari-2.5.yaml.mako new file mode 100644 index 00000000..52de39fc --- /dev/null +++ b/sahara_tests/scenario/defaults/ambari-2.5.yaml.mako @@ -0,0 +1,69 @@ +<%page args="use_auto_security_group='true', ci_flavor_id='m1.small', medium_flavor_id='m1.medium', availability_zone='nova', volumes_availability_zone='nova'"/> + +clusters: + - plugin_name: ambari + plugin_version: '2.5' + image: ${ambari_25_image} + node_group_templates: + - name: master + flavor: ${medium_flavor_id} + node_processes: + - Ambari + - MapReduce History Server + - Spark History Server + - NameNode + - ResourceManager + - SecondaryNameNode + - YARN Timeline Server + - ZooKeeper + - Kafka Broker + auto_security_group: ${use_auto_security_group} + - name: master-edp + flavor: ${ci_flavor_id} + node_processes: + - Hive Metastore + - HiveServer + - Oozie + auto_security_group: ${use_auto_security_group} + - name: worker + flavor: ${ci_flavor_id} + node_processes: + - DataNode + - NodeManager + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + cluster_template: + name: ambari21 + node_group_templates: + master: 1 + master-edp: 1 + worker: 3 + cluster_configs: + HDFS: + dfs.datanode.du.reserved: 0 + custom_checks: + check_kafka: + zookeeper_process: ZooKeeper + kafka_process: Kafka Broker + spark_flow: + - type: Spark + main_lib: + type: database + source: edp-examples/edp-spark/spark-kafka-example.jar + args: + - '{zookeeper_list}' + - '{topic}' + - '{timeout}' + timeout: 30 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: + - java_job + - spark_pi diff --git a/sahara_tests/scenario/defaults/ambari-2.6.yaml.mako b/sahara_tests/scenario/defaults/ambari-2.6.yaml.mako new file mode 100644 index 00000000..ddbbe66e --- /dev/null +++ b/sahara_tests/scenario/defaults/ambari-2.6.yaml.mako @@ -0,0 +1,69 @@ +<%page args="use_auto_security_group='true', ci_flavor_id='m1.small', medium_flavor_id='m1.medium', availability_zone='nova', volumes_availability_zone='nova'"/> + +clusters: + - plugin_name: ambari + plugin_version: '2.6' + image: ${ambari_26_image} + node_group_templates: + - name: master + flavor: ${medium_flavor_id} + node_processes: + - Ambari + - MapReduce History Server + - Spark History Server + - NameNode + - ResourceManager + - SecondaryNameNode + - YARN Timeline Server + - ZooKeeper + - Kafka Broker + auto_security_group: ${use_auto_security_group} + - name: master-edp + flavor: ${ci_flavor_id} + node_processes: + - Hive Metastore + - HiveServer + - Oozie + auto_security_group: ${use_auto_security_group} + - name: worker + flavor: ${ci_flavor_id} + node_processes: + - DataNode + - NodeManager + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + cluster_template: + name: ambari21 + node_group_templates: + master: 1 + master-edp: 1 + worker: 3 + cluster_configs: + HDFS: + dfs.datanode.du.reserved: 0 + custom_checks: + check_kafka: + zookeeper_process: ZooKeeper + kafka_process: Kafka Broker + spark_flow: + - type: Spark + main_lib: + type: database + source: edp-examples/edp-spark/spark-kafka-example.jar + args: + - '{zookeeper_list}' + - '{topic}' + - '{timeout}' + timeout: 30 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: + - java_job + - spark_pi diff --git a/sahara_tests/scenario/defaults/cdh-5.5.0.yaml.mako b/sahara_tests/scenario/defaults/cdh-5.11.0.yaml.mako similarity index 87% rename from sahara_tests/scenario/defaults/cdh-5.5.0.yaml.mako rename to sahara_tests/scenario/defaults/cdh-5.11.0.yaml.mako index 7d4b5b15..85e393d4 100644 --- a/sahara_tests/scenario/defaults/cdh-5.5.0.yaml.mako +++ b/sahara_tests/scenario/defaults/cdh-5.11.0.yaml.mako @@ -1,9 +1,9 @@ -<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small', medium_flavor_id='m1.medium', large_flavor_id='m1.large', availability_zone='nova', volumes_availability_zone='nova'"/> +<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small', large_flavor_id='m1.large', availability_zone='nova', volumes_availability_zone='nova'"/> clusters: - plugin_name: cdh - plugin_version: 5.5.0 - image: ${cdh_550_image} + plugin_version: 5.11.0 + image: ${cdh_5110_image} node_group_templates: - name: worker-dn flavor: ${ci_flavor_id} @@ -43,7 +43,7 @@ clusters: is_proxy_gateway: ${is_proxy_gateway} auto_security_group: ${use_auto_security_group} - name: master-core - flavor: ${medium_flavor_id} + flavor: ${large_flavor_id} node_processes: - HDFS_NAMENODE - YARN_RESOURCEMANAGER @@ -52,7 +52,7 @@ clusters: - ZOOKEEPER_SERVER auto_security_group: ${use_auto_security_group} - name: master-additional - flavor: ${medium_flavor_id} + flavor: ${large_flavor_id} node_processes: - OOZIE_SERVER - YARN_JOBHISTORY @@ -62,7 +62,7 @@ clusters: - HIVE_SERVER2 - SPARK_YARN_HISTORY_SERVER auto_security_group: ${use_auto_security_group} - # In 5.5 the defaults of following configs are too large, + # In 5.11 the defaults of following configs are too large, # restrict them to save memory for scenario testing. node_configs: HIVEMETASTORE: @@ -70,7 +70,7 @@ clusters: HIVESERVER: hiveserver2_java_heapsize: 2147483648 cluster_template: - name: cdh550 + name: cdh5110 node_group_templates: manager: 1 master-core: 1 diff --git a/sahara_tests/scenario/defaults/rocky/ambari-2.3.yaml.mako b/sahara_tests/scenario/defaults/rocky/ambari-2.3.yaml.mako new file mode 100644 index 00000000..a19de68a --- /dev/null +++ b/sahara_tests/scenario/defaults/rocky/ambari-2.3.yaml.mako @@ -0,0 +1,69 @@ +<%page args="use_auto_security_group='true', ci_flavor_id='m1.small', medium_flavor_id='m1.medium', availability_zone='nova', volumes_availability_zone='nova'"/> + +clusters: + - plugin_name: ambari + plugin_version: '2.3' + image: ${ambari_23_image} + node_group_templates: + - name: master + flavor: ${medium_flavor_id} + node_processes: + - Ambari + - MapReduce History Server + - Spark History Server + - NameNode + - ResourceManager + - SecondaryNameNode + - YARN Timeline Server + - ZooKeeper + - Kafka Broker + auto_security_group: ${use_auto_security_group} + - name: master-edp + flavor: ${ci_flavor_id} + node_processes: + - Hive Metastore + - HiveServer + - Oozie + auto_security_group: ${use_auto_security_group} + - name: worker + flavor: ${ci_flavor_id} + node_processes: + - DataNode + - NodeManager + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + cluster_template: + name: ambari21 + node_group_templates: + master: 1 + master-edp: 1 + worker: 3 + cluster_configs: + HDFS: + dfs.datanode.du.reserved: 0 + custom_checks: + check_kafka: + zookeeper_process: ZooKeeper + kafka_process: Kafka Broker + spark_flow: + - type: Spark + main_lib: + type: database + source: edp-examples/edp-spark/spark-kafka-example.jar + args: + - '{zookeeper_list}' + - '{topic}' + - '{timeout}' + timeout: 30 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: + - java_job + - spark_pi diff --git a/sahara_tests/scenario/defaults/rocky/ambari-2.4.yaml.mako b/sahara_tests/scenario/defaults/rocky/ambari-2.4.yaml.mako new file mode 100644 index 00000000..08e04c7c --- /dev/null +++ b/sahara_tests/scenario/defaults/rocky/ambari-2.4.yaml.mako @@ -0,0 +1,69 @@ +<%page args="use_auto_security_group='true', ci_flavor_id='m1.small', medium_flavor_id='m1.medium', availability_zone='nova', volumes_availability_zone='nova'"/> + +clusters: + - plugin_name: ambari + plugin_version: '2.4' + image: ${ambari_24_image} + node_group_templates: + - name: master + flavor: ${medium_flavor_id} + node_processes: + - Ambari + - MapReduce History Server + - Spark History Server + - NameNode + - ResourceManager + - SecondaryNameNode + - YARN Timeline Server + - ZooKeeper + - Kafka Broker + auto_security_group: ${use_auto_security_group} + - name: master-edp + flavor: ${ci_flavor_id} + node_processes: + - Hive Metastore + - HiveServer + - Oozie + auto_security_group: ${use_auto_security_group} + - name: worker + flavor: ${ci_flavor_id} + node_processes: + - DataNode + - NodeManager + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + cluster_template: + name: ambari21 + node_group_templates: + master: 1 + master-edp: 1 + worker: 3 + cluster_configs: + HDFS: + dfs.datanode.du.reserved: 0 + custom_checks: + check_kafka: + zookeeper_process: ZooKeeper + kafka_process: Kafka Broker + spark_flow: + - type: Spark + main_lib: + type: database + source: edp-examples/edp-spark/spark-kafka-example.jar + args: + - '{zookeeper_list}' + - '{topic}' + - '{timeout}' + timeout: 30 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: + - java_job + - spark_pi diff --git a/sahara_tests/scenario/defaults/rocky/ambari-2.5.yaml.mako b/sahara_tests/scenario/defaults/rocky/ambari-2.5.yaml.mako new file mode 100644 index 00000000..52de39fc --- /dev/null +++ b/sahara_tests/scenario/defaults/rocky/ambari-2.5.yaml.mako @@ -0,0 +1,69 @@ +<%page args="use_auto_security_group='true', ci_flavor_id='m1.small', medium_flavor_id='m1.medium', availability_zone='nova', volumes_availability_zone='nova'"/> + +clusters: + - plugin_name: ambari + plugin_version: '2.5' + image: ${ambari_25_image} + node_group_templates: + - name: master + flavor: ${medium_flavor_id} + node_processes: + - Ambari + - MapReduce History Server + - Spark History Server + - NameNode + - ResourceManager + - SecondaryNameNode + - YARN Timeline Server + - ZooKeeper + - Kafka Broker + auto_security_group: ${use_auto_security_group} + - name: master-edp + flavor: ${ci_flavor_id} + node_processes: + - Hive Metastore + - HiveServer + - Oozie + auto_security_group: ${use_auto_security_group} + - name: worker + flavor: ${ci_flavor_id} + node_processes: + - DataNode + - NodeManager + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + cluster_template: + name: ambari21 + node_group_templates: + master: 1 + master-edp: 1 + worker: 3 + cluster_configs: + HDFS: + dfs.datanode.du.reserved: 0 + custom_checks: + check_kafka: + zookeeper_process: ZooKeeper + kafka_process: Kafka Broker + spark_flow: + - type: Spark + main_lib: + type: database + source: edp-examples/edp-spark/spark-kafka-example.jar + args: + - '{zookeeper_list}' + - '{topic}' + - '{timeout}' + timeout: 30 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: + - java_job + - spark_pi diff --git a/sahara_tests/scenario/defaults/rocky/ambari-2.6.yaml.mako b/sahara_tests/scenario/defaults/rocky/ambari-2.6.yaml.mako new file mode 100644 index 00000000..ddbbe66e --- /dev/null +++ b/sahara_tests/scenario/defaults/rocky/ambari-2.6.yaml.mako @@ -0,0 +1,69 @@ +<%page args="use_auto_security_group='true', ci_flavor_id='m1.small', medium_flavor_id='m1.medium', availability_zone='nova', volumes_availability_zone='nova'"/> + +clusters: + - plugin_name: ambari + plugin_version: '2.6' + image: ${ambari_26_image} + node_group_templates: + - name: master + flavor: ${medium_flavor_id} + node_processes: + - Ambari + - MapReduce History Server + - Spark History Server + - NameNode + - ResourceManager + - SecondaryNameNode + - YARN Timeline Server + - ZooKeeper + - Kafka Broker + auto_security_group: ${use_auto_security_group} + - name: master-edp + flavor: ${ci_flavor_id} + node_processes: + - Hive Metastore + - HiveServer + - Oozie + auto_security_group: ${use_auto_security_group} + - name: worker + flavor: ${ci_flavor_id} + node_processes: + - DataNode + - NodeManager + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + cluster_template: + name: ambari21 + node_group_templates: + master: 1 + master-edp: 1 + worker: 3 + cluster_configs: + HDFS: + dfs.datanode.du.reserved: 0 + custom_checks: + check_kafka: + zookeeper_process: ZooKeeper + kafka_process: Kafka Broker + spark_flow: + - type: Spark + main_lib: + type: database + source: edp-examples/edp-spark/spark-kafka-example.jar + args: + - '{zookeeper_list}' + - '{topic}' + - '{timeout}' + timeout: 30 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: + - java_job + - spark_pi diff --git a/sahara_tests/scenario/defaults/rocky/cdh-5.11.0.yaml.mako b/sahara_tests/scenario/defaults/rocky/cdh-5.11.0.yaml.mako new file mode 100644 index 00000000..85e393d4 --- /dev/null +++ b/sahara_tests/scenario/defaults/rocky/cdh-5.11.0.yaml.mako @@ -0,0 +1,94 @@ +<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small', large_flavor_id='m1.large', availability_zone='nova', volumes_availability_zone='nova'"/> + +clusters: + - plugin_name: cdh + plugin_version: 5.11.0 + image: ${cdh_5110_image} + node_group_templates: + - name: worker-dn + flavor: ${ci_flavor_id} + node_processes: + - HDFS_DATANODE + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + node_configs: + &ng_configs + DATANODE: + dfs_datanode_du_reserved: 0 + - name: worker-nm + flavor: ${ci_flavor_id} + node_processes: + - YARN_NODEMANAGER + auto_security_group: ${use_auto_security_group} + - name: worker-nm-dn + flavor: ${ci_flavor_id} + node_processes: + - YARN_NODEMANAGER + - HDFS_DATANODE + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + node_configs: + *ng_configs + - name: manager + flavor: ${large_flavor_id} + node_processes: + - CLOUDERA_MANAGER + - KMS + is_proxy_gateway: ${is_proxy_gateway} + auto_security_group: ${use_auto_security_group} + - name: master-core + flavor: ${large_flavor_id} + node_processes: + - HDFS_NAMENODE + - YARN_RESOURCEMANAGER + - SENTRY_SERVER + - YARN_NODEMANAGER + - ZOOKEEPER_SERVER + auto_security_group: ${use_auto_security_group} + - name: master-additional + flavor: ${large_flavor_id} + node_processes: + - OOZIE_SERVER + - YARN_JOBHISTORY + - YARN_NODEMANAGER + - HDFS_SECONDARYNAMENODE + - HIVE_METASTORE + - HIVE_SERVER2 + - SPARK_YARN_HISTORY_SERVER + auto_security_group: ${use_auto_security_group} + # In 5.11 the defaults of following configs are too large, + # restrict them to save memory for scenario testing. + node_configs: + HIVEMETASTORE: + hive_metastore_java_heapsize: 2147483648 + HIVESERVER: + hiveserver2_java_heapsize: 2147483648 + cluster_template: + name: cdh5110 + node_group_templates: + manager: 1 + master-core: 1 + master-additional: 1 + worker-nm-dn: 1 + worker-nm: 1 + worker-dn: 1 + cluster_configs: + HDFS: + dfs_replication: 1 + cluster: + name: ${cluster_name} + scenario: + - run_jobs + - sentry + edp_jobs_flow: + - pig_job + - mapreduce_job + - mapreduce_streaming_job + - java_job + - spark_wordcount diff --git a/sahara_tests/scenario/defaults/rocky/cdh-5.13.0.yaml.mako b/sahara_tests/scenario/defaults/rocky/cdh-5.13.0.yaml.mako new file mode 100644 index 00000000..986654ab --- /dev/null +++ b/sahara_tests/scenario/defaults/rocky/cdh-5.13.0.yaml.mako @@ -0,0 +1,94 @@ +<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small', large_flavor_id='m1.large', availability_zone='nova', volumes_availability_zone='nova'"/> + +clusters: + - plugin_name: cdh + plugin_version: 5.13.0 + image: ${cdh_5130_image} + node_group_templates: + - name: worker-dn + flavor: ${ci_flavor_id} + node_processes: + - HDFS_DATANODE + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + node_configs: + &ng_configs + DATANODE: + dfs_datanode_du_reserved: 0 + - name: worker-nm + flavor: ${ci_flavor_id} + node_processes: + - YARN_NODEMANAGER + auto_security_group: ${use_auto_security_group} + - name: worker-nm-dn + flavor: ${ci_flavor_id} + node_processes: + - YARN_NODEMANAGER + - HDFS_DATANODE + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + node_configs: + *ng_configs + - name: manager + flavor: ${large_flavor_id} + node_processes: + - CLOUDERA_MANAGER + - KMS + is_proxy_gateway: ${is_proxy_gateway} + auto_security_group: ${use_auto_security_group} + - name: master-core + flavor: ${large_flavor_id} + node_processes: + - HDFS_NAMENODE + - YARN_RESOURCEMANAGER + - SENTRY_SERVER + - YARN_NODEMANAGER + - ZOOKEEPER_SERVER + auto_security_group: ${use_auto_security_group} + - name: master-additional + flavor: ${large_flavor_id} + node_processes: + - OOZIE_SERVER + - YARN_JOBHISTORY + - YARN_NODEMANAGER + - HDFS_SECONDARYNAMENODE + - HIVE_METASTORE + - HIVE_SERVER2 + - SPARK_YARN_HISTORY_SERVER + auto_security_group: ${use_auto_security_group} + # In >=5.9 the defaults of following configs are too large, + # restrict them to save memory for scenario testing. + node_configs: + HIVEMETASTORE: + hive_metastore_java_heapsize: 2147483648 + HIVESERVER: + hiveserver2_java_heapsize: 2147483648 + cluster_template: + name: cdh5130 + node_group_templates: + manager: 1 + master-core: 1 + master-additional: 1 + worker-nm-dn: 1 + worker-nm: 1 + worker-dn: 1 + cluster_configs: + HDFS: + dfs_replication: 1 + cluster: + name: ${cluster_name} + scenario: + - run_jobs + - sentry + edp_jobs_flow: + - pig_job + - mapreduce_job + - mapreduce_streaming_job + - java_job + - spark_wordcount diff --git a/sahara_tests/scenario/defaults/cdh-5.7.0.yaml.mako b/sahara_tests/scenario/defaults/rocky/cdh-5.9.0.yaml.mako similarity index 95% rename from sahara_tests/scenario/defaults/cdh-5.7.0.yaml.mako rename to sahara_tests/scenario/defaults/rocky/cdh-5.9.0.yaml.mako index 89c9c142..e5c6c92c 100644 --- a/sahara_tests/scenario/defaults/cdh-5.7.0.yaml.mako +++ b/sahara_tests/scenario/defaults/rocky/cdh-5.9.0.yaml.mako @@ -2,8 +2,8 @@ clusters: - plugin_name: cdh - plugin_version: 5.7.0 - image: ${cdh_570_image} + plugin_version: 5.9.0 + image: ${cdh_590_image} node_group_templates: - name: worker-dn flavor: ${ci_flavor_id} @@ -62,7 +62,7 @@ clusters: - HIVE_SERVER2 - SPARK_YARN_HISTORY_SERVER auto_security_group: ${use_auto_security_group} - # In 5.7 the defaults of following configs are too large, + # In 5.9 the defaults of following configs are too large, # restrict them to save memory for scenario testing. node_configs: HIVEMETASTORE: @@ -70,7 +70,7 @@ clusters: HIVESERVER: hiveserver2_java_heapsize: 2147483648 cluster_template: - name: cdh570 + name: cdh590 node_group_templates: manager: 1 master-core: 1 diff --git a/sahara_tests/scenario/defaults/rocky/mapr-5.2.0.mrv2.yaml.mako b/sahara_tests/scenario/defaults/rocky/mapr-5.2.0.mrv2.yaml.mako new file mode 100644 index 00000000..777a73ab --- /dev/null +++ b/sahara_tests/scenario/defaults/rocky/mapr-5.2.0.mrv2.yaml.mako @@ -0,0 +1,56 @@ +<%page args="use_auto_security_group='true', mapr_master_flavor_id='mapr.master', mapr_worker_flavor_id='mapr.worker'"/> + +clusters: + - plugin_name: mapr + plugin_version: 5.2.0.mrv2 + image: ${mapr_520mrv2_image} + node_group_templates: + - name: master + flavor: + name: ${mapr_master_flavor_id} + vcpus: 4 + ram: 8192 + root_disk: 80 + ephemeral_disk: 40 + node_processes: + - Metrics + - Webserver + - ZooKeeper + - HTTPFS + - Oozie + - FileServer + - CLDB + - Flume + - Hue + - NodeManager + - HistoryServer + - ResourceManager + - HiveServer2 + - HiveMetastore + - Sqoop2-Client + - Sqoop2-Server + auto_security_group: ${use_auto_security_group} + - name: worker + flavor: + name: ${mapr_worker_flavor_id} + vcpus: 2 + ram: 4096 + root_disk: 40 + ephemeral_disk: 40 + node_processes: + - NodeManager + - FileServer + auto_security_group: ${use_auto_security_group} + cluster_template: + name: mapr520mrv2 + node_group_templates: + master: 1 + worker: 1 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: + - mapr diff --git a/sahara_tests/scenario/defaults/spark-1.6.0.yaml.mako b/sahara_tests/scenario/defaults/rocky/spark-2.2.yaml.mako similarity index 91% rename from sahara_tests/scenario/defaults/spark-1.6.0.yaml.mako rename to sahara_tests/scenario/defaults/rocky/spark-2.2.yaml.mako index c3ad84c8..e0a0e38f 100644 --- a/sahara_tests/scenario/defaults/spark-1.6.0.yaml.mako +++ b/sahara_tests/scenario/defaults/rocky/spark-2.2.yaml.mako @@ -2,8 +2,8 @@ clusters: - plugin_name: spark - plugin_version: 1.6.0 - image: ${spark_160_image} + plugin_version: '2.2' + image: ${spark_22_image} node_group_templates: - name: master flavor: ${ci_flavor_id} @@ -19,7 +19,7 @@ clusters: - slave auto_security_group: ${use_auto_security_group} cluster_template: - name: spark160 + name: spark220 node_group_templates: master: 1 worker: 1 diff --git a/sahara_tests/scenario/defaults/spark-2.1.0.yaml.mako b/sahara_tests/scenario/defaults/rocky/spark-2.3.yaml.mako similarity index 91% rename from sahara_tests/scenario/defaults/spark-2.1.0.yaml.mako rename to sahara_tests/scenario/defaults/rocky/spark-2.3.yaml.mako index 94264c1c..134b5732 100644 --- a/sahara_tests/scenario/defaults/spark-2.1.0.yaml.mako +++ b/sahara_tests/scenario/defaults/rocky/spark-2.3.yaml.mako @@ -2,8 +2,8 @@ clusters: - plugin_name: spark - plugin_version: 2.1.0 - image: ${spark_210_image} + plugin_version: '2.3' + image: ${spark_22_image} node_group_templates: - name: master flavor: ${ci_flavor_id} @@ -19,7 +19,7 @@ clusters: - slave auto_security_group: ${use_auto_security_group} cluster_template: - name: spark210 + name: spark220 node_group_templates: master: 1 worker: 1 diff --git a/sahara_tests/scenario/defaults/rocky/storm-1.1.0.yaml.mako b/sahara_tests/scenario/defaults/rocky/storm-1.1.0.yaml.mako new file mode 100644 index 00000000..cff00c53 --- /dev/null +++ b/sahara_tests/scenario/defaults/rocky/storm-1.1.0.yaml.mako @@ -0,0 +1,37 @@ +<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small', medium_flavor_id='m1.medium'"/> + +clusters: + - plugin_name: storm + plugin_version: 1.1.0 + image: ${storm_110_image} + node_group_templates: + - name: master + flavor: ${ci_flavor_id} + node_processes: + - nimbus + auto_security_group: ${use_auto_security_group} + is_proxy_gateway: ${is_proxy_gateway} + - name: worker + flavor: ${ci_flavor_id} + node_processes: + - supervisor + auto_security_group: ${use_auto_security_group} + - name: zookeeper + flavor: ${medium_flavor_id} + node_processes: + - zookeeper + auto_security_group: ${use_auto_security_group} + cluster_template: + name: storm110 + node_group_templates: + master: 1 + worker: 1 + zookeeper: 1 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + scenario: + - scale diff --git a/sahara_tests/scenario/defaults/rocky/storm-1.2.0.yaml.mako b/sahara_tests/scenario/defaults/rocky/storm-1.2.0.yaml.mako new file mode 100644 index 00000000..23eb7ae8 --- /dev/null +++ b/sahara_tests/scenario/defaults/rocky/storm-1.2.0.yaml.mako @@ -0,0 +1,37 @@ +<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small', medium_flavor_id='m1.medium'"/> + +clusters: + - plugin_name: storm + plugin_version: 1.2.0 + image: ${storm_120_image} + node_group_templates: + - name: master + flavor: ${ci_flavor_id} + node_processes: + - nimbus + auto_security_group: ${use_auto_security_group} + is_proxy_gateway: ${is_proxy_gateway} + - name: worker + flavor: ${ci_flavor_id} + node_processes: + - supervisor + auto_security_group: ${use_auto_security_group} + - name: zookeeper + flavor: ${medium_flavor_id} + node_processes: + - zookeeper + auto_security_group: ${use_auto_security_group} + cluster_template: + name: storm110 + node_group_templates: + master: 1 + worker: 1 + zookeeper: 1 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + scenario: + - scale diff --git a/sahara_tests/scenario/defaults/rocky/vanilla-2.7.1.yaml.mako b/sahara_tests/scenario/defaults/rocky/vanilla-2.7.1.yaml.mako new file mode 100644 index 00000000..77d70baf --- /dev/null +++ b/sahara_tests/scenario/defaults/rocky/vanilla-2.7.1.yaml.mako @@ -0,0 +1,85 @@ +<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small', availability_zone='nova', volumes_availability_zone='nova'"/> + +clusters: + - plugin_name: vanilla + plugin_version: 2.7.1 + image: ${vanilla_271_image} + node_group_templates: + - name: worker-dn-nm + flavor: ${ci_flavor_id} + node_processes: + - datanode + - nodemanager + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + - name: worker-nm + flavor: ${ci_flavor_id} + node_processes: + - nodemanager + auto_security_group: ${use_auto_security_group} + - name: worker-dn + flavor: ${ci_flavor_id} + node_processes: + - datanode + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + - name: master-rm-nn-hvs-sp + flavor: ${ci_flavor_id} + node_processes: + - namenode + - resourcemanager + - hiveserver + - nodemanager + - spark history server + auto_security_group: ${use_auto_security_group} + - name: master-oo-hs-sn + flavor: ${ci_flavor_id} + node_processes: + - oozie + - historyserver + - secondarynamenode + - nodemanager + auto_security_group: ${use_auto_security_group} + is_proxy_gateway: ${is_proxy_gateway} + cluster_template: + name: vanilla271 + node_group_templates: + master-rm-nn-hvs-sp: 1 + master-oo-hs-sn: 1 + worker-dn-nm: 2 + worker-dn: 1 + worker-nm: 1 + cluster_configs: + HDFS: + dfs.replication: 1 + cluster: + name: ${cluster_name} + scaling: + - operation: resize + node_group: worker-dn-nm + size: 1 + - operation: resize + node_group: worker-dn + size: 0 + - operation: resize + node_group: worker-nm + size: 0 + - operation: add + node_group: worker-dn + size: 1 + - operation: add + node_group: worker-nm + size: 2 + edp_jobs_flow: + - pig_job + - mapreduce_job + - mapreduce_streaming_job + - java_job + - hive_job + - spark_wordcount diff --git a/sahara_tests/scenario/defaults/rocky/vanilla-2.7.5.yaml.mako b/sahara_tests/scenario/defaults/rocky/vanilla-2.7.5.yaml.mako new file mode 100644 index 00000000..c2570e9f --- /dev/null +++ b/sahara_tests/scenario/defaults/rocky/vanilla-2.7.5.yaml.mako @@ -0,0 +1,84 @@ +<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small', cluster_name='vanilla-275', availability_zone='nova', volumes_availability_zone='nova'"/> + +clusters: + - plugin_name: vanilla + plugin_version: 2.7.5 + image: ${vanilla_275_image} + node_group_templates: + - name: worker-dn-nm + flavor: ${ci_flavor_id} + node_processes: + - datanode + - nodemanager + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + - name: worker-nm + flavor: ${ci_flavor_id} + node_processes: + - nodemanager + auto_security_group: ${use_auto_security_group} + - name: worker-dn + flavor: ${ci_flavor_id} + node_processes: + - datanode + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + - name: master-rm-nn-hvs-sp + flavor: ${ci_flavor_id} + node_processes: + - namenode + - resourcemanager + - hiveserver + - nodemanager + - spark history server + auto_security_group: ${use_auto_security_group} + - name: master-oo-hs-sn + flavor: ${ci_flavor_id} + node_processes: + - oozie + - historyserver + - secondarynamenode + - nodemanager + auto_security_group: ${use_auto_security_group} + is_proxy_gateway: ${is_proxy_gateway} + cluster_template: + node_group_templates: + master-rm-nn-hvs-sp: 1 + master-oo-hs-sn: 1 + worker-dn-nm: 2 + worker-dn: 1 + worker-nm: 1 + cluster_configs: + HDFS: + dfs.replication: 1 + cluster: + name: ${cluster_name} + scaling: + - operation: resize + node_group: worker-dn-nm + size: 1 + - operation: resize + node_group: worker-dn + size: 0 + - operation: resize + node_group: worker-nm + size: 0 + - operation: add + node_group: worker-dn + size: 1 + - operation: add + node_group: worker-nm + size: 2 + edp_jobs_flow: + - pig_job + - mapreduce_job + - mapreduce_streaming_job + - java_job + - hive_job + - spark_wordcount diff --git a/sahara_tests/scenario/defaults/rocky/vanilla-2.8.2.yaml.mako b/sahara_tests/scenario/defaults/rocky/vanilla-2.8.2.yaml.mako new file mode 100644 index 00000000..34512993 --- /dev/null +++ b/sahara_tests/scenario/defaults/rocky/vanilla-2.8.2.yaml.mako @@ -0,0 +1,84 @@ +<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small', cluster_name='vanilla-282', availability_zone='nova', volumes_availability_zone='nova'"/> + +clusters: + - plugin_name: vanilla + plugin_version: 2.8.2 + image: ${vanilla_282_image} + node_group_templates: + - name: worker-dn-nm + flavor: ${ci_flavor_id} + node_processes: + - datanode + - nodemanager + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + - name: worker-nm + flavor: ${ci_flavor_id} + node_processes: + - nodemanager + auto_security_group: ${use_auto_security_group} + - name: worker-dn + flavor: ${ci_flavor_id} + node_processes: + - datanode + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + - name: master-rm-nn-hvs-sp + flavor: ${ci_flavor_id} + node_processes: + - namenode + - resourcemanager + - hiveserver + - nodemanager + - spark history server + auto_security_group: ${use_auto_security_group} + - name: master-oo-hs-sn + flavor: ${ci_flavor_id} + node_processes: + - oozie + - historyserver + - secondarynamenode + - nodemanager + auto_security_group: ${use_auto_security_group} + is_proxy_gateway: ${is_proxy_gateway} + cluster_template: + node_group_templates: + master-rm-nn-hvs-sp: 1 + master-oo-hs-sn: 1 + worker-dn-nm: 2 + worker-dn: 1 + worker-nm: 1 + cluster_configs: + HDFS: + dfs.replication: 1 + cluster: + name: ${cluster_name} + scaling: + - operation: resize + node_group: worker-dn-nm + size: 1 + - operation: resize + node_group: worker-dn + size: 0 + - operation: resize + node_group: worker-nm + size: 0 + - operation: add + node_group: worker-dn + size: 1 + - operation: add + node_group: worker-nm + size: 2 + edp_jobs_flow: + - pig_job + - mapreduce_job + - mapreduce_streaming_job + - java_job + - hive_job + - spark_wordcount diff --git a/sahara_tests/scenario/defaults/spark-2.3.yaml.mako b/sahara_tests/scenario/defaults/spark-2.3.yaml.mako new file mode 100644 index 00000000..134b5732 --- /dev/null +++ b/sahara_tests/scenario/defaults/spark-2.3.yaml.mako @@ -0,0 +1,37 @@ +<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small'"/> + +clusters: + - plugin_name: spark + plugin_version: '2.3' + image: ${spark_22_image} + node_group_templates: + - name: master + flavor: ${ci_flavor_id} + node_processes: + - master + - namenode + auto_security_group: ${use_auto_security_group} + is_proxy_gateway: ${is_proxy_gateway} + - name: worker + flavor: ${ci_flavor_id} + node_processes: + - datanode + - slave + auto_security_group: ${use_auto_security_group} + cluster_template: + name: spark220 + node_group_templates: + master: 1 + worker: 1 + cluster_configs: + HDFS: + dfs.replication: 1 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: + - spark_pi + - spark_wordcount diff --git a/sahara_tests/scenario/defaults/storm-1.2.0.yaml.mako b/sahara_tests/scenario/defaults/storm-1.2.0.yaml.mako new file mode 100644 index 00000000..23eb7ae8 --- /dev/null +++ b/sahara_tests/scenario/defaults/storm-1.2.0.yaml.mako @@ -0,0 +1,37 @@ +<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small', medium_flavor_id='m1.medium'"/> + +clusters: + - plugin_name: storm + plugin_version: 1.2.0 + image: ${storm_120_image} + node_group_templates: + - name: master + flavor: ${ci_flavor_id} + node_processes: + - nimbus + auto_security_group: ${use_auto_security_group} + is_proxy_gateway: ${is_proxy_gateway} + - name: worker + flavor: ${ci_flavor_id} + node_processes: + - supervisor + auto_security_group: ${use_auto_security_group} + - name: zookeeper + flavor: ${medium_flavor_id} + node_processes: + - zookeeper + auto_security_group: ${use_auto_security_group} + cluster_template: + name: storm110 + node_group_templates: + master: 1 + worker: 1 + zookeeper: 1 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + scenario: + - scale diff --git a/sahara_tests/scenario/defaults/vanilla-2.7.5.yaml.mako b/sahara_tests/scenario/defaults/vanilla-2.7.5.yaml.mako new file mode 100644 index 00000000..c2570e9f --- /dev/null +++ b/sahara_tests/scenario/defaults/vanilla-2.7.5.yaml.mako @@ -0,0 +1,84 @@ +<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small', cluster_name='vanilla-275', availability_zone='nova', volumes_availability_zone='nova'"/> + +clusters: + - plugin_name: vanilla + plugin_version: 2.7.5 + image: ${vanilla_275_image} + node_group_templates: + - name: worker-dn-nm + flavor: ${ci_flavor_id} + node_processes: + - datanode + - nodemanager + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + - name: worker-nm + flavor: ${ci_flavor_id} + node_processes: + - nodemanager + auto_security_group: ${use_auto_security_group} + - name: worker-dn + flavor: ${ci_flavor_id} + node_processes: + - datanode + volumes_per_node: 2 + volumes_size: 2 + availability_zone: ${availability_zone} + volumes_availability_zone: ${volumes_availability_zone} + auto_security_group: ${use_auto_security_group} + - name: master-rm-nn-hvs-sp + flavor: ${ci_flavor_id} + node_processes: + - namenode + - resourcemanager + - hiveserver + - nodemanager + - spark history server + auto_security_group: ${use_auto_security_group} + - name: master-oo-hs-sn + flavor: ${ci_flavor_id} + node_processes: + - oozie + - historyserver + - secondarynamenode + - nodemanager + auto_security_group: ${use_auto_security_group} + is_proxy_gateway: ${is_proxy_gateway} + cluster_template: + node_group_templates: + master-rm-nn-hvs-sp: 1 + master-oo-hs-sn: 1 + worker-dn-nm: 2 + worker-dn: 1 + worker-nm: 1 + cluster_configs: + HDFS: + dfs.replication: 1 + cluster: + name: ${cluster_name} + scaling: + - operation: resize + node_group: worker-dn-nm + size: 1 + - operation: resize + node_group: worker-dn + size: 0 + - operation: resize + node_group: worker-nm + size: 0 + - operation: add + node_group: worker-dn + size: 1 + - operation: add + node_group: worker-nm + size: 2 + edp_jobs_flow: + - pig_job + - mapreduce_job + - mapreduce_streaming_job + - java_job + - hive_job + - spark_wordcount