Refresh templates (also for Queens)

Add few new tamplates (Spark 2.2, Vanilla 2.8.2). Add a template directory for Queens, remove Newton templates and adapt the unit tests which were relying on the Newton templates. Change-Id: I9234af30e5070f71b69d41b6724eae70ce2085ea
2018-02-22 12:38:03 +01:00 · 2018-02-22 12:38:03 +01:00 · 415421f2c2
parent a2027280cd
commit 415421f2c2
16 changed files with 461 additions and 72 deletions
--- a/releasenotes/notes/test-templates-queens-901f1af3cf80f98d.yaml
+++ b/releasenotes/notes/test-templates-queens-901f1af3cf80f98d.yaml
@ -0,0 +1,13 @@
+---
+prelude: >
+    Queens test templates are now available, while Newton
+    test templates have been removed.
+features:
+  - A folder with scenario templates for Queens was added.
+    It is a subset of the templates in the main directory.
+  - The available templates now supports Spark 2.2
+    and Vanilla 2.8.2.
+deprecations:
+  - The Newton-specific job templates have been removed.
+    This means that starting from this release Newton is
+    not supported (it has been EOL for a while).
--- a/sahara_tests/scenario/defaults/newton/mapr-5.1.0.mrv2.yaml.mako
+++ b/sahara_tests/scenario/defaults/newton/mapr-5.1.0.mrv2.yaml.mako
@ -1,56 +0,0 @@
-<%page args="use_auto_security_group='true', mapr_master_flavor_id='mapr.master', mapr_worker_flavor_id='mapr.worker'"/>
-
-clusters:
-  - plugin_name: mapr
-    plugin_version: 5.1.0.mrv2
-    image: ${mapr_510mrv2_image}
-    node_group_templates:
-      - name: master
-        flavor:
-          name: ${mapr_master_flavor_id}
-          vcpus: 4
-          ram: 8192
-          root_disk: 80
-          ephemeral_disk: 40
-        node_processes:
-          - Metrics
-          - Webserver
-          - ZooKeeper
-          - HTTPFS
-          - Oozie
-          - FileServer
-          - CLDB
-          - Flume
-          - Hue
-          - NodeManager
-          - HistoryServer
-          - ResourceManager
-          - HiveServer2
-          - HiveMetastore
-          - Sqoop2-Client
-          - Sqoop2-Server
-        auto_security_group: ${use_auto_security_group}
-      - name: worker
-        flavor:
-          name: ${mapr_worker_flavor_id}
-          vcpus: 2
-          ram: 4096
-          root_disk: 40
-          ephemeral_disk: 40
-        node_processes:
-          - NodeManager
-          - FileServer
-        auto_security_group: ${use_auto_security_group}
-    cluster_template:
-      name: mapr510mrv2
-      node_group_templates:
-        master: 1
-        worker: 1
-    cluster:
-      name: ${cluster_name}
-    scaling:
-      - operation: add
-        node_group: worker
-        size: 1
-    edp_jobs_flow:
-      - mapr
--- a/sahara_tests/scenario/defaults/queens/ambari-2.4.yaml.mako
+++ b/sahara_tests/scenario/defaults/queens/ambari-2.4.yaml.mako
@ -3,7 +3,7 @@
 clusters:
  - plugin_name: ambari
    plugin_version: '2.4'
-    image: ${ambari_22_image}
+    image: ${ambari_24_image}
    node_group_templates:
      - name: master
        flavor: ${medium_flavor_id}
--- a/sahara_tests/scenario/defaults/queens/cdh-5.11.0.yaml.mako
+++ b/sahara_tests/scenario/defaults/queens/cdh-5.11.0.yaml.mako
@ -0,0 +1,94 @@
+<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small', large_flavor_id='m1.large', availability_zone='nova', volumes_availability_zone='nova'"/>
+
+clusters:
+  - plugin_name: cdh
+    plugin_version: 5.11.0
+    image: ${cdh_5110_image}
+    node_group_templates:
+      - name: worker-dn
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - HDFS_DATANODE
+        volumes_per_node: 2
+        volumes_size: 2
+        availability_zone: ${availability_zone}
+        volumes_availability_zone: ${volumes_availability_zone}
+        auto_security_group: ${use_auto_security_group}
+        node_configs:
+          &ng_configs
+          DATANODE:
+            dfs_datanode_du_reserved: 0
+      - name: worker-nm
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - YARN_NODEMANAGER
+        auto_security_group: ${use_auto_security_group}
+      - name: worker-nm-dn
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - YARN_NODEMANAGER
+          - HDFS_DATANODE
+        volumes_per_node: 2
+        volumes_size: 2
+        availability_zone: ${availability_zone}
+        volumes_availability_zone: ${volumes_availability_zone}
+        auto_security_group: ${use_auto_security_group}
+        node_configs:
+          *ng_configs
+      - name: manager
+        flavor: ${large_flavor_id}
+        node_processes:
+          - CLOUDERA_MANAGER
+          - KMS
+        is_proxy_gateway: ${is_proxy_gateway}
+        auto_security_group: ${use_auto_security_group}
+      - name: master-core
+        flavor: ${large_flavor_id}
+        node_processes:
+          - HDFS_NAMENODE
+          - YARN_RESOURCEMANAGER
+          - SENTRY_SERVER
+          - YARN_NODEMANAGER
+          - ZOOKEEPER_SERVER
+        auto_security_group: ${use_auto_security_group}
+      - name: master-additional
+        flavor: ${large_flavor_id}
+        node_processes:
+          - OOZIE_SERVER
+          - YARN_JOBHISTORY
+          - YARN_NODEMANAGER
+          - HDFS_SECONDARYNAMENODE
+          - HIVE_METASTORE
+          - HIVE_SERVER2
+          - SPARK_YARN_HISTORY_SERVER
+        auto_security_group: ${use_auto_security_group}
+        # In 5.11 the defaults of following configs are too large,
+        # restrict them to save memory for scenario testing.
+        node_configs:
+            HIVEMETASTORE:
+                hive_metastore_java_heapsize: 2147483648
+            HIVESERVER:
+                hiveserver2_java_heapsize: 2147483648
+    cluster_template:
+      name: cdh5110
+      node_group_templates:
+        manager: 1
+        master-core: 1
+        master-additional: 1
+        worker-nm-dn: 1
+        worker-nm: 1
+        worker-dn: 1
+      cluster_configs:
+        HDFS:
+          dfs_replication: 1
+    cluster:
+      name: ${cluster_name}
+    scenario:
+      - run_jobs
+      - sentry
+    edp_jobs_flow:
+      - pig_job
+      - mapreduce_job
+      - mapreduce_streaming_job
+      - java_job
+      - spark_wordcount
--- a/sahara_tests/scenario/defaults/queens/cdh-5.7.0.yaml.mako
+++ b/sahara_tests/scenario/defaults/queens/cdh-5.7.0.yaml.mako
--- a/sahara_tests/scenario/defaults/queens/cdh-5.9.0.yaml.mako
+++ b/sahara_tests/scenario/defaults/queens/cdh-5.9.0.yaml.mako
@ -0,0 +1,94 @@
+<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small', large_flavor_id='m1.large', availability_zone='nova', volumes_availability_zone='nova'"/>
+
+clusters:
+  - plugin_name: cdh
+    plugin_version: 5.9.0
+    image: ${cdh_590_image}
+    node_group_templates:
+      - name: worker-dn
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - HDFS_DATANODE
+        volumes_per_node: 2
+        volumes_size: 2
+        availability_zone: ${availability_zone}
+        volumes_availability_zone: ${volumes_availability_zone}
+        auto_security_group: ${use_auto_security_group}
+        node_configs:
+          &ng_configs
+          DATANODE:
+            dfs_datanode_du_reserved: 0
+      - name: worker-nm
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - YARN_NODEMANAGER
+        auto_security_group: ${use_auto_security_group}
+      - name: worker-nm-dn
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - YARN_NODEMANAGER
+          - HDFS_DATANODE
+        volumes_per_node: 2
+        volumes_size: 2
+        availability_zone: ${availability_zone}
+        volumes_availability_zone: ${volumes_availability_zone}
+        auto_security_group: ${use_auto_security_group}
+        node_configs:
+          *ng_configs
+      - name: manager
+        flavor: ${large_flavor_id}
+        node_processes:
+          - CLOUDERA_MANAGER
+          - KMS
+        is_proxy_gateway: ${is_proxy_gateway}
+        auto_security_group: ${use_auto_security_group}
+      - name: master-core
+        flavor: ${large_flavor_id}
+        node_processes:
+          - HDFS_NAMENODE
+          - YARN_RESOURCEMANAGER
+          - SENTRY_SERVER
+          - YARN_NODEMANAGER
+          - ZOOKEEPER_SERVER
+        auto_security_group: ${use_auto_security_group}
+      - name: master-additional
+        flavor: ${large_flavor_id}
+        node_processes:
+          - OOZIE_SERVER
+          - YARN_JOBHISTORY
+          - YARN_NODEMANAGER
+          - HDFS_SECONDARYNAMENODE
+          - HIVE_METASTORE
+          - HIVE_SERVER2
+          - SPARK_YARN_HISTORY_SERVER
+        auto_security_group: ${use_auto_security_group}
+        # In 5.9 the defaults of following configs are too large,
+        # restrict them to save memory for scenario testing.
+        node_configs:
+            HIVEMETASTORE:
+                hive_metastore_java_heapsize: 2147483648
+            HIVESERVER:
+                hiveserver2_java_heapsize: 2147483648
+    cluster_template:
+      name: cdh590
+      node_group_templates:
+        manager: 1
+        master-core: 1
+        master-additional: 1
+        worker-nm-dn: 1
+        worker-nm: 1
+        worker-dn: 1
+      cluster_configs:
+        HDFS:
+          dfs_replication: 1
+    cluster:
+      name: ${cluster_name}
+    scenario:
+      - run_jobs
+      - sentry
+    edp_jobs_flow:
+      - pig_job
+      - mapreduce_job
+      - mapreduce_streaming_job
+      - java_job
+      - spark_wordcount
--- a/sahara_tests/scenario/defaults/queens/mapr-5.2.0.mrv2.yaml.mako
+++ b/sahara_tests/scenario/defaults/queens/mapr-5.2.0.mrv2.yaml.mako
--- a/sahara_tests/scenario/defaults/queens/spark-2.1.0.yaml.mako
+++ b/sahara_tests/scenario/defaults/queens/spark-2.1.0.yaml.mako
@ -2,8 +2,8 @@

 clusters:
  - plugin_name: spark
-    plugin_version: 1.6.0
-    image: ${spark_160_image}
+    plugin_version: 2.1.0
+    image: ${spark_210_image}
    node_group_templates:
      - name: master
        flavor: ${ci_flavor_id}
@ -19,7 +19,7 @@ clusters:
          - slave
        auto_security_group: ${use_auto_security_group}
    cluster_template:
-      name: spark160
+      name: spark210
      node_group_templates:
        master: 1
        worker: 1
--- a/sahara_tests/scenario/defaults/queens/spark-2.2.yaml.mako
+++ b/sahara_tests/scenario/defaults/queens/spark-2.2.yaml.mako
@ -0,0 +1,37 @@
+<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small'"/>
+
+clusters:
+  - plugin_name: spark
+    plugin_version: '2.2'
+    image: ${spark_22_image}
+    node_group_templates:
+      - name: master
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - master
+          - namenode
+        auto_security_group: ${use_auto_security_group}
+        is_proxy_gateway: ${is_proxy_gateway}
+      - name: worker
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - datanode
+          - slave
+        auto_security_group: ${use_auto_security_group}
+    cluster_template:
+      name: spark220
+      node_group_templates:
+        master: 1
+        worker: 1
+      cluster_configs:
+        HDFS:
+          dfs.replication: 1
+    cluster:
+      name: ${cluster_name}
+    scaling:
+      - operation: add
+        node_group: worker
+        size: 1
+    edp_jobs_flow:
+      - spark_pi
+      - spark_wordcount
--- a/sahara_tests/scenario/defaults/queens/storm-1.1.0.yaml.mako
+++ b/sahara_tests/scenario/defaults/queens/storm-1.1.0.yaml.mako
@ -2,8 +2,8 @@

 clusters:
  - plugin_name: storm
-    plugin_version: 1.0.1
-    image: ${storm_101_image}
+    plugin_version: 1.1.0
+    image: ${storm_110_image}
    node_group_templates:
      - name: master
        flavor: ${ci_flavor_id}
@ -22,7 +22,7 @@ clusters:
          - zookeeper
        auto_security_group: ${use_auto_security_group}
    cluster_template:
-      name: storm101
+      name: storm110
      node_group_templates:
        master: 1
        worker: 1
--- a/sahara_tests/scenario/defaults/queens/vanilla-2.7.1.yaml.mako
+++ b/sahara_tests/scenario/defaults/queens/vanilla-2.7.1.yaml.mako
--- a/sahara_tests/scenario/defaults/queens/vanilla-2.8.2.yaml.mako
+++ b/sahara_tests/scenario/defaults/queens/vanilla-2.8.2.yaml.mako
@ -0,0 +1,84 @@
+<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small', cluster_name='vanilla-282', availability_zone='nova', volumes_availability_zone='nova'"/>
+
+clusters:
+  - plugin_name: vanilla
+    plugin_version: 2.8.2
+    image: ${vanilla_282_image}
+    node_group_templates:
+      - name: worker-dn-nm
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - datanode
+          - nodemanager
+        volumes_per_node: 2
+        volumes_size: 2
+        availability_zone: ${availability_zone}
+        volumes_availability_zone: ${volumes_availability_zone}
+        auto_security_group: ${use_auto_security_group}
+      - name: worker-nm
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - nodemanager
+        auto_security_group: ${use_auto_security_group}
+      - name: worker-dn
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - datanode
+        volumes_per_node: 2
+        volumes_size: 2
+        availability_zone: ${availability_zone}
+        volumes_availability_zone: ${volumes_availability_zone}
+        auto_security_group: ${use_auto_security_group}
+      - name: master-rm-nn-hvs-sp
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - namenode
+          - resourcemanager
+          - hiveserver
+          - nodemanager
+          - spark history server
+        auto_security_group: ${use_auto_security_group}
+      - name: master-oo-hs-sn
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - oozie
+          - historyserver
+          - secondarynamenode
+          - nodemanager
+        auto_security_group: ${use_auto_security_group}
+        is_proxy_gateway: ${is_proxy_gateway}
+    cluster_template:
+      node_group_templates:
+        master-rm-nn-hvs-sp: 1
+        master-oo-hs-sn: 1
+        worker-dn-nm: 2
+        worker-dn: 1
+        worker-nm: 1
+      cluster_configs:
+        HDFS:
+          dfs.replication: 1
+    cluster:
+      name: ${cluster_name}
+    scaling:
+      - operation: resize
+        node_group: worker-dn-nm
+        size: 1
+      - operation: resize
+        node_group: worker-dn
+        size: 0
+      - operation: resize
+        node_group: worker-nm
+        size: 0
+      - operation: add
+        node_group: worker-dn
+        size: 1
+      - operation: add
+        node_group: worker-nm
+        size: 2
+    edp_jobs_flow:
+      - pig_job
+      - mapreduce_job
+      - mapreduce_streaming_job
+      - java_job
+      - hive_job
+      - spark_wordcount
--- a/sahara_tests/scenario/defaults/spark-2.2.yaml.mako
+++ b/sahara_tests/scenario/defaults/spark-2.2.yaml.mako
@ -0,0 +1,37 @@
+<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small'"/>
+
+clusters:
+  - plugin_name: spark
+    plugin_version: '2.2'
+    image: ${spark_22_image}
+    node_group_templates:
+      - name: master
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - master
+          - namenode
+        auto_security_group: ${use_auto_security_group}
+        is_proxy_gateway: ${is_proxy_gateway}
+      - name: worker
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - datanode
+          - slave
+        auto_security_group: ${use_auto_security_group}
+    cluster_template:
+      name: spark220
+      node_group_templates:
+        master: 1
+        worker: 1
+      cluster_configs:
+        HDFS:
+          dfs.replication: 1
+    cluster:
+      name: ${cluster_name}
+    scaling:
+      - operation: add
+        node_group: worker
+        size: 1
+    edp_jobs_flow:
+      - spark_pi
+      - spark_wordcount
--- a/sahara_tests/scenario/defaults/vanilla-2.8.2.yaml.mako
+++ b/sahara_tests/scenario/defaults/vanilla-2.8.2.yaml.mako
@ -0,0 +1,84 @@
+<%page args="is_proxy_gateway='true', use_auto_security_group='true', ci_flavor_id='m1.small', cluster_name='vanilla-282', availability_zone='nova', volumes_availability_zone='nova'"/>
+
+clusters:
+  - plugin_name: vanilla
+    plugin_version: 2.8.2
+    image: ${vanilla_282_image}
+    node_group_templates:
+      - name: worker-dn-nm
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - datanode
+          - nodemanager
+        volumes_per_node: 2
+        volumes_size: 2
+        availability_zone: ${availability_zone}
+        volumes_availability_zone: ${volumes_availability_zone}
+        auto_security_group: ${use_auto_security_group}
+      - name: worker-nm
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - nodemanager
+        auto_security_group: ${use_auto_security_group}
+      - name: worker-dn
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - datanode
+        volumes_per_node: 2
+        volumes_size: 2
+        availability_zone: ${availability_zone}
+        volumes_availability_zone: ${volumes_availability_zone}
+        auto_security_group: ${use_auto_security_group}
+      - name: master-rm-nn-hvs-sp
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - namenode
+          - resourcemanager
+          - hiveserver
+          - nodemanager
+          - spark history server
+        auto_security_group: ${use_auto_security_group}
+      - name: master-oo-hs-sn
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - oozie
+          - historyserver
+          - secondarynamenode
+          - nodemanager
+        auto_security_group: ${use_auto_security_group}
+        is_proxy_gateway: ${is_proxy_gateway}
+    cluster_template:
+      node_group_templates:
+        master-rm-nn-hvs-sp: 1
+        master-oo-hs-sn: 1
+        worker-dn-nm: 2
+        worker-dn: 1
+        worker-nm: 1
+      cluster_configs:
+        HDFS:
+          dfs.replication: 1
+    cluster:
+      name: ${cluster_name}
+    scaling:
+      - operation: resize
+        node_group: worker-dn-nm
+        size: 1
+      - operation: resize
+        node_group: worker-dn
+        size: 0
+      - operation: resize
+        node_group: worker-nm
+        size: 0
+      - operation: add
+        node_group: worker-dn
+        size: 1
+      - operation: add
+        node_group: worker-nm
+        size: 2
+    edp_jobs_flow:
+      - pig_job
+      - mapreduce_job
+      - mapreduce_streaming_job
+      - java_job
+      - hive_job
+      - spark_wordcount
--- a/sahara_tests/unit/scenario/templatevars_complete.ini
+++ b/sahara_tests/unit/scenario/templatevars_complete.ini
@ -4,7 +4,7 @@ network_private_name: private
 network_public_name: public
 vanilla_26_image: centos_sahara_vanilla_hadoop_2_6_latest
 vanilla_271_image: vanilla271
-spark_160_image: spark
+spark_22_image: spark
 cdh_550_image: cdh550
 cluster_name: cluster
 ci_flavor_id: '2'
--- a/sahara_tests/unit/scenario/test_runner.py
+++ b/sahara_tests/unit/scenario/test_runner.py
@ -380,7 +380,7 @@ class RunnerUnitTest(testtools.TestCase):
        sys.argv = ['sahara_tests/scenario/runner.py',
                    '-V',
                    'sahara_tests/unit/scenario/templatevars_complete.ini',
-                    '-p', 'spark', '-v', '1.6.0', '-r', 'newton',
+                    '-p', 'spark', '-v', '2.2', '-r', 'queens',
                    '--os-username', 'demo', '--os-password', 'demopwd',
                    '--os-project-name', 'demo',
                    '--os-auth-url', 'http://127.0.0.1:5000/v2']
@ -388,7 +388,7 @@ class RunnerUnitTest(testtools.TestCase):
        self.assertEqual('spark',
                         mock_validate.call_args[0][0]['clusters'][0][
                             'plugin_name'])
-        self.assertEqual('1.6.0',
+        self.assertEqual('2.2',
                         mock_validate.call_args[0][0]['clusters'][0][
                             'plugin_version'])

@ -399,7 +399,7 @@ class RunnerUnitTest(testtools.TestCase):
        sys.argv = ['sahara_tests/scenario/runner.py',
                    '-V',
                    'sahara_tests/unit/scenario/templatevars_complete.ini',
-                    '-p', 'spark', '-v', '1.6.0', '--release', 'newton',
+                    '-p', 'spark', '-v', '2.2', '--release', 'queens',
                    '--count', '4',
                    '--os-username', 'demo', '--os-password', 'demopwd',
                    '--os-project-name', 'demo',
@ -413,15 +413,17 @@ class RunnerUnitTest(testtools.TestCase):
        sys.argv = ['sahara_tests/scenario/runner.py',
                    '-V',
                    'sahara_tests/unit/scenario/templatevars_complete.ini',
-                    'sahara_tests/scenario/defaults/newton',
+                    'sahara_tests/scenario/defaults/queens',
                    'sahara_tests/scenario/defaults/edp.yaml.mako',
                    '--os-username', 'demo', '--os-password', 'demopwd',
                    '--os-project-name', 'demo',
                    '--os-auth-url', 'http://127.0.0.1:5000/v2', '--args',
-                    'ambari_22_image:ambari', 'fake_plugin_image:fake',
-                    'mapr_510mrv2_image:mapr', 'mapr_520mrv2_image:mapr',
-                    'cdh_570_image:cdh', 'spark_160_image:spark',
-                    'storm_101_image:storm']
+                    'ambari_24_image:ambari', 'fake_plugin_image:fake',
+                    'mapr_520mrv2_image:mapr', 'cdh_570_image:cdh',
+                    'cdh_590_image:cdh', 'cdh_5110_image:cdh',
+                    'spark_210_image:spark', 'spark_22_image:spark',
+                    'storm_101_image:storm', 'storm_110_image:storm',
+                    'vanilla_282_image:vanilla']
        runner.main()

    @mock.patch('sahara_tests.scenario.validation.validate')