summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZuul <zuul@review.openstack.org>2018-07-12 00:33:16 +0000
committerGerrit Code Review <review@openstack.org>2018-07-12 00:33:16 +0000
commit2a6ddfee995977ce59cd88b4d4922f5550b7ee16 (patch)
tree0893fd3b520f59c596748b152bd0dbf1041d7ca2
parent8716202480f640848ba8afae5b45c44c8511dd29 (diff)
parente1a36ee28cf9b60d740300fb952d8928f48f8069 (diff)
Merge "Updating Spark versions"
-rw-r--r--releasenotes/notes/spark-2.3-0277fe9feae6668a.yaml3
-rw-r--r--sahara/plugins/spark/edp_engine.py2
-rw-r--r--sahara/plugins/spark/plugin.py4
-rw-r--r--sahara/tests/unit/plugins/spark/test_plugin.py18
4 files changed, 24 insertions, 3 deletions
diff --git a/releasenotes/notes/spark-2.3-0277fe9feae6668a.yaml b/releasenotes/notes/spark-2.3-0277fe9feae6668a.yaml
new file mode 100644
index 0000000..d3c24cc
--- /dev/null
+++ b/releasenotes/notes/spark-2.3-0277fe9feae6668a.yaml
@@ -0,0 +1,3 @@
1---
2upgrade:
3 - Adding Spark 2.3 to supported plugins list.
diff --git a/sahara/plugins/spark/edp_engine.py b/sahara/plugins/spark/edp_engine.py
index 5720283..1ca3791 100644
--- a/sahara/plugins/spark/edp_engine.py
+++ b/sahara/plugins/spark/edp_engine.py
@@ -26,7 +26,7 @@ from sahara.service.edp.spark import engine as edp_engine
26 26
27class EdpEngine(edp_engine.SparkJobEngine): 27class EdpEngine(edp_engine.SparkJobEngine):
28 28
29 edp_base_version = "1.3.1" 29 edp_base_version = "1.6.0"
30 30
31 def __init__(self, cluster): 31 def __init__(self, cluster):
32 super(EdpEngine, self).__init__(cluster) 32 super(EdpEngine, self).__init__(cluster)
diff --git a/sahara/plugins/spark/plugin.py b/sahara/plugins/spark/plugin.py
index 4c4bd98..19d56a0 100644
--- a/sahara/plugins/spark/plugin.py
+++ b/sahara/plugins/spark/plugin.py
@@ -63,7 +63,7 @@ class SparkProvider(p.ProvisioningPluginBase):
63 deprecated = {'enabled': {'status': True}, 63 deprecated = {'enabled': {'status': True},
64 'deprecated': {'status': True}} 64 'deprecated': {'status': True}}
65 result = {'plugin_labels': copy.deepcopy(default)} 65 result = {'plugin_labels': copy.deepcopy(default)}
66 stable_versions = ['2.2'] 66 stable_versions = ['2.3', '2.2']
67 result['version_labels'] = { 67 result['version_labels'] = {
68 version: copy.deepcopy( 68 version: copy.deepcopy(
69 default if version in stable_versions else deprecated 69 default if version in stable_versions else deprecated
@@ -72,7 +72,7 @@ class SparkProvider(p.ProvisioningPluginBase):
72 return result 72 return result
73 73
74 def get_versions(self): 74 def get_versions(self):
75 return ['2.2', '2.1.0', '1.6.0'] 75 return ['2.3', '2.2', '2.1.0', '1.6.0']
76 76
77 def get_configs(self, hadoop_version): 77 def get_configs(self, hadoop_version):
78 return c_helper.get_plugin_configs() 78 return c_helper.get_plugin_configs()
diff --git a/sahara/tests/unit/plugins/spark/test_plugin.py b/sahara/tests/unit/plugins/spark/test_plugin.py
index f9d506b..d768207 100644
--- a/sahara/tests/unit/plugins/spark/test_plugin.py
+++ b/sahara/tests/unit/plugins/spark/test_plugin.py
@@ -68,6 +68,14 @@ class SparkPluginTest(base.SaharaWithDbTestCase):
68 self._test_engine('2.2', edp.JOB_TYPE_SHELL, 68 self._test_engine('2.2', edp.JOB_TYPE_SHELL,
69 engine.SparkShellJobEngine) 69 engine.SparkShellJobEngine)
70 70
71 def test_plugin23_edp_engine(self):
72 self._test_engine('2.3', edp.JOB_TYPE_SPARK,
73 engine.SparkJobEngine)
74
75 def test_plugin23_shell_engine(self):
76 self._test_engine('2.3', edp.JOB_TYPE_SHELL,
77 engine.SparkShellJobEngine)
78
71 def _test_engine(self, version, job_type, eng): 79 def _test_engine(self, version, job_type, eng):
72 cluster_dict = self._init_cluster_dict(version) 80 cluster_dict = self._init_cluster_dict(version)
73 81
@@ -172,6 +180,8 @@ class SparkProviderTest(base.SaharaTestCase):
172 res['2.1.0']) 180 res['2.1.0'])
173 self.assertEqual([edp.JOB_TYPE_SHELL, edp.JOB_TYPE_SPARK], 181 self.assertEqual([edp.JOB_TYPE_SHELL, edp.JOB_TYPE_SPARK],
174 res['2.2']) 182 res['2.2'])
183 self.assertEqual([edp.JOB_TYPE_SHELL, edp.JOB_TYPE_SPARK],
184 res['2.3'])
175 185
176 def test_edp_config_hints(self): 186 def test_edp_config_hints(self):
177 provider = pl.SparkProvider() 187 provider = pl.SparkProvider()
@@ -199,3 +209,11 @@ class SparkProviderTest(base.SaharaTestCase):
199 res = provider.get_edp_config_hints(edp.JOB_TYPE_SHELL, "2.2") 209 res = provider.get_edp_config_hints(edp.JOB_TYPE_SHELL, "2.2")
200 self.assertEqual({'args': [], 'configs': {}, 'params': {}}, 210 self.assertEqual({'args': [], 'configs': {}, 'params': {}},
201 res['job_config']) 211 res['job_config'])
212
213 res = provider.get_edp_config_hints(edp.JOB_TYPE_SPARK, "2.3")
214 self.assertEqual({'args': [], 'configs': []},
215 res['job_config'])
216
217 res = provider.get_edp_config_hints(edp.JOB_TYPE_SHELL, "2.3")
218 self.assertEqual({'args': [], 'configs': {}, 'params': {}},
219 res['job_config'])