openstack
/
sahara
Code Issues Proposed changes

Upgrading Spark to version 2.2 

Adding newest version of Spark to sahara

Change-Id: Ia85e6f28d90789279fd319074ce0e230ffa98be4
This commit is contained in:
Telles Nobrega 2017-12-06 11:25:03 -03:00 committed by Telles Mota Vidal Nóbrega
parent bc2f5f03e2
commit 7140495c70
3 changed files with 25 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
releasenotes/notes/spark-2.2-d7c3a84bd52f735a.yaml Normal file
View File

@ -0,0 +1,3 @@
---
features:
- Adding Spark version 2.2 to Sahara.

4
sahara/plugins/spark/plugin.py
View File

@ -63,7 +63,7 @@ class SparkProvider(p.ProvisioningPluginBase):
deprecated = {'enabled': {'status': True},
'deprecated': {'status': True}}
result = {'plugin_labels': copy.deepcopy(default)}
stable_versions = ['2.1.0', '1.6.0']
stable_versions = ['2.2', '2.1.0', '1.6.0']
result['version_labels'] = {
version: copy.deepcopy(
default if version in stable_versions else deprecated
@ -72,7 +72,7 @@ class SparkProvider(p.ProvisioningPluginBase):
return result
def get_versions(self):
return ['2.1.0', '1.6.0', '1.3.1']
return ['2.2', '2.1.0', '1.6.0', '1.3.1']
def get_configs(self, hadoop_version):
return c_helper.get_plugin_configs()

22
sahara/tests/unit/plugins/spark/test_plugin.py
View File

@ -80,10 +80,18 @@ class SparkPluginTest(base.SaharaWithDbTestCase):
self._test_engine('2.1.0', edp.JOB_TYPE_SPARK,
engine.SparkJobEngine)
def test_plugin22_shell_engine(self):
def test_plugin21_shell_engine(self):
self._test_engine('2.1.0', edp.JOB_TYPE_SHELL,
engine.SparkShellJobEngine)
def test_plugin22_edp_engine(self):
self._test_engine('2.2', edp.JOB_TYPE_SPARK,
engine.SparkJobEngine)
def test_plugin22_shell_engine(self):
self._test_engine('2.2', edp.JOB_TYPE_SHELL,
engine.SparkShellJobEngine)
def _test_engine(self, version, job_type, eng):
cluster_dict = self._init_cluster_dict(version)
@ -201,6 +209,8 @@ class SparkProviderTest(base.SaharaTestCase):
res['1.6.0'])
self.assertEqual([edp.JOB_TYPE_SHELL, edp.JOB_TYPE_SPARK],
res['2.1.0'])
self.assertEqual([edp.JOB_TYPE_SHELL, edp.JOB_TYPE_SPARK],
res['2.2'])
def test_edp_config_hints(self):
provider = pl.SparkProvider()
@ -225,6 +235,14 @@ class SparkProviderTest(base.SaharaTestCase):
self.assertEqual({'args': [], 'configs': []},
res['job_config'])
res = provider.get_edp_config_hints(edp.JOB_TYPE_SPARK, "2.1.0")
res = provider.get_edp_config_hints(edp.JOB_TYPE_SHELL, "2.1.0")
self.assertEqual({'args': [], 'configs': {}, 'params': {}},
res['job_config'])
res = provider.get_edp_config_hints(edp.JOB_TYPE_SPARK, "2.2")
self.assertEqual({'args': [], 'configs': []},
res['job_config'])
res = provider.get_edp_config_hints(edp.JOB_TYPE_SHELL, "2.2")
self.assertEqual({'args': [], 'configs': {}, 'params': {}},
res['job_config'])