Merge "Updating Spark versions"

This commit is contained in:
Zuul 2018-07-12 00:33:16 +00:00 committed by Gerrit Code Review
commit cd377cfbd8
3 changed files with 21 additions and 3 deletions

View File

@ -26,7 +26,7 @@ from sahara.service.edp.spark import engine as edp_engine
class EdpEngine(edp_engine.SparkJobEngine):
edp_base_version = "1.3.1"
edp_base_version = "1.6.0"
def __init__(self, cluster):
super(EdpEngine, self).__init__(cluster)

View File

@ -63,7 +63,7 @@ class SparkProvider(p.ProvisioningPluginBase):
deprecated = {'enabled': {'status': True},
'deprecated': {'status': True}}
result = {'plugin_labels': copy.deepcopy(default)}
stable_versions = ['2.2']
stable_versions = ['2.3', '2.2']
result['version_labels'] = {
version: copy.deepcopy(
default if version in stable_versions else deprecated
@ -72,7 +72,7 @@ class SparkProvider(p.ProvisioningPluginBase):
return result
def get_versions(self):
return ['2.2', '2.1.0', '1.6.0']
return ['2.3', '2.2', '2.1.0', '1.6.0']
def get_configs(self, hadoop_version):
return c_helper.get_plugin_configs()

View File

@ -68,6 +68,14 @@ class SparkPluginTest(base.SaharaWithDbTestCase):
self._test_engine('2.2', edp.JOB_TYPE_SHELL,
engine.SparkShellJobEngine)
def test_plugin23_edp_engine(self):
self._test_engine('2.3', edp.JOB_TYPE_SPARK,
engine.SparkJobEngine)
def test_plugin23_shell_engine(self):
self._test_engine('2.3', edp.JOB_TYPE_SHELL,
engine.SparkShellJobEngine)
def _test_engine(self, version, job_type, eng):
cluster_dict = self._init_cluster_dict(version)
@ -172,6 +180,8 @@ class SparkProviderTest(base.SaharaTestCase):
res['2.1.0'])
self.assertEqual([edp.JOB_TYPE_SHELL, edp.JOB_TYPE_SPARK],
res['2.2'])
self.assertEqual([edp.JOB_TYPE_SHELL, edp.JOB_TYPE_SPARK],
res['2.3'])
def test_edp_config_hints(self):
provider = pl.SparkProvider()
@ -199,3 +209,11 @@ class SparkProviderTest(base.SaharaTestCase):
res = provider.get_edp_config_hints(edp.JOB_TYPE_SHELL, "2.2")
self.assertEqual({'args': [], 'configs': {}, 'params': {}},
res['job_config'])
res = provider.get_edp_config_hints(edp.JOB_TYPE_SPARK, "2.3")
self.assertEqual({'args': [], 'configs': []},
res['job_config'])
res = provider.get_edp_config_hints(edp.JOB_TYPE_SHELL, "2.3")
self.assertEqual({'args': [], 'configs': {}, 'params': {}},
res['job_config'])