diff --git a/sahara/tests/unit/plugins/cdh/base_config_helper_test.py b/sahara/tests/unit/plugins/cdh/base_config_helper_test.py new file mode 100644 index 0000000000..076e808a24 --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/base_config_helper_test.py @@ -0,0 +1,187 @@ +# Copyright (c) 2016 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_serialization import jsonutils as json + +from sahara.tests.unit import base +from sahara.tests.unit.plugins.cdh import utils as ctu +from sahara.utils import files as f + + +json_files = [ + 'hdfs-service.json', + 'hdfs-namenode.json', + 'hdfs-datanode.json', + 'hdfs-secondarynamenode.json', + 'hdfs-gateway.json', + 'hdfs-journalnode.json', + 'yarn-service.json', + 'yarn-resourcemanager.json', + 'yarn-nodemanager.json', + 'yarn-jobhistory.json', + 'yarn-gateway.json', + 'oozie-service.json', + 'oozie-oozie_server.json', + 'hive-service.json', + 'hive-hivemetastore.json', + 'hive-hiveserver2.json', + 'hive-webhcat.json', + 'hue-service.json', + 'hue-hue_server.json', + 'spark-service.json', + 'spark-spark_yarn_history_server.json', + 'zookeeper-service.json', + 'zookeeper-server.json', + 'hbase-service.json', + 'hbase-master.json', + 'hbase-regionserver.json', + 'flume-service.json', + 'flume-agent.json', + 'sentry-service.json', + 'sentry-sentry_server.json', + 'solr-service.json', + 'solr-solr_server.json', + 'sqoop-service.json', + 'sqoop-sqoop_server.json', + 'ks_indexer-service.json', + 'ks_indexer-hbase_indexer.json', + 'impala-service.json', + 'impala-catalogserver.json', + 'impala-impalad.json', + 'impala-statestore.json', + 'kms-service.json', + 'kms-kms.json', + 'kafka-kafka_broker.json', + 'kafka-kafka_mirror_maker.json', + 'kafka-service.json' +] + + +class TestConfigHelper(base.SaharaTestCase): + + def setUp(self): + super(TestConfigHelper, self).setUp() + self.path_to_config = None + self.c_h = None + + def test_get_ng_plugin_configs(self): + actual_configs = self.c_h._get_ng_plugin_configs() + + expected_configs = [] + for json_file in json_files: + expected_configs += json.loads( + f.get_file_text(self.path_to_config + json_file)) + + # compare names + expected_names = set(i['name'] for i in expected_configs) + actual_names = set(i.to_dict()['name'] for i in actual_configs) + self.assertEqual(expected_names, actual_names) + + def test_get_cdh5_repo_url(self): + cluster = ctu.get_fake_cluster(cluster_configs={}) + self.assertEqual(self.c_h.CDH5_REPO_URL.default_value, + self.c_h.get_cdh5_repo_url(cluster)) + + cluster = ctu.get_fake_cluster( + cluster_configs={ + 'general': {self.c_h.CDH5_REPO_URL.name: 'spam'}}) + self.assertEqual('spam', self.c_h.get_cdh5_repo_url(cluster)) + + def test_get_cdh5_key_url(self): + cluster = ctu.get_fake_cluster(cluster_configs={}) + self.assertEqual(self.c_h.CDH5_REPO_KEY_URL.default_value, + self.c_h.get_cdh5_key_url(cluster)) + + cluster = ctu.get_fake_cluster( + cluster_configs={ + 'general': {self.c_h.CDH5_REPO_KEY_URL.name: 'spam'}}) + self.assertEqual('spam', self.c_h.get_cdh5_key_url(cluster)) + + def test_get_cm5_repo_url(self): + cluster = ctu.get_fake_cluster(cluster_configs={}) + self.assertEqual(self.c_h.CM5_REPO_URL.default_value, + self.c_h.get_cm5_repo_url(cluster)) + + cluster = ctu.get_fake_cluster( + cluster_configs={ + 'general': {self.c_h.CM5_REPO_URL.name: 'spam'}}) + self.assertEqual('spam', self.c_h.get_cm5_repo_url(cluster)) + + def test_get_cm5_key_url(self): + cluster = ctu.get_fake_cluster(cluster_configs={}) + self.assertEqual(self.c_h.CM5_REPO_KEY_URL.default_value, + self.c_h.get_cm5_key_url(cluster)) + + cluster = ctu.get_fake_cluster( + cluster_configs={ + 'general': {self.c_h.CM5_REPO_KEY_URL.name: 'spam'}}) + self.assertEqual('spam', self.c_h.get_cm5_key_url(cluster)) + + def test_is_swift_enabled(self): + cluster = ctu.get_fake_cluster(cluster_configs={}) + self.assertTrue(self.c_h.is_swift_enabled(cluster)) + + cluster = ctu.get_fake_cluster( + cluster_configs={ + 'general': {self.c_h.ENABLE_SWIFT.name: False}}) + self.assertFalse(self.c_h.is_swift_enabled(cluster)) + + def test_get_swift_lib_url(self): + cluster = ctu.get_fake_cluster(cluster_configs={}) + self.assertEqual(self.c_h.DEFAULT_SWIFT_LIB_URL, + self.c_h.get_swift_lib_url(cluster)) + + cluster = ctu.get_fake_cluster( + cluster_configs={ + 'general': {self.c_h.SWIFT_LIB_URL.name: 'spam'}}) + self.assertEqual('spam', self.c_h.get_swift_lib_url(cluster)) + + def test_is_hbase_common_lib_enabled(self): + cluster = ctu.get_fake_cluster(cluster_configs={}) + self.assertTrue(self.c_h.is_hbase_common_lib_enabled(cluster)) + + cluster = ctu.get_fake_cluster( + cluster_configs={ + 'general': {self.c_h.ENABLE_HBASE_COMMON_LIB.name: False}}) + self.assertFalse(self.c_h.is_hbase_common_lib_enabled(cluster)) + + def test_get_extjs_lib_url(self): + cluster = ctu.get_fake_cluster(cluster_configs={}) + self.assertEqual(self.c_h.DEFAULT_EXTJS_LIB_URL, + self.c_h.get_extjs_lib_url(cluster)) + + cluster = ctu.get_fake_cluster( + cluster_configs={ + 'general': {self.c_h.EXTJS_LIB_URL.name: 'spam'}}) + self.assertEqual('spam', self.c_h.get_extjs_lib_url(cluster)) + + def test_get_kms_key_url(self): + cluster = ctu.get_fake_cluster(cluster_configs={}) + self.assertEqual(self.c_h.KMS_REPO_KEY_URL.default_value, + self.c_h.get_kms_key_url(cluster)) + + cluster = ctu.get_fake_cluster( + cluster_configs={ + 'general': {self.c_h.KMS_REPO_KEY_URL.name: 'spam'}}) + self.assertEqual('spam', self.c_h.get_kms_key_url(cluster)) + + def test_get_required_anti_affinity(self): + cluster = ctu.get_fake_cluster(cluster_configs={}) + self.assertTrue(self.c_h.get_required_anti_affinity(cluster)) + + cluster = ctu.get_fake_cluster( + cluster_configs={ + 'general': {self.c_h.REQUIRE_ANTI_AFFINITY.name: False}}) + self.assertFalse(self.c_h.get_required_anti_affinity(cluster)) diff --git a/sahara/tests/unit/plugins/cdh/base_validation_tests.py b/sahara/tests/unit/plugins/cdh/base_validation_tests.py index a6842be316..bcb438f58e 100644 --- a/sahara/tests/unit/plugins/cdh/base_validation_tests.py +++ b/sahara/tests/unit/plugins/cdh/base_validation_tests.py @@ -23,6 +23,9 @@ from sahara.tests.unit import testutils as tu icce = ex.InvalidComponentCountException rsme = ex.RequiredServiceMissingException +icte = ex.InvalidClusterTopology +nnce = ex.NameNodeHAConfigurationError +rmce = ex.ResourceManagerHAConfigurationError def make_ng_dict_with_inst(counter, name, flavor, @@ -169,6 +172,94 @@ class BaseValidationTestCase(base.SaharaTestCase): [None, {}] ] + disable_anti_affinity = {'cluster_configs': {'general': { + 'Require Anti Affinity': False}}} + cases += [ + [None, {'HDFS_JOURNALNODE': 3, + 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], + [rsme, {'HDFS_JOURNALNODE': 3, + 'ZOOKEEPER_SERVER': 0}, [], disable_anti_affinity], + [icce, {'HDFS_JOURNALNODE': 4, + 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], + [icce, {'HDFS_JOURNALNODE': 2, + 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], + [nnce, {'HDFS_JOURNALNODE': 3, 'ZOOKEEPER_SERVER': 1}, [], + {'anti_affinity': ['HDFS_SECONDARYNAMENODE']}], + [nnce, {'HDFS_JOURNALNODE': 3, 'ZOOKEEPER_SERVER': 1}, [], + {'anti_affinity': ['HDFS_NAMENODE']}], + [None, {'HDFS_JOURNALNODE': 3, 'ZOOKEEPER_SERVER': 1}, [], + {'anti_affinity': ['HDFS_NAMENODE', 'HDFS_SECONDARYNAMENODE']}], + [None, {'YARN_STANDBYRM': 1, + 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], + [icce, {'YARN_STANDBYRM': 2, + 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], + [rsme, {'YARN_STANDBYRM': 1, + 'ZOOKEEPER_SERVER': 0}, [], disable_anti_affinity], + [rmce, {'YARN_STANDBYRM': 1, 'ZOOKEEPER_SERVER': 1}, [], + {'anti_affinity': ['YARN_RESOURCEMANAGER']}], + [rmce, {'YARN_STANDBYRM': 1, 'ZOOKEEPER_SERVER': 1}, [], + {'anti_affinity': ['YARN_STANDBYRM']}], + [None, {'YARN_STANDBYRM': 1, 'ZOOKEEPER_SERVER': 1}, [], + {'anti_affinity': ['YARN_STANDBYRM', 'YARN_RESOURCEMANAGER']}], + ] + + cases += [ + [None, {'FLUME_AGENT': 1}], + [icce, {'ZOOKEEPER_SERVER': 1, 'SENTRY_SERVER': 2}], + [None, {'ZOOKEEPER_SERVER': 1, 'SENTRY_SERVER': 1}], + [rsme, {'ZOOKEEPER_SERVER': 0, 'SENTRY_SERVER': 1}], + [None, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1}], + [rsme, {'ZOOKEEPER_SERVER': 0, 'SOLR_SERVER': 1}], + [None, {'YARN_NODEMANAGER': 1, 'YARN_JOBHISTORY': 1, + 'SQOOP_SERVER': 1}], + [rsme, {'YARN_NODEMANAGER': 0, 'YARN_JOBHISTORY': 1, + 'SQOOP_SERVER': 1}], + [rsme, {'YARN_NODEMANAGER': 1, 'YARN_JOBHISTORY': 0, + 'SQOOP_SERVER': 1}], + # HBASE_MASTER AND HBASE_REGIONSERVER depend circularly + [None, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1, + 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, + 'HBASE_REGIONSERVER': 1}], + [rsme, {'ZOOKEEPER_SERVER': 0, 'SOLR_SERVER': 1, + 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, + 'HBASE_REGIONSERVER': 1}], + [rsme, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 0, + 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, + 'HBASE_REGIONSERVER': 1}], + [rsme, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1, + 'HBASE_MASTER': 0, 'HBASE_INDEXER': 1}], + ] + + worker_with_implama = ('worker_ng', 1, ['HDFS_DATANODE', + 'YARN_NODEMANAGER', + 'IMPALAD'], 3) + cases += [ + [None, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, + 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1, + 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, + [worker_with_implama]], + [icte, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, + 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1}, + []], + [icte, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, + 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1}, + [worker_with_implama]], + [rsme, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 0, + 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1, + 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, + [worker_with_implama]], + [rsme, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, + 'HIVE_METASTORE': 0, 'HIVE_SERVER2': 1, + 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, + [worker_with_implama]] + + ] + + cases += [ + [None, {'KMS': 1}], + [icce, {'KMS': 2}] + ] + return cases def test_validate_cluster_creating(self): diff --git a/sahara/tests/unit/plugins/cdh/v5_5_0/test_config_helper.py b/sahara/tests/unit/plugins/cdh/v5_5_0/test_config_helper.py new file mode 100644 index 0000000000..9348d0c623 --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/v5_5_0/test_config_helper.py @@ -0,0 +1,25 @@ +# Copyright (c) 2016 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.plugins.cdh.v5_5_0 import config_helper +from sahara.tests.unit.plugins.cdh import base_config_helper_test as bcht + + +class TestConfigHelperV550(bcht.TestConfigHelper): + + def setUp(self): + super(TestConfigHelperV550, self).setUp() + self.c_h = config_helper.ConfigHelperV550() + self.path_to_config = 'plugins/cdh/v5_5_0/resources/' diff --git a/sahara/tests/unit/plugins/cdh/v5_5_0/test_plugin_utils.py b/sahara/tests/unit/plugins/cdh/v5_5_0/test_plugin_utils.py new file mode 100644 index 0000000000..29e4c37935 --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/v5_5_0/test_plugin_utils.py @@ -0,0 +1,25 @@ +# Copyright (c) 2015 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.plugins.cdh.v5_5_0 import plugin_utils as pu +from sahara.tests.unit.plugins.cdh import base_plugin_utils_test + + +class TestPluginUtilsV550(base_plugin_utils_test.TestPluginUtilsHigherThanV5): + + def setUp(self): + super(TestPluginUtilsV550, self).setUp() + self.plug_utils = pu.PluginUtilsV550() + self.version = "v5_5_0" diff --git a/sahara/tests/unit/plugins/cdh/v5_5_0/test_validation.py b/sahara/tests/unit/plugins/cdh/v5_5_0/test_validation.py index c832b4e9e1..133fcc4494 100644 --- a/sahara/tests/unit/plugins/cdh/v5_5_0/test_validation.py +++ b/sahara/tests/unit/plugins/cdh/v5_5_0/test_validation.py @@ -14,110 +14,11 @@ # limitations under the License. from sahara.plugins.cdh.v5_5_0 import validation -from sahara.plugins import exceptions as ex from sahara.tests.unit.plugins.cdh import base_validation_tests as bvt -icte = ex.InvalidClusterTopology -nnce = ex.NameNodeHAConfigurationError -icce = ex.InvalidComponentCountException -rsme = ex.RequiredServiceMissingException -rmce = ex.ResourceManagerHAConfigurationError - class ValidationTestCase(bvt.BaseValidationTestCase): def setUp(self): super(ValidationTestCase, self).setUp() self.module = validation.ValidatorV550 - - def _get_test_cases(self): - cases = super(ValidationTestCase, self)._get_test_cases() - - disable_anti_affinity = {'cluster_configs': {'general': { - 'Require Anti Affinity': False}}} - cases += [ - [None, {'HDFS_JOURNALNODE': 3, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [rsme, {'HDFS_JOURNALNODE': 3, - 'ZOOKEEPER_SERVER': 0}, [], disable_anti_affinity], - [icce, {'HDFS_JOURNALNODE': 4, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [icce, {'HDFS_JOURNALNODE': 2, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [nnce, {'HDFS_JOURNALNODE': 3, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['HDFS_SECONDARYNAMENODE']}], - [nnce, {'HDFS_JOURNALNODE': 3, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['HDFS_NAMENODE']}], - [None, {'HDFS_JOURNALNODE': 3, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['HDFS_NAMENODE', 'HDFS_SECONDARYNAMENODE']}], - [None, {'YARN_STANDBYRM': 1, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [icce, {'YARN_STANDBYRM': 2, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [rsme, {'YARN_STANDBYRM': 1, - 'ZOOKEEPER_SERVER': 0}, [], disable_anti_affinity], - [rmce, {'YARN_STANDBYRM': 1, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['YARN_RESOURCEMANAGER']}], - [rmce, {'YARN_STANDBYRM': 1, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['YARN_STANDBYRM']}], - [None, {'YARN_STANDBYRM': 1, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['YARN_STANDBYRM', 'YARN_RESOURCEMANAGER']}], - ] - - cases += [ - [None, {'FLUME_AGENT': 1}], - [icce, {'ZOOKEEPER_SERVER': 1, 'SENTRY_SERVER': 2}], - [None, {'ZOOKEEPER_SERVER': 1, 'SENTRY_SERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 0, 'SENTRY_SERVER': 1}], - [None, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 0, 'SOLR_SERVER': 1}], - [None, {'YARN_NODEMANAGER': 1, 'YARN_JOBHISTORY': 1, - 'SQOOP_SERVER': 1}], - [rsme, {'YARN_NODEMANAGER': 0, 'YARN_JOBHISTORY': 1, - 'SQOOP_SERVER': 1}], - [rsme, {'YARN_NODEMANAGER': 1, 'YARN_JOBHISTORY': 0, - 'SQOOP_SERVER': 1}], - # HBASE_MASTER AND HBASE_REGIONSERVER depend circularly - [None, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1, - 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, - 'HBASE_REGIONSERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 0, 'SOLR_SERVER': 1, - 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, - 'HBASE_REGIONSERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 0, - 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, - 'HBASE_REGIONSERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1, - 'HBASE_MASTER': 0, 'HBASE_INDEXER': 1}], - ] - - worker_with_implama = ('worker_ng', 1, ['HDFS_DATANODE', - 'YARN_NODEMANAGER', - 'IMPALAD'], 3) - cases += [ - [None, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1, - 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, - [worker_with_implama]], - [icte, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1}, - []], - [icte, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1}, - [worker_with_implama]], - [rsme, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 0, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1, - 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, - [worker_with_implama]], - [rsme, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 0, 'HIVE_SERVER2': 1, - 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, - [worker_with_implama]] - - ] - - cases += [ - [None, {'KMS': 1}], - [icce, {'KMS': 2}] - ] - return cases diff --git a/sahara/tests/unit/plugins/cdh/v5_7_0/test_config_helper.py b/sahara/tests/unit/plugins/cdh/v5_7_0/test_config_helper.py new file mode 100644 index 0000000000..52b96ac4a3 --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/v5_7_0/test_config_helper.py @@ -0,0 +1,25 @@ +# Copyright (c) 2016 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.plugins.cdh.v5_7_0 import config_helper +from sahara.tests.unit.plugins.cdh import base_config_helper_test as bcht + + +class TestConfigHelperV570(bcht.TestConfigHelper): + + def setUp(self): + super(TestConfigHelperV570, self).setUp() + self.c_h = config_helper.ConfigHelperV570() + self.path_to_config = 'plugins/cdh/v5_7_0/resources/' diff --git a/sahara/tests/unit/plugins/cdh/v5_7_0/test_deploy.py b/sahara/tests/unit/plugins/cdh/v5_7_0/test_deploy.py new file mode 100644 index 0000000000..62ecdff6a7 --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/v5_7_0/test_deploy.py @@ -0,0 +1,222 @@ +# Copyright (c) 2016 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from sahara.plugins.cdh.v5_7_0 import deploy +from sahara.tests.unit import base + + +class DeployCDHV570(base.SaharaTestCase): + + def setUp(self): + super(DeployCDHV570, self).setUp() + self.master = mock.MagicMock() + self.master.node_group.node_processes = [ + "HDFS_NAMENODE", "YARN_RESOURCEMANAGER", "CLOUDERA_MANAGER", + "SENTRY_SERVER", "YARN_NODEMANAGER", "ZOOKEEPER_SERVER", + "OOZIE_SERVER", "YARN_JOBHISTORY", "HDFS_SECONDARYNAMENODE", + "HIVE_METASTORE", "HIVE_SERVER2", "SPARK_YARN_HISTORY_SERVER", + "HBASE_MASTER", "HBASE_REGIONSERVER", "HUE_SERVER", "KMS", + "FLUME_AGENT", "SOLR_SERVER", "SQOOP_SERVER", "IMPALA_STATESTORE", + "IMPALA_CATALOGSERVER", "IMPALAD", "KEY_VALUE_STORE_INDEXER", + ] + self.worker = mock.MagicMock() + self.worker.node_group.node_processes = [ + "HDFS_DATANODE", "HDFS_JOURNALNODE", "JOURNALNODE", + "YARN_NODEMANAGER", "YARN_STANDBYRM", + ] + self.instances = [self.master, self.worker] + self.cluster = mock.MagicMock() + + self.is_cdh_exists = mock.patch( + "sahara.plugins.cdh.commands.is_pre_installed_cdh", + return_value=False) + self.is_cdh_exists.start() + self._create_facade = mock.patch( + "sahara.db.sqlalchemy.api._create_facade_lazily") + self._create_facade.start() + + def tearDown(self): + self.is_cdh_exists.stop() + self._create_facade.stop() + super(DeployCDHV570, self).tearDown() + + @mock.patch("sahara.plugins.utils.get_instances") + @mock.patch("sahara.plugins.cdh.v5_7_0.deploy.CU") + def test_configure_cluster(self, mock_cu, mock_get_instances): + mock_get_instances.return_value = self.instances + + deploy.configure_cluster(self.cluster) + + mock_cu.pu.configure_os.assert_called_once_with(self.instances) + mock_cu.pu.install_packages.assert_called_once_with(self.instances, + deploy.PACKAGES) + mock_cu.pu.start_cloudera_agents.assert_called_once_with( + self.instances) + mock_cu.pu.start_cloudera_manager.assert_called_once_with(self.cluster) + mock_cu.update_cloudera_password.assert_called_once_with(self.cluster) + mock_cu.await_agents.assert_called_once_with(self.cluster, + self.instances) + mock_cu.create_mgmt_service.assert_called_once_with(self.cluster) + mock_cu.create_services.assert_called_once_with(self.cluster) + mock_cu.configure_services.assert_called_once_with(self.cluster) + mock_cu.configure_instances.assert_called_once_with(self.instances, + self.cluster) + mock_cu.deploy_configs.assert_called_once_with(self.cluster) + + @mock.patch("sahara.plugins.cdh.v5_7_0.deploy.CU") + def test__start_roles(self, mock_cu): + + deploy._start_roles(self.cluster, self.instances) + + mock_cu.get_service_by_role.assert_any_call('DATANODE', + instance=self.worker) + mock_cu.get_service_by_role.assert_any_call('NODEMANAGER', + instance=self.master) + mock_cu.get_service_by_role.assert_any_call('NODEMANAGER', + instance=self.worker) + self.assertEqual(mock_cu.start_roles.call_count, 3) + + @mock.patch("sahara.plugins.cdh.v5_7_0.deploy._start_roles") + @mock.patch("sahara.plugins.cdh.v5_7_0.deploy.CU") + def test_scale_cluster(self, mock_cu, mock__start_roles): + deploy.scale_cluster(self.cluster, None) + self.assertEqual(mock_cu.call_count, 0) + self.assertEqual(mock__start_roles.call_count, 0) + + deploy.scale_cluster(self.cluster, self.instances) + + mock_cu.pu.configure_os.assert_called_once_with(self.instances) + mock_cu.pu.install_packages.assert_called_once_with(self.instances, + deploy.PACKAGES) + mock_cu.pu.start_cloudera_agents.assert_called_once_with( + self.instances) + mock_cu.await_agents.assert_called_once_with(self.cluster, + self.instances) + mock_cu.configure_instances.assert_called_once_with(self.instances, + self.cluster) + mock_cu.update_configs.assert_called_once_with(self.instances) + mock_cu.pu.configure_swift.assert_called_once_with(self.cluster, + self.instances) + mock_cu.refresh_datanodes.assert_called_once_with(self.cluster) + mock__start_roles.assert_called_once_with(self.cluster, + self.instances) + + @mock.patch("sahara.plugins.cdh.v5_7_0.deploy.CU") + def test_decommission_cluster(self, mock_cu): + deploy.decommission_cluster(self.cluster, self.instances) + dns = [] + dns_2 = [] + nms = [] + nms_2 = [] + for i in self.instances: + if 'HDFS_DATANODE' in i.node_group.node_processes: + dns.append(mock_cu.pu.get_role_name(i, 'DATANODE')) + dns_2.append(mock_cu.pu.get_role_name(i, 'HDFS_GATEWAY')) + if 'YARN_NODEMANAGER' in i.node_group.node_processes: + nms.append(mock_cu.pu.get_role_name(i, 'NODEMANAGER')) + nms_2.append(mock_cu.pu.get_role_name(i, 'YARN_GATEWAY')) + mock_cu.decommission_nodes.assert_any_call( + self.cluster, 'DATANODE', dns, dns_2) + mock_cu.decommission_nodes.assert_any_call( + self.cluster, 'NODEMANAGER', nms, nms_2) + mock_cu.delete_instances.assert_called_once_with(self.cluster, + self.instances) + mock_cu.refresh_datanodes.assert_called_once_with(self.cluster) + mock_cu.refresh_yarn_nodes.assert_called_once_with(self.cluster) + + @mock.patch("sahara.plugins.cdh.v5_7_0.deploy.CU") + def test__prepare_cluster(self, mock_cu): + deploy._prepare_cluster(self.cluster) + + mock_cu.pu.install_extjs.assert_called_once_with(self.cluster) + mock_cu.pu.configure_hive.assert_called_once_with(self.cluster) + mock_cu.pu.configure_sentry.assert_called_once_with(self.cluster) + + @mock.patch("sahara.service.edp.hdfs_helper.create_hbase_common_lib") + @mock.patch("sahara.plugins.cdh.v5_7_0.deploy.CU") + def test__finish_cluster_starting(self, mock_cu, mock_create_hbase): + deploy._finish_cluster_starting(self.cluster) + mock_cu.pu.put_hive_hdfs_xml.assert_called_once_with(self.cluster) + self.assertTrue(mock_create_hbase.called) + mock_cu.start_service.assert_called_once_with( + mock_cu.get_service_by_role('AGENT', self.cluster)) + + @mock.patch("sahara.plugins.cdh.v5_7_0.deploy._finish_cluster_starting") + @mock.patch("sahara.plugins.cdh.v5_7_0.deploy._prepare_cluster") + @mock.patch("sahara.plugins.cdh.v5_7_0.deploy.CU") + def test_start_cluster(self, mock_cu, mock_prepare, mock_finish): + jns_count = 0 + for i in self.instances: + if "HDFS_JOURNALNODE" in i.node_group.node_processes: + jns_count += 1 + mock_cu.pu.get_jns.return_value.__len__.return_value = jns_count + + deploy.start_cluster(self.cluster) + + mock_prepare.assert_called_once_with(self.cluster) + mock_cu.first_run.assert_called_once_with(self.cluster) + mock_cu.pu.configure_swift.assert_called_once_with(self.cluster) + if jns_count > 0: + mock_cu.enable_namenode_ha.assert_called_once_with(self.cluster) + mock_cu.update_role_config.assert_any_call( + mock_cu.pu.get_secondarynamenode(self.cluster), + 'HDFS_NAMENODE' + ) + mock_cu.enable_resourcemanager_ha.assert_called_once_with(self.cluster) + mock_cu.update_role_config.assert_any_call( + mock_cu.pu.get_stdb_rm(self.cluster), 'YARN_STANDBYRM') + mock_finish.assert_called_once_with(self.cluster) + + def test_get_open_ports(self): + master_ports = [ + 9000, + 7180, 7182, 7183, 7432, 7184, 8084, 8086, 10101, + 9997, 9996, 8087, 9998, 9999, 8085, 9995, 9994, + 8020, 8022, 50070, 50470, + 50090, 50495, + 8030, 8031, 8032, 8033, 8088, + 8040, 8041, 8042, + 10020, 19888, + 9083, + 10000, + 8888, + 11000, 11001, + 18088, + 2181, 3181, 4181, 9010, + 60000, + 60020, + 41414, + 8038, + 8983, 8984, + 8005, 12000, + 25020, 26000, + 25010, 24000, + 21050, 21000, 23000, 25000, 28000, 22000, + 16000, 16001 + ] + deploy.get_open_ports(self.master.node_group) + self.assertItemsEqual(master_ports, + deploy.get_open_ports(self.master.node_group)) + worker_ports = [ + 9000, + 50010, 1004, 50075, 1006, 50020, + 8480, 8481, 8485, + 8040, 8041, 8042, + 8030, 8031, 8032, 8033, 8088 + ] + self.assertItemsEqual(worker_ports, + deploy.get_open_ports(self.worker.node_group)) diff --git a/sahara/tests/unit/plugins/cdh/v5_7_0/test_plugin_utils.py b/sahara/tests/unit/plugins/cdh/v5_7_0/test_plugin_utils.py new file mode 100644 index 0000000000..0a023422c7 --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/v5_7_0/test_plugin_utils.py @@ -0,0 +1,25 @@ +# Copyright (c) 2015 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.plugins.cdh.v5_7_0 import plugin_utils as pu +from sahara.tests.unit.plugins.cdh import base_plugin_utils_test + + +class TestPluginUtilsV570(base_plugin_utils_test.TestPluginUtilsHigherThanV5): + + def setUp(self): + super(TestPluginUtilsV570, self).setUp() + self.plug_utils = pu.PluginUtilsV570() + self.version = "v5_7_0" diff --git a/sahara/tests/unit/plugins/cdh/v5_7_0/test_validation.py b/sahara/tests/unit/plugins/cdh/v5_7_0/test_validation.py index bbeb2ccc07..845fa271e7 100644 --- a/sahara/tests/unit/plugins/cdh/v5_7_0/test_validation.py +++ b/sahara/tests/unit/plugins/cdh/v5_7_0/test_validation.py @@ -14,110 +14,11 @@ # limitations under the License. from sahara.plugins.cdh.v5_7_0 import validation -from sahara.plugins import exceptions as ex from sahara.tests.unit.plugins.cdh import base_validation_tests as bvt -icte = ex.InvalidClusterTopology -nnce = ex.NameNodeHAConfigurationError -icce = ex.InvalidComponentCountException -rsme = ex.RequiredServiceMissingException -rmce = ex.ResourceManagerHAConfigurationError - class ValidationTestCase(bvt.BaseValidationTestCase): def setUp(self): super(ValidationTestCase, self).setUp() self.module = validation.ValidatorV570 - - def _get_test_cases(self): - cases = super(ValidationTestCase, self)._get_test_cases() - - disable_anti_affinity = {'cluster_configs': {'general': { - 'Require Anti Affinity': False}}} - cases += [ - [None, {'HDFS_JOURNALNODE': 3, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [rsme, {'HDFS_JOURNALNODE': 3, - 'ZOOKEEPER_SERVER': 0}, [], disable_anti_affinity], - [icce, {'HDFS_JOURNALNODE': 4, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [icce, {'HDFS_JOURNALNODE': 2, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [nnce, {'HDFS_JOURNALNODE': 3, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['HDFS_SECONDARYNAMENODE']}], - [nnce, {'HDFS_JOURNALNODE': 3, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['HDFS_NAMENODE']}], - [None, {'HDFS_JOURNALNODE': 3, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['HDFS_NAMENODE', 'HDFS_SECONDARYNAMENODE']}], - [None, {'YARN_STANDBYRM': 1, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [icce, {'YARN_STANDBYRM': 2, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [rsme, {'YARN_STANDBYRM': 1, - 'ZOOKEEPER_SERVER': 0}, [], disable_anti_affinity], - [rmce, {'YARN_STANDBYRM': 1, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['YARN_RESOURCEMANAGER']}], - [rmce, {'YARN_STANDBYRM': 1, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['YARN_STANDBYRM']}], - [None, {'YARN_STANDBYRM': 1, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['YARN_STANDBYRM', 'YARN_RESOURCEMANAGER']}], - ] - - cases += [ - [None, {'FLUME_AGENT': 1}], - [icce, {'ZOOKEEPER_SERVER': 1, 'SENTRY_SERVER': 2}], - [None, {'ZOOKEEPER_SERVER': 1, 'SENTRY_SERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 0, 'SENTRY_SERVER': 1}], - [None, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 0, 'SOLR_SERVER': 1}], - [None, {'YARN_NODEMANAGER': 1, 'YARN_JOBHISTORY': 1, - 'SQOOP_SERVER': 1}], - [rsme, {'YARN_NODEMANAGER': 0, 'YARN_JOBHISTORY': 1, - 'SQOOP_SERVER': 1}], - [rsme, {'YARN_NODEMANAGER': 1, 'YARN_JOBHISTORY': 0, - 'SQOOP_SERVER': 1}], - # HBASE_MASTER AND HBASE_REGIONSERVER depend circularly - [None, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1, - 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, - 'HBASE_REGIONSERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 0, 'SOLR_SERVER': 1, - 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, - 'HBASE_REGIONSERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 0, - 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, - 'HBASE_REGIONSERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1, - 'HBASE_MASTER': 0, 'HBASE_INDEXER': 1}], - ] - - worker_with_implama = ('worker_ng', 1, ['HDFS_DATANODE', - 'YARN_NODEMANAGER', - 'IMPALAD'], 3) - cases += [ - [None, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1, - 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, - [worker_with_implama]], - [icte, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1}, - []], - [icte, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1}, - [worker_with_implama]], - [rsme, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 0, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1, - 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, - [worker_with_implama]], - [rsme, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 0, 'HIVE_SERVER2': 1, - 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, - [worker_with_implama]] - - ] - - cases += [ - [None, {'KMS': 1}], - [icce, {'KMS': 2}] - ] - return cases diff --git a/sahara/tests/unit/plugins/cdh/v5_9_0/test_config_helper.py b/sahara/tests/unit/plugins/cdh/v5_9_0/test_config_helper.py new file mode 100644 index 0000000000..ac2582e019 --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/v5_9_0/test_config_helper.py @@ -0,0 +1,25 @@ +# Copyright (c) 2016 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.plugins.cdh.v5_9_0 import config_helper +from sahara.tests.unit.plugins.cdh import base_config_helper_test as bcht + + +class TestConfigHelperV590(bcht.TestConfigHelper): + + def setUp(self): + super(TestConfigHelperV590, self).setUp() + self.c_h = config_helper.ConfigHelperV590() + self.path_to_config = 'plugins/cdh/v5_9_0/resources/' diff --git a/sahara/tests/unit/plugins/cdh/v5_9_0/test_deploy.py b/sahara/tests/unit/plugins/cdh/v5_9_0/test_deploy.py new file mode 100644 index 0000000000..61c84d9902 --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/v5_9_0/test_deploy.py @@ -0,0 +1,222 @@ +# Copyright (c) 2016 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from sahara.plugins.cdh.v5_9_0 import deploy +from sahara.tests.unit import base + + +class DeployCDHV590(base.SaharaTestCase): + + def setUp(self): + super(DeployCDHV590, self).setUp() + self.master = mock.MagicMock() + self.master.node_group.node_processes = [ + "HDFS_NAMENODE", "YARN_RESOURCEMANAGER", "CLOUDERA_MANAGER", + "SENTRY_SERVER", "YARN_NODEMANAGER", "ZOOKEEPER_SERVER", + "OOZIE_SERVER", "YARN_JOBHISTORY", "HDFS_SECONDARYNAMENODE", + "HIVE_METASTORE", "HIVE_SERVER2", "SPARK_YARN_HISTORY_SERVER", + "HBASE_MASTER", "HBASE_REGIONSERVER", "HUE_SERVER", "KMS", + "FLUME_AGENT", "SOLR_SERVER", "SQOOP_SERVER", "IMPALA_STATESTORE", + "IMPALA_CATALOGSERVER", "IMPALAD", "KEY_VALUE_STORE_INDEXER", + ] + self.worker = mock.MagicMock() + self.worker.node_group.node_processes = [ + "HDFS_DATANODE", "HDFS_JOURNALNODE", "JOURNALNODE", + "YARN_NODEMANAGER", "YARN_STANDBYRM", + ] + self.instances = [self.master, self.worker] + self.cluster = mock.MagicMock() + + self.is_cdh_exists = mock.patch( + "sahara.plugins.cdh.commands.is_pre_installed_cdh", + return_value=False) + self.is_cdh_exists.start() + self._create_facade = mock.patch( + "sahara.db.sqlalchemy.api._create_facade_lazily") + self._create_facade.start() + + def tearDown(self): + self.is_cdh_exists.stop() + self._create_facade.stop() + super(DeployCDHV590, self).tearDown() + + @mock.patch("sahara.plugins.utils.get_instances") + @mock.patch("sahara.plugins.cdh.v5_9_0.deploy.CU") + def test_configure_cluster(self, mock_cu, mock_get_instances): + mock_get_instances.return_value = self.instances + + deploy.configure_cluster(self.cluster) + + mock_cu.pu.configure_os.assert_called_once_with(self.instances) + mock_cu.pu.install_packages.assert_called_once_with(self.instances, + deploy.PACKAGES) + mock_cu.pu.start_cloudera_agents.assert_called_once_with( + self.instances) + mock_cu.pu.start_cloudera_manager.assert_called_once_with(self.cluster) + mock_cu.update_cloudera_password.assert_called_once_with(self.cluster) + mock_cu.await_agents.assert_called_once_with(self.cluster, + self.instances) + mock_cu.create_mgmt_service.assert_called_once_with(self.cluster) + mock_cu.create_services.assert_called_once_with(self.cluster) + mock_cu.configure_services.assert_called_once_with(self.cluster) + mock_cu.configure_instances.assert_called_once_with(self.instances, + self.cluster) + mock_cu.deploy_configs.assert_called_once_with(self.cluster) + + @mock.patch("sahara.plugins.cdh.v5_9_0.deploy.CU") + def test__start_roles(self, mock_cu): + + deploy._start_roles(self.cluster, self.instances) + + mock_cu.get_service_by_role.assert_any_call('DATANODE', + instance=self.worker) + mock_cu.get_service_by_role.assert_any_call('NODEMANAGER', + instance=self.master) + mock_cu.get_service_by_role.assert_any_call('NODEMANAGER', + instance=self.worker) + self.assertEqual(mock_cu.start_roles.call_count, 3) + + @mock.patch("sahara.plugins.cdh.v5_9_0.deploy._start_roles") + @mock.patch("sahara.plugins.cdh.v5_9_0.deploy.CU") + def test_scale_cluster(self, mock_cu, mock__start_roles): + deploy.scale_cluster(self.cluster, None) + self.assertEqual(mock_cu.call_count, 0) + self.assertEqual(mock__start_roles.call_count, 0) + + deploy.scale_cluster(self.cluster, self.instances) + + mock_cu.pu.configure_os.assert_called_once_with(self.instances) + mock_cu.pu.install_packages.assert_called_once_with(self.instances, + deploy.PACKAGES) + mock_cu.pu.start_cloudera_agents.assert_called_once_with( + self.instances) + mock_cu.await_agents.assert_called_once_with(self.cluster, + self.instances) + mock_cu.configure_instances.assert_called_once_with(self.instances, + self.cluster) + mock_cu.update_configs.assert_called_once_with(self.instances) + mock_cu.pu.configure_swift.assert_called_once_with(self.cluster, + self.instances) + mock_cu.refresh_datanodes.assert_called_once_with(self.cluster) + mock__start_roles.assert_called_once_with(self.cluster, + self.instances) + + @mock.patch("sahara.plugins.cdh.v5_9_0.deploy.CU") + def test_decommission_cluster(self, mock_cu): + deploy.decommission_cluster(self.cluster, self.instances) + dns = [] + dns_2 = [] + nms = [] + nms_2 = [] + for i in self.instances: + if 'HDFS_DATANODE' in i.node_group.node_processes: + dns.append(mock_cu.pu.get_role_name(i, 'DATANODE')) + dns_2.append(mock_cu.pu.get_role_name(i, 'HDFS_GATEWAY')) + if 'YARN_NODEMANAGER' in i.node_group.node_processes: + nms.append(mock_cu.pu.get_role_name(i, 'NODEMANAGER')) + nms_2.append(mock_cu.pu.get_role_name(i, 'YARN_GATEWAY')) + mock_cu.decommission_nodes.assert_any_call( + self.cluster, 'DATANODE', dns, dns_2) + mock_cu.decommission_nodes.assert_any_call( + self.cluster, 'NODEMANAGER', nms, nms_2) + mock_cu.delete_instances.assert_called_once_with(self.cluster, + self.instances) + mock_cu.refresh_datanodes.assert_called_once_with(self.cluster) + mock_cu.refresh_yarn_nodes.assert_called_once_with(self.cluster) + + @mock.patch("sahara.plugins.cdh.v5_9_0.deploy.CU") + def test__prepare_cluster(self, mock_cu): + deploy._prepare_cluster(self.cluster) + + mock_cu.pu.install_extjs.assert_called_once_with(self.cluster) + mock_cu.pu.configure_hive.assert_called_once_with(self.cluster) + mock_cu.pu.configure_sentry.assert_called_once_with(self.cluster) + + @mock.patch("sahara.service.edp.hdfs_helper.create_hbase_common_lib") + @mock.patch("sahara.plugins.cdh.v5_9_0.deploy.CU") + def test__finish_cluster_starting(self, mock_cu, mock_create_hbase): + deploy._finish_cluster_starting(self.cluster) + mock_cu.pu.put_hive_hdfs_xml.assert_called_once_with(self.cluster) + self.assertTrue(mock_create_hbase.called) + mock_cu.start_service.assert_called_once_with( + mock_cu.get_service_by_role('AGENT', self.cluster)) + + @mock.patch("sahara.plugins.cdh.v5_9_0.deploy._finish_cluster_starting") + @mock.patch("sahara.plugins.cdh.v5_9_0.deploy._prepare_cluster") + @mock.patch("sahara.plugins.cdh.v5_9_0.deploy.CU") + def test_start_cluster(self, mock_cu, mock_prepare, mock_finish): + jns_count = 0 + for i in self.instances: + if "HDFS_JOURNALNODE" in i.node_group.node_processes: + jns_count += 1 + mock_cu.pu.get_jns.return_value.__len__.return_value = jns_count + + deploy.start_cluster(self.cluster) + + mock_prepare.assert_called_once_with(self.cluster) + mock_cu.first_run.assert_called_once_with(self.cluster) + mock_cu.pu.configure_swift.assert_called_once_with(self.cluster) + if jns_count > 0: + mock_cu.enable_namenode_ha.assert_called_once_with(self.cluster) + mock_cu.update_role_config.assert_any_call( + mock_cu.pu.get_secondarynamenode(self.cluster), + 'HDFS_NAMENODE' + ) + mock_cu.enable_resourcemanager_ha.assert_called_once_with(self.cluster) + mock_cu.update_role_config.assert_any_call( + mock_cu.pu.get_stdb_rm(self.cluster), 'YARN_STANDBYRM') + mock_finish.assert_called_once_with(self.cluster) + + def test_get_open_ports(self): + master_ports = [ + 9000, + 7180, 7182, 7183, 7432, 7184, 8084, 8086, 10101, + 9997, 9996, 8087, 9998, 9999, 8085, 9995, 9994, + 8020, 8022, 50070, 50470, + 50090, 50495, + 8030, 8031, 8032, 8033, 8088, + 8040, 8041, 8042, + 10020, 19888, + 9083, + 10000, + 8888, + 11000, 11001, + 18088, + 2181, 3181, 4181, 9010, + 60000, + 60020, + 41414, + 8038, + 8983, 8984, + 8005, 12000, + 25020, 26000, + 25010, 24000, + 21050, 21000, 23000, 25000, 28000, 22000, + 16000, 16001 + ] + deploy.get_open_ports(self.master.node_group) + self.assertItemsEqual(master_ports, + deploy.get_open_ports(self.master.node_group)) + worker_ports = [ + 9000, + 50010, 1004, 50075, 1006, 50020, + 8480, 8481, 8485, + 8040, 8041, 8042, + 8030, 8031, 8032, 8033, 8088 + ] + self.assertItemsEqual(worker_ports, + deploy.get_open_ports(self.worker.node_group)) diff --git a/sahara/tests/unit/plugins/cdh/v5_9_0/test_plugin_utils.py b/sahara/tests/unit/plugins/cdh/v5_9_0/test_plugin_utils.py new file mode 100644 index 0000000000..7e59eb4769 --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/v5_9_0/test_plugin_utils.py @@ -0,0 +1,25 @@ +# Copyright (c) 2015 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.plugins.cdh.v5_9_0 import plugin_utils as pu +from sahara.tests.unit.plugins.cdh import base_plugin_utils_test + + +class TestPluginUtilsV590(base_plugin_utils_test.TestPluginUtilsHigherThanV5): + + def setUp(self): + super(TestPluginUtilsV590, self).setUp() + self.plug_utils = pu.PluginUtilsV590() + self.version = "v5_9_0" diff --git a/sahara/tests/unit/plugins/cdh/v5_9_0/test_validation.py b/sahara/tests/unit/plugins/cdh/v5_9_0/test_validation.py index cda26cb74d..1864f063f8 100644 --- a/sahara/tests/unit/plugins/cdh/v5_9_0/test_validation.py +++ b/sahara/tests/unit/plugins/cdh/v5_9_0/test_validation.py @@ -14,110 +14,11 @@ # limitations under the License. from sahara.plugins.cdh.v5_9_0 import validation -from sahara.plugins import exceptions as ex from sahara.tests.unit.plugins.cdh import base_validation_tests as bvt -icte = ex.InvalidClusterTopology -nnce = ex.NameNodeHAConfigurationError -icce = ex.InvalidComponentCountException -rsme = ex.RequiredServiceMissingException -rmce = ex.ResourceManagerHAConfigurationError - class ValidationTestCase(bvt.BaseValidationTestCase): def setUp(self): super(ValidationTestCase, self).setUp() self.module = validation.ValidatorV590 - - def _get_test_cases(self): - cases = super(ValidationTestCase, self)._get_test_cases() - - disable_anti_affinity = {'cluster_configs': {'general': { - 'Require Anti Affinity': False}}} - cases += [ - [None, {'HDFS_JOURNALNODE': 3, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [rsme, {'HDFS_JOURNALNODE': 3, - 'ZOOKEEPER_SERVER': 0}, [], disable_anti_affinity], - [icce, {'HDFS_JOURNALNODE': 4, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [icce, {'HDFS_JOURNALNODE': 2, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [nnce, {'HDFS_JOURNALNODE': 3, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['HDFS_SECONDARYNAMENODE']}], - [nnce, {'HDFS_JOURNALNODE': 3, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['HDFS_NAMENODE']}], - [None, {'HDFS_JOURNALNODE': 3, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['HDFS_NAMENODE', 'HDFS_SECONDARYNAMENODE']}], - [None, {'YARN_STANDBYRM': 1, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [icce, {'YARN_STANDBYRM': 2, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [rsme, {'YARN_STANDBYRM': 1, - 'ZOOKEEPER_SERVER': 0}, [], disable_anti_affinity], - [rmce, {'YARN_STANDBYRM': 1, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['YARN_RESOURCEMANAGER']}], - [rmce, {'YARN_STANDBYRM': 1, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['YARN_STANDBYRM']}], - [None, {'YARN_STANDBYRM': 1, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['YARN_STANDBYRM', 'YARN_RESOURCEMANAGER']}], - ] - - cases += [ - [None, {'FLUME_AGENT': 1}], - [icce, {'ZOOKEEPER_SERVER': 1, 'SENTRY_SERVER': 2}], - [None, {'ZOOKEEPER_SERVER': 1, 'SENTRY_SERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 0, 'SENTRY_SERVER': 1}], - [None, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 0, 'SOLR_SERVER': 1}], - [None, {'YARN_NODEMANAGER': 1, 'YARN_JOBHISTORY': 1, - 'SQOOP_SERVER': 1}], - [rsme, {'YARN_NODEMANAGER': 0, 'YARN_JOBHISTORY': 1, - 'SQOOP_SERVER': 1}], - [rsme, {'YARN_NODEMANAGER': 1, 'YARN_JOBHISTORY': 0, - 'SQOOP_SERVER': 1}], - # HBASE_MASTER AND HBASE_REGIONSERVER depend circularly - [None, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1, - 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, - 'HBASE_REGIONSERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 0, 'SOLR_SERVER': 1, - 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, - 'HBASE_REGIONSERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 0, - 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, - 'HBASE_REGIONSERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1, - 'HBASE_MASTER': 0, 'HBASE_INDEXER': 1}], - ] - - worker_with_implama = ('worker_ng', 1, ['HDFS_DATANODE', - 'YARN_NODEMANAGER', - 'IMPALAD'], 3) - cases += [ - [None, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1, - 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, - [worker_with_implama]], - [icte, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1}, - []], - [icte, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1}, - [worker_with_implama]], - [rsme, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 0, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1, - 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, - [worker_with_implama]], - [rsme, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 0, 'HIVE_SERVER2': 1, - 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, - [worker_with_implama]] - - ] - - cases += [ - [None, {'KMS': 1}], - [icce, {'KMS': 2}] - ] - return cases