Add CDH validation for attached volume size

CDH plugin has a DATANODE configuration parameter
called "dfs_datanode_du_reserved". If user sets cinder volume
as the storage location and sets volumes size less than the value
of the parameter, HDFS would not work. The cluster deployment
would be Error in "Starting" Step because of "Oozie Share Lib Upload
Failed".
This patch adds a validation in "Validating" Step before spawning
Cluster.

BTW, If this situation (volume size less than "du_reserved")
happen in Vanilla, Spark plugin, the deploy can finish successfully,
but the HDFS can't be used when upload file to HDFS.

Closes-Bug: #1709071
Change-Id: I840559d54ca474299eeec9812dc1573ab305bc1b

sahara/plugins/cdh/plugin_utils.py

@@ -444,6 +444,10 @@ class AbstractPluginUtils(object):
             conf = cluster.cluster_configs
             if service in conf and name in conf[service]:
                 return types.transform_to_num(conf[service][name])
+            for node_group in cluster.node_groups:
+                conf = node_group.node_configs
+                if service in conf and name in conf[service]:
+                    return types.transform_to_num(conf[service][name])
         for config in configs:
             if config.applicable_target == service and config.name == name:
                 return types.transform_to_num(config.default_value)

sahara/plugins/cdh/validation.py

@@ -66,6 +66,14 @@ class Validator(object):
                 _('Number of datanodes must be not'
                   ' less than dfs_replication.'))
 
+        du_reserved = cls.PU.get_config_value(
+            'DATANODE', 'dfs_datanode_du_reserved', cluster)
+        du_reserved = du_reserved/1073741824.
+        for node_group in cluster.node_groups:
+            volume_size = node_group.volumes_size
+            if volume_size and volume_size < du_reserved:
+                raise ex.InvalidVolumeSizeException(volume_size, du_reserved)
+
         rm_count = cls.get_inst_count(cluster, 'YARN_RESOURCEMANAGER')
         if rm_count > 1:
             raise ex.InvalidComponentCountException('YARN_RESOURCEMANAGER',

sahara/plugins/exceptions.py

@@ -178,3 +178,18 @@ class AllValidationsFailedError(ImageValidationError):
         message = self.sub_message % data
 
         super(AllValidationsFailedError, self).__init__(message)
+
+
+class InvalidVolumeSizeException(e.SaharaException):
+    """Exception indicating invalid configuration of components in a cluster.
+
+    """
+
+    def __init__(self, volume_size, reserved):
+        message = _("Volume size: %(volume_size)s GB should be greater than "
+                    "value of \"dfs_datanode_du_reserved\": %(reserved)s GB")
+        self.message = message % {"volume_size": volume_size,
+                                  "reserved": reserved}
+        self.code = "INVALID_CONFIGURATION"
+
+        super(InvalidVolumeSizeException, self).__init__()

sahara/tests/unit/plugins/cdh/base_validation_tests.py

@@ -21,6 +21,7 @@ from sahara.plugins import exceptions as ex
 from sahara.tests.unit import base
 from sahara.tests.unit import testutils as tu
 
+ivse = ex.InvalidVolumeSizeException
 icce = ex.InvalidComponentCountException
 rsme = ex.RequiredServiceMissingException
 icte = ex.InvalidClusterTopology
@@ -28,9 +29,9 @@ nnce = ex.NameNodeHAConfigurationError
 rmce = ex.ResourceManagerHAConfigurationError
 
 
-def make_ng_dict_with_inst(counter, name, flavor,
-                           processes, count, instances=None,
-                           **kwargs):
+def make_ng_dict_with_inst(counter, name, flavor, processes, count,
+                           instances=None, volumes_size=None,
+                           node_configs=None, **kwargs):
     if not instances:
         instances = []
         for i in range(count):
@@ -39,8 +40,8 @@ def make_ng_dict_with_inst(counter, name, flavor,
                                          "fake_inst{0}".format(n),
                                          management_ip='1.2.3.{0}'.format(n))
             instances.append(instance)
-    return tu.make_ng_dict(name, flavor, processes,
-                           count, instances, **kwargs)
+    return tu.make_ng_dict(name, flavor, processes, count, instances,
+                           volumes_size, node_configs, **kwargs)
 
 
 def get_fake_cluster_with_process(processes=None,
@@ -139,6 +140,14 @@ class BaseValidationTestCase(base.SaharaTestCase):
                 {'cluster_configs': {'HDFS': {'dfs_replication': 3}}}],
             [icce, {}, [],
                 {'cluster_configs': {'HDFS': {'dfs_replication': 4}}}],
+            [ivse, {}, [(
+                'worker_ng_vol', 1, ['HDFS_DATANODE', 'YARN_NODEMANAGER'],
+                3, None, 20,
+                {'DATANODE': {'dfs_datanode_du_reserved': 22548578304}})]],
+            [None, {}, [(
+                'worker_ng_vol', 1, ['HDFS_DATANODE', 'YARN_NODEMANAGER'],
+                3, None, 22,
+                {'DATANODE': {'dfs_datanode_du_reserved': 22548578304}})]],
             [None, {'YARN_RESOURCEMANAGER': 1}],
             [icce, {'YARN_RESOURCEMANAGER': 2}],
             [None, {'YARN_JOBHISTORY': 1}],

sahara/tests/unit/testutils.py

@@ -28,11 +28,14 @@ def create_cluster(name, tenant, plugin, version, node_groups, **kwargs):
     return r.ClusterResource(dct)
 
 
-def make_ng_dict(name, flavor, processes, count, instances=None, **kwargs):
+def make_ng_dict(name, flavor, processes, count, instances=None,
+                 volumes_size=None, node_configs=None, **kwargs):
+    node_configs = node_configs or {}
     instances = instances or []
     dct = {'id': uuidutils.generate_uuid(), 'name': name,
-           'flavor_id': flavor, 'node_processes': processes,
-           'count': count, 'instances': instances, 'node_configs': {},
+           'volumes_size': volumes_size, 'flavor_id': flavor,
+           'node_processes': processes, 'count': count,
+           'instances': instances, 'node_configs': node_configs,
            'security_groups': None, 'auto_security_group': False,
            'availability_zone': None, 'volumes_availability_zone': None,
            'open_ports': [], 'is_proxy_gateway': False,