From 47f3ae6eac3ddaab7a52ca62c59e521d9fa6ffa3 Mon Sep 17 00:00:00 2001 From: Alexander Ignatov Date: Tue, 9 Jul 2013 03:47:39 +0400 Subject: [PATCH] Cluster scaling validation added * Added schema * Checks all posible scaling wrong flows * Unigied nodegroup basic checks * Remove validation checks from api functions Implements blueprint savanna-rest-api-1-0-validation Change-Id: Ia16300a08cae3edd04c2b06b4154c598b6122cfd --- etc/rest-api-samples/scale_cluster.json | 3 - savanna/service/api.py | 46 ------- savanna/service/validations/base.py | 116 ++++++++++++++---- .../service/validations/clusters_scaling.py | 67 +++++++++- .../validations/node_group_templates.py | 12 +- 5 files changed, 153 insertions(+), 91 deletions(-) diff --git a/etc/rest-api-samples/scale_cluster.json b/etc/rest-api-samples/scale_cluster.json index 1b9db90d08..57b02e33c8 100644 --- a/etc/rest-api-samples/scale_cluster.json +++ b/etc/rest-api-samples/scale_cluster.json @@ -4,15 +4,12 @@ "node_group_template_id": "9e1225e0-b7ce-425b-9555-3a61166167c8", "count":1, "name": "test_ng" - } ], - "resize_node_groups":[ { "name" : "worker", "count":2 - } ] } \ No newline at end of file diff --git a/savanna/service/api.py b/savanna/service/api.py index 61088b0d64..aad3a6bc99 100644 --- a/savanna/service/api.py +++ b/savanna/service/api.py @@ -49,7 +49,6 @@ def scale_cluster(cluster_id, data): try: context.model_update(cluster, status='Validating') - _validate_cluster(cluster, plugin, additional) plugin.validate_scaling(cluster, to_be_enlarged, additional) except Exception: with excutils.save_and_reraise_exception(): @@ -70,9 +69,6 @@ def create_cluster(values): cluster = s.create_cluster(values) plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name) - # TODO(slukjanov): validate configs and etc. - _validate(cluster, plugin) - # validating cluster try: context.model_update(cluster, status='Validating') @@ -188,48 +184,6 @@ def convert_to_cluster_template(plugin_name, version, config_file): return s.persist_cluster_template(ct) -# TODO(aignatov): Will be removed after implementing scaling validation -def _validate(cluster, plugin): - # Validate that user configs are not included in plugin configs set - pl_confs = _get_plugin_configs(cluster, plugin) - for ng in cluster.node_groups: - _validate_node_group(pl_confs, ng) - - -# TODO(aignatov): Will be removed after implementing scaling validation -def _validate_cluster(cluster, plugin, node_groups): - # Validate that user configs are not included in plugin configs set - pl_confs = _get_plugin_configs(cluster, plugin) - for ng in node_groups: - ng.cluster = cluster - _validate_node_group(pl_confs, ng) - ng.cluster = None - - -# TODO(aignatov): Will be removed after implementing scaling validation -def _get_plugin_configs(cluster, plugin): - pl_confs = {} - for config in plugin.get_configs(cluster.hadoop_version): - if pl_confs.get(config.applicable_target): - pl_confs[config.applicable_target].append(config.name) - else: - pl_confs[config.applicable_target] = [config.name] - return pl_confs - - -# TODO(aignatov): Will be removed after implementing scaling validation -def _validate_node_group(pl_confs, node_group): - for app_target, configs in node_group.configuration.items(): - if app_target not in pl_confs: - raise RuntimeError("Plugin doesn't contain applicable " - "target '%s'" % app_target) - for name, values in configs.items(): - if name not in pl_confs[app_target]: - raise RuntimeError("Plugin's applicable target '%s' " - "doesn't contain config with name '%s'" - % (app_target, name)) - - def construct_ngs_for_scaling(additional_node_groups): additional = {} for ng in additional_node_groups: diff --git a/savanna/service/validations/base.py b/savanna/service/validations/base.py index c0d0bd34c8..0a958bc566 100644 --- a/savanna/service/validations/base.py +++ b/savanna/service/validations/base.py @@ -55,28 +55,6 @@ def check_image_exists(image_id): % image_id) -def check_flavor_exists(flavor_id): - try: - nova.client().flavors.get(flavor_id) - except nova_ex.NotFound: - raise ex.InvalidException("Requested flavor '%s' not found" - % flavor_id) - - -def check_node_processes(plugin_name, version, node_processes): - if len(set(node_processes)) != len(node_processes): - raise ex.InvalidException("Duplicates in node processes " - "have been detected") - plugin_procesess = [] - for process in plugin_base.PLUGINS.get_plugin( - plugin_name).get_node_processes(version).values(): - plugin_procesess += process - - if not set(node_processes).issubset(set(plugin_procesess)): - raise ex.InvalidException("Plugin supports the following " - "node procesess: %s" % plugin_procesess) - - def check_node_group_configs(plugin_name, hadoop_version, ng_configs, plugin_configs=None): # TODO(aignatov): Should have scope and config type validations @@ -103,11 +81,59 @@ def check_all_configurations(data): if data.get('node_groups'): for ng in data['node_groups']: - if ng.get('node_configs'): - check_node_group_configs(data['plugin_name'], - data['hadoop_version'], - ng['node_configs'], - plugin_configs=pl_confs) + check_node_group_basic_fields(data['plugin_name'], + data['hadoop_version'], + ng, pl_confs) + +## NodeGroup related checks + + +def check_node_group_basic_fields(plugin_name, hadoop_version, ng, + plugin_configs=None): + + if ng.get('node_group_template_id'): + check_node_group_template_exists(ng['node_group_template_id']) + + if ng.get('node_configs'): + check_node_group_configs(plugin_name, hadoop_version, + ng['node_configs'], plugin_configs) + if ng.get('flavor_id'): + check_flavor_exists(ng['flavor_id']) + + if ng.get('node_processes'): + check_node_processes(plugin_name, hadoop_version, ng['node_processes']) + + if ng.get('image_id'): + check_image_exists(ng['image_id']) + + +def check_flavor_exists(flavor_id): + try: + nova.client().flavors.get(flavor_id) + except nova_ex.NotFound: + raise ex.InvalidException("Requested flavor '%s' not found" + % flavor_id) + + +def check_node_processes(plugin_name, version, node_processes): + if len(set(node_processes)) != len(node_processes): + raise ex.InvalidException("Duplicates in node processes " + "have been detected") + plugin_procesess = [] + for process in plugin_base.PLUGINS.get_plugin( + plugin_name).get_node_processes(version).values(): + plugin_procesess += process + + if not set(node_processes).issubset(set(plugin_procesess)): + raise ex.InvalidException("Plugin supports the following " + "node procesess: %s" % plugin_procesess) + + +def check_duplicates_node_groups_names(node_groups): + ng_names = [ng['name'] for ng in node_groups] + if len(set(ng_names)) < len(node_groups): + raise ex.InvalidException("Duplicates in node group names " + "are detected") ## Cluster creation related checks @@ -139,3 +165,39 @@ def check_node_group_template_unique_name(name): if name in [t.name for t in api.get_node_group_templates()]: raise ex.NameAlreadyExistsException("NodeGroup template with name '%s'" " already exists" % name) + + +def check_node_group_template_exists(ng_tmpl_id): + if not api.get_node_group_templates(id=ng_tmpl_id): + raise ex.InvalidException("NodeGroup template with id '%s'" + " doesn't exist" % ng_tmpl_id) + + +## Cluster scaling + +def check_resize(cluster, r_node_groups): + cluster_ng_names = [ng.name for ng in cluster.node_groups] + + check_duplicates_node_groups_names(r_node_groups) + + for ng in r_node_groups: + if ng['name'] not in cluster_ng_names: + raise ex.InvalidException("Cluster doesn't contain node group " + "with name '%s'" % ng['name']) + + +def check_add_node_groups(cluster, add_node_groups): + cluster_ng_names = [ng.name for ng in cluster.node_groups] + + check_duplicates_node_groups_names(add_node_groups) + + pl_confs = _get_plugin_configs(cluster.plugin_name, cluster.hadoop_version) + + for ng in add_node_groups: + if ng['name'] in cluster_ng_names: + raise ex.InvalidException("Can't add new nodegroup. Cluster " + "already has nodegroup with name '%s'" + % ng['name']) + + check_node_group_basic_fields(cluster.plugin_name, + cluster.hadoop_version, ng, pl_confs) diff --git a/savanna/service/validations/clusters_scaling.py b/savanna/service/validations/clusters_scaling.py index f7061d4a54..b3be2fcbd4 100644 --- a/savanna/service/validations/clusters_scaling.py +++ b/savanna/service/validations/clusters_scaling.py @@ -13,16 +13,73 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy + import savanna.exceptions as ex -from savanna.plugins import base as plugin_base +import savanna.plugins.base as plugin_base +import savanna.service.api as api +import savanna.service.validations.base as b +import savanna.service.validations.cluster_templates as cl_t -CLUSTER_SCALING_SCHEMA = None +def _build_node_groups_schema(): + schema = copy.deepcopy(cl_t.CLUSTER_TEMPLATE_SCHEMA) + return schema['properties']['node_groups'] + + +CLUSTER_SCALING_SCHEMA = { + "type": "object", + "properties": { + "resize_node_groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + }, + "count": { + "type": "integer", + "minimum": 1, + }, + }, + "additionalProperties": False, + "required": [ + "name", + "count", + ] + }, + "minItems": 1 + }, + "add_node_groups": _build_node_groups_schema(), + }, + "additionalProperties": False, + "anyOf": [ + { + "required": ["resize_node_groups"] + }, + { + "required": ["add_node_groups"] + } + ] + +} def check_cluster_scaling(data, cluster_id, **kwargs): - plugin_name = data['plugin_name'] - if not plugin_base.PLUGINS.is_plugin_implements(plugin_name, 'convert'): + cluster = api.get_cluster(id=cluster_id) + if not plugin_base.PLUGINS.is_plugin_implements(cluster.plugin_name, + 'scale_cluster'): raise ex.InvalidException( "Requested plugin '%s' doesn't support cluster scaling feature" - % plugin_name) + % cluster.plugin_name) + + if cluster.status != 'Active': + raise ex.InvalidException("Cluster cannot be scaled not in 'Active' " + "status. Cluster status: " + cluster.status) + + if data.get("resize_node_groups"): + b.check_resize(cluster, data['resize_node_groups']) + + if data.get("add_node_groups"): + b.check_add_node_groups(cluster, data['add_node_groups']) diff --git a/savanna/service/validations/node_group_templates.py b/savanna/service/validations/node_group_templates.py index e4c0bc92bf..98027159c3 100644 --- a/savanna/service/validations/node_group_templates.py +++ b/savanna/service/validations/node_group_templates.py @@ -78,13 +78,5 @@ def check_node_group_template_create(data, **kwargs): b.check_plugin_name_exists(data['plugin_name']) b.check_plugin_supports_version(data['plugin_name'], data['hadoop_version']) - b.check_flavor_exists(data['flavor_id']) - b.check_node_processes(data['plugin_name'], data['hadoop_version'], - data['node_processes']) - - if data.get('image_id'): - b.check_image_exists(data['image_id']) - - if data.get('node_configs'): - b.check_node_group_configs(data['plugin_name'], data['hadoop_version'], - data['node_configs']) + b.check_node_group_basic_fields(data['plugin_name'], + data['hadoop_version'], data)