From a23cc5e94fb97d03e9c14529192999107449a2c2 Mon Sep 17 00:00:00 2001 From: Ilya Kutukov Date: Tue, 28 Jun 2016 21:45:12 +0300 Subject: [PATCH] Graph list now working for all levels, not only cluster Before this patch it was possible only to view list of graphs that are related to cluster level of given env and no more. Now graphs list for env is returning all graphs related to it (release, plugins and cluster level). Now the following commands are possible: fuel2 graph list [--env ENV_ID] [--cluster] [--plugins] [--release] All options working as filter narrowing list to the given environment related graphs and/or graphs levels. Change-Id: I006cf6767a9bf0d89af5026728bd13ddc42c4aa8 Partial-Bug: #1563851 Closes-Bug: #1621585 --- fuelclient/commands/graph.py | 62 +++++-- fuelclient/objects/environment.py | 14 ++ .../unit/v2/cli/test_deployment_graph.py | 17 +- .../unit/v2/lib/test_deployment_graph.py | 153 ++++++++++++++- fuelclient/v1/graph.py | 175 ++++++++++++++---- 5 files changed, 359 insertions(+), 62 deletions(-) diff --git a/fuelclient/commands/graph.py b/fuelclient/commands/graph.py index c97e34a8..252c2052 100644 --- a/fuelclient/commands/graph.py +++ b/fuelclient/commands/graph.py @@ -329,26 +329,52 @@ class GraphList(base.BaseListCommand): def get_parser(self, prog_name): parser = super(GraphList, self).get_parser(prog_name) - parser.add_argument('-e', - '--env', - type=int, - required=True, - help='Id of the environment') + parser.add_argument( + '-e', + '--env', + type=int, + help='Id of the environment' + ) + parser.add_argument( + '--cluster', + dest='filters', + action='append_const', + const='cluster', + help='Include cluster-specific graphs' + ) + parser.add_argument( + '--plugins', + dest='filters', + action='append_const', + const='plugins', + help='Include plugins-specific graphs' + ) + parser.add_argument( + '--release', + dest='filters', + action='append_const', + const='release', + help='Include release-specific graphs' + ) return parser - def take_action(self, parsed_args): - data = self.client.list( - env_id=parsed_args.env - ) - # format fields + def take_action(self, args): + data = self.client.list(env_id=args.env, filters=args.filters) + + # make table context applying special formatting to data copy + display_data = [] for d in data: - d['relations'] = "\n".join( - 'as "{type}" to {model}(ID={model_id})' - .format(**r) for r in d['relations'] - ) - d['tasks'] = ', '.join(sorted(t['id'] for t in d['tasks'])) - data = data_utils.get_display_data_multi(self.columns, data) - scolumn_ids = [self.columns.index(col) - for col in parsed_args.sort_columns] + d = d.copy() + d.update({ + 'relations': "\n".join( + 'as "{type}" to {model}(ID={model_id})'.format(**r) + for r in d['relations'] + ), + 'tasks': ', '.join(sorted(t['id'] for t in d['tasks'])) + }) + display_data.append(d) + + data = data_utils.get_display_data_multi(self.columns, display_data) + scolumn_ids = [self.columns.index(col) for col in args.sort_columns] data.sort(key=lambda x: [x[scolumn_id] for scolumn_id in scolumn_ids]) return self.columns, data diff --git a/fuelclient/objects/environment.py b/fuelclient/objects/environment.py index 48d2c105..e39f234d 100644 --- a/fuelclient/objects/environment.py +++ b/fuelclient/objects/environment.py @@ -667,3 +667,17 @@ class Environment(BaseObject): """ return self.connection.post_request(self._get_ip_addrs_url(), vip_kwargs) + + def get_enabled_plugins(self): + """Get list of enabled plugins ids. + + :returns: plugins ids list + :rtype: list[int] + """ + attrs = self.get_attributes()['editable'] + enabled_plugins_ids = [] + for attr_name in attrs: + metadata = attrs[attr_name].get('metadata', {}) + if metadata.get('class') == 'plugin' and metadata.get('enabled'): + enabled_plugins_ids.append(metadata['chosen_id']) + return enabled_plugins_ids diff --git a/fuelclient/tests/unit/v2/cli/test_deployment_graph.py b/fuelclient/tests/unit/v2/cli/test_deployment_graph.py index f27db788..a9dad4f3 100644 --- a/fuelclient/tests/unit/v2/cli/test_deployment_graph.py +++ b/fuelclient/tests/unit/v2/cli/test_deployment_graph.py @@ -264,11 +264,24 @@ class TestGraphActions(test_engine.BaseCLITest): 'id': 1 } ] - self.exec_command('graph list --env 1') + self.exec_command( + 'graph list --env 1 --release --plugins --cluster') self.m_get_client.assert_called_once_with('graph', mock.ANY) - self.m_client.list.assert_called_once_with(env_id=1) self.assertIn('1', m_stdout.getvalue()) self.assertIn('updated-graph-name', m_stdout.getvalue()) self.assertIn('custom-graph', m_stdout.getvalue()) self.assertIn('test-task2', m_stdout.getvalue()) + + self.exec_command('graph list --release') + self.exec_command('graph list --plugins') + self.exec_command('graph list --cluster') + self.exec_command('graph list') + + self.m_client.list.assert_has_calls([ + mock.call(env_id=1, filters=['release', 'plugins', 'cluster']), + mock.call(env_id=None, filters=['release']), + mock.call(env_id=None, filters=['plugins']), + mock.call(env_id=None, filters=['cluster']), + mock.call(env_id=None, filters=None) + ]) diff --git a/fuelclient/tests/unit/v2/lib/test_deployment_graph.py b/fuelclient/tests/unit/v2/lib/test_deployment_graph.py index e045ad5d..1584c6b4 100644 --- a/fuelclient/tests/unit/v2/lib/test_deployment_graph.py +++ b/fuelclient/tests/unit/v2/lib/test_deployment_graph.py @@ -19,7 +19,8 @@ import yaml import fuelclient from fuelclient.tests.unit.v2.lib import test_api -from fuelclient.tests.utils import fake_task +from fuelclient.tests import utils + TASKS_YAML = '''- id: custom-task-1 type: puppet @@ -104,7 +105,7 @@ class TestDeploymentGraphFacade(test_api.BaseLibTest): def test_new_graph_run_wo_params(self): matcher_execute = self.m_request.post( '/api/v1/graphs/execute/', - json=fake_task.get_fake_task(cluster=370)) + json=utils.fake_task.get_fake_task(cluster=370)) # this is required to form running task info self.m_request.get( '/api/v1/nodes/?cluster_id=370', @@ -121,7 +122,7 @@ class TestDeploymentGraphFacade(test_api.BaseLibTest): def test_new_graph_run_with_parameters(self): matcher_execute = self.m_request.post( '/api/v1/graphs/execute/', - json=fake_task.get_fake_task(cluster=370)) + json=utils.fake_task.get_fake_task(cluster=370)) # this is required to form running task info self.m_request.get( '/api/v1/nodes/?cluster_id=370', @@ -152,13 +153,144 @@ class TestDeploymentGraphFacade(test_api.BaseLibTest): matcher_execute.last_request.json() ) - def test_graphs_list(self): - matcher_get = self.m_request.get( - '/api/v1/clusters/1/deployment_graphs/', - json=[] + def test_env_graphs_list(self): + release_id = 101 + env_id = 11 + fake_env = utils.get_fake_env(release_id=release_id, env_id=env_id) + enabled_plugin_id = 331 + self.m_request.get( + '/api/v1/clusters/{}/'.format(env_id), + json=fake_env + ) + + self.m_request.get( + '/api/v1/clusters/{}/attributes'.format(env_id), + json={ + 'editable': { + 'test-plugin-1': { + 'metadata': { + 'class': 'plugin', + 'enabled': True, + 'chosen_id': enabled_plugin_id + } + }, + 'test-plugin-2': { + 'metadata': { + 'class': 'plugin', + 'enabled': False, + } + } + } + } + ) + + release_graphs = [ + { + "tasks": [], + "id": 1, + "relations": [ + { + "model_id": release_id, + "model": "release", + "type": "default" + } + ], + "name": None + } + ] + enabled_plugin_graphs = [ + { + "tasks": [], + "id": 2, + "relations": [ + { + "model_id": enabled_plugin_id, + "model": "plugin", + "type": "default" + } + ], + "name": None + } + ] + cluster_graphs = [ + { + "tasks": [], + "id": 3, + "relations": [ + { + "model_id": env_id, + "model": "cluster", + "type": "default" + } + ], + "name": None + } + ] + + all_env_graphs = \ + release_graphs + cluster_graphs + enabled_plugin_graphs + + not_this_env_cluster_graphs = [ + { + "tasks": [], + "id": 4, + "relations": [ + { + "model_id": env_id + 1, + "model": "cluster", + "type": "default" + } + ], + "name": None + } + ] + + self.m_request.get( + '/api/v1/releases/{}/deployment_graphs/'.format(release_id), + json=release_graphs + ) + + self.m_request.get( + '/api/v1/plugins/{}/deployment_graphs/'.format(enabled_plugin_id), + json=enabled_plugin_graphs + ) + + self.m_request.get( + '/api/v1/clusters/{}/deployment_graphs/'.format(env_id), + json=cluster_graphs + ) + + self.m_request.get( + '/api/v1/graphs/'.format(env_id), + json=all_env_graphs + not_this_env_cluster_graphs + ) + + self.assertItemsEqual( + all_env_graphs, self.client.list(env_id) + ) + self.assertItemsEqual( + release_graphs, self.client.list(env_id, filters=['release']) + ) + + self.assertItemsEqual( + enabled_plugin_graphs, + self.client.list(env_id, filters=['plugins']) + ) + + self.assertItemsEqual( + cluster_graphs, + self.client.list(env_id, filters=['cluster']) + ) + + self.assertItemsEqual( + all_env_graphs + not_this_env_cluster_graphs, + self.client.list() + ) + + self.assertItemsEqual( + cluster_graphs + not_this_env_cluster_graphs + release_graphs, + self.client.list(filters=['cluster', 'release']) ) - self.client.list(1) - self.assertTrue(matcher_get.called) def test_graphs_download_all(self): matcher_get = self.m_request.get( @@ -191,7 +323,8 @@ class TestDeploymentGraphFacade(test_api.BaseLibTest): def test_graphs_download_cluster(self): matcher_get = self.m_request.get( - '/api/v1/clusters/1/deployment_graphs/custom_graph', + '/api/v1/clusters/1/deployment_tasks/own/' + '?graph_type=custom_graph', json=[{'tasks': []}] ) self.client.download(env_id=1, level='cluster', diff --git a/fuelclient/v1/graph.py b/fuelclient/v1/graph.py index 378102c4..f091abdf 100644 --- a/fuelclient/v1/graph.py +++ b/fuelclient/v1/graph.py @@ -29,44 +29,46 @@ class GraphClient(base_v1.BaseV1Client): related_graph_api_path = "{related_model}/{related_model_id}" \ "/deployment_graphs/{graph_type}" + graphs_list_api = "graphs/" cluster_deploy_api_path = "graphs/execute/" - merged_cluster_tasks_api_path = "clusters/{env_id}/deployment_tasks" \ - "/?graph_type={graph_type}" + cluster_own_tasks_api_path = "clusters/{env_id}/deployment_tasks/own/" - merged_plugins_tasks_api_path = "clusters/{env_id}/deployment_tasks" \ - "/plugins/?graph_type={graph_type}" + merged_cluster_tasks_api_path = "clusters/{env_id}/deployment_tasks/" - cluster_release_tasks_api_path = "clusters/{env_id}/deployment_tasks" \ - "/release/?graph_type={graph_type}" + merged_plugins_tasks_api_path = "clusters/{env_id}/deployment_tasks/" \ + "plugins/" + + cluster_release_tasks_api_path = "clusters/{env_id}/deployment_tasks/" \ + "release/" def update_graph_for_model( - self, data, related_model, related_model_id, graph_type=None): + self, data, related_model, related_model_id, graph_type): return self.connection.put_request( self.related_graph_api_path.format( related_model=related_model, related_model_id=related_model_id, - graph_type=graph_type or ""), + graph_type=graph_type), data ) def create_graph_for_model( - self, data, related_model, related_model_id, graph_type=None): + self, data, related_model, related_model_id, graph_type): return self.connection.post_request( self.related_graph_api_path.format( related_model=related_model, related_model_id=related_model_id, - graph_type=graph_type or ""), + graph_type=graph_type), data ) def get_graph_for_model( - self, related_model, related_model_id, graph_type=None): + self, related_model, related_model_id, graph_type): return self.connection.get_request( self.related_graph_api_path.format( related_model=related_model, related_model_id=related_model_id, - graph_type=graph_type or "")) + graph_type=graph_type)) def upload(self, data, related_model, related_id, graph_type): # create or update @@ -119,49 +121,158 @@ class GraphClient(base_v1.BaseV1Client): return objects.DeployTask.init_with_data(deploy_data) def get_merged_cluster_tasks(self, env_id, graph_type=None): + params = {} + if graph_type is not None: + params['graph_type'] = graph_type + return self.connection.get_request( - self.merged_cluster_tasks_api_path.format( - env_id=env_id, - graph_type=graph_type or "")) + self.merged_cluster_tasks_api_path.format(env_id=env_id), + params=params + ) def get_merged_plugins_tasks(self, env_id, graph_type=None): + params = {} + if graph_type is not None: + params['graph_type'] = graph_type + return self.connection.get_request( - self.merged_plugins_tasks_api_path.format( - env_id=env_id, - graph_type=graph_type or "")) + self.merged_plugins_tasks_api_path.format(env_id=env_id), + params=params + ) def get_release_tasks_for_cluster(self, env_id, graph_type=None): + params = {} + if graph_type is not None: + params['graph_type'] = graph_type + return self.connection.get_request( - self.cluster_release_tasks_api_path.format( - env_id=env_id, - graph_type=graph_type or "")) + self.cluster_release_tasks_api_path.format(env_id=env_id), + params=params + ) + + def get_own_tasks_for_cluster(self, env_id, graph_type=None): + params = {} + if graph_type is not None: + params['graph_type'] = graph_type + + return self.connection.get_request( + self.cluster_own_tasks_api_path.format(env_id=env_id), + params=params + ) def download(self, env_id, level, graph_type): tasks_levels = { 'all': lambda: self.get_merged_cluster_tasks( env_id=env_id, graph_type=graph_type), - 'cluster': lambda: self.get_graph_for_model( - related_model='clusters', - related_model_id=env_id, - graph_type=graph_type)[0].get('tasks', []), + 'cluster': lambda: self.get_own_tasks_for_cluster( + env_id=env_id, graph_type=graph_type), 'plugins': lambda: self.get_merged_plugins_tasks( - env_id=env_id, - graph_type=graph_type), + env_id=env_id, graph_type=graph_type), 'release': lambda: self.get_release_tasks_for_cluster( - env_id=env_id, - graph_type=graph_type) + env_id=env_id, graph_type=graph_type) } return tasks_levels[level]() - def list(self, env_id): - # todo(ikutukov): extend lists to support all models + def get_env_release_graphs_list(self, env_id): + """Get list of graphs related to the environment's release. + + :param env_id: environment ID + :type env_id: int + :return: list of graphs records + :rtype: list[dict] + """ + data = self.get_by_id(env_id) + release_id = data['release_id'] + return self.connection.get_request( + self.related_graphs_list_api_path.format( + related_model='releases', + related_model_id=release_id + ), params={'fetch_related': '0'} + ) + + def get_env_cluster_graphs_list(self, env_id, fetch_related=True): + """Get list of graphs related to the environment. + + :param env_id: environment ID + :type env_id: int + :param fetch_related: fetch graphs related to + cluster plugins and release + :type fetch_related: bool + + :return: list of graphs records + :rtype: list[dict] + """ return self.connection.get_request( self.related_graphs_list_api_path.format( related_model='clusters', - related_model_id=env_id)) + related_model_id=env_id, + ), params={'fetch_related': '1' if fetch_related else '0'} + ) + + def get_env_plugins_graphs_list(self, env_id): + """Get list of graphs related to plugins active for the + + given environment. + + :param env_id: environment ID + :type env_id: int + :return: list of graphs records + :rtype: list[dict] + """ + env = objects.Environment(env_id) + enabled_plugins_ids = env.get_enabled_plugins() + result = [] + for plugin_id in enabled_plugins_ids: + result += self.connection.get_request( + self.related_graphs_list_api_path.format( + related_model='plugins', + related_model_id=plugin_id + ), params={'fetch_related': '0'} + ) + return result + + def get_all_graphs_list(self): + return self.connection.get_request(self.graphs_list_api) + + def list(self, env_id=None, filters=None): + """Get graphs list. + + If all filter flags are set to False, then it fill be considered as + 'show all' and all filter flags will be toggled to True. + + :param env_id: environment ID + :type env_id: int + :param filters: the name of models which graphs will be included + to result + :return: list of graphs records + :rtype: list[dict] + """ + # we cannot use dict here, because order is important + handlers = ( + ('release', self.get_env_release_graphs_list), + ('plugins', self.get_env_plugins_graphs_list), + ('cluster', self.get_env_cluster_graphs_list) + ) + + graphs_list = [] + filters = filters and set(filters) + + if env_id: + for relation, handler in handlers: + if not filters or relation in filters: + graphs_list.extend(handler(env_id=env_id)) + else: + all_graphs_list = self.get_all_graphs_list() + for graph in all_graphs_list: + for relation in graph['relations']: + if not filters or relation['model'] in filters: + graphs_list.append(graph) + break + + return graphs_list def get_client(connection):