From 564d89d13479b93a0c00b37f9289b75f22306d55 Mon Sep 17 00:00:00 2001 From: Alexander Kislitsky Date: Tue, 22 Dec 2015 12:56:45 +0300 Subject: [PATCH] OSWL resource status representation in report fixed If resource removed and added we show only one record in the report with status True in is_added, is_removed. Change-Id: I783414ee4c23d07e02d13e76eb19fe3a2b3db80e Closes-Bug: #1526320 --- .../api/resources/utils/oswl_stats_to_csv.py | 40 +++-- .../resources/utils/test_oswl_stats_to_csv.py | 146 ++++++++++++++++++ 2 files changed, 173 insertions(+), 13 deletions(-) diff --git a/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py b/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py index b2a282b..f8f71f8 100644 --- a/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py +++ b/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py @@ -133,19 +133,27 @@ class OswlStatsToCsv(object): oswl) resource_data = oswl.resource_data current = resource_data.get('current', []) + added = resource_data.get('added', []) + modified = resource_data.get('modified', []) removed = resource_data.get('removed', []) - # Filtering id, time only data - removed = itertools.ifilter(lambda x: len(x) > 2, removed) + # Filtering wrong formatted removed data + # delivered by old Fuel versions + removed = [res for res in removed if len(res) > 2] # Extracting ids or oswl resources - added_ids = set(item['id'] for item in - resource_data.get('added', [])) - modified_ids = set(item['id'] for item in - resource_data.get('removed', [])) - removed_ids = set(item['id'] for item in - resource_data.get('modified', [])) + added_ids = set(item['id'] for item in added) + modified_ids = set(item['id'] for item in modified) + removed_ids = set(item['id'] for item in removed) - for resource in itertools.chain(current, removed): + # If resource removed and added several times it would + # be present in current and removed. We should exclude + # duplicates from flatten resources of the same + # resource. + current_ids = set(item['id'] for item in current) + finally_removed = (res for res in removed + if res['id'] not in current_ids) + + for resource in itertools.chain(current, finally_removed): flatten_resource = export_utils.get_flatten_data( resource_keys_paths, {resource_type: resource}) additional_info = self.get_additional_resource_info( @@ -205,11 +213,17 @@ class OswlStatsToCsv(object): idx = export_utils.get_index(oswl, *self.OSWL_INDEX_FIELDS) # We can have duplication of the oswls in the DB with the same - # checksum but with different external_id. We shouldn't add - # the same oswl into horizon if it already present in it. + # checksum but with different external_id. Checksum is calculated + # by the resource_data['current'] only. Thus we shouldn't add + # the same oswl into horizon if it already present in it and + # has no differences in checksum and added, modified, removed + # resources. old_oswl = horizon.get(idx) - if old_oswl is None or \ - old_oswl.resource_checksum != oswl.resource_checksum: + if ( + old_oswl is None or + old_oswl.resource_checksum != oswl.resource_checksum or + old_oswl.resource_data != oswl.resource_data + ): horizon[idx] = oswl def fill_date_gaps(self, oswls, to_date): diff --git a/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py b/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py index b57e402..3459388 100644 --- a/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py +++ b/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py @@ -707,3 +707,149 @@ class OswlStatsToCsvTest(OswlTest, DbTest): # Checking every flatten_resources contain fuel_release_info self.assertTrue(all(d[fuel_release_pos] for d in flatten_resources)) + + def test_all_resource_statuses_are_shown(self): + exporter = OswlStatsToCsv() + resource_type = consts.OSWL_RESOURCE_TYPES.vm + updated_time_str = datetime.utcnow().time().isoformat() + oswls_saved = [ + OpenStackWorkloadStats( + master_node_uid='x', + external_id=1, + cluster_id=1, + created_date=(datetime.utcnow().date() - + timedelta(days=8)), + updated_time=datetime.utcnow().time(), + resource_type=resource_type, + resource_checksum='checksum', + resource_data={'current': [{'id': 1, 'status': 'enabled', + 'tenant_id': 'first'}], + 'added': [{'id': 1, 'time': updated_time_str}], + 'modified': [], 'removed': []} + ), + # Removing and adding back the same resource. + OpenStackWorkloadStats( + master_node_uid='x', + external_id=2, + cluster_id=1, + created_date=(datetime.utcnow().date() - + timedelta(days=6)), + updated_time=datetime.utcnow().time(), + resource_type=resource_type, + resource_checksum='checksum', + resource_data={ + 'current': [{'id': 1, 'status': 'enabled', + 'tenant_id': 'second'}], + 'added': [{'id': 1, 'time': updated_time_str}], + 'modified': [], + 'removed': [{'id': 1, 'status': 'enabled', + 'time': updated_time_str, + 'tenant_id': 'second'}]} + ), + # Changing and restoring back resource + OpenStackWorkloadStats( + master_node_uid='x', + external_id=3, + cluster_id=1, + created_date=(datetime.utcnow().date() - + timedelta(days=4)), + updated_time=datetime.utcnow().time(), + resource_type=resource_type, + resource_checksum='checksum', + resource_data={ + 'current': [{'id': 1, 'enabled': True, + 'tenant_id': 'third'}], + 'added': [], + 'modified': [ + {'id': 1, 'enabled': False, 'time': updated_time_str}, + {'id': 1, 'enabled': True, 'time': updated_time_str}, + ], + 'removed': [] + } + ), + # Resource modified and finally deleted + OpenStackWorkloadStats( + master_node_uid='x', + external_id=4, + cluster_id=1, + created_date=(datetime.utcnow().date() - + timedelta(days=2)), + updated_time=datetime.utcnow().time(), + resource_type=resource_type, + resource_checksum='another_checksum', + resource_data={ + 'current': [], + 'added': [], + 'modified': [ + {'id': 1, 'enabled': False, 'time': updated_time_str}, + {'id': 1, 'enabled': True, 'time': updated_time_str}, + ], + 'removed': [{'id': 1, 'enabled': True, + 'tenant_id': 'fourth'}] + } + ), + ] + for oswl in oswls_saved: + db.session.add(oswl) + self.get_saved_inst_structs(oswls_saved, creation_date_range=(0, 0)) + + with app.test_request_context(): + oswls = get_oswls(resource_type) + + oswls_seamless = list(exporter.fill_date_gaps( + oswls, datetime.utcnow().date())) + + oswl_keys_paths, resource_keys_paths, csv_keys_paths = \ + exporter.get_resource_keys_paths(resource_type) + + flatten_resources = list(exporter.get_flatten_resources( + resource_type, oswl_keys_paths, resource_keys_paths, + oswls_seamless)) + + # Expected oswls num: 2 for 'first', 2 for 'second', 2 for 'third' + # and only one for finally removed 'fourth' + expected_oswls_num = 7 + self.assertEqual(expected_oswls_num, len(flatten_resources)) + + is_added_pos = csv_keys_paths.index([resource_type, 'is_added']) + is_modified_pos = csv_keys_paths.index([resource_type, 'is_modified']) + is_removed_pos = csv_keys_paths.index([resource_type, 'is_removed']) + tenant_id_pos = csv_keys_paths.index([resource_type, 'tenant_id']) + + def check_resource_state(resource, tenant_id, is_added, + is_modified, is_removed): + self.assertEquals(is_added, resource[is_added_pos]) + self.assertEquals(is_modified, resource[is_modified_pos]) + self.assertEquals(is_removed, resource[is_removed_pos]) + self.assertEquals(tenant_id, resource[tenant_id_pos]) + + # The fist oswl status True only in is_added + check_resource_state(flatten_resources[0], 'first', + True, False, False) + + # The first oswl status on the next day is False for + # is_added, is_modified, is_removed + check_resource_state(flatten_resources[1], 'first', + False, False, False) + + # The second oswl status True in is_added, is_modified + check_resource_state(flatten_resources[2], 'second', + True, False, True) + + # The second oswl status on the next day is False for + # is_added, is_modified, is_removed + check_resource_state(flatten_resources[3], 'second', + False, False, False) + + # The third oswl status True only in is_modified + check_resource_state(flatten_resources[4], 'third', + False, True, False) + + # The third oswl status on the next day is False for + # is_added, is_modified, is_removed + check_resource_state(flatten_resources[5], 'third', + False, False, False) + + # The fourth oswl status True in is_modified, is_deleted + check_resource_state(flatten_resources[6], 'fourth', + False, True, True)