OSWL resource status representation in report fixed
If resource removed and added we show only one record in the report with status True in is_added, is_removed. Change-Id: I783414ee4c23d07e02d13e76eb19fe3a2b3db80e Closes-Bug: #1526320
This commit is contained in:
parent
c1fe2d34d9
commit
564d89d134
|
@ -133,19 +133,27 @@ class OswlStatsToCsv(object):
|
||||||
oswl)
|
oswl)
|
||||||
resource_data = oswl.resource_data
|
resource_data = oswl.resource_data
|
||||||
current = resource_data.get('current', [])
|
current = resource_data.get('current', [])
|
||||||
|
added = resource_data.get('added', [])
|
||||||
|
modified = resource_data.get('modified', [])
|
||||||
removed = resource_data.get('removed', [])
|
removed = resource_data.get('removed', [])
|
||||||
# Filtering id, time only data
|
# Filtering wrong formatted removed data
|
||||||
removed = itertools.ifilter(lambda x: len(x) > 2, removed)
|
# delivered by old Fuel versions
|
||||||
|
removed = [res for res in removed if len(res) > 2]
|
||||||
|
|
||||||
# Extracting ids or oswl resources
|
# Extracting ids or oswl resources
|
||||||
added_ids = set(item['id'] for item in
|
added_ids = set(item['id'] for item in added)
|
||||||
resource_data.get('added', []))
|
modified_ids = set(item['id'] for item in modified)
|
||||||
modified_ids = set(item['id'] for item in
|
removed_ids = set(item['id'] for item in removed)
|
||||||
resource_data.get('removed', []))
|
|
||||||
removed_ids = set(item['id'] for item in
|
|
||||||
resource_data.get('modified', []))
|
|
||||||
|
|
||||||
for resource in itertools.chain(current, removed):
|
# If resource removed and added several times it would
|
||||||
|
# be present in current and removed. We should exclude
|
||||||
|
# duplicates from flatten resources of the same
|
||||||
|
# resource.
|
||||||
|
current_ids = set(item['id'] for item in current)
|
||||||
|
finally_removed = (res for res in removed
|
||||||
|
if res['id'] not in current_ids)
|
||||||
|
|
||||||
|
for resource in itertools.chain(current, finally_removed):
|
||||||
flatten_resource = export_utils.get_flatten_data(
|
flatten_resource = export_utils.get_flatten_data(
|
||||||
resource_keys_paths, {resource_type: resource})
|
resource_keys_paths, {resource_type: resource})
|
||||||
additional_info = self.get_additional_resource_info(
|
additional_info = self.get_additional_resource_info(
|
||||||
|
@ -205,11 +213,17 @@ class OswlStatsToCsv(object):
|
||||||
idx = export_utils.get_index(oswl, *self.OSWL_INDEX_FIELDS)
|
idx = export_utils.get_index(oswl, *self.OSWL_INDEX_FIELDS)
|
||||||
|
|
||||||
# We can have duplication of the oswls in the DB with the same
|
# We can have duplication of the oswls in the DB with the same
|
||||||
# checksum but with different external_id. We shouldn't add
|
# checksum but with different external_id. Checksum is calculated
|
||||||
# the same oswl into horizon if it already present in it.
|
# by the resource_data['current'] only. Thus we shouldn't add
|
||||||
|
# the same oswl into horizon if it already present in it and
|
||||||
|
# has no differences in checksum and added, modified, removed
|
||||||
|
# resources.
|
||||||
old_oswl = horizon.get(idx)
|
old_oswl = horizon.get(idx)
|
||||||
if old_oswl is None or \
|
if (
|
||||||
old_oswl.resource_checksum != oswl.resource_checksum:
|
old_oswl is None or
|
||||||
|
old_oswl.resource_checksum != oswl.resource_checksum or
|
||||||
|
old_oswl.resource_data != oswl.resource_data
|
||||||
|
):
|
||||||
horizon[idx] = oswl
|
horizon[idx] = oswl
|
||||||
|
|
||||||
def fill_date_gaps(self, oswls, to_date):
|
def fill_date_gaps(self, oswls, to_date):
|
||||||
|
|
|
@ -707,3 +707,149 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
|
||||||
|
|
||||||
# Checking every flatten_resources contain fuel_release_info
|
# Checking every flatten_resources contain fuel_release_info
|
||||||
self.assertTrue(all(d[fuel_release_pos] for d in flatten_resources))
|
self.assertTrue(all(d[fuel_release_pos] for d in flatten_resources))
|
||||||
|
|
||||||
|
def test_all_resource_statuses_are_shown(self):
|
||||||
|
exporter = OswlStatsToCsv()
|
||||||
|
resource_type = consts.OSWL_RESOURCE_TYPES.vm
|
||||||
|
updated_time_str = datetime.utcnow().time().isoformat()
|
||||||
|
oswls_saved = [
|
||||||
|
OpenStackWorkloadStats(
|
||||||
|
master_node_uid='x',
|
||||||
|
external_id=1,
|
||||||
|
cluster_id=1,
|
||||||
|
created_date=(datetime.utcnow().date() -
|
||||||
|
timedelta(days=8)),
|
||||||
|
updated_time=datetime.utcnow().time(),
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_checksum='checksum',
|
||||||
|
resource_data={'current': [{'id': 1, 'status': 'enabled',
|
||||||
|
'tenant_id': 'first'}],
|
||||||
|
'added': [{'id': 1, 'time': updated_time_str}],
|
||||||
|
'modified': [], 'removed': []}
|
||||||
|
),
|
||||||
|
# Removing and adding back the same resource.
|
||||||
|
OpenStackWorkloadStats(
|
||||||
|
master_node_uid='x',
|
||||||
|
external_id=2,
|
||||||
|
cluster_id=1,
|
||||||
|
created_date=(datetime.utcnow().date() -
|
||||||
|
timedelta(days=6)),
|
||||||
|
updated_time=datetime.utcnow().time(),
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_checksum='checksum',
|
||||||
|
resource_data={
|
||||||
|
'current': [{'id': 1, 'status': 'enabled',
|
||||||
|
'tenant_id': 'second'}],
|
||||||
|
'added': [{'id': 1, 'time': updated_time_str}],
|
||||||
|
'modified': [],
|
||||||
|
'removed': [{'id': 1, 'status': 'enabled',
|
||||||
|
'time': updated_time_str,
|
||||||
|
'tenant_id': 'second'}]}
|
||||||
|
),
|
||||||
|
# Changing and restoring back resource
|
||||||
|
OpenStackWorkloadStats(
|
||||||
|
master_node_uid='x',
|
||||||
|
external_id=3,
|
||||||
|
cluster_id=1,
|
||||||
|
created_date=(datetime.utcnow().date() -
|
||||||
|
timedelta(days=4)),
|
||||||
|
updated_time=datetime.utcnow().time(),
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_checksum='checksum',
|
||||||
|
resource_data={
|
||||||
|
'current': [{'id': 1, 'enabled': True,
|
||||||
|
'tenant_id': 'third'}],
|
||||||
|
'added': [],
|
||||||
|
'modified': [
|
||||||
|
{'id': 1, 'enabled': False, 'time': updated_time_str},
|
||||||
|
{'id': 1, 'enabled': True, 'time': updated_time_str},
|
||||||
|
],
|
||||||
|
'removed': []
|
||||||
|
}
|
||||||
|
),
|
||||||
|
# Resource modified and finally deleted
|
||||||
|
OpenStackWorkloadStats(
|
||||||
|
master_node_uid='x',
|
||||||
|
external_id=4,
|
||||||
|
cluster_id=1,
|
||||||
|
created_date=(datetime.utcnow().date() -
|
||||||
|
timedelta(days=2)),
|
||||||
|
updated_time=datetime.utcnow().time(),
|
||||||
|
resource_type=resource_type,
|
||||||
|
resource_checksum='another_checksum',
|
||||||
|
resource_data={
|
||||||
|
'current': [],
|
||||||
|
'added': [],
|
||||||
|
'modified': [
|
||||||
|
{'id': 1, 'enabled': False, 'time': updated_time_str},
|
||||||
|
{'id': 1, 'enabled': True, 'time': updated_time_str},
|
||||||
|
],
|
||||||
|
'removed': [{'id': 1, 'enabled': True,
|
||||||
|
'tenant_id': 'fourth'}]
|
||||||
|
}
|
||||||
|
),
|
||||||
|
]
|
||||||
|
for oswl in oswls_saved:
|
||||||
|
db.session.add(oswl)
|
||||||
|
self.get_saved_inst_structs(oswls_saved, creation_date_range=(0, 0))
|
||||||
|
|
||||||
|
with app.test_request_context():
|
||||||
|
oswls = get_oswls(resource_type)
|
||||||
|
|
||||||
|
oswls_seamless = list(exporter.fill_date_gaps(
|
||||||
|
oswls, datetime.utcnow().date()))
|
||||||
|
|
||||||
|
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
|
||||||
|
exporter.get_resource_keys_paths(resource_type)
|
||||||
|
|
||||||
|
flatten_resources = list(exporter.get_flatten_resources(
|
||||||
|
resource_type, oswl_keys_paths, resource_keys_paths,
|
||||||
|
oswls_seamless))
|
||||||
|
|
||||||
|
# Expected oswls num: 2 for 'first', 2 for 'second', 2 for 'third'
|
||||||
|
# and only one for finally removed 'fourth'
|
||||||
|
expected_oswls_num = 7
|
||||||
|
self.assertEqual(expected_oswls_num, len(flatten_resources))
|
||||||
|
|
||||||
|
is_added_pos = csv_keys_paths.index([resource_type, 'is_added'])
|
||||||
|
is_modified_pos = csv_keys_paths.index([resource_type, 'is_modified'])
|
||||||
|
is_removed_pos = csv_keys_paths.index([resource_type, 'is_removed'])
|
||||||
|
tenant_id_pos = csv_keys_paths.index([resource_type, 'tenant_id'])
|
||||||
|
|
||||||
|
def check_resource_state(resource, tenant_id, is_added,
|
||||||
|
is_modified, is_removed):
|
||||||
|
self.assertEquals(is_added, resource[is_added_pos])
|
||||||
|
self.assertEquals(is_modified, resource[is_modified_pos])
|
||||||
|
self.assertEquals(is_removed, resource[is_removed_pos])
|
||||||
|
self.assertEquals(tenant_id, resource[tenant_id_pos])
|
||||||
|
|
||||||
|
# The fist oswl status True only in is_added
|
||||||
|
check_resource_state(flatten_resources[0], 'first',
|
||||||
|
True, False, False)
|
||||||
|
|
||||||
|
# The first oswl status on the next day is False for
|
||||||
|
# is_added, is_modified, is_removed
|
||||||
|
check_resource_state(flatten_resources[1], 'first',
|
||||||
|
False, False, False)
|
||||||
|
|
||||||
|
# The second oswl status True in is_added, is_modified
|
||||||
|
check_resource_state(flatten_resources[2], 'second',
|
||||||
|
True, False, True)
|
||||||
|
|
||||||
|
# The second oswl status on the next day is False for
|
||||||
|
# is_added, is_modified, is_removed
|
||||||
|
check_resource_state(flatten_resources[3], 'second',
|
||||||
|
False, False, False)
|
||||||
|
|
||||||
|
# The third oswl status True only in is_modified
|
||||||
|
check_resource_state(flatten_resources[4], 'third',
|
||||||
|
False, True, False)
|
||||||
|
|
||||||
|
# The third oswl status on the next day is False for
|
||||||
|
# is_added, is_modified, is_removed
|
||||||
|
check_resource_state(flatten_resources[5], 'third',
|
||||||
|
False, False, False)
|
||||||
|
|
||||||
|
# The fourth oswl status True in is_modified, is_deleted
|
||||||
|
check_resource_state(flatten_resources[6], 'fourth',
|
||||||
|
False, True, True)
|
||||||
|
|
Loading…
Reference in New Issue