Fuel version from cluster used in OSWL reports
Wrong source for Fuel version was used in the OSWL report. We used the version of Fuel installed on master node, instead of cluster version. Thus in case of upgrade master node we had different fuel versions in the clusters and OSWL reports for the same master node uid. Now we are using Fuel version from OSWL.version_info or from the cluster data. Fuel version from cluster is used also in the plugins report. Change-Id: I114beae261686fb6e4d77504dd102b722a43e611 Closes-Bug: #1547565
This commit is contained in:
parent
154a24df0f
commit
9ce6b3f412
|
@ -179,6 +179,7 @@ def get_oswls_query(resource_type, from_date=None, to_date=None):
|
|||
IS.creation_date.label('installation_created_date'),
|
||||
IS.modification_date.label('installation_updated_date'),
|
||||
IS.structure['fuel_release'].label('fuel_release_from_inst_info'),
|
||||
IS.structure['clusters'].label('clusters'),
|
||||
IS.is_filtered).\
|
||||
join(IS, IS.master_node_uid == OSWS.master_node_uid).\
|
||||
filter(OSWS.resource_type == resource_type).\
|
||||
|
|
|
@ -95,25 +95,69 @@ class OswlStatsToCsv(object):
|
|||
|
||||
return result
|
||||
|
||||
def handle_empty_version_info(self, oswl):
|
||||
def _add_oswl_to_clusters_versions_cache(self, oswl, clusters_versions):
|
||||
"""Adds oswl clusters info into clusters_versions cache.
|
||||
|
||||
:param oswl: OSWL DB object
|
||||
:type oswl: fuel_analytics.api.db.model.OpenStackWorkloadStats
|
||||
:param clusters_versions: cache for saving cluster versions with
|
||||
structure {mn_uid: {cluster_id: fuel_release}}
|
||||
:type clusters_versions: dict
|
||||
"""
|
||||
|
||||
mn_uid = oswl.master_node_uid
|
||||
|
||||
# Result of csv_exporter.get_oswls_query contains info about all
|
||||
# clusters in the installation. Thus we need to add clusters data
|
||||
# into the cache only once for specified master_node_uid.
|
||||
if mn_uid in clusters_versions:
|
||||
return
|
||||
|
||||
if oswl.clusters is None:
|
||||
return
|
||||
|
||||
clusters_versions[mn_uid] = {}
|
||||
|
||||
for cluster in oswl.clusters:
|
||||
fuel_release = cluster.get('fuel_release')
|
||||
if fuel_release:
|
||||
clusters_versions[mn_uid][cluster['id']] = fuel_release
|
||||
|
||||
def handle_empty_version_info(self, oswl, clusters_versions):
|
||||
"""Handles empty version info in oswl object
|
||||
|
||||
For OSWLs with empty version_info data we compose version_info
|
||||
from InstallationStructure data and assign it to oswl object.
|
||||
If we extract OpenStack release, os, name from
|
||||
InstallationStructure.structure.clusters we have performance
|
||||
degradation on fetching all clusters data in csv_exporter.get_oswls
|
||||
thus only fuel_release will be used in case of empty version_info.
|
||||
We bound InstallationStructure.structure.clusters to the oswl
|
||||
and extract fuel_release from clusters data. If fuel_release
|
||||
info doesn't provided by clusters data then
|
||||
InstallationStructure.structure.fuel_release is used.
|
||||
|
||||
:param oswl: OSWL DB object
|
||||
:type oswl: fuel_analytics.api.db.model.OpenStackWorkloadStats
|
||||
:param clusters_versions: cache for saving cluster versions with
|
||||
structure {mn_uid: {cluster_id: fuel_release}}
|
||||
:type clusters_versions: dict
|
||||
"""
|
||||
if oswl.version_info:
|
||||
return
|
||||
|
||||
fuel_release = oswl.fuel_release_from_inst_info or {}
|
||||
self._add_oswl_to_clusters_versions_cache(oswl, clusters_versions)
|
||||
|
||||
mn_uid = oswl.master_node_uid
|
||||
cluster_id = oswl.cluster_id
|
||||
|
||||
# Fetching fuel_release info from cache
|
||||
fuel_release = clusters_versions.get(mn_uid, {}).get(cluster_id)
|
||||
|
||||
# If clusters data doesn't contain fuel_release info we are using
|
||||
# info from installation info
|
||||
if fuel_release is None:
|
||||
fuel_release = oswl.fuel_release_from_inst_info or {}
|
||||
fuel_release = fuel_release.get('release')
|
||||
|
||||
oswl.version_info = {
|
||||
'fuel_release': fuel_release.get('release')
|
||||
'fuel_release': fuel_release
|
||||
}
|
||||
|
||||
def get_flatten_resources(self, resource_type, oswl_keys_paths,
|
||||
|
@ -126,9 +170,15 @@ class OswlStatsToCsv(object):
|
|||
:return: generator on flatten resources info collection
|
||||
"""
|
||||
app.logger.debug("Getting OSWL flatten %s info started", resource_type)
|
||||
|
||||
# Cache for saving cluster versions. Cache is used only if version_info
|
||||
# is not provided in the oswl.
|
||||
# Structure: {mn_uid: {cluster_id: fuel_release}}
|
||||
clusters_versions = {}
|
||||
|
||||
for oswl in oswls:
|
||||
try:
|
||||
self.handle_empty_version_info(oswl)
|
||||
self.handle_empty_version_info(oswl, clusters_versions)
|
||||
flatten_oswl = export_utils.get_flatten_data(oswl_keys_paths,
|
||||
oswl)
|
||||
resource_data = oswl.resource_data
|
||||
|
|
|
@ -86,9 +86,8 @@ class StatsToCsv(object):
|
|||
|
||||
plugin_key_paths = export_utils.get_keys_paths(plugin_skeleton)
|
||||
structure_key_paths = [['master_node_uid'],
|
||||
['structure', 'fuel_packages'],
|
||||
['structure', 'fuel_release', 'release']]
|
||||
cluster_key_paths = [['cluster_id']]
|
||||
['structure', 'fuel_packages']]
|
||||
cluster_key_paths = [['cluster_id'], ['cluster_fuel_version']]
|
||||
result_key_paths = plugin_key_paths + cluster_key_paths + \
|
||||
structure_key_paths
|
||||
app.logger.debug("Plugin keys paths got")
|
||||
|
@ -202,6 +201,8 @@ class StatsToCsv(object):
|
|||
|
||||
for cluster in clusters:
|
||||
cluster['cluster_id'] = cluster['id']
|
||||
cluster['cluster_fuel_version'] = \
|
||||
cluster.get('fuel_version')
|
||||
flatten_cluster = export_utils.get_flatten_data(
|
||||
cluster_keys_paths, cluster)
|
||||
plugins = cluster.pop('installed_plugins', [])
|
||||
|
|
|
@ -28,6 +28,7 @@ from fuel_analytics.test.base import DbTest
|
|||
from fuel_analytics.api.app import app
|
||||
from fuel_analytics.api.app import db
|
||||
from fuel_analytics.api.common import consts
|
||||
from fuel_analytics.api.db.model import InstallationStructure
|
||||
from fuel_analytics.api.db.model import OpenStackWorkloadStats
|
||||
from fuel_analytics.api.resources.csv_exporter import get_oswls
|
||||
from fuel_analytics.api.resources.csv_exporter import get_oswls_query
|
||||
|
@ -853,3 +854,95 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
|
|||
# The fourth oswl status True in is_modified, is_deleted
|
||||
check_resource_state(flatten_resources[6], 'fourth',
|
||||
False, True, True)
|
||||
|
||||
def test_fuel_version_from_clusters_data_is_used(self):
|
||||
master_node_uid = 'x'
|
||||
exporter = OswlStatsToCsv()
|
||||
resource_type = consts.OSWL_RESOURCE_TYPES.vm
|
||||
version_from_cluster = '7.0'
|
||||
version_from_version_info = '9.0'
|
||||
version_from_installation_info = '8.0'
|
||||
installation_date = datetime.utcnow().date() - timedelta(days=3)
|
||||
|
||||
# Upgraded Fuel and not upgraded cluster
|
||||
structure = InstallationStructure(
|
||||
master_node_uid=master_node_uid,
|
||||
structure={
|
||||
'fuel_release': {'release': version_from_installation_info},
|
||||
'clusters_num': 2,
|
||||
'clusters': [
|
||||
{'id': 1, 'fuel_release': version_from_cluster},
|
||||
{'id': 2}
|
||||
],
|
||||
'unallocated_nodes_num_range': 0,
|
||||
'allocated_nodes_num': 0
|
||||
},
|
||||
creation_date=installation_date,
|
||||
is_filtered=False
|
||||
)
|
||||
db.session.add(structure)
|
||||
|
||||
oswls = [
|
||||
OpenStackWorkloadStats(
|
||||
master_node_uid=master_node_uid,
|
||||
external_id=1,
|
||||
cluster_id=1,
|
||||
created_date=installation_date,
|
||||
updated_time=datetime.utcnow().time(),
|
||||
resource_type=resource_type,
|
||||
resource_checksum='info_from_cluster',
|
||||
resource_data={'current': [{'id': 1, 'status': 'enabled'}],
|
||||
'added': [], 'modified': [], 'removed': []},
|
||||
version_info=None
|
||||
),
|
||||
OpenStackWorkloadStats(
|
||||
master_node_uid=master_node_uid,
|
||||
external_id=3,
|
||||
cluster_id=1,
|
||||
created_date=installation_date + timedelta(days=1),
|
||||
updated_time=datetime.utcnow().time(),
|
||||
resource_type=resource_type,
|
||||
resource_checksum='info_from_version_info',
|
||||
resource_data={'current': [{'id': 1}],
|
||||
'added': [], 'modified': [], 'removed': []},
|
||||
version_info={'fuel_release': version_from_version_info}
|
||||
),
|
||||
OpenStackWorkloadStats(
|
||||
master_node_uid=master_node_uid,
|
||||
external_id=2,
|
||||
cluster_id=2,
|
||||
created_date=installation_date + timedelta(days=2),
|
||||
updated_time=datetime.utcnow().time(),
|
||||
resource_type=resource_type,
|
||||
resource_checksum='info_from_installation_info',
|
||||
resource_data={'current': [{'id': 1}],
|
||||
'added': [], 'modified': [], 'removed': []},
|
||||
version_info=None
|
||||
)
|
||||
]
|
||||
for oswl in oswls:
|
||||
db.session.add(oswl)
|
||||
|
||||
with app.test_request_context():
|
||||
oswls_data = list(get_oswls(resource_type))
|
||||
|
||||
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
|
||||
exporter.get_resource_keys_paths(resource_type)
|
||||
fuel_release_pos = csv_keys_paths.index(
|
||||
['version_info', 'fuel_release'])
|
||||
flatten_resources = list(exporter.get_flatten_resources(
|
||||
resource_type, oswl_keys_paths, resource_keys_paths, oswls_data))
|
||||
|
||||
self.assertEqual(len(oswls), len(flatten_resources))
|
||||
|
||||
# Checking release info fetched from cluster
|
||||
self.assertEqual(version_from_cluster,
|
||||
flatten_resources[0][fuel_release_pos])
|
||||
|
||||
# Checking release info fetched from oswl.version_info
|
||||
self.assertEqual(version_from_version_info,
|
||||
flatten_resources[1][fuel_release_pos])
|
||||
|
||||
# Checking release info fetched from installation info
|
||||
self.assertEqual(version_from_installation_info,
|
||||
flatten_resources[2][fuel_release_pos])
|
||||
|
|
|
@ -36,14 +36,13 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest):
|
|||
exporter = StatsToCsv()
|
||||
_, _, _, csv_keys_paths = exporter.get_plugin_keys_paths()
|
||||
self.assertTrue(['cluster_id'] in csv_keys_paths)
|
||||
self.assertTrue(['cluster_fuel_version'] in csv_keys_paths)
|
||||
self.assertTrue(['master_node_uid'] in csv_keys_paths)
|
||||
self.assertTrue(['name'] in csv_keys_paths)
|
||||
self.assertTrue(['version'] in csv_keys_paths)
|
||||
self.assertTrue(['fuel_version'] in csv_keys_paths)
|
||||
self.assertTrue(['package_version'] in csv_keys_paths)
|
||||
self.assertTrue(['structure', 'fuel_packages'] in csv_keys_paths)
|
||||
self.assertTrue(['structure', 'fuel_release', 'release'] in
|
||||
csv_keys_paths)
|
||||
|
||||
def test_get_flatten_plugins(self):
|
||||
installations_num = 10
|
||||
|
@ -100,7 +99,8 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest):
|
|||
self.assertEqual(num - 1, len(list(flatten_plugins)))
|
||||
|
||||
def test_fuel_release_info_in_flatten_plugins(self):
|
||||
release = '8.0'
|
||||
inst_fuel_version = '8.0'
|
||||
cluster_fuel_version = '7.0'
|
||||
packages = ['z', 'a', 'c']
|
||||
inst_structures = [
|
||||
model.InstallationStructure(
|
||||
|
@ -108,10 +108,11 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest):
|
|||
creation_date=datetime.datetime.utcnow(),
|
||||
is_filtered=False,
|
||||
structure={
|
||||
'fuel_release': {'release': release},
|
||||
'fuel_release': {'release': inst_fuel_version},
|
||||
'fuel_packages': packages,
|
||||
'clusters': [{
|
||||
'id': 1, 'nodes': [],
|
||||
'fuel_version': cluster_fuel_version,
|
||||
'installed_plugins': [{
|
||||
'name': 'plugin_a',
|
||||
'version': 'plugin_version_0',
|
||||
|
@ -133,10 +134,10 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest):
|
|||
flatten_plugins = exporter.get_flatten_plugins(
|
||||
structure_paths, cluster_paths, plugins_paths, inst_structures)
|
||||
|
||||
pos_release = csv_paths.index(['structure', 'fuel_release',
|
||||
'release'])
|
||||
pos_fuel_version = csv_paths.index(['cluster_fuel_version'])
|
||||
pos_packages = csv_paths.index(['structure', 'fuel_packages'])
|
||||
for flatten_plugin in flatten_plugins:
|
||||
self.assertEqual(release, flatten_plugin[pos_release])
|
||||
self.assertEqual(cluster_fuel_version,
|
||||
flatten_plugin[pos_fuel_version])
|
||||
self.assertEqual(' '.join(packages),
|
||||
flatten_plugin[pos_packages])
|
||||
|
|
Loading…
Reference in New Issue