openstack
/
fuel-stats
Code Issues Proposed changes

Fuel version from cluster used in OSWL reports 

Wrong source for Fuel version was used in the OSWL report.
We used the version of Fuel installed on master node, instead of
cluster version. Thus in case of upgrade master node we had
different fuel versions in the clusters and OSWL reports for
the same master node uid.

Now we are using Fuel version from OSWL.version_info or from
the cluster data. Fuel version from cluster is used also in the
plugins report.

Change-Id: I114beae261686fb6e4d77504dd102b722a43e611
Closes-Bug: #1547565
This commit is contained in:
Alexander Kislitsky 2016-02-29 12:51:45 +03:00
parent 154a24df0f
commit 9ce6b3f412
5 changed files with 164 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
analytics/fuel_analytics/api/resources/csv_exporter.py
View File

@ -179,6 +179,7 @@ def get_oswls_query(resource_type, from_date=None, to_date=None):
IS.creation_date.label('installation_created_date'),
IS.modification_date.label('installation_updated_date'),
IS.structure['fuel_release'].label('fuel_release_from_inst_info'),
IS.structure['clusters'].label('clusters'),
IS.is_filtered).\
join(IS, IS.master_node_uid == OSWS.master_node_uid).\
filter(OSWS.resource_type == resource_type).\

66
analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py
View File

@ -95,25 +95,69 @@ class OswlStatsToCsv(object):
return result
def handle_empty_version_info(self, oswl):
def _add_oswl_to_clusters_versions_cache(self, oswl, clusters_versions):
"""Adds oswl clusters info into clusters_versions cache.
:param oswl: OSWL DB object
:type oswl: fuel_analytics.api.db.model.OpenStackWorkloadStats
:param clusters_versions: cache for saving cluster versions with
structure {mn_uid: {cluster_id: fuel_release}}
:type clusters_versions: dict
"""
mn_uid = oswl.master_node_uid
# Result of csv_exporter.get_oswls_query contains info about all
# clusters in the installation. Thus we need to add clusters data
# into the cache only once for specified master_node_uid.
if mn_uid in clusters_versions:
return
if oswl.clusters is None:
return
clusters_versions[mn_uid] = {}
for cluster in oswl.clusters:
fuel_release = cluster.get('fuel_release')
if fuel_release:
clusters_versions[mn_uid][cluster['id']] = fuel_release
def handle_empty_version_info(self, oswl, clusters_versions):
"""Handles empty version info in oswl object
For OSWLs with empty version_info data we compose version_info
from InstallationStructure data and assign it to oswl object.
If we extract OpenStack release, os, name from
InstallationStructure.structure.clusters we have performance
degradation on fetching all clusters data in csv_exporter.get_oswls
thus only fuel_release will be used in case of empty version_info.
We bound InstallationStructure.structure.clusters to the oswl
and extract fuel_release from clusters data. If fuel_release
info doesn't provided by clusters data then
InstallationStructure.structure.fuel_release is used.
:param oswl: OSWL DB object
:type oswl: fuel_analytics.api.db.model.OpenStackWorkloadStats
:param clusters_versions: cache for saving cluster versions with
structure {mn_uid: {cluster_id: fuel_release}}
:type clusters_versions: dict
"""
if oswl.version_info:
return
fuel_release = oswl.fuel_release_from_inst_info or {}
self._add_oswl_to_clusters_versions_cache(oswl, clusters_versions)
mn_uid = oswl.master_node_uid
cluster_id = oswl.cluster_id
# Fetching fuel_release info from cache
fuel_release = clusters_versions.get(mn_uid, {}).get(cluster_id)
# If clusters data doesn't contain fuel_release info we are using
# info from installation info
if fuel_release is None:
fuel_release = oswl.fuel_release_from_inst_info or {}
fuel_release = fuel_release.get('release')
oswl.version_info = {
'fuel_release': fuel_release.get('release')
'fuel_release': fuel_release
}
def get_flatten_resources(self, resource_type, oswl_keys_paths,
@ -126,9 +170,15 @@ class OswlStatsToCsv(object):
:return: generator on flatten resources info collection
"""
app.logger.debug("Getting OSWL flatten %s info started", resource_type)
# Cache for saving cluster versions. Cache is used only if version_info
# is not provided in the oswl.
# Structure: {mn_uid: {cluster_id: fuel_release}}
clusters_versions = {}
for oswl in oswls:
try:
self.handle_empty_version_info(oswl)
self.handle_empty_version_info(oswl, clusters_versions)
flatten_oswl = export_utils.get_flatten_data(oswl_keys_paths,
oswl)
resource_data = oswl.resource_data

7
analytics/fuel_analytics/api/resources/utils/stats_to_csv.py
View File

@ -86,9 +86,8 @@ class StatsToCsv(object):
plugin_key_paths = export_utils.get_keys_paths(plugin_skeleton)
structure_key_paths = [['master_node_uid'],
['structure', 'fuel_packages'],
['structure', 'fuel_release', 'release']]
cluster_key_paths = [['cluster_id']]
['structure', 'fuel_packages']]
cluster_key_paths = [['cluster_id'], ['cluster_fuel_version']]
result_key_paths = plugin_key_paths + cluster_key_paths + \
structure_key_paths
app.logger.debug("Plugin keys paths got")
@ -202,6 +201,8 @@ class StatsToCsv(object):
for cluster in clusters:
cluster['cluster_id'] = cluster['id']
cluster['cluster_fuel_version'] = \
cluster.get('fuel_version')
flatten_cluster = export_utils.get_flatten_data(
cluster_keys_paths, cluster)
plugins = cluster.pop('installed_plugins', [])

93
analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py
View File

@ -28,6 +28,7 @@ from fuel_analytics.test.base import DbTest
from fuel_analytics.api.app import app
from fuel_analytics.api.app import db
from fuel_analytics.api.common import consts
from fuel_analytics.api.db.model import InstallationStructure
from fuel_analytics.api.db.model import OpenStackWorkloadStats
from fuel_analytics.api.resources.csv_exporter import get_oswls
from fuel_analytics.api.resources.csv_exporter import get_oswls_query
@ -853,3 +854,95 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
# The fourth oswl status True in is_modified, is_deleted
check_resource_state(flatten_resources[6], 'fourth',
False, True, True)
def test_fuel_version_from_clusters_data_is_used(self):
master_node_uid = 'x'
exporter = OswlStatsToCsv()
resource_type = consts.OSWL_RESOURCE_TYPES.vm
version_from_cluster = '7.0'
version_from_version_info = '9.0'
version_from_installation_info = '8.0'
installation_date = datetime.utcnow().date() - timedelta(days=3)
# Upgraded Fuel and not upgraded cluster
structure = InstallationStructure(
master_node_uid=master_node_uid,
structure={
'fuel_release': {'release': version_from_installation_info},
'clusters_num': 2,
'clusters': [
{'id': 1, 'fuel_release': version_from_cluster},
{'id': 2}
],
'unallocated_nodes_num_range': 0,
'allocated_nodes_num': 0
},
creation_date=installation_date,
is_filtered=False
)
db.session.add(structure)
oswls = [
OpenStackWorkloadStats(
master_node_uid=master_node_uid,
external_id=1,
cluster_id=1,
created_date=installation_date,
updated_time=datetime.utcnow().time(),
resource_type=resource_type,
resource_checksum='info_from_cluster',
resource_data={'current': [{'id': 1, 'status': 'enabled'}],
'added': [], 'modified': [], 'removed': []},
version_info=None
),
OpenStackWorkloadStats(
master_node_uid=master_node_uid,
external_id=3,
cluster_id=1,
created_date=installation_date + timedelta(days=1),
updated_time=datetime.utcnow().time(),
resource_type=resource_type,
resource_checksum='info_from_version_info',
resource_data={'current': [{'id': 1}],
'added': [], 'modified': [], 'removed': []},
version_info={'fuel_release': version_from_version_info}
),
OpenStackWorkloadStats(
master_node_uid=master_node_uid,
external_id=2,
cluster_id=2,
created_date=installation_date + timedelta(days=2),
updated_time=datetime.utcnow().time(),
resource_type=resource_type,
resource_checksum='info_from_installation_info',
resource_data={'current': [{'id': 1}],
'added': [], 'modified': [], 'removed': []},
version_info=None
)
]
for oswl in oswls:
db.session.add(oswl)
with app.test_request_context():
oswls_data = list(get_oswls(resource_type))
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
exporter.get_resource_keys_paths(resource_type)
fuel_release_pos = csv_keys_paths.index(
['version_info', 'fuel_release'])
flatten_resources = list(exporter.get_flatten_resources(
resource_type, oswl_keys_paths, resource_keys_paths, oswls_data))
self.assertEqual(len(oswls), len(flatten_resources))
# Checking release info fetched from cluster
self.assertEqual(version_from_cluster,
flatten_resources[0][fuel_release_pos])
# Checking release info fetched from oswl.version_info
self.assertEqual(version_from_version_info,
flatten_resources[1][fuel_release_pos])
# Checking release info fetched from installation info
self.assertEqual(version_from_installation_info,
flatten_resources[2][fuel_release_pos])

15
analytics/fuel_analytics/test/api/resources/utils/test_plugins_to_csv.py
View File

@ -36,14 +36,13 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest):
exporter = StatsToCsv()
_, _, _, csv_keys_paths = exporter.get_plugin_keys_paths()
self.assertTrue(['cluster_id'] in csv_keys_paths)
self.assertTrue(['cluster_fuel_version'] in csv_keys_paths)
self.assertTrue(['master_node_uid'] in csv_keys_paths)
self.assertTrue(['name'] in csv_keys_paths)
self.assertTrue(['version'] in csv_keys_paths)
self.assertTrue(['fuel_version'] in csv_keys_paths)
self.assertTrue(['package_version'] in csv_keys_paths)
self.assertTrue(['structure', 'fuel_packages'] in csv_keys_paths)
self.assertTrue(['structure', 'fuel_release', 'release'] in
csv_keys_paths)
def test_get_flatten_plugins(self):
installations_num = 10
@ -100,7 +99,8 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest):
self.assertEqual(num - 1, len(list(flatten_plugins)))
def test_fuel_release_info_in_flatten_plugins(self):
release = '8.0'
inst_fuel_version = '8.0'
cluster_fuel_version = '7.0'
packages = ['z', 'a', 'c']
inst_structures = [
model.InstallationStructure(
@ -108,10 +108,11 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest):
creation_date=datetime.datetime.utcnow(),
is_filtered=False,
structure={
'fuel_release': {'release': release},
'fuel_release': {'release': inst_fuel_version},
'fuel_packages': packages,
'clusters': [{
'id': 1, 'nodes': [],
'fuel_version': cluster_fuel_version,
'installed_plugins': [{
'name': 'plugin_a',
'version': 'plugin_version_0',
@ -133,10 +134,10 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest):
flatten_plugins = exporter.get_flatten_plugins(
structure_paths, cluster_paths, plugins_paths, inst_structures)
pos_release = csv_paths.index(['structure', 'fuel_release',
'release'])
pos_fuel_version = csv_paths.index(['cluster_fuel_version'])
pos_packages = csv_paths.index(['structure', 'fuel_packages'])
for flatten_plugin in flatten_plugins:
self.assertEqual(release, flatten_plugin[pos_release])
self.assertEqual(cluster_fuel_version,
flatten_plugin[pos_fuel_version])
self.assertEqual(' '.join(packages),
flatten_plugin[pos_packages])