From 9ce6b3f412d302d5ab3b720afb3fa8ad575d1a32 Mon Sep 17 00:00:00 2001 From: Alexander Kislitsky Date: Mon, 29 Feb 2016 12:51:45 +0300 Subject: [PATCH] Fuel version from cluster used in OSWL reports Wrong source for Fuel version was used in the OSWL report. We used the version of Fuel installed on master node, instead of cluster version. Thus in case of upgrade master node we had different fuel versions in the clusters and OSWL reports for the same master node uid. Now we are using Fuel version from OSWL.version_info or from the cluster data. Fuel version from cluster is used also in the plugins report. Change-Id: I114beae261686fb6e4d77504dd102b722a43e611 Closes-Bug: #1547565 --- .../api/resources/csv_exporter.py | 1 + .../api/resources/utils/oswl_stats_to_csv.py | 66 +++++++++++-- .../api/resources/utils/stats_to_csv.py | 7 +- .../resources/utils/test_oswl_stats_to_csv.py | 93 +++++++++++++++++++ .../resources/utils/test_plugins_to_csv.py | 15 +-- 5 files changed, 164 insertions(+), 18 deletions(-) diff --git a/analytics/fuel_analytics/api/resources/csv_exporter.py b/analytics/fuel_analytics/api/resources/csv_exporter.py index 75aabcd..7aa3376 100644 --- a/analytics/fuel_analytics/api/resources/csv_exporter.py +++ b/analytics/fuel_analytics/api/resources/csv_exporter.py @@ -179,6 +179,7 @@ def get_oswls_query(resource_type, from_date=None, to_date=None): IS.creation_date.label('installation_created_date'), IS.modification_date.label('installation_updated_date'), IS.structure['fuel_release'].label('fuel_release_from_inst_info'), + IS.structure['clusters'].label('clusters'), IS.is_filtered).\ join(IS, IS.master_node_uid == OSWS.master_node_uid).\ filter(OSWS.resource_type == resource_type).\ diff --git a/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py b/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py index f8f71f8..bc69c1b 100644 --- a/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py +++ b/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py @@ -95,25 +95,69 @@ class OswlStatsToCsv(object): return result - def handle_empty_version_info(self, oswl): + def _add_oswl_to_clusters_versions_cache(self, oswl, clusters_versions): + """Adds oswl clusters info into clusters_versions cache. + + :param oswl: OSWL DB object + :type oswl: fuel_analytics.api.db.model.OpenStackWorkloadStats + :param clusters_versions: cache for saving cluster versions with + structure {mn_uid: {cluster_id: fuel_release}} + :type clusters_versions: dict + """ + + mn_uid = oswl.master_node_uid + + # Result of csv_exporter.get_oswls_query contains info about all + # clusters in the installation. Thus we need to add clusters data + # into the cache only once for specified master_node_uid. + if mn_uid in clusters_versions: + return + + if oswl.clusters is None: + return + + clusters_versions[mn_uid] = {} + + for cluster in oswl.clusters: + fuel_release = cluster.get('fuel_release') + if fuel_release: + clusters_versions[mn_uid][cluster['id']] = fuel_release + + def handle_empty_version_info(self, oswl, clusters_versions): """Handles empty version info in oswl object For OSWLs with empty version_info data we compose version_info from InstallationStructure data and assign it to oswl object. - If we extract OpenStack release, os, name from - InstallationStructure.structure.clusters we have performance - degradation on fetching all clusters data in csv_exporter.get_oswls - thus only fuel_release will be used in case of empty version_info. + We bound InstallationStructure.structure.clusters to the oswl + and extract fuel_release from clusters data. If fuel_release + info doesn't provided by clusters data then + InstallationStructure.structure.fuel_release is used. :param oswl: OSWL DB object :type oswl: fuel_analytics.api.db.model.OpenStackWorkloadStats + :param clusters_versions: cache for saving cluster versions with + structure {mn_uid: {cluster_id: fuel_release}} + :type clusters_versions: dict """ if oswl.version_info: return - fuel_release = oswl.fuel_release_from_inst_info or {} + self._add_oswl_to_clusters_versions_cache(oswl, clusters_versions) + + mn_uid = oswl.master_node_uid + cluster_id = oswl.cluster_id + + # Fetching fuel_release info from cache + fuel_release = clusters_versions.get(mn_uid, {}).get(cluster_id) + + # If clusters data doesn't contain fuel_release info we are using + # info from installation info + if fuel_release is None: + fuel_release = oswl.fuel_release_from_inst_info or {} + fuel_release = fuel_release.get('release') + oswl.version_info = { - 'fuel_release': fuel_release.get('release') + 'fuel_release': fuel_release } def get_flatten_resources(self, resource_type, oswl_keys_paths, @@ -126,9 +170,15 @@ class OswlStatsToCsv(object): :return: generator on flatten resources info collection """ app.logger.debug("Getting OSWL flatten %s info started", resource_type) + + # Cache for saving cluster versions. Cache is used only if version_info + # is not provided in the oswl. + # Structure: {mn_uid: {cluster_id: fuel_release}} + clusters_versions = {} + for oswl in oswls: try: - self.handle_empty_version_info(oswl) + self.handle_empty_version_info(oswl, clusters_versions) flatten_oswl = export_utils.get_flatten_data(oswl_keys_paths, oswl) resource_data = oswl.resource_data diff --git a/analytics/fuel_analytics/api/resources/utils/stats_to_csv.py b/analytics/fuel_analytics/api/resources/utils/stats_to_csv.py index 1b68f06..cf18387 100644 --- a/analytics/fuel_analytics/api/resources/utils/stats_to_csv.py +++ b/analytics/fuel_analytics/api/resources/utils/stats_to_csv.py @@ -86,9 +86,8 @@ class StatsToCsv(object): plugin_key_paths = export_utils.get_keys_paths(plugin_skeleton) structure_key_paths = [['master_node_uid'], - ['structure', 'fuel_packages'], - ['structure', 'fuel_release', 'release']] - cluster_key_paths = [['cluster_id']] + ['structure', 'fuel_packages']] + cluster_key_paths = [['cluster_id'], ['cluster_fuel_version']] result_key_paths = plugin_key_paths + cluster_key_paths + \ structure_key_paths app.logger.debug("Plugin keys paths got") @@ -202,6 +201,8 @@ class StatsToCsv(object): for cluster in clusters: cluster['cluster_id'] = cluster['id'] + cluster['cluster_fuel_version'] = \ + cluster.get('fuel_version') flatten_cluster = export_utils.get_flatten_data( cluster_keys_paths, cluster) plugins = cluster.pop('installed_plugins', []) diff --git a/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py b/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py index 3459388..590c861 100644 --- a/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py +++ b/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py @@ -28,6 +28,7 @@ from fuel_analytics.test.base import DbTest from fuel_analytics.api.app import app from fuel_analytics.api.app import db from fuel_analytics.api.common import consts +from fuel_analytics.api.db.model import InstallationStructure from fuel_analytics.api.db.model import OpenStackWorkloadStats from fuel_analytics.api.resources.csv_exporter import get_oswls from fuel_analytics.api.resources.csv_exporter import get_oswls_query @@ -853,3 +854,95 @@ class OswlStatsToCsvTest(OswlTest, DbTest): # The fourth oswl status True in is_modified, is_deleted check_resource_state(flatten_resources[6], 'fourth', False, True, True) + + def test_fuel_version_from_clusters_data_is_used(self): + master_node_uid = 'x' + exporter = OswlStatsToCsv() + resource_type = consts.OSWL_RESOURCE_TYPES.vm + version_from_cluster = '7.0' + version_from_version_info = '9.0' + version_from_installation_info = '8.0' + installation_date = datetime.utcnow().date() - timedelta(days=3) + + # Upgraded Fuel and not upgraded cluster + structure = InstallationStructure( + master_node_uid=master_node_uid, + structure={ + 'fuel_release': {'release': version_from_installation_info}, + 'clusters_num': 2, + 'clusters': [ + {'id': 1, 'fuel_release': version_from_cluster}, + {'id': 2} + ], + 'unallocated_nodes_num_range': 0, + 'allocated_nodes_num': 0 + }, + creation_date=installation_date, + is_filtered=False + ) + db.session.add(structure) + + oswls = [ + OpenStackWorkloadStats( + master_node_uid=master_node_uid, + external_id=1, + cluster_id=1, + created_date=installation_date, + updated_time=datetime.utcnow().time(), + resource_type=resource_type, + resource_checksum='info_from_cluster', + resource_data={'current': [{'id': 1, 'status': 'enabled'}], + 'added': [], 'modified': [], 'removed': []}, + version_info=None + ), + OpenStackWorkloadStats( + master_node_uid=master_node_uid, + external_id=3, + cluster_id=1, + created_date=installation_date + timedelta(days=1), + updated_time=datetime.utcnow().time(), + resource_type=resource_type, + resource_checksum='info_from_version_info', + resource_data={'current': [{'id': 1}], + 'added': [], 'modified': [], 'removed': []}, + version_info={'fuel_release': version_from_version_info} + ), + OpenStackWorkloadStats( + master_node_uid=master_node_uid, + external_id=2, + cluster_id=2, + created_date=installation_date + timedelta(days=2), + updated_time=datetime.utcnow().time(), + resource_type=resource_type, + resource_checksum='info_from_installation_info', + resource_data={'current': [{'id': 1}], + 'added': [], 'modified': [], 'removed': []}, + version_info=None + ) + ] + for oswl in oswls: + db.session.add(oswl) + + with app.test_request_context(): + oswls_data = list(get_oswls(resource_type)) + + oswl_keys_paths, resource_keys_paths, csv_keys_paths = \ + exporter.get_resource_keys_paths(resource_type) + fuel_release_pos = csv_keys_paths.index( + ['version_info', 'fuel_release']) + flatten_resources = list(exporter.get_flatten_resources( + resource_type, oswl_keys_paths, resource_keys_paths, oswls_data)) + + self.assertEqual(len(oswls), len(flatten_resources)) + + # Checking release info fetched from cluster + self.assertEqual(version_from_cluster, + flatten_resources[0][fuel_release_pos]) + + # Checking release info fetched from oswl.version_info + self.assertEqual(version_from_version_info, + flatten_resources[1][fuel_release_pos]) + + # Checking release info fetched from installation info + self.assertEqual(version_from_installation_info, + flatten_resources[2][fuel_release_pos]) diff --git a/analytics/fuel_analytics/test/api/resources/utils/test_plugins_to_csv.py b/analytics/fuel_analytics/test/api/resources/utils/test_plugins_to_csv.py index 699f3d2..f9f1267 100644 --- a/analytics/fuel_analytics/test/api/resources/utils/test_plugins_to_csv.py +++ b/analytics/fuel_analytics/test/api/resources/utils/test_plugins_to_csv.py @@ -36,14 +36,13 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest): exporter = StatsToCsv() _, _, _, csv_keys_paths = exporter.get_plugin_keys_paths() self.assertTrue(['cluster_id'] in csv_keys_paths) + self.assertTrue(['cluster_fuel_version'] in csv_keys_paths) self.assertTrue(['master_node_uid'] in csv_keys_paths) self.assertTrue(['name'] in csv_keys_paths) self.assertTrue(['version'] in csv_keys_paths) self.assertTrue(['fuel_version'] in csv_keys_paths) self.assertTrue(['package_version'] in csv_keys_paths) self.assertTrue(['structure', 'fuel_packages'] in csv_keys_paths) - self.assertTrue(['structure', 'fuel_release', 'release'] in - csv_keys_paths) def test_get_flatten_plugins(self): installations_num = 10 @@ -100,7 +99,8 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest): self.assertEqual(num - 1, len(list(flatten_plugins))) def test_fuel_release_info_in_flatten_plugins(self): - release = '8.0' + inst_fuel_version = '8.0' + cluster_fuel_version = '7.0' packages = ['z', 'a', 'c'] inst_structures = [ model.InstallationStructure( @@ -108,10 +108,11 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest): creation_date=datetime.datetime.utcnow(), is_filtered=False, structure={ - 'fuel_release': {'release': release}, + 'fuel_release': {'release': inst_fuel_version}, 'fuel_packages': packages, 'clusters': [{ 'id': 1, 'nodes': [], + 'fuel_version': cluster_fuel_version, 'installed_plugins': [{ 'name': 'plugin_a', 'version': 'plugin_version_0', @@ -133,10 +134,10 @@ class PluginsToCsvExportTest(InstStructureTest, DbTest): flatten_plugins = exporter.get_flatten_plugins( structure_paths, cluster_paths, plugins_paths, inst_structures) - pos_release = csv_paths.index(['structure', 'fuel_release', - 'release']) + pos_fuel_version = csv_paths.index(['cluster_fuel_version']) pos_packages = csv_paths.index(['structure', 'fuel_packages']) for flatten_plugin in flatten_plugins: - self.assertEqual(release, flatten_plugin[pos_release]) + self.assertEqual(cluster_fuel_version, + flatten_plugin[pos_fuel_version]) self.assertEqual(' '.join(packages), flatten_plugin[pos_packages])