From c1fe2d34d9c3d8200383e55a342c5d9ea56ec5ad Mon Sep 17 00:00:00 2001 From: Alexander Kislitsky Date: Tue, 15 Dec 2015 15:32:13 +0300 Subject: [PATCH] Version info handled for OSWLs Version info data handled for OpenStackWorkloadStats. If master node updated and the version info in InstallationStructure changed we have correct version info in OSWLs stats report. Fixes in collector: - version_info added to DB model - version_info added to API protocol - tests for version_info added Fixes in analytics: - version_info added to DB model - version_info fields added to CSV - field installation structure release info removed from CSV - version_info building on the fly for old OSWLs added Partial-Bug: #1525902 Change-Id: I27d9d65517b4d5a7c6125b889fb0d1ba1ea213cf --- analytics/fuel_analytics/api/db/model.py | 1 + .../api/resources/csv_exporter.py | 3 +- .../api/resources/utils/oswl_stats_to_csv.py | 24 +++++- .../api/resources/utils/skeleton.py | 8 +- .../resources/utils/test_oswl_stats_to_csv.py | 78 ++++++++++++++++++- .../4f46e2c07565_version_info_added.py | 44 +++++++++++ collector/collector/api/db/model.py | 1 + collector/collector/api/schemas/oswl.json | 10 +++ .../test/resources/test_oswl_stats.py | 49 ++++++++++++ 9 files changed, 212 insertions(+), 6 deletions(-) create mode 100644 collector/collector/api/db/migrations/versions/4f46e2c07565_version_info_added.py diff --git a/analytics/fuel_analytics/api/db/model.py b/analytics/fuel_analytics/api/db/model.py index dc4c601..b06ca67 100644 --- a/analytics/fuel_analytics/api/db/model.py +++ b/analytics/fuel_analytics/api/db/model.py @@ -28,6 +28,7 @@ class OpenStackWorkloadStats(db.Model): resource_type = db.Column(db.Text) resource_data = db.Column(JSON) resource_checksum = db.Column(db.Text) + version_info = db.Column(JSON) class InstallationStructure(db.Model): diff --git a/analytics/fuel_analytics/api/resources/csv_exporter.py b/analytics/fuel_analytics/api/resources/csv_exporter.py index 6cbf49b..47375d9 100644 --- a/analytics/fuel_analytics/api/resources/csv_exporter.py +++ b/analytics/fuel_analytics/api/resources/csv_exporter.py @@ -175,9 +175,10 @@ def get_oswls_query(resource_type, from_date=None, to_date=None): OSWS.created_date, # for checking if row is duplicated in CSV OSWS.created_date.label('stats_on_date'), # for showing in CSV OSWS.resource_type, OSWS.resource_data, OSWS.resource_checksum, + OSWS.version_info, IS.creation_date.label('installation_created_date'), IS.modification_date.label('installation_updated_date'), - IS.structure['fuel_release'].label('fuel_release'), + IS.structure['fuel_release'].label('fuel_release_from_inst_info'), IS.is_filtered).\ join(IS, IS.master_node_uid == OSWS.master_node_uid).\ filter(OSWS.resource_type == resource_type).\ diff --git a/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py b/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py index a7e09d5..b2a282b 100644 --- a/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py +++ b/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py @@ -95,6 +95,27 @@ class OswlStatsToCsv(object): return result + def handle_empty_version_info(self, oswl): + """Handles empty version info in oswl object + + For OSWLs with empty version_info data we compose version_info + from InstallationStructure data and assign it to oswl object. + If we extract OpenStack release, os, name from + InstallationStructure.structure.clusters we have performance + degradation on fetching all clusters data in csv_exporter.get_oswls + thus only fuel_release will be used in case of empty version_info. + + :param oswl: OSWL DB object + :type oswl: fuel_analytics.api.db.model.OpenStackWorkloadStats + """ + if oswl.version_info: + return + + fuel_release = oswl.fuel_release_from_inst_info or {} + oswl.version_info = { + 'fuel_release': fuel_release.get('release') + } + def get_flatten_resources(self, resource_type, oswl_keys_paths, resource_keys_paths, oswls): """Gets flatten vms data @@ -107,8 +128,7 @@ class OswlStatsToCsv(object): app.logger.debug("Getting OSWL flatten %s info started", resource_type) for oswl in oswls: try: - fuel_release = oswl.fuel_release or {} - setattr(oswl, 'release', fuel_release.get('release')) + self.handle_empty_version_info(oswl) flatten_oswl = export_utils.get_flatten_data(oswl_keys_paths, oswl) resource_data = oswl.resource_data diff --git a/analytics/fuel_analytics/api/resources/utils/skeleton.py b/analytics/fuel_analytics/api/resources/utils/skeleton.py index 6171480..3a061be 100644 --- a/analytics/fuel_analytics/api/resources/utils/skeleton.py +++ b/analytics/fuel_analytics/api/resources/utils/skeleton.py @@ -152,7 +152,13 @@ OSWL_SKELETONS = { 'cluster_id': None, 'stats_on_date': None, 'resource_type': None, - 'release': None, + 'version_info': { + 'fuel_release': None, + 'release_version': None, + 'release_os': None, + 'release_name': None, + 'environment_version': None + } }, consts.OSWL_RESOURCE_TYPES.vm: { 'id': None, diff --git a/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py b/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py index 73bea8a..b57e402 100644 --- a/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py +++ b/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py @@ -45,7 +45,14 @@ class OswlStatsToCsvTest(OswlTest, DbTest): exporter.get_resource_keys_paths(resource_type) self.assertNotIn(['external_id'], oswl_keys_paths) self.assertNotIn(['updated_time'], oswl_keys_paths) - self.assertIn(['release'], oswl_keys_paths) + self.assertNotIn(['release'], oswl_keys_paths) + self.assertIn(['version_info', 'fuel_release'], oswl_keys_paths) + self.assertIn(['version_info', 'release_version'], + oswl_keys_paths) + self.assertIn(['version_info', 'release_name'], oswl_keys_paths) + self.assertIn(['version_info', 'release_os'], oswl_keys_paths) + self.assertIn(['version_info', 'environment_version'], + oswl_keys_paths) self.assertIn([resource_type, 'id'], resource_keys_paths) self.assertIn([resource_type, 'is_added'], csv_keys_paths) self.assertIn([resource_type, 'is_modified'], csv_keys_paths) @@ -579,7 +586,8 @@ class OswlStatsToCsvTest(OswlTest, DbTest): exporter.get_resource_keys_paths(resource_type) # Checking release value in flatten resources - release_pos = csv_keys_paths.index(['release']) + release_pos = csv_keys_paths.index( + ['version_info', 'fuel_release']) flatten_resources = exporter.get_flatten_resources( resource_type, oswl_keys_paths, resource_keys_paths, oswls) for flatten_resource in flatten_resources: @@ -633,3 +641,69 @@ class OswlStatsToCsvTest(OswlTest, DbTest): # Checking only old oswl in seamless_oswls for o in oswls_seamless: self.assertEqual(old_created_date, o.created_date) + + def test_version_info_in_flatten_resource(self): + exporter = OswlStatsToCsv() + resource_type = consts.OSWL_RESOURCE_TYPES.vm + oswls_saved = [ + OpenStackWorkloadStats( + master_node_uid='x', + external_id=1, + cluster_id=1, + created_date=datetime.utcnow().date(), + updated_time=datetime.utcnow().time(), + resource_type=resource_type, + resource_checksum='no_version_info', + resource_data={'current': [{'id': 1}]} + ), + OpenStackWorkloadStats( + master_node_uid='y', + external_id=2, + cluster_id=2, + created_date=datetime.utcnow().date(), + updated_time=datetime.utcnow().time(), + resource_type=resource_type, + resource_checksum='empty_version_info', + resource_data={'current': [{'id': 1}]}, + version_info={} + ), + OpenStackWorkloadStats( + master_node_uid='z', + external_id=3, + cluster_id=3, + created_date=datetime.utcnow().date(), + updated_time=datetime.utcnow().time(), + resource_type=resource_type, + resource_checksum='with_version_info', + resource_data={'current': [{'id': 1}]}, + version_info={ + 'fuel_release': 'fr', + 'release_version': 'osr', + 'release_os': 'osos', + 'release_name': 'osn', + 'environment_version': '7.0' + } + ), + ] + for oswl in oswls_saved: + db.session.add(oswl) + self.get_saved_inst_structs(oswls_saved, creation_date_range=(0, 0)) + + with app.test_request_context(): + oswls = list(get_oswls(resource_type)) + + oswl_keys_paths, resource_keys_paths, csv_keys_paths = \ + exporter.get_resource_keys_paths(resource_type) + fuel_release_pos = csv_keys_paths.index( + ['version_info', 'fuel_release']) + flatten_resources = list(exporter.get_flatten_resources( + resource_type, oswl_keys_paths, resource_keys_paths, oswls)) + + # Checking all oswls are in flatten resources + external_uid_pos = csv_keys_paths.index(['master_node_uid']) + expected_uids = set([oswl.master_node_uid for oswl in oswls]) + actual_uids = set([d[external_uid_pos] for d in flatten_resources]) + self.assertEqual(expected_uids, actual_uids) + + # Checking every flatten_resources contain fuel_release_info + self.assertTrue(all(d[fuel_release_pos] for d in flatten_resources)) diff --git a/collector/collector/api/db/migrations/versions/4f46e2c07565_version_info_added.py b/collector/collector/api/db/migrations/versions/4f46e2c07565_version_info_added.py new file mode 100644 index 0000000..e683b2d --- /dev/null +++ b/collector/collector/api/db/migrations/versions/4f46e2c07565_version_info_added.py @@ -0,0 +1,44 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""version_info added + +Revision ID: 4f46e2c07565 +Revises: 278885b460cd +Create Date: 2015-12-15 11:51:56.237567 + +""" + +# revision identifiers, used by Alembic. +revision = '4f46e2c07565' +down_revision = '278885b460cd' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column( + 'oswl_stats', + sa.Column('version_info', postgresql.JSON(), nullable=True) + ) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('oswl_stats', 'version_info') + ### end Alembic commands ### diff --git a/collector/collector/api/db/model.py b/collector/collector/api/db/model.py index b29da2a..e8b88e0 100644 --- a/collector/collector/api/db/model.py +++ b/collector/collector/api/db/model.py @@ -64,3 +64,4 @@ class OpenStackWorkloadStats(db.Model): ) resource_data = db.Column(JSON, nullable=True) resource_checksum = db.Column(db.Text, nullable=False) + version_info = db.Column(JSON, nullable=True) diff --git a/collector/collector/api/schemas/oswl.json b/collector/collector/api/schemas/oswl.json index 173673e..acaa420 100644 --- a/collector/collector/api/schemas/oswl.json +++ b/collector/collector/api/schemas/oswl.json @@ -34,6 +34,16 @@ }, "required": ["added", "current", "removed", "modified"], "additionalProperties": false + }, + "version_info": { + "type": "object", + "properties": { + "fuel_version": {"type": ["string", "null"]}, + "release_version": {"type": ["string", "null"]}, + "release_name": {"type": ["string", "null"]}, + "release_os": {"type": ["string", "null"]}, + "environment_version": {"type": ["string", "null"]} + } } }, "required": ["master_node_uid", "id", "cluster_id", diff --git a/collector/collector/test/resources/test_oswl_stats.py b/collector/collector/test/resources/test_oswl_stats.py index 88b7aa8..78ff346 100644 --- a/collector/collector/test/resources/test_oswl_stats.py +++ b/collector/collector/test/resources/test_oswl_stats.py @@ -80,6 +80,26 @@ class TestOswlStats(DbTest): 'modified': [] } + }, + { + 'master_node_uid': 'x', + 'cluster_id': 1, + 'id': 3, + 'resource_type': consts.OSWL_RESOURCE_TYPES.vm, + 'resource_checksum': 'xx', + 'created_date': datetime.utcnow().date().isoformat(), + 'updated_time': datetime.utcnow().time().isoformat(), + 'resource_data': { + 'added': [{'id': 1, 'time': 343434343}], + 'current': [{'id': 'xxx', 'status': 'down'}], + 'removed': [], + 'modified': [] + + }, + 'version_info': { + 'fuel_version': '7.0', + 'openstack_version': None, + } } ]] for oswls in oswls_sets: @@ -112,6 +132,22 @@ class TestOswlStats(DbTest): } for i in xrange(oswls_num)] + def generate_oswls_with_version_info(self, oswls_num): + oswls = self.generate_dumb_oswls(oswls_num) + version_info_variants = [ + {}, {'fuel_version': None}, {'fuel_version': "7.0"}, + {'release_version': None}, {'release_version': "2015-xx-yy"}, + {'release_os': None}, {'release_os': "OSos"}, + {'release_name': None}, {'release_name': "OSname"}, + {'environment_version': None}, {'environment_version': "OSname"}, + {'fuel_version': 'w', 'release_version': 'x', + 'release_name': 'y', 'release_os': 'z', + 'environment_version': '8.0'} + ] + for oswl in oswls: + oswl['version_info'] = random.choice(version_info_variants) + return oswls + def test_existed_oswls_filtering(self): oswls_num = 10 dicts = self.generate_dumb_oswls(oswls_num) @@ -277,3 +313,16 @@ class TestOswlStats(DbTest): for oswl_stat in resp_data['oswl_stats']: self.assertNotEqual(oswl_stat['status'], consts.OSWL_STATUSES.failed) + + def test_post_oswls_with_version_info(self): + oswls_num = 30 + expected_oswls = self.generate_oswls_with_version_info(oswls_num) + resp = self.post( + '/api/v1/oswl_stats/', + {'oswl_stats': expected_oswls} + ) + self.check_response_ok(resp) + resp_data = json.loads(resp.data) + oswls_actual_num = db.session.query(OSWL).count() + self.assertEqual(oswls_num, oswls_actual_num) + self.assertEqual(len(resp_data['oswl_stats']), oswls_actual_num)