diff --git a/analytics/fuel_analytics/api/db/model.py b/analytics/fuel_analytics/api/db/model.py index dc4c601..b06ca67 100644 --- a/analytics/fuel_analytics/api/db/model.py +++ b/analytics/fuel_analytics/api/db/model.py @@ -28,6 +28,7 @@ class OpenStackWorkloadStats(db.Model): resource_type = db.Column(db.Text) resource_data = db.Column(JSON) resource_checksum = db.Column(db.Text) + version_info = db.Column(JSON) class InstallationStructure(db.Model): diff --git a/analytics/fuel_analytics/api/resources/csv_exporter.py b/analytics/fuel_analytics/api/resources/csv_exporter.py index 6cbf49b..47375d9 100644 --- a/analytics/fuel_analytics/api/resources/csv_exporter.py +++ b/analytics/fuel_analytics/api/resources/csv_exporter.py @@ -175,9 +175,10 @@ def get_oswls_query(resource_type, from_date=None, to_date=None): OSWS.created_date, # for checking if row is duplicated in CSV OSWS.created_date.label('stats_on_date'), # for showing in CSV OSWS.resource_type, OSWS.resource_data, OSWS.resource_checksum, + OSWS.version_info, IS.creation_date.label('installation_created_date'), IS.modification_date.label('installation_updated_date'), - IS.structure['fuel_release'].label('fuel_release'), + IS.structure['fuel_release'].label('fuel_release_from_inst_info'), IS.is_filtered).\ join(IS, IS.master_node_uid == OSWS.master_node_uid).\ filter(OSWS.resource_type == resource_type).\ diff --git a/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py b/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py index a7e09d5..b2a282b 100644 --- a/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py +++ b/analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py @@ -95,6 +95,27 @@ class OswlStatsToCsv(object): return result + def handle_empty_version_info(self, oswl): + """Handles empty version info in oswl object + + For OSWLs with empty version_info data we compose version_info + from InstallationStructure data and assign it to oswl object. + If we extract OpenStack release, os, name from + InstallationStructure.structure.clusters we have performance + degradation on fetching all clusters data in csv_exporter.get_oswls + thus only fuel_release will be used in case of empty version_info. + + :param oswl: OSWL DB object + :type oswl: fuel_analytics.api.db.model.OpenStackWorkloadStats + """ + if oswl.version_info: + return + + fuel_release = oswl.fuel_release_from_inst_info or {} + oswl.version_info = { + 'fuel_release': fuel_release.get('release') + } + def get_flatten_resources(self, resource_type, oswl_keys_paths, resource_keys_paths, oswls): """Gets flatten vms data @@ -107,8 +128,7 @@ class OswlStatsToCsv(object): app.logger.debug("Getting OSWL flatten %s info started", resource_type) for oswl in oswls: try: - fuel_release = oswl.fuel_release or {} - setattr(oswl, 'release', fuel_release.get('release')) + self.handle_empty_version_info(oswl) flatten_oswl = export_utils.get_flatten_data(oswl_keys_paths, oswl) resource_data = oswl.resource_data diff --git a/analytics/fuel_analytics/api/resources/utils/skeleton.py b/analytics/fuel_analytics/api/resources/utils/skeleton.py index 6171480..3a061be 100644 --- a/analytics/fuel_analytics/api/resources/utils/skeleton.py +++ b/analytics/fuel_analytics/api/resources/utils/skeleton.py @@ -152,7 +152,13 @@ OSWL_SKELETONS = { 'cluster_id': None, 'stats_on_date': None, 'resource_type': None, - 'release': None, + 'version_info': { + 'fuel_release': None, + 'release_version': None, + 'release_os': None, + 'release_name': None, + 'environment_version': None + } }, consts.OSWL_RESOURCE_TYPES.vm: { 'id': None, diff --git a/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py b/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py index 73bea8a..b57e402 100644 --- a/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py +++ b/analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py @@ -45,7 +45,14 @@ class OswlStatsToCsvTest(OswlTest, DbTest): exporter.get_resource_keys_paths(resource_type) self.assertNotIn(['external_id'], oswl_keys_paths) self.assertNotIn(['updated_time'], oswl_keys_paths) - self.assertIn(['release'], oswl_keys_paths) + self.assertNotIn(['release'], oswl_keys_paths) + self.assertIn(['version_info', 'fuel_release'], oswl_keys_paths) + self.assertIn(['version_info', 'release_version'], + oswl_keys_paths) + self.assertIn(['version_info', 'release_name'], oswl_keys_paths) + self.assertIn(['version_info', 'release_os'], oswl_keys_paths) + self.assertIn(['version_info', 'environment_version'], + oswl_keys_paths) self.assertIn([resource_type, 'id'], resource_keys_paths) self.assertIn([resource_type, 'is_added'], csv_keys_paths) self.assertIn([resource_type, 'is_modified'], csv_keys_paths) @@ -579,7 +586,8 @@ class OswlStatsToCsvTest(OswlTest, DbTest): exporter.get_resource_keys_paths(resource_type) # Checking release value in flatten resources - release_pos = csv_keys_paths.index(['release']) + release_pos = csv_keys_paths.index( + ['version_info', 'fuel_release']) flatten_resources = exporter.get_flatten_resources( resource_type, oswl_keys_paths, resource_keys_paths, oswls) for flatten_resource in flatten_resources: @@ -633,3 +641,69 @@ class OswlStatsToCsvTest(OswlTest, DbTest): # Checking only old oswl in seamless_oswls for o in oswls_seamless: self.assertEqual(old_created_date, o.created_date) + + def test_version_info_in_flatten_resource(self): + exporter = OswlStatsToCsv() + resource_type = consts.OSWL_RESOURCE_TYPES.vm + oswls_saved = [ + OpenStackWorkloadStats( + master_node_uid='x', + external_id=1, + cluster_id=1, + created_date=datetime.utcnow().date(), + updated_time=datetime.utcnow().time(), + resource_type=resource_type, + resource_checksum='no_version_info', + resource_data={'current': [{'id': 1}]} + ), + OpenStackWorkloadStats( + master_node_uid='y', + external_id=2, + cluster_id=2, + created_date=datetime.utcnow().date(), + updated_time=datetime.utcnow().time(), + resource_type=resource_type, + resource_checksum='empty_version_info', + resource_data={'current': [{'id': 1}]}, + version_info={} + ), + OpenStackWorkloadStats( + master_node_uid='z', + external_id=3, + cluster_id=3, + created_date=datetime.utcnow().date(), + updated_time=datetime.utcnow().time(), + resource_type=resource_type, + resource_checksum='with_version_info', + resource_data={'current': [{'id': 1}]}, + version_info={ + 'fuel_release': 'fr', + 'release_version': 'osr', + 'release_os': 'osos', + 'release_name': 'osn', + 'environment_version': '7.0' + } + ), + ] + for oswl in oswls_saved: + db.session.add(oswl) + self.get_saved_inst_structs(oswls_saved, creation_date_range=(0, 0)) + + with app.test_request_context(): + oswls = list(get_oswls(resource_type)) + + oswl_keys_paths, resource_keys_paths, csv_keys_paths = \ + exporter.get_resource_keys_paths(resource_type) + fuel_release_pos = csv_keys_paths.index( + ['version_info', 'fuel_release']) + flatten_resources = list(exporter.get_flatten_resources( + resource_type, oswl_keys_paths, resource_keys_paths, oswls)) + + # Checking all oswls are in flatten resources + external_uid_pos = csv_keys_paths.index(['master_node_uid']) + expected_uids = set([oswl.master_node_uid for oswl in oswls]) + actual_uids = set([d[external_uid_pos] for d in flatten_resources]) + self.assertEqual(expected_uids, actual_uids) + + # Checking every flatten_resources contain fuel_release_info + self.assertTrue(all(d[fuel_release_pos] for d in flatten_resources)) diff --git a/collector/collector/api/db/migrations/versions/4f46e2c07565_version_info_added.py b/collector/collector/api/db/migrations/versions/4f46e2c07565_version_info_added.py new file mode 100644 index 0000000..e683b2d --- /dev/null +++ b/collector/collector/api/db/migrations/versions/4f46e2c07565_version_info_added.py @@ -0,0 +1,44 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""version_info added + +Revision ID: 4f46e2c07565 +Revises: 278885b460cd +Create Date: 2015-12-15 11:51:56.237567 + +""" + +# revision identifiers, used by Alembic. +revision = '4f46e2c07565' +down_revision = '278885b460cd' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column( + 'oswl_stats', + sa.Column('version_info', postgresql.JSON(), nullable=True) + ) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('oswl_stats', 'version_info') + ### end Alembic commands ### diff --git a/collector/collector/api/db/model.py b/collector/collector/api/db/model.py index b29da2a..e8b88e0 100644 --- a/collector/collector/api/db/model.py +++ b/collector/collector/api/db/model.py @@ -64,3 +64,4 @@ class OpenStackWorkloadStats(db.Model): ) resource_data = db.Column(JSON, nullable=True) resource_checksum = db.Column(db.Text, nullable=False) + version_info = db.Column(JSON, nullable=True) diff --git a/collector/collector/api/schemas/oswl.json b/collector/collector/api/schemas/oswl.json index 173673e..acaa420 100644 --- a/collector/collector/api/schemas/oswl.json +++ b/collector/collector/api/schemas/oswl.json @@ -34,6 +34,16 @@ }, "required": ["added", "current", "removed", "modified"], "additionalProperties": false + }, + "version_info": { + "type": "object", + "properties": { + "fuel_version": {"type": ["string", "null"]}, + "release_version": {"type": ["string", "null"]}, + "release_name": {"type": ["string", "null"]}, + "release_os": {"type": ["string", "null"]}, + "environment_version": {"type": ["string", "null"]} + } } }, "required": ["master_node_uid", "id", "cluster_id", diff --git a/collector/collector/test/resources/test_oswl_stats.py b/collector/collector/test/resources/test_oswl_stats.py index 88b7aa8..78ff346 100644 --- a/collector/collector/test/resources/test_oswl_stats.py +++ b/collector/collector/test/resources/test_oswl_stats.py @@ -80,6 +80,26 @@ class TestOswlStats(DbTest): 'modified': [] } + }, + { + 'master_node_uid': 'x', + 'cluster_id': 1, + 'id': 3, + 'resource_type': consts.OSWL_RESOURCE_TYPES.vm, + 'resource_checksum': 'xx', + 'created_date': datetime.utcnow().date().isoformat(), + 'updated_time': datetime.utcnow().time().isoformat(), + 'resource_data': { + 'added': [{'id': 1, 'time': 343434343}], + 'current': [{'id': 'xxx', 'status': 'down'}], + 'removed': [], + 'modified': [] + + }, + 'version_info': { + 'fuel_version': '7.0', + 'openstack_version': None, + } } ]] for oswls in oswls_sets: @@ -112,6 +132,22 @@ class TestOswlStats(DbTest): } for i in xrange(oswls_num)] + def generate_oswls_with_version_info(self, oswls_num): + oswls = self.generate_dumb_oswls(oswls_num) + version_info_variants = [ + {}, {'fuel_version': None}, {'fuel_version': "7.0"}, + {'release_version': None}, {'release_version': "2015-xx-yy"}, + {'release_os': None}, {'release_os': "OSos"}, + {'release_name': None}, {'release_name': "OSname"}, + {'environment_version': None}, {'environment_version': "OSname"}, + {'fuel_version': 'w', 'release_version': 'x', + 'release_name': 'y', 'release_os': 'z', + 'environment_version': '8.0'} + ] + for oswl in oswls: + oswl['version_info'] = random.choice(version_info_variants) + return oswls + def test_existed_oswls_filtering(self): oswls_num = 10 dicts = self.generate_dumb_oswls(oswls_num) @@ -277,3 +313,16 @@ class TestOswlStats(DbTest): for oswl_stat in resp_data['oswl_stats']: self.assertNotEqual(oswl_stat['status'], consts.OSWL_STATUSES.failed) + + def test_post_oswls_with_version_info(self): + oswls_num = 30 + expected_oswls = self.generate_oswls_with_version_info(oswls_num) + resp = self.post( + '/api/v1/oswl_stats/', + {'oswl_stats': expected_oswls} + ) + self.check_response_ok(resp) + resp_data = json.loads(resp.data) + oswls_actual_num = db.session.query(OSWL).count() + self.assertEqual(oswls_num, oswls_actual_num) + self.assertEqual(len(resp_data['oswl_stats']), oswls_actual_num)