Version info handled for OSWLs
Version info data handled for OpenStackWorkloadStats. If master node updated and the version info in InstallationStructure changed we have correct version info in OSWLs stats report. Fixes in collector: - version_info added to DB model - version_info added to API protocol - tests for version_info added Fixes in analytics: - version_info added to DB model - version_info fields added to CSV - field installation structure release info removed from CSV - version_info building on the fly for old OSWLs added Partial-Bug: #1525902 Change-Id: I27d9d65517b4d5a7c6125b889fb0d1ba1ea213cf
This commit is contained in:
parent
4c6b98d126
commit
c1fe2d34d9
|
@ -28,6 +28,7 @@ class OpenStackWorkloadStats(db.Model):
|
|||
resource_type = db.Column(db.Text)
|
||||
resource_data = db.Column(JSON)
|
||||
resource_checksum = db.Column(db.Text)
|
||||
version_info = db.Column(JSON)
|
||||
|
||||
|
||||
class InstallationStructure(db.Model):
|
||||
|
|
|
@ -175,9 +175,10 @@ def get_oswls_query(resource_type, from_date=None, to_date=None):
|
|||
OSWS.created_date, # for checking if row is duplicated in CSV
|
||||
OSWS.created_date.label('stats_on_date'), # for showing in CSV
|
||||
OSWS.resource_type, OSWS.resource_data, OSWS.resource_checksum,
|
||||
OSWS.version_info,
|
||||
IS.creation_date.label('installation_created_date'),
|
||||
IS.modification_date.label('installation_updated_date'),
|
||||
IS.structure['fuel_release'].label('fuel_release'),
|
||||
IS.structure['fuel_release'].label('fuel_release_from_inst_info'),
|
||||
IS.is_filtered).\
|
||||
join(IS, IS.master_node_uid == OSWS.master_node_uid).\
|
||||
filter(OSWS.resource_type == resource_type).\
|
||||
|
|
|
@ -95,6 +95,27 @@ class OswlStatsToCsv(object):
|
|||
|
||||
return result
|
||||
|
||||
def handle_empty_version_info(self, oswl):
|
||||
"""Handles empty version info in oswl object
|
||||
|
||||
For OSWLs with empty version_info data we compose version_info
|
||||
from InstallationStructure data and assign it to oswl object.
|
||||
If we extract OpenStack release, os, name from
|
||||
InstallationStructure.structure.clusters we have performance
|
||||
degradation on fetching all clusters data in csv_exporter.get_oswls
|
||||
thus only fuel_release will be used in case of empty version_info.
|
||||
|
||||
:param oswl: OSWL DB object
|
||||
:type oswl: fuel_analytics.api.db.model.OpenStackWorkloadStats
|
||||
"""
|
||||
if oswl.version_info:
|
||||
return
|
||||
|
||||
fuel_release = oswl.fuel_release_from_inst_info or {}
|
||||
oswl.version_info = {
|
||||
'fuel_release': fuel_release.get('release')
|
||||
}
|
||||
|
||||
def get_flatten_resources(self, resource_type, oswl_keys_paths,
|
||||
resource_keys_paths, oswls):
|
||||
"""Gets flatten vms data
|
||||
|
@ -107,8 +128,7 @@ class OswlStatsToCsv(object):
|
|||
app.logger.debug("Getting OSWL flatten %s info started", resource_type)
|
||||
for oswl in oswls:
|
||||
try:
|
||||
fuel_release = oswl.fuel_release or {}
|
||||
setattr(oswl, 'release', fuel_release.get('release'))
|
||||
self.handle_empty_version_info(oswl)
|
||||
flatten_oswl = export_utils.get_flatten_data(oswl_keys_paths,
|
||||
oswl)
|
||||
resource_data = oswl.resource_data
|
||||
|
|
|
@ -152,7 +152,13 @@ OSWL_SKELETONS = {
|
|||
'cluster_id': None,
|
||||
'stats_on_date': None,
|
||||
'resource_type': None,
|
||||
'release': None,
|
||||
'version_info': {
|
||||
'fuel_release': None,
|
||||
'release_version': None,
|
||||
'release_os': None,
|
||||
'release_name': None,
|
||||
'environment_version': None
|
||||
}
|
||||
},
|
||||
consts.OSWL_RESOURCE_TYPES.vm: {
|
||||
'id': None,
|
||||
|
|
|
@ -45,7 +45,14 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
|
|||
exporter.get_resource_keys_paths(resource_type)
|
||||
self.assertNotIn(['external_id'], oswl_keys_paths)
|
||||
self.assertNotIn(['updated_time'], oswl_keys_paths)
|
||||
self.assertIn(['release'], oswl_keys_paths)
|
||||
self.assertNotIn(['release'], oswl_keys_paths)
|
||||
self.assertIn(['version_info', 'fuel_release'], oswl_keys_paths)
|
||||
self.assertIn(['version_info', 'release_version'],
|
||||
oswl_keys_paths)
|
||||
self.assertIn(['version_info', 'release_name'], oswl_keys_paths)
|
||||
self.assertIn(['version_info', 'release_os'], oswl_keys_paths)
|
||||
self.assertIn(['version_info', 'environment_version'],
|
||||
oswl_keys_paths)
|
||||
self.assertIn([resource_type, 'id'], resource_keys_paths)
|
||||
self.assertIn([resource_type, 'is_added'], csv_keys_paths)
|
||||
self.assertIn([resource_type, 'is_modified'], csv_keys_paths)
|
||||
|
@ -579,7 +586,8 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
|
|||
exporter.get_resource_keys_paths(resource_type)
|
||||
|
||||
# Checking release value in flatten resources
|
||||
release_pos = csv_keys_paths.index(['release'])
|
||||
release_pos = csv_keys_paths.index(
|
||||
['version_info', 'fuel_release'])
|
||||
flatten_resources = exporter.get_flatten_resources(
|
||||
resource_type, oswl_keys_paths, resource_keys_paths, oswls)
|
||||
for flatten_resource in flatten_resources:
|
||||
|
@ -633,3 +641,69 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
|
|||
# Checking only old oswl in seamless_oswls
|
||||
for o in oswls_seamless:
|
||||
self.assertEqual(old_created_date, o.created_date)
|
||||
|
||||
def test_version_info_in_flatten_resource(self):
|
||||
exporter = OswlStatsToCsv()
|
||||
resource_type = consts.OSWL_RESOURCE_TYPES.vm
|
||||
oswls_saved = [
|
||||
OpenStackWorkloadStats(
|
||||
master_node_uid='x',
|
||||
external_id=1,
|
||||
cluster_id=1,
|
||||
created_date=datetime.utcnow().date(),
|
||||
updated_time=datetime.utcnow().time(),
|
||||
resource_type=resource_type,
|
||||
resource_checksum='no_version_info',
|
||||
resource_data={'current': [{'id': 1}]}
|
||||
),
|
||||
OpenStackWorkloadStats(
|
||||
master_node_uid='y',
|
||||
external_id=2,
|
||||
cluster_id=2,
|
||||
created_date=datetime.utcnow().date(),
|
||||
updated_time=datetime.utcnow().time(),
|
||||
resource_type=resource_type,
|
||||
resource_checksum='empty_version_info',
|
||||
resource_data={'current': [{'id': 1}]},
|
||||
version_info={}
|
||||
),
|
||||
OpenStackWorkloadStats(
|
||||
master_node_uid='z',
|
||||
external_id=3,
|
||||
cluster_id=3,
|
||||
created_date=datetime.utcnow().date(),
|
||||
updated_time=datetime.utcnow().time(),
|
||||
resource_type=resource_type,
|
||||
resource_checksum='with_version_info',
|
||||
resource_data={'current': [{'id': 1}]},
|
||||
version_info={
|
||||
'fuel_release': 'fr',
|
||||
'release_version': 'osr',
|
||||
'release_os': 'osos',
|
||||
'release_name': 'osn',
|
||||
'environment_version': '7.0'
|
||||
}
|
||||
),
|
||||
]
|
||||
for oswl in oswls_saved:
|
||||
db.session.add(oswl)
|
||||
self.get_saved_inst_structs(oswls_saved, creation_date_range=(0, 0))
|
||||
|
||||
with app.test_request_context():
|
||||
oswls = list(get_oswls(resource_type))
|
||||
|
||||
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
|
||||
exporter.get_resource_keys_paths(resource_type)
|
||||
fuel_release_pos = csv_keys_paths.index(
|
||||
['version_info', 'fuel_release'])
|
||||
flatten_resources = list(exporter.get_flatten_resources(
|
||||
resource_type, oswl_keys_paths, resource_keys_paths, oswls))
|
||||
|
||||
# Checking all oswls are in flatten resources
|
||||
external_uid_pos = csv_keys_paths.index(['master_node_uid'])
|
||||
expected_uids = set([oswl.master_node_uid for oswl in oswls])
|
||||
actual_uids = set([d[external_uid_pos] for d in flatten_resources])
|
||||
self.assertEqual(expected_uids, actual_uids)
|
||||
|
||||
# Checking every flatten_resources contain fuel_release_info
|
||||
self.assertTrue(all(d[fuel_release_pos] for d in flatten_resources))
|
||||
|
|
|
@ -0,0 +1,44 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""version_info added
|
||||
|
||||
Revision ID: 4f46e2c07565
|
||||
Revises: 278885b460cd
|
||||
Create Date: 2015-12-15 11:51:56.237567
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '4f46e2c07565'
|
||||
down_revision = '278885b460cd'
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
|
||||
def upgrade():
|
||||
### commands auto generated by Alembic - please adjust! ###
|
||||
op.add_column(
|
||||
'oswl_stats',
|
||||
sa.Column('version_info', postgresql.JSON(), nullable=True)
|
||||
)
|
||||
### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade():
|
||||
### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_column('oswl_stats', 'version_info')
|
||||
### end Alembic commands ###
|
|
@ -64,3 +64,4 @@ class OpenStackWorkloadStats(db.Model):
|
|||
)
|
||||
resource_data = db.Column(JSON, nullable=True)
|
||||
resource_checksum = db.Column(db.Text, nullable=False)
|
||||
version_info = db.Column(JSON, nullable=True)
|
||||
|
|
|
@ -34,6 +34,16 @@
|
|||
},
|
||||
"required": ["added", "current", "removed", "modified"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"version_info": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"fuel_version": {"type": ["string", "null"]},
|
||||
"release_version": {"type": ["string", "null"]},
|
||||
"release_name": {"type": ["string", "null"]},
|
||||
"release_os": {"type": ["string", "null"]},
|
||||
"environment_version": {"type": ["string", "null"]}
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["master_node_uid", "id", "cluster_id",
|
||||
|
|
|
@ -80,6 +80,26 @@ class TestOswlStats(DbTest):
|
|||
'modified': []
|
||||
|
||||
}
|
||||
},
|
||||
{
|
||||
'master_node_uid': 'x',
|
||||
'cluster_id': 1,
|
||||
'id': 3,
|
||||
'resource_type': consts.OSWL_RESOURCE_TYPES.vm,
|
||||
'resource_checksum': 'xx',
|
||||
'created_date': datetime.utcnow().date().isoformat(),
|
||||
'updated_time': datetime.utcnow().time().isoformat(),
|
||||
'resource_data': {
|
||||
'added': [{'id': 1, 'time': 343434343}],
|
||||
'current': [{'id': 'xxx', 'status': 'down'}],
|
||||
'removed': [],
|
||||
'modified': []
|
||||
|
||||
},
|
||||
'version_info': {
|
||||
'fuel_version': '7.0',
|
||||
'openstack_version': None,
|
||||
}
|
||||
}
|
||||
]]
|
||||
for oswls in oswls_sets:
|
||||
|
@ -112,6 +132,22 @@ class TestOswlStats(DbTest):
|
|||
}
|
||||
for i in xrange(oswls_num)]
|
||||
|
||||
def generate_oswls_with_version_info(self, oswls_num):
|
||||
oswls = self.generate_dumb_oswls(oswls_num)
|
||||
version_info_variants = [
|
||||
{}, {'fuel_version': None}, {'fuel_version': "7.0"},
|
||||
{'release_version': None}, {'release_version': "2015-xx-yy"},
|
||||
{'release_os': None}, {'release_os': "OSos"},
|
||||
{'release_name': None}, {'release_name': "OSname"},
|
||||
{'environment_version': None}, {'environment_version': "OSname"},
|
||||
{'fuel_version': 'w', 'release_version': 'x',
|
||||
'release_name': 'y', 'release_os': 'z',
|
||||
'environment_version': '8.0'}
|
||||
]
|
||||
for oswl in oswls:
|
||||
oswl['version_info'] = random.choice(version_info_variants)
|
||||
return oswls
|
||||
|
||||
def test_existed_oswls_filtering(self):
|
||||
oswls_num = 10
|
||||
dicts = self.generate_dumb_oswls(oswls_num)
|
||||
|
@ -277,3 +313,16 @@ class TestOswlStats(DbTest):
|
|||
for oswl_stat in resp_data['oswl_stats']:
|
||||
self.assertNotEqual(oswl_stat['status'],
|
||||
consts.OSWL_STATUSES.failed)
|
||||
|
||||
def test_post_oswls_with_version_info(self):
|
||||
oswls_num = 30
|
||||
expected_oswls = self.generate_oswls_with_version_info(oswls_num)
|
||||
resp = self.post(
|
||||
'/api/v1/oswl_stats/',
|
||||
{'oswl_stats': expected_oswls}
|
||||
)
|
||||
self.check_response_ok(resp)
|
||||
resp_data = json.loads(resp.data)
|
||||
oswls_actual_num = db.session.query(OSWL).count()
|
||||
self.assertEqual(oswls_num, oswls_actual_num)
|
||||
self.assertEqual(len(resp_data['oswl_stats']), oswls_actual_num)
|
||||
|
|
Loading…
Reference in New Issue