openstack
/
fuel-stats
Code Issues Proposed changes

Fetching info about master node liveness 

Creation and modification dates of stats from
installation_structures table are added into get_oswls function.
Now we are selecting only required fields in get_oswls funciton.
Oswl resource_checksum and id fields removed from CSV report.

Change-Id: Ib0f00ed2bac8758988b512cc067ce74a66b7f3d8
Blueprint: export-stats-to-csv
This commit is contained in:
Alexander Kislitsky 2015-02-16 18:44:21 +03:00
parent d77da05ff6
commit 92ed03cac4
6 changed files with 98 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
analytics/fuel_analytics/api/db/model.py
View File

@ -28,3 +28,12 @@ class OpenStackWorkloadStats(db.Model):
resource_type = db.Column(db.Text)
resource_data = db.Column(JSON)
resource_checksum = db.Column(db.Text)
class InstallationStructure(db.Model):
__tablename__ = 'installation_structures'
id = db.Column(db.Integer, primary_key=True)
master_node_uid = db.Column(db.String, nullable=False, unique=True)
structure = db.Column(JSON)
creation_date = db.Column(db.DateTime)
modification_date = db.Column(db.DateTime)

19
analytics/fuel_analytics/api/resources/csv_exporter.py
View File

@ -17,7 +17,8 @@ from flask import Response
from fuel_analytics.api.app import app
from fuel_analytics.api.app import db
from fuel_analytics.api.db.model import OpenStackWorkloadStats
from fuel_analytics.api.db.model import InstallationStructure as IS
from fuel_analytics.api.db.model import OpenStackWorkloadStats as OSWS
from fuel_analytics.api.resources.utils.es_client import ElasticSearchClient
from fuel_analytics.api.resources.utils.oswl_stats_to_csv import OswlStatsToCsv
from fuel_analytics.api.resources.utils.stats_to_csv import StatsToCsv
@ -43,13 +44,21 @@ def clusters_to_csv():
return Response(result, mimetype='text/csv', headers=headers)
def get_oswls_query(resource_type):
return db.session.query(
OSWS.master_node_uid, OSWS.cluster_id, OSWS.created_date,
OSWS.updated_time, OSWS.resource_type, OSWS.resource_data,
IS.creation_date.label('installation_created_date'),
IS.modification_date.label('installation_updated_date')).\
join(IS, IS.master_node_uid == OSWS.master_node_uid).\
filter(OSWS.resource_type == resource_type).\
order_by(OSWS.created_date)
def get_oswls(resource_type, yield_per=1000):
app.logger.debug("Fetching %s oswls with yeld per %d",
resource_type, yield_per)
return db.session.query(OpenStackWorkloadStats).filter(
OpenStackWorkloadStats.resource_type == resource_type).\
order_by(OpenStackWorkloadStats.created_date).\
yield_per(yield_per)
return get_oswls_query(resource_type).yield_per(yield_per)
@bp.route('/<resource_type>', methods=['GET'])

4
analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py
View File

@ -16,7 +16,6 @@ import itertools
import six
from fuel_analytics.api.app import app
from fuel_analytics.api.common import consts
from fuel_analytics.api.resources.utils import export_utils
from fuel_analytics.api.resources.utils.export_utils import get_keys_paths
from fuel_analytics.api.resources.utils.skeleton import OSWL_SKELETONS
@ -98,6 +97,3 @@ class OswlStatsToCsv(object):
app.logger.info("Export oswls %s info into CSV finished",
resource_type)
return result
def export_vms(self, oswls):
return self.export(consts.OSWL_RESOURCE_TYPES.vm, oswls)

2
analytics/fuel_analytics/api/resources/utils/skeleton.py
View File

@ -110,12 +110,10 @@ INSTALLATION_INFO_SKELETON = {
OSWL_SKELETONS = {
'general': {
'id': None,
'master_node_uid': None,
'cluster_id': None,
'created_date': None,
'resource_type': None,
'resource_checksum': None,
},
consts.OSWL_RESOURCE_TYPES.vm: {
'id': None,

47
analytics/fuel_analytics/test/api/resources/utils/oswl_test.py
View File

@ -23,7 +23,9 @@ import uuid
from fuel_analytics.test.base import BaseTest
from fuel_analytics.api.app import db
from fuel_analytics.api.common import consts
from fuel_analytics.api.db.model import InstallationStructure
from fuel_analytics.api.db.model import OpenStackWorkloadStats
@ -151,3 +153,48 @@ class OswlTest(BaseTest):
)
current_mn_stats -= 1
yield obj
def get_saved_oswls(self, num, resource_type, *args, **kwargs):
oswls = self.generate_oswls(num, resource_type, *args, **kwargs)
result = []
for oswl in oswls:
db.session.add(oswl)
result.append(oswl)
db.session.commit()
return result
def generate_inst_structs(self, oswls,
creation_date_range=(2, 10),
modification_date_range=(2, 5),
is_modified_date_nullable=True):
mn_uids = set()
for oswl in oswls:
if oswl.master_node_uid not in mn_uids:
creation_date = (datetime.utcnow() - timedelta(
days=random.randint(*creation_date_range))).\
date().isoformat()
if random.choice((False, is_modified_date_nullable)):
modification_date = None
else:
modification_date = (datetime.utcnow() - timedelta(
days=random.randint(*modification_date_range))).\
date().isoformat()
obj = InstallationStructure(
master_node_uid=oswl.master_node_uid,
creation_date=creation_date,
modification_date=modification_date,
structure='',
)
mn_uids.add(oswl.master_node_uid)
yield obj
def get_saved_inst_structs(self, oswls, *args, **kwargs):
inst_structs = self.generate_inst_structs(oswls, *args, **kwargs)
result = []
for inst_struct in inst_structs:
db.session.add(inst_struct)
result.append(inst_struct)
db.session.commit()
return result

34
analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py
View File

@ -24,6 +24,7 @@ from fuel_analytics.test.base import DbTest
from fuel_analytics.api.app import db
from fuel_analytics.api.common import consts
from fuel_analytics.api.resources.csv_exporter import get_oswls
from fuel_analytics.api.resources.csv_exporter import get_oswls_query
from fuel_analytics.api.resources.utils.oswl_stats_to_csv import OswlStatsToCsv
@ -94,15 +95,36 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
num = 200
for resource_type in self.RESOURCE_TYPES:
# Saving data for true JSON loading from DB object
gen_oswls = self.generate_oswls(num, resource_type)
for oswl in gen_oswls:
db.session.add(oswl)
db.session.commit()
oswls = get_oswls(resource_type)
oswls_saved = self.get_saved_oswls(num, resource_type)
# Saving installation structures for proper oswls filtering
self.get_saved_inst_structs(oswls_saved)
# Checking oswls filtered properly
oswls = list(get_oswls(resource_type))
self.assertEqual(num, len(oswls))
# Checking export
result = exporter.export(resource_type, oswls)
self.assertTrue(isinstance(result, types.GeneratorType))
output = six.StringIO(list(result))
reader = csv.reader(output)
for _ in reader:
pass
def test_get_oswls_query(self):
num = 2
for resource_type in self.RESOURCE_TYPES[0:1]:
# Fetching oswls count
count_before = get_oswls_query(resource_type).count()
# Generating oswls without installation info
oswls = self.get_saved_oswls(num, resource_type)
# Checking count of fetched oswls is not changed
count_after = get_oswls_query(resource_type).count()
self.assertEqual(count_before, count_after)
# Saving inst structures
self.get_saved_inst_structs(oswls)
# Checking count of fetched oswls is changed
count_after = get_oswls_query(resource_type).count()
self.assertEqual(num + count_before, count_after)