openstack
/
fuel-stats
Code Issues Proposed changes

Split reports by release 

PoC for splitting fuel-stats reports by releases list.

Change-Id: I51fdadea161cb37a631a7e3638746caebb1130a1
This commit is contained in:
Alexander Kislitsky 2016-07-05 20:08:42 +03:00
parent ce4e26fc12
commit ebea961467
7 changed files with 165 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
analytics/fuel_analytics/api/db/model.py
View File

@ -39,6 +39,7 @@ class InstallationStructure(db.Model):
creation_date = db.Column(db.DateTime)
modification_date = db.Column(db.DateTime)
is_filtered = db.Column(db.Boolean)
release = db.Column(db.Text)
class ActionLog(db.Model):

71
analytics/fuel_analytics/api/resources/csv_exporter.py
View File

@ -66,6 +66,13 @@ def get_to_date():
default_value=datetime.utcnow().date())
def get_list_values(param_name, delimiter=','):
if param_name not in request.args:
return None
value = request.args.get(param_name).strip()
return [v.strip() for v in value.split(delimiter)]
def get_inst_structures_query(from_date=None, to_date=None, fields=()):
"""Composes query for fetching not filtered installation
structures info with filtering by from and to dates and
@ -207,12 +214,14 @@ def plugins_to_csv():
return Response(result, mimetype='text/csv', headers=headers)
def get_oswls_query(resource_type, from_date=None, to_date=None):
def get_oswls_query(resource_type, from_date=None, to_date=None,
release=None):
"""Composes query for fetching oswls with installation
info creation and update dates with ordering by created_date
:param resource_type: resource type
:param from_date: filter from date
:param to_date: filter to date
:param release: filter by release
:return: SQLAlchemy query
"""
query = db.session.query(
@ -232,20 +241,22 @@ def get_oswls_query(resource_type, from_date=None, to_date=None):
query = query.filter(OSWS.created_date >= from_date)
if to_date is not None:
query = query.filter(OSWS.created_date <= to_date)
if release is not None:
query = query.filter(IS.release == release)
# For proper handling of paging we must use additional ordering by id.
# In other case we will lose some OSWLs form the execution result.
query = query.order_by(OSWS.created_date, OSWS.id)
return query
def get_oswls(resource_type):
def get_oswls(resource_type, release=None):
yield_per = app.config['CSV_DB_YIELD_PER']
app.logger.debug("Fetching %s oswls with yield per %d",
resource_type, yield_per)
from_date = get_from_date()
to_date = get_to_date()
query = get_oswls_query(resource_type, from_date=from_date,
to_date=to_date)
to_date=to_date, release=release)
return query.yield_per(yield_per)
@ -279,27 +290,53 @@ def _add_oswl_to_clusters_versions_cache(inst_structure, clusters_versions):
clusters_versions[mn_uid][cluster['id']] = version_info
def get_oswls_reports(resource_type, releases, from_date, to_date,
clusters_version_info):
exporter = OswlStatsToCsv()
if releases is None:
releases = [None]
for release in releases:
app.logger.debug("Getting report '%s' for release: '%s'",
resource_type, release)
oswls = get_oswls_query(resource_type, from_date=from_date,
to_date=to_date, release=release)
report = exporter.export(resource_type, oswls, to_date,
clusters_version_info)
if release is None:
file_name = '{0}.csv'.format(resource_type)
else:
file_name = '{0}_{1}.csv'.format(resource_type, release)
app.logger.debug("Report '%s' for release '%s' got",
resource_type, release)
yield report, file_name
@bp.route('/<resource_type>', methods=['GET'])
def oswl_to_csv(resource_type):
app.logger.debug("Handling oswl_to_csv get request for resource %s",
resource_type)
exporter = OswlStatsToCsv()
oswls = get_oswls(resource_type)
releases = get_list_values('releases')
from_date = get_from_date()
to_date = get_to_date()
clusters_version_info = get_clusters_version_info()
result = exporter.export(resource_type, oswls, get_to_date(),
clusters_version_info)
# NOTE: result - is generator, but streaming can not work with some
# WSGI middlewares: http://flask.pocoo.org/docs/0.10/patterns/streaming/
app.logger.debug("Request oswl_to_csv for resource %s handled",
resource_type)
reports = get_oswls_reports(resource_type, releases,
from_date, to_date, clusters_version_info)
file_name = '{}_from{}_to{}'.format(resource_type, from_date, to_date)
headers = {
'Content-Disposition': 'attachment; filename={}.csv'.format(
resource_type)
'Content-Disposition': 'attachment; filename={}.tar'.format(file_name)
}
return Response(result, mimetype='text/csv', headers=headers)
app.logger.debug("Get oswl_to_csv request for resource %s handled",
resource_type)
return Response(stream_reports_tar(reports),
mimetype='application/x-tar', headers=headers)
def get_resources_types():
@ -379,6 +416,10 @@ def stream_reports_tar(reports):
tar_stream.seek(io.SEEK_SET)
yield tar_stream.getvalue()
import time
time.sleep(5)
app.logger.debug("HHHHHRRRRRR!!!")
tar_stream.seek(io.SEEK_SET)
tar_stream.truncate()

23
analytics/fuel_analytics/test/api/resources/utils/test_csv_exporter.py
View File

@ -28,12 +28,10 @@ from fuel_analytics.api.errors import DateExtractionError
from fuel_analytics.api.resources import csv_exporter as ce
from fuel_analytics.api.resources.csv_exporter import extract_date
from fuel_analytics.api.resources.csv_exporter import get_action_logs
from fuel_analytics.api.resources.csv_exporter import get_from_date
from fuel_analytics.api.resources.csv_exporter import get_inst_structures
from fuel_analytics.api.resources.csv_exporter import get_inst_structures_query
from fuel_analytics.api.resources.csv_exporter import get_oswls_query
from fuel_analytics.api.resources.csv_exporter import get_resources_types
from fuel_analytics.api.resources.csv_exporter import get_to_date
from fuel_analytics.api.resources.utils.stats_to_csv import ActionLogInfo
from fuel_analytics.api.resources.utils.stats_to_csv import StatsToCsv
@ -74,24 +72,35 @@ class CsvExporterTest(OswlTest, DbTest):
with app.test_request_context():
expected = datetime.utcnow().date() - \
timedelta(days=app.config['CSV_DEFAULT_FROM_DATE_DAYS'])
actual = get_from_date()
actual = ce.get_from_date()
self.assertEqual(expected, actual)
with app.test_request_context('/?from_date=2015-02-24'):
expected = datetime(2015, 2, 24).date()
actual = get_from_date()
actual = ce.get_from_date()
self.assertEqual(expected, actual)
def test_to_date(self):
def test_get_to_date(self):
with app.test_request_context():
actual = get_to_date()
actual = ce.get_to_date()
self.assertEqual(datetime.utcnow().date(), actual)
with app.test_request_context('/?to_date=2015-02-25'):
expected = datetime(2015, 2, 25).date()
actual = get_to_date()
actual = ce.get_to_date()
self.assertEqual(expected, actual)
def test_get_list_values(self):
with app.test_request_context():
self.assertIsNone(ce.get_list_values('param'))
with app.test_request_context('/?param='):
self.assertItemsEqual([''], ce.get_list_values('param'))
with app.test_request_context('/?param= '):
self.assertItemsEqual([''], ce.get_list_values('param'))
with app.test_request_context('/?param=a, b , c'):
self.assertItemsEqual(['a', 'b', 'c'],
ce.get_list_values('param'))
def test_get_oswls_query_with_dates(self):
num = 20
for resource_type in self.RESOURCE_TYPES:

61
collector/collector/api/db/migrations/versions/24081e26a283_release_column_addded_to_installation.py Normal file
View File

@ -0,0 +1,61 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Release column added to installation_structures
Revision ID: 24081e26a283
Revises: 2ec36f35eeaa
Create Date: 2016-06-23 18:53:01.431773
"""
# revision identifiers, used by Alembic.
revision = '24081e26a283'
down_revision = '2ec36f35eeaa'
from alembic import op
import sqlalchemy as sa
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column(
'installation_structures',
sa.Column('release', sa.Text(), nullable=True)
)
op.create_index(
op.f('ix_installation_structures_release'),
'installation_structures',
['release'],
unique=False
)
set_release = sa.sql.text(
"UPDATE installation_structures "
"SET release = structure->'fuel_release'->>'release'"
)
connection = op.get_bind()
connection.execute(set_release)
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_index(
op.f('ix_installation_structures_release'),
table_name='installation_structures'
)
op.drop_column('installation_structures', 'release')
### end Alembic commands ###

1
collector/collector/api/db/model.py
View File

@ -43,6 +43,7 @@ class InstallationStructure(db.Model):
creation_date = db.Column(db.DateTime)
modification_date = db.Column(db.DateTime)
is_filtered = db.Column(db.Boolean, default=False, index=True)
release = db.Column(db.Text, index=True)
class OpenStackWorkloadStats(db.Model):

5
collector/collector/api/resources/installation_structure.py
View File

@ -52,6 +52,7 @@ def post():
obj.modification_date = datetime.utcnow()
status_code = 200
obj.is_filtered = _is_filtered(structure)
obj.release = get_release(structure)
obj.structure = structure
db.session.add(obj)
return status_code, {'status': 'ok'}
@ -133,3 +134,7 @@ def _is_filtered(structure):
packages, filtered_by_packages)
return filtered_by_build_id or filtered_by_packages
def get_release(structure):
return structure.get('fuel_release', {}).get('release')

25
collector/collector/test/resources/test_installation_structure.py
View File

@ -507,3 +507,28 @@ class TestInstallationStructure(DbTest):
filtering_rules = {tuple(sorted(packages)): from_dt_str}
self.assertFalse(_is_filtered_by_build_info(
packages, filtering_rules))
def test_release_column(self):
master_node_uid = 'x'
release = 'release'
struct = {
'master_node_uid': master_node_uid,
'fuel_release': {
'release': release,
'feature_groups': [],
'api': 'v1'
},
'allocated_nodes_num': 0,
'unallocated_nodes_num': 0,
'clusters_num': 0,
'clusters': []
}
resp = self.post(
'/api/v1/installation_structure/',
{'installation_structure': struct}
)
self.check_response_ok(resp, codes=(201,))
obj = db.session.query(InstallationStructure).filter(
InstallationStructure.master_node_uid == master_node_uid).one()
self.assertEqual(struct, obj.structure)
self.assertEqual(release, obj.release)