From e200196347d5860bbc510c43598e5dad0831d82c Mon Sep 17 00:00:00 2001 From: Alexander Kislitsky Date: Fri, 8 Apr 2016 14:57:14 +0300 Subject: [PATCH] Connection hanging issue fixed On huge reports data we can have 500 HTTP error by timeout limitation. In case of streaming data from the DB cursor we had hanged DB operation inside the open transaction. For fix the issue releasing of SQLAlchemy session was added to collector and analytics apps. Additional changes: - We stop to migrate action logs data to Elasticsearch due to no one UI report uses this data. - Parameters fixed in OSWLs 'export' call in 'all_reports'. Change-Id: I009081a2304b36fb4ab3705b6fa09a8a3df5710b Closes-Bug: #1564427 --- analytics/fuel_analytics/api/app.py | 5 +++++ analytics/fuel_analytics/api/resources/csv_exporter.py | 4 ++-- collector/collector/api/app.py | 5 +++++ migration/manage_migration.py | 7 ++++++- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/analytics/fuel_analytics/api/app.py b/analytics/fuel_analytics/api/app.py index 04d79e1..31ff5ef 100644 --- a/analytics/fuel_analytics/api/app.py +++ b/analytics/fuel_analytics/api/app.py @@ -39,3 +39,8 @@ def date_parsing_error(error): @app.errorhandler(NoResultFound) def db_object_not_found(error): return make_response(six.text_type(error), 404) + + +@app.teardown_appcontext +def shutdown_session(exception=None): + db.session.remove() diff --git a/analytics/fuel_analytics/api/resources/csv_exporter.py b/analytics/fuel_analytics/api/resources/csv_exporter.py index 576b2ae..dab13c7 100644 --- a/analytics/fuel_analytics/api/resources/csv_exporter.py +++ b/analytics/fuel_analytics/api/resources/csv_exporter.py @@ -312,8 +312,8 @@ def get_all_reports(from_date, to_date, clusters_version_info): app.logger.debug("Getting report '%s'", resource_type) oswls = get_oswls_query(resource_type, from_date=from_date, to_date=to_date) - report = oswl_exporter.export(resource_type, oswls, - clusters_version_info, to_date) + report = oswl_exporter.export(resource_type, oswls, to_date, + clusters_version_info) app.logger.debug("Report '%s' got", resource_type) yield report, '{}.csv'.format(resource_type) diff --git a/collector/collector/api/app.py b/collector/collector/api/app.py index 665d7c7..e7d771b 100644 --- a/collector/collector/api/app.py +++ b/collector/collector/api/app.py @@ -100,3 +100,8 @@ def server_error(error): ), 500 ) + + +@app.teardown_appcontext +def shutdown_session(exception=None): + db.session.remove() diff --git a/migration/manage_migration.py b/migration/manage_migration.py index b20077a..df6998d 100755 --- a/migration/manage_migration.py +++ b/migration/manage_migration.py @@ -44,7 +44,12 @@ def execute(params): from migration.migrator import Migrator migrator = Migrator() if params.action == 'migrate': - migrator.migrate_action_logs() + # We don't use action logs in the Elasticsearch reports. + # Action logs table contains huge number of action logs, + # thus we temporarily comment this migration. + # TODO (akislitsky): uncomment this when ActionLogs + # reports will be introduced into Fuel stats web UI. + # migrator.migrate_action_logs() migrator.migrate_installation_structure() elif params.action == 'remove_indices': migrator.remove_indices()