From f4e8fda24c2286116076705d8a089250b338f20a Mon Sep 17 00:00:00 2001 From: SamKirsch10 Date: Mon, 22 Jun 2015 15:42:16 -0600 Subject: [PATCH 01/12] added get specific transform and bug fixes --- .../mysql/transforms_repository.py | 9 ++++- monasca_events_api/v2/transforms.py | 33 +++++++++++++++---- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/monasca_events_api/common/repositories/mysql/transforms_repository.py b/monasca_events_api/common/repositories/mysql/transforms_repository.py index bb5325d..5b6a02e 100644 --- a/monasca_events_api/common/repositories/mysql/transforms_repository.py +++ b/monasca_events_api/common/repositories/mysql/transforms_repository.py @@ -15,7 +15,6 @@ import MySQLdb from oslo_utils import timeutils -from monasca_events_api.common.repositories import exceptions from monasca_events_api.common.repositories import transforms_repository from monasca_events_api.common.repositories.mysql import mysql_repository from monasca_events_api.openstack.common import log @@ -61,6 +60,14 @@ class TransformsRepository(mysql_repository.MySQLRepository, where tenant_id = %s and deleted_at IS NULL""", [tenant_id]) return cursor.fetchall() + def list_transform(self, tenant_id, transform_id): + cnxn, cursor = self._get_cnxn_cursor_tuple() + with cnxn: + cursor.execute("""select * from event_transform + where tenant_id = %s and id = %s and deleted_at IS NULL""", + (tenant_id, transform_id)) + return cursor.fetchall() + def delete_transform(self, tenant_id, transform_id): cnxn, cursor = self._get_cnxn_cursor_tuple() with cnxn: diff --git a/monasca_events_api/v2/transforms.py b/monasca_events_api/v2/transforms.py index 390e713..1199b15 100644 --- a/monasca_events_api/v2/transforms.py +++ b/monasca_events_api/v2/transforms.py @@ -71,11 +71,17 @@ class Transforms(transforms_api_v2.TransformsV2API): res.body = self._create_transform_response(transform_id, transform) res.status = falcon.HTTP_200 - def on_get(self, req, res): - helpers.validate_authorization(req, self._default_authorized_roles) - tenant_id = helpers.get_tenant_id(req) - res.body = self._list_transforms(tenant_id) - res.status = falcon.HTTP_200 + def on_get(self, req, res, transform_id=None): + if transform_id: + helpers.validate_authorization(req, self._default_authorized_roles) + tenant_id = helpers.get_tenant_id(req) + res.body = self._list_transform(tenant_id, transform_id) + res.status = falcon.HTTP_200 + else: + helpers.validate_authorization(req, self._default_authorized_roles) + tenant_id = helpers.get_tenant_id(req) + res.body = self._list_transforms(tenant_id) + res.status = falcon.HTTP_200 def on_delete(self, req, res, transform_id): helpers.validate_authorization(req, self._default_authorized_roles) @@ -150,14 +156,27 @@ class Transforms(transforms_api_v2.TransformsV2API): try: transforms = self._transforms_repo.list_transforms(tenant_id) for transform in transforms: - transform['specification'] = yaml.safe_dump( - ast.literal_eval(transform['specification'])) + transform['specification'] = yaml.safe_dump(transform['specification']) return json.dumps(transforms, cls=MyEncoder) except repository_exceptions.RepositoryException as ex: LOG.error(ex) raise falcon.HTTPInternalServerError('Service unavailable', ex.message) + def _list_transform(self, tenant_id, transform_id): + try: + transform = self._transforms_repo.list_transform(tenant_id, + transform_id)[0] + transform['specification'] = yaml.safe_dump( + transform['specification']) + transform_list = list() + transform_list.append(transform) + return json.dumps(transform_list, cls=MyEncoder) + except repository_exceptions.RepositoryException as ex: + LOG.error(ex) + raise falcon.HTTPInternalServerError('Service unavailable', + ex.message) + def _delete_transform(self, tenant_id, transform_id): try: self._transforms_repo.delete_transform(tenant_id, transform_id) From 2dcd40e4736c614fbd276c01526358bc2fd5da51 Mon Sep 17 00:00:00 2001 From: SamKirsch10 Date: Tue, 23 Jun 2015 11:03:02 -0600 Subject: [PATCH 02/12] the real limit fixes --- .../common/repositories/mysql/streams_repository.py | 3 +++ .../repositories/mysql/transforms_repository.py | 13 ++++++++++--- monasca_events_api/v2/transforms.py | 8 +++++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/monasca_events_api/common/repositories/mysql/streams_repository.py b/monasca_events_api/common/repositories/mysql/streams_repository.py index 89d2574..b485e19 100644 --- a/monasca_events_api/common/repositories/mysql/streams_repository.py +++ b/monasca_events_api/common/repositories/mysql/streams_repository.py @@ -94,6 +94,9 @@ class StreamsRepository(mysql_repository.MySQLRepository, else: order_by_clause = " order by sd.created_at " limit_clause = "" + if limit: + limit_clause = " limit %s" + parms.append(int(limit)) query = select_clause + where_clause + order_by_clause + limit_clause diff --git a/monasca_events_api/common/repositories/mysql/transforms_repository.py b/monasca_events_api/common/repositories/mysql/transforms_repository.py index 5b6a02e..8347b76 100644 --- a/monasca_events_api/common/repositories/mysql/transforms_repository.py +++ b/monasca_events_api/common/repositories/mysql/transforms_repository.py @@ -53,11 +53,18 @@ class TransformsRepository(mysql_repository.MySQLRepository, else: raise e - def list_transforms(self, tenant_id): + def list_transforms(self, tenant_id, limit=None): cnxn, cursor = self._get_cnxn_cursor_tuple() with cnxn: - cursor.execute("""select * from event_transform - where tenant_id = %s and deleted_at IS NULL""", [tenant_id]) + if limit: + query = ("""select * from event_transform + where tenant_id = "{}" and deleted_at IS NULL limit {}""" + .format(tenant_id, limit)) + cursor.execute(query) + else: + cursor.execute("""select * from event_transform + where tenant_id = %s and deleted_at IS NULL""", + [tenant_id]) return cursor.fetchall() def list_transform(self, tenant_id, transform_id): diff --git a/monasca_events_api/v2/transforms.py b/monasca_events_api/v2/transforms.py index 1199b15..9433480 100644 --- a/monasca_events_api/v2/transforms.py +++ b/monasca_events_api/v2/transforms.py @@ -80,7 +80,8 @@ class Transforms(transforms_api_v2.TransformsV2API): else: helpers.validate_authorization(req, self._default_authorized_roles) tenant_id = helpers.get_tenant_id(req) - res.body = self._list_transforms(tenant_id) + limit = helpers.get_query_param(req, 'limit') + res.body = self._list_transforms(tenant_id, limit) res.status = falcon.HTTP_200 def on_delete(self, req, res, transform_id): @@ -152,9 +153,10 @@ class Transforms(transforms_api_v2.TransformsV2API): 'specification': specification, 'enabled': enabled} return json.dumps(response) - def _list_transforms(self, tenant_id): + def _list_transforms(self, tenant_id, limit): try: - transforms = self._transforms_repo.list_transforms(tenant_id) + transforms = self._transforms_repo.list_transforms(tenant_id, + limit) for transform in transforms: transform['specification'] = yaml.safe_dump(transform['specification']) return json.dumps(transforms, cls=MyEncoder) From e19c5d411994e2a7bfacbb144302db7372eaf0ca Mon Sep 17 00:00:00 2001 From: SamKirsch10 Date: Tue, 23 Jun 2015 15:15:29 -0600 Subject: [PATCH 03/12] fix pagination helper --- .../mysql/transforms_repository.py | 25 ++++++--- monasca_events_api/v2/common/helpers.py | 53 ++++++++++--------- monasca_events_api/v2/stream_definitions.py | 2 +- monasca_events_api/v2/transforms.py | 9 ++-- 4 files changed, 53 insertions(+), 36 deletions(-) diff --git a/monasca_events_api/common/repositories/mysql/transforms_repository.py b/monasca_events_api/common/repositories/mysql/transforms_repository.py index 8347b76..920df5b 100644 --- a/monasca_events_api/common/repositories/mysql/transforms_repository.py +++ b/monasca_events_api/common/repositories/mysql/transforms_repository.py @@ -53,17 +53,30 @@ class TransformsRepository(mysql_repository.MySQLRepository, else: raise e - def list_transforms(self, tenant_id, limit=None): + def list_transforms(self, tenant_id, limit=None, offset=None): cnxn, cursor = self._get_cnxn_cursor_tuple() with cnxn: if limit: + if offset: + query = ("""select * from event_transform + where tenant_id = "{}" and id > "{}" and deleted_at + IS NULL order by id limit {}""" + .format(tenant_id, offset, limit)) + else: + query = ("""select * from event_transform + where tenant_id = "{}" and deleted_at + IS NULL order by id limit {}""" + .format(tenant_id, limit)) + cursor.execute(query) + elif offset: query = ("""select * from event_transform - where tenant_id = "{}" and deleted_at IS NULL limit {}""" - .format(tenant_id, limit)) + where tenant_id = "{}" and id > "{}" and deleted_at + IS NULL order by id""" + .format(tenant_id, offset)) cursor.execute(query) else: cursor.execute("""select * from event_transform - where tenant_id = %s and deleted_at IS NULL""", + where tenant_id = %s and deleted_at IS NULL order by id""", [tenant_id]) return cursor.fetchall() @@ -71,8 +84,8 @@ class TransformsRepository(mysql_repository.MySQLRepository, cnxn, cursor = self._get_cnxn_cursor_tuple() with cnxn: cursor.execute("""select * from event_transform - where tenant_id = %s and id = %s and deleted_at IS NULL""", - (tenant_id, transform_id)) + where tenant_id = %s and id = %s and deleted_at + IS NULL order by id """,(tenant_id, transform_id)) return cursor.fetchall() def delete_transform(self, tenant_id, transform_id): diff --git a/monasca_events_api/v2/common/helpers.py b/monasca_events_api/v2/common/helpers.py index f05cbb8..28badd2 100644 --- a/monasca_events_api/v2/common/helpers.py +++ b/monasca_events_api/v2/common/helpers.py @@ -15,6 +15,7 @@ import datetime import json import urlparse +import urllib import falcon import simplejson @@ -244,46 +245,46 @@ def get_query_period(req): raise falcon.HTTPBadRequest('Bad request', ex.message) -def paginate(resource, uri, offset): +def paginate(resource, uri): - if offset is not None: + limit = constants.PAGE_LIMIT + parsed_uri = urlparse.urlparse(uri) - if resource: + self_link = build_base_uri(parsed_uri) + print("RESOURCE LENGTH = {}".format(len(resource))) + print("LIMIT = {}".format(limit)) + if resource and len(resource) >= limit: - if len(resource) >= constants.PAGE_LIMIT: + if 'timestamp' in resource[limit - 1]: + new_offset = resource[limit - 1]['timestamp'] - new_offset = resource[-1]['id'] + if 'id' in resource[limit - 1]: + new_offset = resource[limit - 1]['id'] - parsed_uri = urlparse.urlparse(uri) + next_link = build_base_uri(parsed_uri) - next_link = build_base_uri(parsed_uri) + new_query_params = [u'offset' + '=' + urllib.quote( + new_offset.encode('utf8'), safe='')] - new_query_params = [u'offset' + '=' + str(new_offset).decode( - 'utf8')] + if new_query_params: + next_link += '?' + '&'.join(new_query_params) - for query_param in parsed_uri.query.split('&'): - query_param_name, query_param_val = query_param.split('=') - if query_param_name.lower() != 'offset': - new_query_params.append(query_param) + resource = {u'links': ([{u'rel': u'self', + u'href': self_link.decode('utf8')}, + {u'rel': u'next', + u'href': next_link.decode('utf8')}]), + u'elements': resource[:limit]} - next_link += '?' + '&'.join(new_query_params) + else: - resource = {u'links': - [{u'rel': u'self', u'href': uri.decode('utf8')}, - {u'rel': u'next', - u'href': next_link.decode('utf8')}], - u'elements': resource} - - else: - - resource = {u'links': - [{u'rel': u'self', u'href': uri.decode('utf8')}], - u'elements': resource} + resource = {u'links': ([{u'rel': u'self', + u'href': self_link.decode('utf8')}]), + u'elements': resource} return resource -def paginate_measurement(measurement, uri, offset): +def paginate_measurement(measurement, uri): if offset is not None: diff --git a/monasca_events_api/v2/stream_definitions.py b/monasca_events_api/v2/stream_definitions.py index 1dd0e46..2744d4c 100644 --- a/monasca_events_api/v2/stream_definitions.py +++ b/monasca_events_api/v2/stream_definitions.py @@ -229,7 +229,7 @@ class StreamDefinitions(stream_definitions_api_v2.StreamDefinitionsV2API): helpers.add_links_to_resource(sd, req_uri) result.append(sd) - result = helpers.paginate(result, req_uri, offset) + result = helpers.paginate(result, req_uri) return result diff --git a/monasca_events_api/v2/transforms.py b/monasca_events_api/v2/transforms.py index 9433480..3aabf00 100644 --- a/monasca_events_api/v2/transforms.py +++ b/monasca_events_api/v2/transforms.py @@ -81,7 +81,10 @@ class Transforms(transforms_api_v2.TransformsV2API): helpers.validate_authorization(req, self._default_authorized_roles) tenant_id = helpers.get_tenant_id(req) limit = helpers.get_query_param(req, 'limit') - res.body = self._list_transforms(tenant_id, limit) + offset = helpers.normalize_offset(helpers.get_query_param( + req, + 'offset')) + res.body = self._list_transforms(tenant_id, limit, offset) res.status = falcon.HTTP_200 def on_delete(self, req, res, transform_id): @@ -153,10 +156,10 @@ class Transforms(transforms_api_v2.TransformsV2API): 'specification': specification, 'enabled': enabled} return json.dumps(response) - def _list_transforms(self, tenant_id, limit): + def _list_transforms(self, tenant_id, limit, offset): try: transforms = self._transforms_repo.list_transforms(tenant_id, - limit) + limit, offset) for transform in transforms: transform['specification'] = yaml.safe_dump(transform['specification']) return json.dumps(transforms, cls=MyEncoder) From ae327a1e3a9435d9c7853c93c89ee92e093ce996 Mon Sep 17 00:00:00 2001 From: SamKirsch10 Date: Tue, 23 Jun 2015 16:02:14 -0600 Subject: [PATCH 04/12] lots of bug fixes for the limit and transform query cleanup --- .../mysql/transforms_repository.py | 51 +++++++++++++------ monasca_events_api/v2/common/helpers.py | 4 +- monasca_events_api/v2/events.py | 2 +- monasca_events_api/v2/transforms.py | 10 ++-- 4 files changed, 45 insertions(+), 22 deletions(-) diff --git a/monasca_events_api/common/repositories/mysql/transforms_repository.py b/monasca_events_api/common/repositories/mysql/transforms_repository.py index 920df5b..586322c 100644 --- a/monasca_events_api/common/repositories/mysql/transforms_repository.py +++ b/monasca_events_api/common/repositories/mysql/transforms_repository.py @@ -26,8 +26,13 @@ LOG = log.getLogger(__name__) class TransformsRepository(mysql_repository.MySQLRepository, transforms_repository.TransformsRepository): + + base_query = """select * from event_transform where deleted_at IS NULL""" + order_by_clause = " order by id" + def create_transforms(self, id, tenant_id, name, description, specification, enabled): + cnxn, cursor = self._get_cnxn_cursor_tuple() with cnxn: now = timeutils.utcnow() @@ -58,26 +63,42 @@ class TransformsRepository(mysql_repository.MySQLRepository, with cnxn: if limit: if offset: - query = ("""select * from event_transform - where tenant_id = "{}" and id > "{}" and deleted_at - IS NULL order by id limit {}""" - .format(tenant_id, offset, limit)) + tenant_id_clause = (" and tenant_id = \"{}\"" + .format(tenant_id)) + + offset_clause = " and id > \"{}\"".format(offset) + + limit_clause = " limit {}".format(limit) + + query = (TransformsRepository.base_query + tenant_id_clause + + offset_clause + TransformsRepository.order_by_clause + limit_clause) else: - query = ("""select * from event_transform - where tenant_id = "{}" and deleted_at - IS NULL order by id limit {}""" - .format(tenant_id, limit)) + tenant_id_clause = (" and tenant_id = \"{}\"" + .format(tenant_id)) + + limit_clause = " limit {}".format(limit) + + query = (TransformsRepository.base_query + tenant_id_clause + + TransformsRepository.order_by_clause + limit_clause) + cursor.execute(query) elif offset: - query = ("""select * from event_transform - where tenant_id = "{}" and id > "{}" and deleted_at - IS NULL order by id""" - .format(tenant_id, offset)) + tenant_id_clause = (" and tenant_id = \"{}\"" + .format(tenant_id)) + + offset_clause = " and id > \"{}\"".format(offset) + + query = (TransformsRepository.base_query + tenant_id_clause + offset_clause + + TransformsRepository.order_by_clause) + cursor.execute(query) else: - cursor.execute("""select * from event_transform - where tenant_id = %s and deleted_at IS NULL order by id""", - [tenant_id]) + tenant_id_clause = (" and tenant_id = \"{}\"" + .format(tenant_id)) + + query = (TransformsRepository.base_query + tenant_id_clause + TransformsRepository.order_by_clause) + print query + cursor.execute(query) return cursor.fetchall() def list_transform(self, tenant_id, transform_id): diff --git a/monasca_events_api/v2/common/helpers.py b/monasca_events_api/v2/common/helpers.py index 28badd2..811edba 100644 --- a/monasca_events_api/v2/common/helpers.py +++ b/monasca_events_api/v2/common/helpers.py @@ -248,11 +248,11 @@ def get_query_period(req): def paginate(resource, uri): limit = constants.PAGE_LIMIT + parsed_uri = urlparse.urlparse(uri) self_link = build_base_uri(parsed_uri) - print("RESOURCE LENGTH = {}".format(len(resource))) - print("LIMIT = {}".format(limit)) + if resource and len(resource) >= limit: if 'timestamp' in resource[limit - 1]: diff --git a/monasca_events_api/v2/events.py b/monasca_events_api/v2/events.py index 1aa6b51..13e9c9a 100644 --- a/monasca_events_api/v2/events.py +++ b/monasca_events_api/v2/events.py @@ -119,7 +119,7 @@ class Events(events_api_v2.EventsV2API): @resource.resource_try_catch_block def _list_events(self, tenant_id, uri, offset, limit): rows = self._events_repo.list_events(tenant_id, offset, limit) - return helpers.paginate(self._build_events(rows), uri, offset) + return helpers.paginate(self._build_events(rows), uri) @resource.resource_try_catch_block def _list_event(self, tenant_id, event_id, uri): diff --git a/monasca_events_api/v2/transforms.py b/monasca_events_api/v2/transforms.py index 3aabf00..26b02d9 100644 --- a/monasca_events_api/v2/transforms.py +++ b/monasca_events_api/v2/transforms.py @@ -75,7 +75,7 @@ class Transforms(transforms_api_v2.TransformsV2API): if transform_id: helpers.validate_authorization(req, self._default_authorized_roles) tenant_id = helpers.get_tenant_id(req) - res.body = self._list_transform(tenant_id, transform_id) + res.body = self._list_transform(tenant_id, transform_id, req.uri) res.status = falcon.HTTP_200 else: helpers.validate_authorization(req, self._default_authorized_roles) @@ -84,7 +84,7 @@ class Transforms(transforms_api_v2.TransformsV2API): offset = helpers.normalize_offset(helpers.get_query_param( req, 'offset')) - res.body = self._list_transforms(tenant_id, limit, offset) + res.body = self._list_transforms(tenant_id, limit, offset, req.uri) res.status = falcon.HTTP_200 def on_delete(self, req, res, transform_id): @@ -156,19 +156,20 @@ class Transforms(transforms_api_v2.TransformsV2API): 'specification': specification, 'enabled': enabled} return json.dumps(response) - def _list_transforms(self, tenant_id, limit, offset): + def _list_transforms(self, tenant_id, limit, offset, uri): try: transforms = self._transforms_repo.list_transforms(tenant_id, limit, offset) for transform in transforms: transform['specification'] = yaml.safe_dump(transform['specification']) + transforms = helpers.paginate(transforms, uri) return json.dumps(transforms, cls=MyEncoder) except repository_exceptions.RepositoryException as ex: LOG.error(ex) raise falcon.HTTPInternalServerError('Service unavailable', ex.message) - def _list_transform(self, tenant_id, transform_id): + def _list_transform(self, tenant_id, transform_id, uri): try: transform = self._transforms_repo.list_transform(tenant_id, transform_id)[0] @@ -176,6 +177,7 @@ class Transforms(transforms_api_v2.TransformsV2API): transform['specification']) transform_list = list() transform_list.append(transform) + transform = helpers.paginate(transform, uri) return json.dumps(transform_list, cls=MyEncoder) except repository_exceptions.RepositoryException as ex: LOG.error(ex) From ab566a74f927ae4df371aac741be5781630abef4 Mon Sep 17 00:00:00 2001 From: SamKirsch10 Date: Mon, 22 Jun 2015 15:42:16 -0600 Subject: [PATCH 05/12] added get specific transform and bug fixes --- .../mysql/transforms_repository.py | 9 ++++- monasca_events_api/v2/transforms.py | 33 +++++++++++++++---- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/monasca_events_api/common/repositories/mysql/transforms_repository.py b/monasca_events_api/common/repositories/mysql/transforms_repository.py index bb5325d..5b6a02e 100644 --- a/monasca_events_api/common/repositories/mysql/transforms_repository.py +++ b/monasca_events_api/common/repositories/mysql/transforms_repository.py @@ -15,7 +15,6 @@ import MySQLdb from oslo_utils import timeutils -from monasca_events_api.common.repositories import exceptions from monasca_events_api.common.repositories import transforms_repository from monasca_events_api.common.repositories.mysql import mysql_repository from monasca_events_api.openstack.common import log @@ -61,6 +60,14 @@ class TransformsRepository(mysql_repository.MySQLRepository, where tenant_id = %s and deleted_at IS NULL""", [tenant_id]) return cursor.fetchall() + def list_transform(self, tenant_id, transform_id): + cnxn, cursor = self._get_cnxn_cursor_tuple() + with cnxn: + cursor.execute("""select * from event_transform + where tenant_id = %s and id = %s and deleted_at IS NULL""", + (tenant_id, transform_id)) + return cursor.fetchall() + def delete_transform(self, tenant_id, transform_id): cnxn, cursor = self._get_cnxn_cursor_tuple() with cnxn: diff --git a/monasca_events_api/v2/transforms.py b/monasca_events_api/v2/transforms.py index 6fee82a..8a4ccaf 100644 --- a/monasca_events_api/v2/transforms.py +++ b/monasca_events_api/v2/transforms.py @@ -71,11 +71,17 @@ class Transforms(transforms_api_v2.TransformsV2API): res.body = self._create_transform_response(transform_id, transform) res.status = falcon.HTTP_200 - def on_get(self, req, res): - helpers.validate_authorization(req, self._default_authorized_roles) - tenant_id = helpers.get_tenant_id(req) - res.body = self._list_transforms(tenant_id) - res.status = falcon.HTTP_200 + def on_get(self, req, res, transform_id=None): + if transform_id: + helpers.validate_authorization(req, self._default_authorized_roles) + tenant_id = helpers.get_tenant_id(req) + res.body = self._list_transform(tenant_id, transform_id) + res.status = falcon.HTTP_200 + else: + helpers.validate_authorization(req, self._default_authorized_roles) + tenant_id = helpers.get_tenant_id(req) + res.body = self._list_transforms(tenant_id) + res.status = falcon.HTTP_200 def on_delete(self, req, res, transform_id): helpers.validate_authorization(req, self._default_authorized_roles) @@ -150,14 +156,27 @@ class Transforms(transforms_api_v2.TransformsV2API): try: transforms = self._transforms_repo.list_transforms(tenant_id) for transform in transforms: - transform['specification'] = yaml.safe_dump( - ast.literal_eval(transform['specification'])) + transform['specification'] = yaml.safe_dump(transform['specification']) return json.dumps(transforms, cls=MyEncoder) except repository_exceptions.RepositoryException as ex: LOG.error(ex) raise falcon.HTTPInternalServerError('Service unavailable', ex.message) + def _list_transform(self, tenant_id, transform_id): + try: + transform = self._transforms_repo.list_transform(tenant_id, + transform_id)[0] + transform['specification'] = yaml.safe_dump( + transform['specification']) + transform_list = list() + transform_list.append(transform) + return json.dumps(transform_list, cls=MyEncoder) + except repository_exceptions.RepositoryException as ex: + LOG.error(ex) + raise falcon.HTTPInternalServerError('Service unavailable', + ex.message) + def _delete_transform(self, tenant_id, transform_id): try: self._transforms_repo.delete_transform(tenant_id, transform_id) From 64f69e6ce7d87a14a2d9a140eef6a86f796d298b Mon Sep 17 00:00:00 2001 From: SamKirsch10 Date: Tue, 23 Jun 2015 11:03:02 -0600 Subject: [PATCH 06/12] the real limit fixes --- .../common/repositories/mysql/streams_repository.py | 3 +++ .../repositories/mysql/transforms_repository.py | 13 ++++++++++--- monasca_events_api/v2/transforms.py | 8 +++++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/monasca_events_api/common/repositories/mysql/streams_repository.py b/monasca_events_api/common/repositories/mysql/streams_repository.py index 89d2574..b485e19 100644 --- a/monasca_events_api/common/repositories/mysql/streams_repository.py +++ b/monasca_events_api/common/repositories/mysql/streams_repository.py @@ -94,6 +94,9 @@ class StreamsRepository(mysql_repository.MySQLRepository, else: order_by_clause = " order by sd.created_at " limit_clause = "" + if limit: + limit_clause = " limit %s" + parms.append(int(limit)) query = select_clause + where_clause + order_by_clause + limit_clause diff --git a/monasca_events_api/common/repositories/mysql/transforms_repository.py b/monasca_events_api/common/repositories/mysql/transforms_repository.py index 5b6a02e..8347b76 100644 --- a/monasca_events_api/common/repositories/mysql/transforms_repository.py +++ b/monasca_events_api/common/repositories/mysql/transforms_repository.py @@ -53,11 +53,18 @@ class TransformsRepository(mysql_repository.MySQLRepository, else: raise e - def list_transforms(self, tenant_id): + def list_transforms(self, tenant_id, limit=None): cnxn, cursor = self._get_cnxn_cursor_tuple() with cnxn: - cursor.execute("""select * from event_transform - where tenant_id = %s and deleted_at IS NULL""", [tenant_id]) + if limit: + query = ("""select * from event_transform + where tenant_id = "{}" and deleted_at IS NULL limit {}""" + .format(tenant_id, limit)) + cursor.execute(query) + else: + cursor.execute("""select * from event_transform + where tenant_id = %s and deleted_at IS NULL""", + [tenant_id]) return cursor.fetchall() def list_transform(self, tenant_id, transform_id): diff --git a/monasca_events_api/v2/transforms.py b/monasca_events_api/v2/transforms.py index 8a4ccaf..0b60724 100644 --- a/monasca_events_api/v2/transforms.py +++ b/monasca_events_api/v2/transforms.py @@ -80,7 +80,8 @@ class Transforms(transforms_api_v2.TransformsV2API): else: helpers.validate_authorization(req, self._default_authorized_roles) tenant_id = helpers.get_tenant_id(req) - res.body = self._list_transforms(tenant_id) + limit = helpers.get_query_param(req, 'limit') + res.body = self._list_transforms(tenant_id, limit) res.status = falcon.HTTP_200 def on_delete(self, req, res, transform_id): @@ -152,9 +153,10 @@ class Transforms(transforms_api_v2.TransformsV2API): 'specification': specification, 'enabled': enabled} return json.dumps(response) - def _list_transforms(self, tenant_id): + def _list_transforms(self, tenant_id, limit): try: - transforms = self._transforms_repo.list_transforms(tenant_id) + transforms = self._transforms_repo.list_transforms(tenant_id, + limit) for transform in transforms: transform['specification'] = yaml.safe_dump(transform['specification']) return json.dumps(transforms, cls=MyEncoder) From 1489fb21bd658ee829582f518bce4cca076c99f5 Mon Sep 17 00:00:00 2001 From: SamKirsch10 Date: Tue, 23 Jun 2015 15:15:29 -0600 Subject: [PATCH 07/12] fix pagination helper --- .../mysql/transforms_repository.py | 25 ++++++--- monasca_events_api/v2/common/helpers.py | 53 ++++++++++--------- monasca_events_api/v2/stream_definitions.py | 2 +- monasca_events_api/v2/transforms.py | 9 ++-- 4 files changed, 53 insertions(+), 36 deletions(-) diff --git a/monasca_events_api/common/repositories/mysql/transforms_repository.py b/monasca_events_api/common/repositories/mysql/transforms_repository.py index 8347b76..920df5b 100644 --- a/monasca_events_api/common/repositories/mysql/transforms_repository.py +++ b/monasca_events_api/common/repositories/mysql/transforms_repository.py @@ -53,17 +53,30 @@ class TransformsRepository(mysql_repository.MySQLRepository, else: raise e - def list_transforms(self, tenant_id, limit=None): + def list_transforms(self, tenant_id, limit=None, offset=None): cnxn, cursor = self._get_cnxn_cursor_tuple() with cnxn: if limit: + if offset: + query = ("""select * from event_transform + where tenant_id = "{}" and id > "{}" and deleted_at + IS NULL order by id limit {}""" + .format(tenant_id, offset, limit)) + else: + query = ("""select * from event_transform + where tenant_id = "{}" and deleted_at + IS NULL order by id limit {}""" + .format(tenant_id, limit)) + cursor.execute(query) + elif offset: query = ("""select * from event_transform - where tenant_id = "{}" and deleted_at IS NULL limit {}""" - .format(tenant_id, limit)) + where tenant_id = "{}" and id > "{}" and deleted_at + IS NULL order by id""" + .format(tenant_id, offset)) cursor.execute(query) else: cursor.execute("""select * from event_transform - where tenant_id = %s and deleted_at IS NULL""", + where tenant_id = %s and deleted_at IS NULL order by id""", [tenant_id]) return cursor.fetchall() @@ -71,8 +84,8 @@ class TransformsRepository(mysql_repository.MySQLRepository, cnxn, cursor = self._get_cnxn_cursor_tuple() with cnxn: cursor.execute("""select * from event_transform - where tenant_id = %s and id = %s and deleted_at IS NULL""", - (tenant_id, transform_id)) + where tenant_id = %s and id = %s and deleted_at + IS NULL order by id """,(tenant_id, transform_id)) return cursor.fetchall() def delete_transform(self, tenant_id, transform_id): diff --git a/monasca_events_api/v2/common/helpers.py b/monasca_events_api/v2/common/helpers.py index f05cbb8..28badd2 100644 --- a/monasca_events_api/v2/common/helpers.py +++ b/monasca_events_api/v2/common/helpers.py @@ -15,6 +15,7 @@ import datetime import json import urlparse +import urllib import falcon import simplejson @@ -244,46 +245,46 @@ def get_query_period(req): raise falcon.HTTPBadRequest('Bad request', ex.message) -def paginate(resource, uri, offset): +def paginate(resource, uri): - if offset is not None: + limit = constants.PAGE_LIMIT + parsed_uri = urlparse.urlparse(uri) - if resource: + self_link = build_base_uri(parsed_uri) + print("RESOURCE LENGTH = {}".format(len(resource))) + print("LIMIT = {}".format(limit)) + if resource and len(resource) >= limit: - if len(resource) >= constants.PAGE_LIMIT: + if 'timestamp' in resource[limit - 1]: + new_offset = resource[limit - 1]['timestamp'] - new_offset = resource[-1]['id'] + if 'id' in resource[limit - 1]: + new_offset = resource[limit - 1]['id'] - parsed_uri = urlparse.urlparse(uri) + next_link = build_base_uri(parsed_uri) - next_link = build_base_uri(parsed_uri) + new_query_params = [u'offset' + '=' + urllib.quote( + new_offset.encode('utf8'), safe='')] - new_query_params = [u'offset' + '=' + str(new_offset).decode( - 'utf8')] + if new_query_params: + next_link += '?' + '&'.join(new_query_params) - for query_param in parsed_uri.query.split('&'): - query_param_name, query_param_val = query_param.split('=') - if query_param_name.lower() != 'offset': - new_query_params.append(query_param) + resource = {u'links': ([{u'rel': u'self', + u'href': self_link.decode('utf8')}, + {u'rel': u'next', + u'href': next_link.decode('utf8')}]), + u'elements': resource[:limit]} - next_link += '?' + '&'.join(new_query_params) + else: - resource = {u'links': - [{u'rel': u'self', u'href': uri.decode('utf8')}, - {u'rel': u'next', - u'href': next_link.decode('utf8')}], - u'elements': resource} - - else: - - resource = {u'links': - [{u'rel': u'self', u'href': uri.decode('utf8')}], - u'elements': resource} + resource = {u'links': ([{u'rel': u'self', + u'href': self_link.decode('utf8')}]), + u'elements': resource} return resource -def paginate_measurement(measurement, uri, offset): +def paginate_measurement(measurement, uri): if offset is not None: diff --git a/monasca_events_api/v2/stream_definitions.py b/monasca_events_api/v2/stream_definitions.py index 1dd0e46..2744d4c 100644 --- a/monasca_events_api/v2/stream_definitions.py +++ b/monasca_events_api/v2/stream_definitions.py @@ -229,7 +229,7 @@ class StreamDefinitions(stream_definitions_api_v2.StreamDefinitionsV2API): helpers.add_links_to_resource(sd, req_uri) result.append(sd) - result = helpers.paginate(result, req_uri, offset) + result = helpers.paginate(result, req_uri) return result diff --git a/monasca_events_api/v2/transforms.py b/monasca_events_api/v2/transforms.py index 0b60724..0a4ea95 100644 --- a/monasca_events_api/v2/transforms.py +++ b/monasca_events_api/v2/transforms.py @@ -81,7 +81,10 @@ class Transforms(transforms_api_v2.TransformsV2API): helpers.validate_authorization(req, self._default_authorized_roles) tenant_id = helpers.get_tenant_id(req) limit = helpers.get_query_param(req, 'limit') - res.body = self._list_transforms(tenant_id, limit) + offset = helpers.normalize_offset(helpers.get_query_param( + req, + 'offset')) + res.body = self._list_transforms(tenant_id, limit, offset) res.status = falcon.HTTP_200 def on_delete(self, req, res, transform_id): @@ -153,10 +156,10 @@ class Transforms(transforms_api_v2.TransformsV2API): 'specification': specification, 'enabled': enabled} return json.dumps(response) - def _list_transforms(self, tenant_id, limit): + def _list_transforms(self, tenant_id, limit, offset): try: transforms = self._transforms_repo.list_transforms(tenant_id, - limit) + limit, offset) for transform in transforms: transform['specification'] = yaml.safe_dump(transform['specification']) return json.dumps(transforms, cls=MyEncoder) From 5e69ab01a54080c394561170116109963accfa2a Mon Sep 17 00:00:00 2001 From: SamKirsch10 Date: Tue, 23 Jun 2015 16:02:14 -0600 Subject: [PATCH 08/12] lots of bug fixes for the limit and transform query cleanup --- .../mysql/transforms_repository.py | 51 +++++++++++++------ monasca_events_api/v2/common/helpers.py | 4 +- monasca_events_api/v2/events.py | 2 +- monasca_events_api/v2/transforms.py | 10 ++-- 4 files changed, 45 insertions(+), 22 deletions(-) diff --git a/monasca_events_api/common/repositories/mysql/transforms_repository.py b/monasca_events_api/common/repositories/mysql/transforms_repository.py index 920df5b..586322c 100644 --- a/monasca_events_api/common/repositories/mysql/transforms_repository.py +++ b/monasca_events_api/common/repositories/mysql/transforms_repository.py @@ -26,8 +26,13 @@ LOG = log.getLogger(__name__) class TransformsRepository(mysql_repository.MySQLRepository, transforms_repository.TransformsRepository): + + base_query = """select * from event_transform where deleted_at IS NULL""" + order_by_clause = " order by id" + def create_transforms(self, id, tenant_id, name, description, specification, enabled): + cnxn, cursor = self._get_cnxn_cursor_tuple() with cnxn: now = timeutils.utcnow() @@ -58,26 +63,42 @@ class TransformsRepository(mysql_repository.MySQLRepository, with cnxn: if limit: if offset: - query = ("""select * from event_transform - where tenant_id = "{}" and id > "{}" and deleted_at - IS NULL order by id limit {}""" - .format(tenant_id, offset, limit)) + tenant_id_clause = (" and tenant_id = \"{}\"" + .format(tenant_id)) + + offset_clause = " and id > \"{}\"".format(offset) + + limit_clause = " limit {}".format(limit) + + query = (TransformsRepository.base_query + tenant_id_clause + + offset_clause + TransformsRepository.order_by_clause + limit_clause) else: - query = ("""select * from event_transform - where tenant_id = "{}" and deleted_at - IS NULL order by id limit {}""" - .format(tenant_id, limit)) + tenant_id_clause = (" and tenant_id = \"{}\"" + .format(tenant_id)) + + limit_clause = " limit {}".format(limit) + + query = (TransformsRepository.base_query + tenant_id_clause + + TransformsRepository.order_by_clause + limit_clause) + cursor.execute(query) elif offset: - query = ("""select * from event_transform - where tenant_id = "{}" and id > "{}" and deleted_at - IS NULL order by id""" - .format(tenant_id, offset)) + tenant_id_clause = (" and tenant_id = \"{}\"" + .format(tenant_id)) + + offset_clause = " and id > \"{}\"".format(offset) + + query = (TransformsRepository.base_query + tenant_id_clause + offset_clause + + TransformsRepository.order_by_clause) + cursor.execute(query) else: - cursor.execute("""select * from event_transform - where tenant_id = %s and deleted_at IS NULL order by id""", - [tenant_id]) + tenant_id_clause = (" and tenant_id = \"{}\"" + .format(tenant_id)) + + query = (TransformsRepository.base_query + tenant_id_clause + TransformsRepository.order_by_clause) + print query + cursor.execute(query) return cursor.fetchall() def list_transform(self, tenant_id, transform_id): diff --git a/monasca_events_api/v2/common/helpers.py b/monasca_events_api/v2/common/helpers.py index 28badd2..811edba 100644 --- a/monasca_events_api/v2/common/helpers.py +++ b/monasca_events_api/v2/common/helpers.py @@ -248,11 +248,11 @@ def get_query_period(req): def paginate(resource, uri): limit = constants.PAGE_LIMIT + parsed_uri = urlparse.urlparse(uri) self_link = build_base_uri(parsed_uri) - print("RESOURCE LENGTH = {}".format(len(resource))) - print("LIMIT = {}".format(limit)) + if resource and len(resource) >= limit: if 'timestamp' in resource[limit - 1]: diff --git a/monasca_events_api/v2/events.py b/monasca_events_api/v2/events.py index 1aa6b51..13e9c9a 100644 --- a/monasca_events_api/v2/events.py +++ b/monasca_events_api/v2/events.py @@ -119,7 +119,7 @@ class Events(events_api_v2.EventsV2API): @resource.resource_try_catch_block def _list_events(self, tenant_id, uri, offset, limit): rows = self._events_repo.list_events(tenant_id, offset, limit) - return helpers.paginate(self._build_events(rows), uri, offset) + return helpers.paginate(self._build_events(rows), uri) @resource.resource_try_catch_block def _list_event(self, tenant_id, event_id, uri): diff --git a/monasca_events_api/v2/transforms.py b/monasca_events_api/v2/transforms.py index 0a4ea95..782fb07 100644 --- a/monasca_events_api/v2/transforms.py +++ b/monasca_events_api/v2/transforms.py @@ -75,7 +75,7 @@ class Transforms(transforms_api_v2.TransformsV2API): if transform_id: helpers.validate_authorization(req, self._default_authorized_roles) tenant_id = helpers.get_tenant_id(req) - res.body = self._list_transform(tenant_id, transform_id) + res.body = self._list_transform(tenant_id, transform_id, req.uri) res.status = falcon.HTTP_200 else: helpers.validate_authorization(req, self._default_authorized_roles) @@ -84,7 +84,7 @@ class Transforms(transforms_api_v2.TransformsV2API): offset = helpers.normalize_offset(helpers.get_query_param( req, 'offset')) - res.body = self._list_transforms(tenant_id, limit, offset) + res.body = self._list_transforms(tenant_id, limit, offset, req.uri) res.status = falcon.HTTP_200 def on_delete(self, req, res, transform_id): @@ -156,19 +156,20 @@ class Transforms(transforms_api_v2.TransformsV2API): 'specification': specification, 'enabled': enabled} return json.dumps(response) - def _list_transforms(self, tenant_id, limit, offset): + def _list_transforms(self, tenant_id, limit, offset, uri): try: transforms = self._transforms_repo.list_transforms(tenant_id, limit, offset) for transform in transforms: transform['specification'] = yaml.safe_dump(transform['specification']) + transforms = helpers.paginate(transforms, uri) return json.dumps(transforms, cls=MyEncoder) except repository_exceptions.RepositoryException as ex: LOG.error(ex) raise falcon.HTTPInternalServerError('Service unavailable', ex.message) - def _list_transform(self, tenant_id, transform_id): + def _list_transform(self, tenant_id, transform_id, uri): try: transform = self._transforms_repo.list_transform(tenant_id, transform_id)[0] @@ -176,6 +177,7 @@ class Transforms(transforms_api_v2.TransformsV2API): transform['specification']) transform_list = list() transform_list.append(transform) + transform = helpers.paginate(transform, uri) return json.dumps(transform_list, cls=MyEncoder) except repository_exceptions.RepositoryException as ex: LOG.error(ex) From ef9065d32ebb66ce0f110b7608cef6f0382fc4d8 Mon Sep 17 00:00:00 2001 From: SamKirsch10 Date: Wed, 24 Jun 2015 13:28:23 -0600 Subject: [PATCH 09/12] bug fixes and pagination fixes --- .../mysql/transforms_repository.py | 2 +- monasca_events_api/v2/events.py | 8 +++++-- monasca_events_api/v2/transforms.py | 21 ++++++++----------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/monasca_events_api/common/repositories/mysql/transforms_repository.py b/monasca_events_api/common/repositories/mysql/transforms_repository.py index 586322c..d875d8d 100644 --- a/monasca_events_api/common/repositories/mysql/transforms_repository.py +++ b/monasca_events_api/common/repositories/mysql/transforms_repository.py @@ -97,7 +97,7 @@ class TransformsRepository(mysql_repository.MySQLRepository, .format(tenant_id)) query = (TransformsRepository.base_query + tenant_id_clause + TransformsRepository.order_by_clause) - print query + cursor.execute(query) return cursor.fetchall() diff --git a/monasca_events_api/v2/events.py b/monasca_events_api/v2/events.py index 13e9c9a..4ae40ba 100644 --- a/monasca_events_api/v2/events.py +++ b/monasca_events_api/v2/events.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +import re + import falcon import collections @@ -61,7 +63,9 @@ class Events(events_api_v2.EventsV2API): if event_id: helpers.validate_authorization(req, self._default_authorized_roles) tenant_id = helpers.get_tenant_id(req) - result = self._list_event(tenant_id, event_id, req.uri) + result = self._list_event(tenant_id, event_id) + helpers.add_links_to_resource( + result[0], re.sub('/' + event_id, '', req.uri)) res.body = helpers.dumpit_utf8(result) res.status = falcon.HTTP_200 else: @@ -122,7 +126,7 @@ class Events(events_api_v2.EventsV2API): return helpers.paginate(self._build_events(rows), uri) @resource.resource_try_catch_block - def _list_event(self, tenant_id, event_id, uri): + def _list_event(self, tenant_id, event_id): rows = self._events_repo.list_event(tenant_id, event_id) return self._build_events(rows) diff --git a/monasca_events_api/v2/transforms.py b/monasca_events_api/v2/transforms.py index 782fb07..1564638 100644 --- a/monasca_events_api/v2/transforms.py +++ b/monasca_events_api/v2/transforms.py @@ -12,7 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -import ast +import re import datetime import json from time import mktime @@ -75,7 +75,10 @@ class Transforms(transforms_api_v2.TransformsV2API): if transform_id: helpers.validate_authorization(req, self._default_authorized_roles) tenant_id = helpers.get_tenant_id(req) - res.body = self._list_transform(tenant_id, transform_id, req.uri) + result = self._list_transform(tenant_id, transform_id, req.uri) + helpers.add_links_to_resource( + result, re.sub('/' + transform_id, '', req.uri)) + res.body = json.dumps(result, cls=MyEncoder) res.status = falcon.HTTP_200 else: helpers.validate_authorization(req, self._default_authorized_roles) @@ -84,7 +87,8 @@ class Transforms(transforms_api_v2.TransformsV2API): offset = helpers.normalize_offset(helpers.get_query_param( req, 'offset')) - res.body = self._list_transforms(tenant_id, limit, offset, req.uri) + result = self._list_transforms(tenant_id, limit, offset, req.uri) + res.body = json.dumps(result, cls=MyEncoder) res.status = falcon.HTTP_200 def on_delete(self, req, res, transform_id): @@ -160,10 +164,8 @@ class Transforms(transforms_api_v2.TransformsV2API): try: transforms = self._transforms_repo.list_transforms(tenant_id, limit, offset) - for transform in transforms: - transform['specification'] = yaml.safe_dump(transform['specification']) transforms = helpers.paginate(transforms, uri) - return json.dumps(transforms, cls=MyEncoder) + return transforms except repository_exceptions.RepositoryException as ex: LOG.error(ex) raise falcon.HTTPInternalServerError('Service unavailable', @@ -173,12 +175,7 @@ class Transforms(transforms_api_v2.TransformsV2API): try: transform = self._transforms_repo.list_transform(tenant_id, transform_id)[0] - transform['specification'] = yaml.safe_dump( - transform['specification']) - transform_list = list() - transform_list.append(transform) - transform = helpers.paginate(transform, uri) - return json.dumps(transform_list, cls=MyEncoder) + return transform except repository_exceptions.RepositoryException as ex: LOG.error(ex) raise falcon.HTTPInternalServerError('Service unavailable', From 41b70b4dd5b3b2f09e3fb3c0b4e0b94b06b224c2 Mon Sep 17 00:00:00 2001 From: Joe Keen Date: Fri, 26 Jun 2015 14:53:01 -0600 Subject: [PATCH 10/12] Refactored transform repo --- .../mysql/transforms_repository.py | 58 ++++++------------- 1 file changed, 17 insertions(+), 41 deletions(-) diff --git a/monasca_events_api/common/repositories/mysql/transforms_repository.py b/monasca_events_api/common/repositories/mysql/transforms_repository.py index 2979353..c08a263 100644 --- a/monasca_events_api/common/repositories/mysql/transforms_repository.py +++ b/monasca_events_api/common/repositories/mysql/transforms_repository.py @@ -25,9 +25,6 @@ LOG = log.getLogger(__name__) class TransformsRepository(mysql_repository.MySQLRepository, transforms_repository.TransformsRepository): - base_query = """select * from event_transform where deleted_at IS NULL""" - order_by_clause = " order by id" - def create_transforms(self, id, tenant_id, name, description, specification, enabled): @@ -57,54 +54,33 @@ class TransformsRepository(mysql_repository.MySQLRepository, raise e def list_transforms(self, tenant_id, limit=None, offset=None): - cnxn, cursor = self._get_cnxn_cursor_tuple() - with cnxn: - if limit: - if offset: - tenant_id_clause = (" and tenant_id = \"{}\"" - .format(tenant_id)) + base_query = """select * from event_transform where deleted_at IS NULL""" + tenant_id_clause = " and tenant_id = \"{}\"".format(tenant_id) + order_by_clause = " order by id" - offset_clause = " and id > \"{}\"".format(offset) + offset_clause = ' ' + if offset: + offset_clause = " and id > \"{}\"".format(offset) - limit_clause = " limit {}".format(limit) + if not limit: + limit = constants.PAGE_LIMIT + limit_clause = " limit {}".format(limit) - query = (TransformsRepository.base_query + tenant_id_clause + - offset_clause + TransformsRepository.order_by_clause + limit_clause) - else: - tenant_id_clause = (" and tenant_id = \"{}\"" - .format(tenant_id)) + query = (base_query + + tenant_id_clause + + offset_clause + + order_by_clause + + limit_clause) - limit_clause = " limit {}".format(limit) - - query = (TransformsRepository.base_query + tenant_id_clause - + TransformsRepository.order_by_clause + limit_clause) - - cursor.execute(query) - elif offset: - tenant_id_clause = (" and tenant_id = \"{}\"" - .format(tenant_id)) - - offset_clause = " and id > \"{}\"".format(offset) - - query = (TransformsRepository.base_query + tenant_id_clause + offset_clause + - TransformsRepository.order_by_clause) - - cursor.execute(query) - else: - tenant_id_clause = (" and tenant_id = \"{}\"" - .format(tenant_id)) - - query = (TransformsRepository.base_query + tenant_id_clause + TransformsRepository.order_by_clause) - - cursor.execute(query) - return cursor.fetchall() + rows = self._execute_query(query, []) + return rows def list_transform(self, tenant_id, transform_id): cnxn, cursor = self._get_cnxn_cursor_tuple() with cnxn: cursor.execute("""select * from event_transform where tenant_id = %s and id = %s and deleted_at - IS NULL order by id """,(tenant_id, transform_id)) + IS NULL order by id """, (tenant_id, transform_id)) return cursor.fetchall() def delete_transform(self, tenant_id, transform_id): From 91c0181f084760b7f3762f5209586c7ddf28b7c0 Mon Sep 17 00:00:00 2001 From: SamKirsch10 Date: Fri, 26 Jun 2015 15:29:01 -0600 Subject: [PATCH 11/12] fixed mysql formatting --- func_test/api_func_test.py | 14 ++++----- .../mysql/transforms_repository.py | 31 +++++++++++++------ 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/func_test/api_func_test.py b/func_test/api_func_test.py index c4ccd9b..58d9013 100644 --- a/func_test/api_func_test.py +++ b/func_test/api_func_test.py @@ -21,7 +21,7 @@ import yaml from monascaclient import ksclient -events_url = "http://192.168.10.4:8082" +events_url = "http://127.0.0.1:8082" def token(): @@ -133,7 +133,7 @@ def test_stream_definition_delete(): data=json.dumps(body), headers=headers) stream_dict = json.loads(stream_resp.text) - stream_id = str(stream_dict[0]['id']) + stream_id = str(stream_dict[0]['elements']['id']) response = requests.delete( url=events_url + "/v2.0/stream-definitions/{}".format( stream_id), @@ -181,7 +181,7 @@ def test_transforms(): data=json.dumps(body), headers=headers) transform_dict = json.loads(response.text) - transform_dict_id = transform_dict[0]['id'] + transform_dict_id = transform_dict['elements'][0]['id'] response = requests.delete( url=events_url + "/v2.0/transforms/{}".format(transform_dict_id), data=json.dumps(body), @@ -189,8 +189,8 @@ def test_transforms(): assert response.status_code == 204 print("DELETE /transforms success") -test_stream_definition_post() -test_stream_definition_get() -test_stream_definition_delete() -test_events_get_all() +# test_stream_definition_post() +# test_stream_definition_get() +# test_stream_definition_delete() +# test_events_get_all() test_transforms() diff --git a/monasca_events_api/common/repositories/mysql/transforms_repository.py b/monasca_events_api/common/repositories/mysql/transforms_repository.py index c08a263..e5fa655 100644 --- a/monasca_events_api/common/repositories/mysql/transforms_repository.py +++ b/monasca_events_api/common/repositories/mysql/transforms_repository.py @@ -18,6 +18,7 @@ from oslo_utils import timeutils from monasca_events_api.common.repositories.mysql import mysql_repository from monasca_events_api.common.repositories import transforms_repository +from monasca_events_api.common.repositories import constants LOG = log.getLogger(__name__) @@ -76,15 +77,25 @@ class TransformsRepository(mysql_repository.MySQLRepository, return rows def list_transform(self, tenant_id, transform_id): - cnxn, cursor = self._get_cnxn_cursor_tuple() - with cnxn: - cursor.execute("""select * from event_transform - where tenant_id = %s and id = %s and deleted_at - IS NULL order by id """, (tenant_id, transform_id)) - return cursor.fetchall() + base_query = """select * from event_transform where deleted_at IS NULL""" + tenant_id_clause = " and tenant_id = \"{}\"".format(tenant_id) + transform_id_clause = " and transform_id = \"{}\"".format(transform_id) + + query = (base_query+ + tenant_id_clause+ + transform_id_clause) + + rows = self._execute_query(query, []) + return rows def delete_transform(self, tenant_id, transform_id): - cnxn, cursor = self._get_cnxn_cursor_tuple() - with cnxn: - cursor.execute("""delete from event_transform - where id = %s and tenant_id = %s""", (transform_id, tenant_id)) + now = timeutils.utcnow() + base_query = "update event_transform set deleted_at = \"{}\" where"\ + .format(now) + tenant_id_clause = " tenant_id = \"{}\"".format(tenant_id) + transform_id_clause = " and id = \"{}\"".format(transform_id) + + query = (base_query + + tenant_id_clause + + transform_id_clause) + self._execute_query(query, []) From 9a35070ad2488361d987ba50e300b5d6c9b3ad15 Mon Sep 17 00:00:00 2001 From: SamKirsch10 Date: Mon, 20 Jul 2015 11:59:24 -0600 Subject: [PATCH 12/12] transform bug fixes in mysql clause and in get single transform --- .../repositories/mysql/transforms_repository.py | 2 +- monasca_events_api/v2/transforms.py | 15 +-------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/monasca_events_api/common/repositories/mysql/transforms_repository.py b/monasca_events_api/common/repositories/mysql/transforms_repository.py index e5fa655..42f15f3 100644 --- a/monasca_events_api/common/repositories/mysql/transforms_repository.py +++ b/monasca_events_api/common/repositories/mysql/transforms_repository.py @@ -79,7 +79,7 @@ class TransformsRepository(mysql_repository.MySQLRepository, def list_transform(self, tenant_id, transform_id): base_query = """select * from event_transform where deleted_at IS NULL""" tenant_id_clause = " and tenant_id = \"{}\"".format(tenant_id) - transform_id_clause = " and transform_id = \"{}\"".format(transform_id) + transform_id_clause = " and id = \"{}\"".format(transform_id) query = (base_query+ tenant_id_clause+ diff --git a/monasca_events_api/v2/transforms.py b/monasca_events_api/v2/transforms.py index 89cbba3..cfd87a9 100644 --- a/monasca_events_api/v2/transforms.py +++ b/monasca_events_api/v2/transforms.py @@ -174,26 +174,13 @@ class Transforms(transforms_api_v2.TransformsV2API): raise falcon.HTTPInternalServerError('Service unavailable', ex.message) - def _list_transform(self, tenant_id, transform_id, uri): - try: - transform = self._transforms_repo.list_transform(tenant_id, - transform_id)[0] - return transform - except repository_exceptions.RepositoryException as ex: - LOG.error(ex) - raise falcon.HTTPInternalServerError('Service unavailable', - ex.message) - def _list_transform(self, tenant_id, transform_id, uri): try: transform = self._transforms_repo.list_transform(tenant_id, transform_id)[0] transform['specification'] = yaml.safe_dump( transform['specification']) - transform_list = list() - transform_list.append(transform) - transform_list = helpers.paginate(transform, uri) - return json.dumps(transform_list, cls=MyEncoder) + return transform except repository_exceptions.RepositoryException as ex: LOG.error(ex) raise falcon.HTTPInternalServerError('Service unavailable',