diff --git a/cloudkitty/storage/v1/sqlalchemy/__init__.py b/cloudkitty/storage/v1/sqlalchemy/__init__.py index 8920e023..938f8138 100644 --- a/cloudkitty/storage/v1/sqlalchemy/__init__.py +++ b/cloudkitty/storage/v1/sqlalchemy/__init__.py @@ -112,9 +112,10 @@ class SQLAlchemyStorage(storage.BaseStorage): if service: q = q.filter( self.frame_model.res_type == service) + # begin and end filters are both needed, do not remove one of them. q = q.filter( - self.frame_model.begin >= begin, - self.frame_model.end <= end, + self.frame_model.begin.between(begin, end), + self.frame_model.end.between(begin, end), self.frame_model.res_type != '_NO_DATA_') if groupby: q = q.group_by(sqlalchemy.sql.text(groupby)) @@ -136,9 +137,10 @@ class SQLAlchemyStorage(storage.BaseStorage): q = utils.model_query( self.frame_model, session) + # begin and end filters are both needed, do not remove one of them. q = q.filter( - self.frame_model.begin >= begin, - self.frame_model.end <= end) + self.frame_model.begin.between(begin, end), + self.frame_model.end.between(begin, end)) tenants = q.distinct().values( self.frame_model.tenant_id) return [tenant.tenant_id for tenant in tenants] @@ -152,9 +154,10 @@ class SQLAlchemyStorage(storage.BaseStorage): q = utils.model_query( self.frame_model, session) + # begin and end filters are both needed, do not remove one of them. q = q.filter( - self.frame_model.begin >= begin, - self.frame_model.end <= end) + self.frame_model.begin.between(begin, end), + self.frame_model.end.between(begin, end)) for filter_name, filter_value in filters.items(): if filter_value: q = q.filter( diff --git a/releasenotes/notes/optimizing-sql-queries-939f48fff1805389.yaml b/releasenotes/notes/optimizing-sql-queries-939f48fff1805389.yaml new file mode 100644 index 00000000..662982b9 --- /dev/null +++ b/releasenotes/notes/optimizing-sql-queries-939f48fff1805389.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + Queries filtering on date fields are slow as they have to parse each row. + There are sometimes millions of rows to parse while only a few thousands + are necessary. + The following patch narrows data to process by filtering more on + frame_model.begin as a firtst step using a `BETWEEN` statement instead + of >= + The BETWEEN statement requires an indexes to be efficient which are + https://review.opendev.org/c/openstack/cloudkitty/+/865435/ \ No newline at end of file