From e3c825a82ceb0b7e67b960ef89b010effdb78254 Mon Sep 17 00:00:00 2001 From: Kushal Agrawal Date: Thu, 25 Jan 2018 10:48:03 +0530 Subject: [PATCH] Total Number of artifacts for given filter Currently there was no way to get the total number of records available during pagination until user reaches the last page. And still he has to keep the track of previous number of records. To overcome this issue we have provided total_count attribute in response structure which provides total number of records available for the given filter. Affected API : GET /artifacts/{artifacts_type} artifact API. For above API our updated response structure contains a new json attribute "total_count" and value will be of integer type. Change-Id: I7608a80a70e90a4be24ebae7961346810d2f3a70 --- glare/api/v1/resource.py | 9 ++++-- glare/db/sqlalchemy/api.py | 31 ++++++++++++++++++- glare/engine.py | 11 ++++--- glare/objects/base.py | 8 +++-- glare/tests/functional/test_database_store.py | 3 +- .../tests/functional/test_sample_artifact.py | 5 ++- glare/tests/unit/api/test_list.py | 4 +++ 7 files changed, 57 insertions(+), 14 deletions(-) diff --git a/glare/api/v1/resource.py b/glare/api/v1/resource.py index d7ad751..5129214 100644 --- a/glare/api/v1/resource.py +++ b/glare/api/v1/resource.py @@ -380,10 +380,12 @@ class ArtifactsController(api_versioning.VersionedResource): versions should be returned in output :return: list of requested artifact definitions """ - artifacts = self.engine.list(req.context, type_name, filters, marker, - limit, sort, latest) + artifacts_data = self.engine.list(req.context, type_name, filters, + marker, limit, sort, latest) + artifacts = artifacts_data["artifacts"] result = {'artifacts': artifacts, - 'type_name': type_name} + 'type_name': type_name, + 'total_count': artifacts_data['total_count']} if len(artifacts) != 0 and len(artifacts) == limit: result['next_marker'] = artifacts[-1]['id'] return result @@ -543,6 +545,7 @@ class ResponseSerializer(api_versioning.VersionedResource, 'artifacts': af_list['artifacts'], 'first': '/artifacts/%s' % type_name, 'schema': '/schemas/%s' % type_name, + 'total_count': af_list['total_count'] } if query: body['first'] = '%s?%s' % (body['first'], query) diff --git a/glare/db/sqlalchemy/api.py b/glare/db/sqlalchemy/api.py index 674aed0..6295cd9 100644 --- a/glare/db/sqlalchemy/api.py +++ b/glare/db/sqlalchemy/api.py @@ -196,7 +196,12 @@ def get_all(context, session, filters=None, marker=None, limit=None, """ artifacts = _get_all( context, session, filters, marker, limit, sort, latest) - return [af.to_dict() for af in artifacts] + total_artifacts_count = get_artifact_count(context, session, filters, + latest) + return { + "artifacts": [af.to_dict() for af in artifacts], + "total_count": total_artifacts_count + } def _apply_latest_filter(context, session, query, @@ -750,3 +755,27 @@ def delete_blob_data(context, uri, session): blob_data_id = uri[6:] session.query( models.ArtifactBlobData).filter_by(id=blob_data_id).delete() + + +def get_artifact_count(context, session, filters=None, latest=False): + + filters = filters or {} + + query = _create_artifact_count_query(context, session) + + basic_conds, tag_conds, prop_conds = _do_query_filters(filters) + + query = _apply_user_filters(query, basic_conds, tag_conds, prop_conds) + + if latest: + query = _apply_latest_filter(context, session, query, + basic_conds, tag_conds, prop_conds) + + return query.all()[0].total_count + + +def _create_artifact_count_query(context, session): + + query = session.query(func.count(models.Artifact.id).label("total_count")) + + return _apply_query_base_filters(query, context) diff --git a/glare/engine.py b/glare/engine.py index bf0847a..7a5a1cc 100644 --- a/glare/engine.py +++ b/glare/engine.py @@ -99,7 +99,7 @@ class Engine(object): lock = self.lock_engine.acquire(context, scope_id) try: - if len(self.list(context, type_name, filters)) > 0: + if self.list(context, type_name, filters).get("total_count") > 0: msg = _("Artifact with this name and version is already " "exists for this scope.") raise exception.Conflict(msg) @@ -332,10 +332,11 @@ class Engine(object): policy.authorize("artifact:list", {}, context) artifact_type = registry.ArtifactRegistry.get_artifact_type(type_name) # return list to the user - af_list = [af.to_dict() - for af in artifact_type.list(context, filters, marker, - limit, sort, latest)] - return af_list + artifacts_data = artifact_type.list(context, filters, marker, + limit, sort, latest) + artifacts_data["artifacts"] = [af.to_dict() + for af in artifacts_data["artifacts"]] + return artifacts_data @staticmethod def _delete_blobs(context, af, blobs): diff --git a/glare/objects/base.py b/glare/objects/base.py index 8310a19..b9ccbc1 100644 --- a/glare/objects/base.py +++ b/glare/objects/base.py @@ -430,9 +430,11 @@ Possible values: if default_filter not in filters: filters.append(default_filter) - return [cls.init_artifact(context, af) - for af in cls.db_api.list( - context, filters, marker, limit, sort, latest)] + artifacts_data = cls.db_api.list(context, filters, marker, limit, + sort, latest) + artifacts_data["artifacts"] = [cls.init_artifact(context, af) + for af in artifacts_data["artifacts"]] + return artifacts_data @classmethod def delete(cls, context, af): diff --git a/glare/tests/functional/test_database_store.py b/glare/tests/functional/test_database_store.py index 47cef3b..c5bf089 100644 --- a/glare/tests/functional/test_database_store.py +++ b/glare/tests/functional/test_database_store.py @@ -44,7 +44,8 @@ default_store = database expected = {'first': '/artifacts/sample_artifact', 'artifacts': [], 'schema': '/schemas/sample_artifact', - 'type_name': 'sample_artifact'} + 'type_name': 'sample_artifact', + 'total_count': 0} self.assertEqual(expected, response) # Create a test artifact diff --git a/glare/tests/functional/test_sample_artifact.py b/glare/tests/functional/test_sample_artifact.py index a3152f3..f0ebde7 100644 --- a/glare/tests/functional/test_sample_artifact.py +++ b/glare/tests/functional/test_sample_artifact.py @@ -45,6 +45,7 @@ class TestList(base.TestArtifact): marker = result['next'] result = self.get(url=marker[10:]) self.assertEqual([art_list[1]], result['artifacts']) + self.assertEqual(5, result['total_count']) # sort by custom marker url = '/sample_artifact?sort=int1:asc&marker=%s' % art_list[1]['id'] @@ -68,6 +69,7 @@ class TestList(base.TestArtifact): marker = result['next'] result = self.get(url=marker[10:]) self.assertEqual([art_list[0]], result['artifacts']) + self.assertEqual(5, result['total_count']) def test_list_base_filters(self): # Create artifact @@ -660,7 +662,8 @@ class TestBlobs(base.TestArtifact): expected = {'first': '/artifacts/sample_artifact', 'artifacts': [], 'schema': '/schemas/sample_artifact', - 'type_name': 'sample_artifact'} + 'type_name': 'sample_artifact', + 'total_count': 0} self.assertEqual(expected, response) # Create a test artifact diff --git a/glare/tests/unit/api/test_list.py b/glare/tests/unit/api/test_list.py index a5f087a..b08a0a4 100644 --- a/glare/tests/unit/api/test_list.py +++ b/glare/tests/unit/api/test_list.py @@ -61,6 +61,7 @@ class TestArtifactList(base.BaseTestArtifactAPI): res = self.controller.list(self.req, 'sample_artifact') self.assertEqual(7, len(res['artifacts'])) self.assertEqual('sample_artifact', res['type_name']) + self.assertEqual(7, res['total_count']) # List all artifacts as an anonymous. Only public artifacts are visible anon_req = self.get_fake_request(user=self.users['anonymous']) @@ -192,6 +193,7 @@ class TestArtifactList(base.BaseTestArtifactAPI): result = self.controller.list(self.req, 'sample_artifact', filters=(), marker=marker, limit=1, sort=sort) self.assertEqual([art_list[1]], result['artifacts']) + self.assertEqual(5, result['total_count']) # sort by custom marker sort = [('int1', 'asc')] @@ -215,11 +217,13 @@ class TestArtifactList(base.BaseTestArtifactAPI): result = self.controller.list(self.req, 'sample_artifact', filters=(), limit=2, sort=sort) self.assertEqual(art_list[4:2:-1], result['artifacts']) + self.assertEqual(5, result['total_count']) marker = result['next_marker'] result = self.controller.list(self.req, 'sample_artifact', filters=(), marker=marker, limit=2, sort=sort) self.assertEqual(art_list[2:0:-1], result['artifacts']) + self.assertEqual(5, result['total_count']) marker = result['next_marker'] result = self.controller.list(self.req, 'sample_artifact', filters=(),