Added paging to list endpoints
In order to accomodate large lists of projects, stories, and tasks in a way that response quickly to user requests, I've added limited support for oslo's paginate_query method. Search responses now also include metadata headers that allow us to manage paging in our result set: X-Limit, X-Marker, X-Total. Using Markers, instead of offsets, comes with a tradeoff - calculating the _current_ page of the result can only be done in client memory by loading all previous records and determining where the marker lives. It probably makes sense to also permit offset, or to only allow offset and determine the marker record based on that. The benefit of using marker-style paging is that - rather than using a paging metaphor, we can implement an 'infinite scroll' UI on long lists of records. Whether that's a good idea remains to be seen. Additional changes - - Page size maximum and default is configurable - I had to change getAllStories to a distinct subselect, because left joins were screwing up our result sets. Change-Id: I058a4182d2b454edbbfb7db3493d94b3bad07b36
This commit is contained in:
Michael Krotscheck
parent
72d10a9464
commit
8565776140
|
|
@ -36,6 +36,10 @@ lock_path = $state_path/lock
|
|||
# OpenId Authentication endpoint
|
||||
# openid_url = https://login.launchpad.net/+openid
|
||||
|
||||
# List paging configuration options.
|
||||
# page_size_maximum = 500
|
||||
# page_size_default = 20
|
||||
|
||||
[database]
|
||||
# This line MUST be changed to actually run storyboard
|
||||
# Example:
|
||||
|
|
|
|||
|
|
@ -13,8 +13,21 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from oslo.config import cfg
|
||||
|
||||
app = {
|
||||
'root': 'storyboard.api.root_controller.RootController',
|
||||
'modules': ['storyboard.api'],
|
||||
'debug': False
|
||||
}
|
||||
|
||||
cfg.CONF.register_opts([
|
||||
cfg.IntOpt('page_size_maximum',
|
||||
default=500,
|
||||
help='The maximum number of results to allow a user to request '
|
||||
'from the API'),
|
||||
cfg.IntOpt('page_size_default',
|
||||
default=20,
|
||||
help='The maximum number of results to allow a user to request '
|
||||
'from the API')
|
||||
])
|
||||
|
|
|
|||
|
|
@ -13,19 +13,20 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from oslo.config import cfg
|
||||
from pecan import response
|
||||
from pecan import rest
|
||||
|
||||
from pecan.secure import secure
|
||||
from wsme.exc import ClientSideError
|
||||
from wsme import types as wtypes
|
||||
|
||||
import wsmeext.pecan as wsme_pecan
|
||||
|
||||
from storyboard.api.auth import authorization_checks as checks
|
||||
from storyboard.api.v1 import base
|
||||
from storyboard.db import api as dbapi
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class Project(base.APIBase):
|
||||
"""The Storyboard Registry describes the open source world as ProjectGroups
|
||||
|
|
@ -79,11 +80,31 @@ class ProjectsController(rest.RestController):
|
|||
status_code=404)
|
||||
|
||||
@secure(checks.guest)
|
||||
@wsme_pecan.wsexpose([Project])
|
||||
def get(self):
|
||||
@wsme_pecan.wsexpose([Project], int, int)
|
||||
def get(self, marker=None, limit=None):
|
||||
"""Retrieve a list of projects.
|
||||
|
||||
:param marker The marker at which the page set should begin. At the
|
||||
moment, this is the unique resource id.
|
||||
:param limit The number of projects to retrieve.
|
||||
"""
|
||||
projects = dbapi.project_get_all()
|
||||
# Boundary check on limit.
|
||||
if limit is None:
|
||||
limit = CONF.page_size_default
|
||||
limit = min(CONF.page_size_maximum, max(1, limit))
|
||||
|
||||
# Resolve the marker record.
|
||||
marker_project = dbapi.project_get(marker)
|
||||
|
||||
projects = dbapi.project_get_all(marker=marker_project, limit=limit)
|
||||
project_count = dbapi.project_get_count()
|
||||
|
||||
# Apply the query response headers.
|
||||
response.headers['X-Limit'] = str(limit)
|
||||
response.headers['X-Total'] = str(project_count)
|
||||
if marker_project:
|
||||
response.headers['X-Marker'] = str(marker_project.id)
|
||||
|
||||
return [Project.from_db_model(p) for p in projects]
|
||||
|
||||
@secure(checks.superuser)
|
||||
|
|
|
|||
|
|
@ -13,20 +13,22 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from oslo.config import cfg
|
||||
from pecan import request
|
||||
from pecan import response
|
||||
from pecan import rest
|
||||
from pecan.secure import secure
|
||||
from storyboard.api.v1.comments import CommentsController
|
||||
from wsme.exc import ClientSideError
|
||||
from wsme import types as wtypes
|
||||
|
||||
import wsmeext.pecan as wsme_pecan
|
||||
|
||||
from storyboard.api.auth import authorization_checks as checks
|
||||
from storyboard.api.v1 import base
|
||||
from storyboard.api.v1.comments import CommentsController
|
||||
from storyboard.db import api as dbapi
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class Story(base.APIBase):
|
||||
"""The Story is the main element of StoryBoard. It represents a user story
|
||||
|
|
@ -82,16 +84,38 @@ class StoriesController(rest.RestController):
|
|||
status_code=404)
|
||||
|
||||
@secure(checks.guest)
|
||||
@wsme_pecan.wsexpose([Story], int)
|
||||
def get_all(self, project_id=None):
|
||||
@wsme_pecan.wsexpose([Story], int, int, int)
|
||||
def get_all(self, project_id=None, marker=None, limit=None):
|
||||
"""Retrieve definitions of all of the stories.
|
||||
|
||||
:param project_id: filter stories by project ID.
|
||||
:param marker The marker at which the page set should begin. At the
|
||||
moment, this is the unique resource id.
|
||||
:param limit The number of stories to retrieve.
|
||||
"""
|
||||
if project_id:
|
||||
stories = dbapi.story_get_all_in_project(project_id)
|
||||
else:
|
||||
stories = dbapi.story_get_all()
|
||||
|
||||
# Boundary check on limit.
|
||||
if limit is None:
|
||||
limit = CONF.page_size_default
|
||||
limit = min(CONF.page_size_maximum, max(1, limit))
|
||||
|
||||
# Resolve the marker record.
|
||||
marker_story = dbapi.story_get(marker)
|
||||
|
||||
if marker_story is None or marker_story.project_id != project_id:
|
||||
marker_story = None
|
||||
|
||||
stories = dbapi.story_get_all(marker=marker_story,
|
||||
limit=limit,
|
||||
project_id=project_id)
|
||||
story_count = dbapi.story_get_count(project_id=project_id)
|
||||
|
||||
# Apply the query response headers.
|
||||
response.headers['X-Limit'] = str(limit)
|
||||
response.headers['X-Total'] = str(story_count)
|
||||
if marker_story:
|
||||
response.headers['X-Marker'] = str(marker_story.id)
|
||||
|
||||
return [Story.from_db_model(s) for s in stories]
|
||||
|
||||
@secure(checks.authenticated)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from oslo.config import cfg
|
||||
from pecan import response
|
||||
from pecan import rest
|
||||
from pecan.secure import secure
|
||||
|
|
@ -24,6 +25,8 @@ from storyboard.api.auth import authorization_checks as checks
|
|||
from storyboard.api.v1 import base
|
||||
from storyboard.db import api as dbapi
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class Task(base.APIBase):
|
||||
"""A Task represents an actionable work item, targeting a specific Project
|
||||
|
|
@ -70,13 +73,37 @@ class TasksController(rest.RestController):
|
|||
status_code=404)
|
||||
|
||||
@secure(checks.guest)
|
||||
@wsme_pecan.wsexpose([Task], int)
|
||||
def get_all(self, story_id=None):
|
||||
@wsme_pecan.wsexpose([Task], int, int, int)
|
||||
def get_all(self, story_id=None, marker=None, limit=None):
|
||||
"""Retrieve definitions of all of the tasks.
|
||||
|
||||
:param story_id: filter tasks by story ID.
|
||||
:param offset The offset within the result set to fetch.
|
||||
:param limit The number of tasks to retrieve.
|
||||
"""
|
||||
tasks = dbapi.task_get_all(story_id=story_id)
|
||||
|
||||
# Boundary check on limit.
|
||||
if limit is None:
|
||||
limit = CONF.page_size_default
|
||||
limit = min(CONF.page_size_maximum, max(1, limit))
|
||||
|
||||
# Resolve the marker record.
|
||||
marker_task = dbapi.task_get(marker)
|
||||
|
||||
if marker_task is None or marker_task.story_id != story_id:
|
||||
marker_task = None
|
||||
|
||||
tasks = dbapi.task_get_all(marker=marker_task,
|
||||
limit=limit,
|
||||
story_id=story_id)
|
||||
task_count = dbapi.task_get_count(story_id=story_id)
|
||||
|
||||
# Apply the query response headers.
|
||||
response.headers['X-Limit'] = str(limit)
|
||||
response.headers['X-Total'] = str(task_count)
|
||||
if marker_task:
|
||||
response.headers['X-Marker'] = str(marker_task.id)
|
||||
|
||||
return [Task.from_db_model(s) for s in tasks]
|
||||
|
||||
@secure(checks.authenticated)
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@
|
|||
# limitations under the License.
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from oslo.config import cfg
|
||||
from pecan import request
|
||||
from pecan import response
|
||||
from pecan import rest
|
||||
|
|
@ -26,6 +28,8 @@ from storyboard.api.auth import authorization_checks as checks
|
|||
from storyboard.api.v1 import base
|
||||
from storyboard.db import api as dbapi
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class User(base.APIBase):
|
||||
"""Represents a user."""
|
||||
|
|
@ -66,12 +70,33 @@ class UsersController(rest.RestController):
|
|||
"""Manages users."""
|
||||
|
||||
@secure(checks.guest)
|
||||
@wsme_pecan.wsexpose([User])
|
||||
def get(self):
|
||||
"""Retrieve definitions of all of the users."""
|
||||
@wsme_pecan.wsexpose([User], int, int)
|
||||
def get(self, marker=None, limit=None):
|
||||
"""Retrieve definitions of all of the users.
|
||||
|
||||
users = dbapi.user_get_all()
|
||||
return [User.from_db_model(user) for user in users]
|
||||
:param marker The marker at which the page set should begin. At the
|
||||
moment, this is the unique resource id..
|
||||
:param limit The number of users to retrieve.
|
||||
"""
|
||||
|
||||
# Boundary check on limit.
|
||||
if limit is None:
|
||||
limit = CONF.page_size_default
|
||||
limit = min(CONF.page_size_maximum, max(1, limit))
|
||||
|
||||
# Resolve the marker record.
|
||||
marker_user = dbapi.user_get(marker)
|
||||
|
||||
users = dbapi.user_get_all(marker=marker_user, limit=limit)
|
||||
user_count = dbapi.user_get_count()
|
||||
|
||||
# Apply the query response headers.
|
||||
response.headers['X-Limit'] = str(limit)
|
||||
response.headers['X-Total'] = str(user_count)
|
||||
if marker_user:
|
||||
response.headers['X-Marker'] = str(marker_user.id)
|
||||
|
||||
return [User.from_db_model(u) for u in users]
|
||||
|
||||
@secure(checks.guest)
|
||||
@wsme_pecan.wsexpose(User, int)
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
# limitations under the License.
|
||||
|
||||
import copy
|
||||
|
||||
from oslo.config import cfg
|
||||
import six
|
||||
|
||||
|
|
@ -21,6 +22,7 @@ from storyboard.common import exception as exc
|
|||
from storyboard.db import models
|
||||
from storyboard.openstack.common.db import exception as db_exc
|
||||
from storyboard.openstack.common.db.sqlalchemy import session as db_session
|
||||
from storyboard.openstack.common.db.sqlalchemy.utils import paginate_query
|
||||
from storyboard.openstack.common import log
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
|
@ -113,12 +115,24 @@ def _entity_get(kls, entity_id, filter_non_public=False):
|
|||
return entity
|
||||
|
||||
|
||||
def _entity_get_all(kls, filter_non_public=False, **kwargs):
|
||||
def _entity_get_all(kls, filter_non_public=False, marker=None, limit=None,
|
||||
**kwargs):
|
||||
|
||||
# Sanity check on input parameters
|
||||
kwargs = dict((k, v) for k, v in kwargs.iteritems() if v)
|
||||
|
||||
query = model_query(kls)
|
||||
entities = query.filter_by(**kwargs).all()
|
||||
if filter_non_public:
|
||||
# Construct the query
|
||||
query = model_query(kls).filter_by(**kwargs)
|
||||
query = paginate_query(query=query,
|
||||
model=kls,
|
||||
limit=limit,
|
||||
sort_keys=['id'],
|
||||
marker=marker,
|
||||
sort_dir='asc')
|
||||
|
||||
# Execute the query
|
||||
entities = query.all()
|
||||
if len(entities) > 0 and filter_non_public:
|
||||
sample_entity = entities[0] if len(entities) > 0 else None
|
||||
public_fields = getattr(sample_entity, "_public_fields", [])
|
||||
|
||||
|
|
@ -128,6 +142,14 @@ def _entity_get_all(kls, filter_non_public=False, **kwargs):
|
|||
return entities
|
||||
|
||||
|
||||
def _entity_get_count(kls, **kwargs):
|
||||
kwargs = dict((k, v) for k, v in kwargs.iteritems() if v)
|
||||
|
||||
count = model_query(kls).filter_by(**kwargs).count()
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def _filter_non_public_fields(entity, public_list=list()):
|
||||
ent_copy = copy.copy(entity)
|
||||
for attr_name, val in six.iteritems(entity.__dict__):
|
||||
|
|
@ -150,7 +172,7 @@ def _entity_create(kls, values):
|
|||
session.add(entity)
|
||||
except db_exc.DBDuplicateEntry:
|
||||
raise exc.DuplicateEntry("Duplicate entry for : %s"
|
||||
% (kls.__name__))
|
||||
% kls.__name__)
|
||||
|
||||
return entity
|
||||
|
||||
|
|
@ -179,8 +201,15 @@ def user_get(user_id, filter_non_public=False):
|
|||
return entity
|
||||
|
||||
|
||||
def user_get_all(filter_non_public=False):
|
||||
return _entity_get_all(models.User, filter_non_public=filter_non_public)
|
||||
def user_get_all(marker=None, limit=None, filter_non_public=False):
|
||||
return _entity_get_all(models.User,
|
||||
marker=marker,
|
||||
limit=limit,
|
||||
filter_non_public=filter_non_public)
|
||||
|
||||
|
||||
def user_get_count():
|
||||
return _entity_get_count(models.User)
|
||||
|
||||
|
||||
def user_get_by_openid(openid):
|
||||
|
|
@ -213,8 +242,16 @@ def project_get(project_id):
|
|||
return _entity_get(models.Project, project_id)
|
||||
|
||||
|
||||
def project_get_all(**kwargs):
|
||||
return _entity_get_all(models.Project, is_active=True)
|
||||
def project_get_all(marker=None, limit=None, **kwargs):
|
||||
return _entity_get_all(models.Project,
|
||||
is_active=True,
|
||||
marker=marker,
|
||||
limit=limit,
|
||||
**kwargs)
|
||||
|
||||
|
||||
def project_get_count(**kwargs):
|
||||
return _entity_get_count(models.Project, is_active=True, **kwargs)
|
||||
|
||||
|
||||
def project_create(values):
|
||||
|
|
@ -239,19 +276,58 @@ def story_get(story_id):
|
|||
return _entity_get(models.Story, story_id)
|
||||
|
||||
|
||||
def story_get_all(project_id=None):
|
||||
def story_get_all(marker=None, limit=None, project_id=None):
|
||||
if project_id:
|
||||
return story_get_all_in_project(project_id)
|
||||
return _story_get_all_in_project(marker=marker,
|
||||
limit=limit,
|
||||
project_id=project_id)
|
||||
else:
|
||||
return _entity_get_all(models.Story, is_active=True)
|
||||
return _entity_get_all(models.Story, is_active=True,
|
||||
marker=marker, limit=limit)
|
||||
|
||||
|
||||
def story_get_all_in_project(project_id):
|
||||
def story_get_count(project_id=None):
|
||||
if project_id:
|
||||
return _story_get_count_in_project(project_id)
|
||||
else:
|
||||
return _entity_get_count(models.Story, is_active=True)
|
||||
|
||||
|
||||
def _story_get_all_in_project(project_id, marker=None, limit=None):
|
||||
session = get_session()
|
||||
|
||||
query = model_query(models.Story, session).join(models.Task)
|
||||
return query.filter(models.Task.project_id == project_id,
|
||||
models.Story.is_active)
|
||||
sub_query = model_query(models.Task.story_id, session) \
|
||||
.filter_by(project_id=project_id, is_active=True) \
|
||||
.distinct(True) \
|
||||
.subquery()
|
||||
|
||||
query = model_query(models.Story, session) \
|
||||
.filter_by(is_active=True) \
|
||||
.join(sub_query, models.Story.tasks)
|
||||
|
||||
query = paginate_query(query=query,
|
||||
model=models.Story,
|
||||
limit=limit,
|
||||
sort_keys=['id'],
|
||||
marker=marker,
|
||||
sort_dir='asc')
|
||||
|
||||
return query.all()
|
||||
|
||||
|
||||
def _story_get_count_in_project(project_id):
|
||||
session = get_session()
|
||||
|
||||
sub_query = model_query(models.Task.story_id, session) \
|
||||
.filter_by(project_id=project_id, is_active=True) \
|
||||
.distinct(True) \
|
||||
.subquery()
|
||||
|
||||
query = model_query(models.Story, session) \
|
||||
.filter_by(is_active=True) \
|
||||
.join(sub_query, models.Story.tasks)
|
||||
|
||||
return query.count()
|
||||
|
||||
|
||||
def story_create(values):
|
||||
|
|
@ -302,8 +378,16 @@ def task_get(task_id):
|
|||
return _entity_get(models.Task, task_id)
|
||||
|
||||
|
||||
def task_get_all(story_id=None):
|
||||
return _entity_get_all(models.Task, story_id=story_id, is_active=True)
|
||||
def task_get_all(marker=None, limit=None, story_id=None):
|
||||
return _entity_get_all(models.Task,
|
||||
marker=marker,
|
||||
limit=limit,
|
||||
story_id=story_id,
|
||||
is_active=True)
|
||||
|
||||
|
||||
def task_get_count(story_id=None):
|
||||
return _entity_get_count(models.Task, story_id=story_id, is_active=True)
|
||||
|
||||
|
||||
def task_create(values):
|
||||
|
|
|
|||
Loading…
Reference in New Issue