implement paging

Change-Id: Id044e0943ed163cf33631b035c3e09f7ace4f668

README.rst

@@ -201,6 +201,7 @@ Additions to the legacy nova's EC2 API include:
 1. VPC API
 2. Filtering
 3. Tags
+4. Paging
 
 Legacy OpenStack release notice
 ===============================

ec2api/api/cloud.py

@@ -525,9 +525,7 @@ class CloudController(object):
             filter (list of filter dict): You can specify filters so that the
                 response includes information for only certain instances.
             max_results (int): The maximum number of items to return.
-                Not used now.
             next_token (str): The token for the next set of items to return.
-                Not used now.
 
         Returns:
             A list of reservations.
@@ -818,9 +816,7 @@ class CloudController(object):
             filter (list of filter dict): You can specify filters so that the
                 response includes information for only certain volumes.
             max_results (int): The maximum number of items to return.
-                Not used now.
             next_token (str): The token for the next set of items to return.
-                Not used now.
 
         Returns:
             A list of volumes.
@@ -852,9 +848,11 @@ class CloudController(object):
         """
 
     @module_and_param_types(snapshot, 'snap_ids', 'strs',
-                            'strs', 'filter')
+                            'strs', 'filter',
+                            'int', 'str')
     def describe_snapshots(self, context, snapshot_id=None, owner=None,
-                           restorable_by=None, filter=None):
+                           restorable_by=None, filter=None,
+                           max_results=None, next_token=None):
         """Describes one or more of the snapshots available to you.
 
         Args:
@@ -868,6 +866,8 @@ class CloudController(object):
                 Not used now.
             filter (list of filter dict): You can specify filters so that the
                 response includes information for only certain snapshots.
+            max_results (int): The maximum number of items to return.
+            next_token (str): The token for the next set of items to return.
 
         Returns:
             A list of snapshots.
@@ -1104,9 +1104,7 @@ class CloudController(object):
             filter (list of filter dict): You can specify filters so that the
                 response includes information for only certain tags.
             max_results (int): The maximum number of items to return.
-                Not used now.
             next_token (str): The token for the next set of items to return.
-                Not used now.
 
         Returns:
             A list of tags.

ec2api/api/common.py

@@ -12,9 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
 import collections
 import fnmatch
 import inspect
+import operator
 
 from oslo_config import cfg
 from oslo_log import log as logging
@@ -23,7 +25,7 @@ from ec2api.api import ec2utils
 from ec2api.api import validator
 from ec2api.db import api as db_api
 from ec2api import exception
-from ec2api.i18n import _LW
+from ec2api.i18n import _, _LW
 
 
 ec2_opts = [
@@ -260,6 +262,7 @@ class UniversalDescriber(object):
     """Abstract Describer class for various Describe implementations."""
 
     KIND = ''
+    SORT_KEY = ''
     FILTER_MAP = {}
 
     def format(self, item=None, os_item=None):
@@ -330,7 +333,34 @@ class UniversalDescriber(object):
             values = [value] if value is not None else []
         return values
 
-    def describe(self, context, ids=None, names=None, filter=None):
+    def get_paged(self, formatted_items, max_results, next_token):
+        self.next_token = None
+        if not max_results and not next_token:
+            return formatted_items
+
+        if max_results and max_results > 1000:
+            max_results = 1000
+        formatted_items = sorted(formatted_items,
+                                 key=operator.itemgetter(self.SORT_KEY))
+
+        next_item = 0
+        if next_token:
+            next_item = int(base64.b64decode(next_token))
+        if next_item:
+            formatted_items = formatted_items[next_item:]
+        if max_results and max_results < len(formatted_items):
+            self.next_token = base64.b64encode(str(next_item + max_results))
+            formatted_items = formatted_items[:max_results]
+
+        return formatted_items
+
+    def describe(self, context, ids=None, names=None, filter=None,
+                 max_results=None, next_token=None):
+        if max_results and max_results < 5:
+            msg = (_('Value ( %s ) for parameter maxResults is invalid. '
+                     'Expecting a value greater than 5.') % max_results)
+            raise exception.InvalidParameterValue(msg)
+
         self.context = context
         self.selective_describe = ids is not None or names is not None
         self.ids = set(ids or [])
@@ -371,7 +401,8 @@ class UniversalDescriber(object):
         if self.ids or self.names:
             params = {'id': next(iter(self.ids or self.names))}
             raise ec2utils.NOT_FOUND_EXCEPTION_MAP[self.KIND](**params)
-        return formatted_items
+
+        return self.get_paged(formatted_items, max_results, next_token)
 
 
 class TaggableItemsDescriber(UniversalDescriber):
@@ -406,7 +437,8 @@ class TaggableItemsDescriber(UniversalDescriber):
             # errors in AWS docs)
             formatted_item['tagSet'] = formatted_tags
 
-    def describe(self, context, ids=None, names=None, filter=None):
+    def describe(self, context, ids=None, names=None, filter=None,
+                 max_results=None, next_token=None):
         if filter:
             for f in filter:
                 if f['name'].startswith('tag:'):
@@ -416,7 +448,8 @@ class TaggableItemsDescriber(UniversalDescriber):
                     f['value'] = [{'key': tag_key,
                                    'value': tag_values}]
         return super(TaggableItemsDescriber, self).describe(
-            context, ids, names, filter)
+            context, ids=ids, names=names, filter=filter,
+            max_results=max_results, next_token=next_token)
 
     def is_filtering_value_found(self, filter_value, value):
         if isinstance(filter_value, dict):
@@ -438,7 +471,13 @@ class TaggableItemsDescriber(UniversalDescriber):
 class NonOpenstackItemsDescriber(UniversalDescriber):
     """Describer class for non-Openstack items Describe implementations."""
 
-    def describe(self, context, ids=None, names=None, filter=None):
+    def describe(self, context, ids=None, names=None, filter=None,
+                 max_results=None, next_token=None):
+        if max_results and max_results < 5:
+            msg = (_('Value ( %s ) for parameter maxResults is invalid. '
+                     'Expecting a value greater than 5.') % max_results)
+            raise exception.InvalidParameterValue(msg)
+
         self.context = context
         self.ids = ids
         self.items = self.get_db_items()
@@ -450,4 +489,5 @@ class NonOpenstackItemsDescriber(UniversalDescriber):
             if (formatted_item and
                     not self.filtered_out(formatted_item, filter)):
                 formatted_items.append(formatted_item)
-        return formatted_items
+
+        return self.get_paged(formatted_items, max_results, next_token)

ec2api/api/instance.py

@@ -212,6 +212,7 @@ def terminate_instances(context, instance_id):
 class InstanceDescriber(common.TaggableItemsDescriber):
 
     KIND = 'i'
+    SORT_KEY = 'instanceId'
     FILTER_MAP = {
         'availability-zone': ('placement', 'availabilityZone'),
         'block-device-mapping.delete-on-termination': [
@@ -382,7 +383,8 @@ class ReservationDescriber(common.NonOpenstackItemsDescriber):
     def get_db_items(self):
         return self.reservations
 
-    def describe(self, context, ids=None, names=None, filter=None):
+    def describe(self, context, ids=None, names=None, filter=None,
+                 max_results=None, next_token=None):
         reservation_filters = []
         instance_filters = []
         for f in filter or []:
@@ -400,7 +402,8 @@ class ReservationDescriber(common.NonOpenstackItemsDescriber):
         try:
             instance_describer = InstanceDescriber()
             formatted_instances = instance_describer.describe(
-                    context, ids=ids, filter=instance_filters)
+                context, ids=ids, filter=instance_filters,
+                max_results=max_results, next_token=next_token)
         except exception.InvalidInstanceIDNotFound:
             _remove_instances(context, instance_describer.obsolete_instances)
             raise
@@ -413,15 +416,28 @@ class ReservationDescriber(common.NonOpenstackItemsDescriber):
         self.suitable_instances = set(i['instanceId']
                                       for i in formatted_instances)
 
-        return super(ReservationDescriber, self).describe(
-                context, filter=reservation_filters)
+        result = super(ReservationDescriber, self).describe(
+            context, filter=reservation_filters)
+        self.next_token = instance_describer.next_token
+        return result
 
 
 def describe_instances(context, instance_id=None, filter=None,
                        max_results=None, next_token=None):
-    formatted_reservations = ReservationDescriber().describe(
-            context, ids=instance_id, filter=filter)
-    return {'reservationSet': formatted_reservations}
+    if instance_id and max_results:
+        msg = _('The parameter instancesSet cannot be used with the parameter '
+                'maxResults')
+        raise exception.InvalidParameterCombination(msg)
+
+    reservation_describer = ReservationDescriber()
+    formatted_reservations = reservation_describer.describe(
+            context, ids=instance_id, filter=filter,
+            max_results=max_results, next_token=next_token)
+
+    result = {'reservationSet': formatted_reservations}
+    if reservation_describer.next_token:
+        result['nextToken'] = reservation_describer.next_token
+    return result
 
 
 def reboot_instances(context, instance_id):

ec2api/api/snapshot.py

@@ -69,6 +69,7 @@ def delete_snapshot(context, snapshot_id):
 class SnapshotDescriber(common.TaggableItemsDescriber):
 
     KIND = 'snap'
+    SORT_KEY = 'snapshotId'
     FILTER_MAP = {'description': 'description',
                   'owner-id': 'ownerId',
                   'progress': 'progress',
@@ -95,10 +96,21 @@ class SnapshotDescriber(common.TaggableItemsDescriber):
 
 
 def describe_snapshots(context, snapshot_id=None, owner=None,
-                       restorable_by=None, filter=None):
-    formatted_snapshots = SnapshotDescriber().describe(
-        context, ids=snapshot_id, filter=filter)
-    return {'snapshotSet': formatted_snapshots}
+                       restorable_by=None, filter=None,
+                       max_results=None, next_token=None):
+    if snapshot_id and max_results:
+        msg = _('The parameter snapshotSet cannot be used with the parameter '
+                'maxResults')
+        raise exception.InvalidParameterCombination(msg)
+
+    snapshot_describer = SnapshotDescriber()
+    formatted_snapshots = snapshot_describer.describe(
+        context, ids=snapshot_id, filter=filter,
+        max_results=max_results, next_token=next_token)
+    result = {'snapshotSet': formatted_snapshots}
+    if snapshot_describer.next_token:
+        result['nextToken'] = snapshot_describer.next_token
+    return result
 
 
 def _format_snapshot(context, snapshot, os_snapshot, volumes={},

ec2api/api/tag.py

@@ -88,10 +88,13 @@ def delete_tags(context, resource_id, tag=None):
 
 class TagDescriber(common.NonOpenstackItemsDescriber):
 
+    SORT_KEY = 'key'
     FILTER_MAP = {'key': 'key',
+                  'tag-key': 'key',
                   'resource-id': 'resourceId',
                   'resource-type': 'resourceType',
-                  'value': 'value'}
+                  'value': 'value',
+                  'tag-value': 'value'}
 
     def get_db_items(self):
         return db_api.get_tags(self.context)
@@ -101,8 +104,13 @@ class TagDescriber(common.NonOpenstackItemsDescriber):
 
 
 def describe_tags(context, filter=None, max_results=None, next_token=None):
-    formatted_tags = TagDescriber().describe(context, filter=filter)
-    return {'tagSet': formatted_tags}
+    tag_describer = TagDescriber()
+    formatted_tags = tag_describer.describe(
+        context, filter=filter, max_results=max_results, next_token=next_token)
+    result = {'tagSet': formatted_tags}
+    if tag_describer.next_token:
+        result['nextToken'] = tag_describer.next_token
+    return result
 
 
 def _format_tag(tag):

ec2api/api/volume.py

@@ -109,6 +109,7 @@ def delete_volume(context, volume_id):
 class VolumeDescriber(common.TaggableItemsDescriber):
 
     KIND = 'vol'
+    SORT_KEY = 'volumeId'
     FILTER_MAP = {'availability-zone': 'availabilityZone',
                   'create-time': 'createTime',
                   'encrypted': 'encrypted',
@@ -141,9 +142,19 @@ class VolumeDescriber(common.TaggableItemsDescriber):
 
 def describe_volumes(context, volume_id=None, filter=None,
                      max_results=None, next_token=None):
-    formatted_volumes = VolumeDescriber().describe(
-        context, ids=volume_id, filter=filter)
-    return {'volumeSet': formatted_volumes}
+    if volume_id and max_results:
+        msg = _('The parameter volumeSet cannot be used with the parameter '
+                'maxResults')
+        raise exception.InvalidParameterCombination(msg)
+
+    volume_describer = VolumeDescriber()
+    formatted_volumes = volume_describer.describe(
+        context, ids=volume_id, filter=filter,
+        max_results=max_results, next_token=next_token)
+    result = {'volumeSet': formatted_volumes}
+    if volume_describer.next_token:
+        result['nextToken'] = volume_describer.next_token
+    return result
 
 
 def _format_volume(context, volume, os_volume, instances={},

ec2api/tests/functional/api/test_security_groups.py

@@ -103,8 +103,8 @@ class SecurityGroupTest(base.EC2TestCase):
         name = data_utils.rand_name('sgName')
         desc = data_utils.rand_name('sgDesc')
         data = self.client.create_security_group(VpcId=self.vpc_id,
-                                                     GroupName=name,
-                                                     Description=desc)
+                                                 GroupName=name,
+                                                 Description=desc)
         group_id = data['GroupId']
         res_clean = self.addResourceCleanUp(self.client.delete_security_group,
                                             GroupId=group_id)

ec2api/tests/functional/base.py

@@ -503,10 +503,14 @@ class EC2TestCase(base.BaseTestCase):
     def assertRaises(self, error_code, fn, rollback_fn=None, **kwargs):
         try:
             fn_data = fn(**kwargs)
-            try:
-                rollback_fn(fn_data)
-            except Exception:
-                LOG.exception()
+            if rollback_fn:
+                try:
+                    rollback_fn(fn_data)
+                except Exception:
+                    LOG.exception('Rollback failed')
+            msg = ("%s hasn't returned exception for params %s"
+                   % (str(fn.__name__), str(kwargs)))
+            raise self.failureException(msg)
         except botocore.exceptions.ClientError as e:
             self.assertEqual(error_code, e.response['Error']['Code'])
 

ec2api/tests/functional/scenario/test_paging.py

@@ -0,0 +1,331 @@
+# Copyright 2015 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_log import log
+from tempest_lib.common.utils import data_utils
+
+from ec2api.tests.functional import base
+from ec2api.tests.functional import config
+from ec2api.tests.functional.scenario import base as scenario_base
+
+CONF = config.CONF
+LOG = log.getLogger(__name__)
+
+
+class TagsPagingTest(scenario_base.BaseScenarioTest):
+
+    # NOTE(andrey-mp): limit for tags for one resource in amazon
+    TAGS_COUNT = 10
+
+    def _create_volume_and_tags(self):
+        data = self.client.create_volume(
+            Size=1, AvailabilityZone=CONF.aws.aws_zone)
+        volume_id = data['VolumeId']
+        self.addResourceCleanUp(self.client.delete_volume, VolumeId=volume_id)
+        self.get_volume_waiter().wait_available(volume_id)
+
+        keys = list()
+        for dummy in xrange(0, self.TAGS_COUNT):
+            key = data_utils.rand_name('key')
+            value = 'aaa' if dummy < 6 else 'bbb'
+            data = self.client.create_tags(Resources=[volume_id],
+                Tags=[{'Key': key, 'Value': value}])
+            keys.append(key)
+
+        return volume_id, keys
+
+    def test_simple_tags_paging_with_many_results(self):
+        volume_id = self._create_volume_and_tags()[0]
+
+        data = self.client.describe_tags(MaxResults=500,
+            Filters=[{'Name': 'resource-id', 'Values': [volume_id]}])
+        self.assertNotIn('NextToken', data)
+        self.assertNotEmpty(data['Tags'])
+        self.assertEqual(self.TAGS_COUNT, len(data['Tags']))
+
+    def test_simple_tags_paging_with_min_results(self):
+        volume_id = self._create_volume_and_tags()[0]
+
+        data = self.client.describe_tags(
+            MaxResults=5,
+            Filters=[{'Name': 'resource-id', 'Values': [volume_id]},
+                     {'Name': 'tag-value', 'Values': ['aaa']}])
+        self.assertIn('NextToken', data)
+        self.assertNotEmpty(data['Tags'])
+
+    def test_tags_paging_second_page_only_with_token(self):
+        volume_id = self._create_volume_and_tags()[0]
+
+        data = self.client.describe_tags(
+            MaxResults=5,
+            Filters=[{'Name': 'resource-id', 'Values': [volume_id]},
+                     {'Name': 'tag-value', 'Values': ['aaa']}])
+        self.assertIn('NextToken', data)
+        self.assertNotEmpty(data['Tags'])
+        data = self.client.describe_tags(
+            NextToken=data['NextToken'],
+            Filters=[{'Name': 'resource-id', 'Values': [volume_id]},
+                     {'Name': 'tag-value', 'Values': ['aaa']}])
+        self.assertNotIn('NextToken', data)
+        self.assertNotEmpty(data['Tags'])
+
+    def test_tags_paging_with_const_filter(self):
+        volume_id = self._create_volume_and_tags()[0]
+
+        data = self.client.describe_tags(
+            MaxResults=5,
+            Filters=[{'Name': 'resource-id', 'Values': [volume_id]},
+                     {'Name': 'tag-value', 'Values': ['aaa']}])
+        self.assertIn('NextToken', data)
+        self.assertNotEmpty(data['Tags'])
+        data = self.client.describe_tags(
+            MaxResults=5, NextToken=data['NextToken'],
+            Filters=[{'Name': 'resource-id', 'Values': [volume_id]},
+                     {'Name': 'tag-value', 'Values': ['aaa']}])
+        self.assertNotIn('NextToken', data)
+        self.assertNotEmpty(data['Tags'])
+
+    def test_tags_paging_with_differenet_filters(self):
+        volume_id = self._create_volume_and_tags()[0]
+
+        data = self.client.describe_tags(
+            MaxResults=5,
+            Filters=[{'Name': 'resource-id', 'Values': [volume_id]},
+                     {'Name': 'tag-value', 'Values': ['aaa']}])
+        self.assertIn('NextToken', data)
+        self.assertNotEmpty(data['Tags'])
+        data = self.client.describe_tags(
+            MaxResults=5, NextToken=data['NextToken'],
+            Filters=[{'Name': 'resource-id', 'Values': [volume_id]}])
+        self.assertNotEmpty(data['Tags'])
+        self.assertIn('bbb', [k.get('Value') for k in data['Tags']])
+
+    def test_tags_paging_with_tags_deletion(self):
+        volume_id, keys = self._create_volume_and_tags()
+
+        data = self.client.describe_tags(MaxResults=5,
+            Filters=[{'Name': 'resource-id', 'Values': [volume_id]}])
+        self.assertIn('NextToken', data)
+        self.assertNotEmpty(data['Tags'])
+        for key in keys:
+            self.client.delete_tags(Resources=[volume_id], Tags=[{'Key': key}])
+        data = self.client.describe_tags(
+            MaxResults=5, NextToken=data['NextToken'],
+            Filters=[{'Name': 'resource-id', 'Values': [volume_id]}])
+        self.assertNotIn('NextToken', data)
+        self.assertEmpty(data['Tags'])
+
+    def test_invalid_max_results(self):
+        self.assertRaises('InvalidParameterValue',
+            self.client.describe_tags, MaxResults=4)
+
+        # NOTE(andrey-mp): value more than 1000 in not invalid
+        # but amazon returns 1000 elements
+        self.client.describe_tags(MaxResults=1100)
+
+
+class VolumesPagingTest(scenario_base.BaseScenarioTest):
+
+    VOLUMES_COUNT = 6
+
+    @classmethod
+    @base.safe_setup
+    def setUpClass(cls):
+        super(VolumesPagingTest, cls).setUpClass()
+        zone = CONF.aws.aws_zone
+        cls.ids = list()
+        for dummy in xrange(0, cls.VOLUMES_COUNT):
+            data = cls.client.create_volume(Size=1, AvailabilityZone=zone)
+            volume_id = data['VolumeId']
+            cls.addResourceCleanUpStatic(cls.client.delete_volume,
+                                         VolumeId=volume_id)
+            cls.ids.append(volume_id)
+        for volume_id in cls.ids:
+            cls.get_volume_waiter().wait_available(volume_id)
+
+    def test_simple_volumes_paging_with_many_results(self):
+        data = self.client.describe_volumes(MaxResults=500)
+        self.assertNotIn('NextToken', data)
+        self.assertNotEmpty(data['Volumes'])
+        self.assertLessEqual(self.VOLUMES_COUNT, len(data['Volumes']))
+
+    def test_simple_volumes_paging_with_min_results(self):
+        data = self.client.describe_volumes(MaxResults=5)
+        self.assertIn('NextToken', data)
+        self.assertNotEmpty(data['Volumes'])
+
+    def test_volumes_paging_second_page(self):
+        data = self.client.describe_volumes(MaxResults=5)
+        self.assertIn('NextToken', data)
+        self.assertNotEmpty(data['Volumes'])
+        data = self.client.describe_volumes(
+            MaxResults=5, NextToken=data['NextToken'])
+        self.assertNotIn('NextToken', data)
+        self.assertNotEmpty(data['Volumes'])
+
+    def test_invalid_paging(self):
+        self.assertRaises('InvalidParameterValue',
+            self.client.describe_volumes, MaxResults=4)
+
+        self.assertRaises('InvalidParameterCombination',
+            self.client.describe_volumes,
+            MaxResults=5, VolumeIds=[self.ids[0]])
+
+    def test_volumes_paging_with_filters(self):
+        data = self.client.describe_volumes(MaxResults=5,
+            Filters=[{'Name': 'volume-id', 'Values': [self.ids[0]]}])
+        self.assertNotEmpty(data['Volumes'])
+        if 'NextToken' in data:
+            # Amazon way
+            data = self.client.describe_volumes(
+                MaxResults=5, NextToken=data['NextToken'],
+                Filters=[{'Name': 'volume-id', 'Values': [self.ids[0]]}])
+            self.assertNotIn('NextToken', data)
+            self.assertEmpty(data['Volumes'])
+
+        data = self.client.describe_volumes(MaxResults=5,
+            Filters=[{'Name': 'volume-id', 'Values': ['vol-*']}])
+        self.assertIn('NextToken', data)
+        self.assertNotEmpty(data['Volumes'])
+        data = self.client.describe_volumes(
+            MaxResults=5, NextToken=data['NextToken'],
+            Filters=[{'Name': 'volume-id', 'Values': ['vol-*']}])
+        self.assertNotEmpty(data['Volumes'])
+
+
+class SnapshotPagingTest(scenario_base.BaseScenarioTest):
+
+    SNAPSHOTS_COUNT = 6
+
+    @classmethod
+    @base.safe_setup
+    def setUpClass(cls):
+        super(SnapshotPagingTest, cls).setUpClass()
+        zone = CONF.aws.aws_zone
+
+        data = cls.client.create_volume(Size=1, AvailabilityZone=zone)
+        volume_id = data['VolumeId']
+        cls.addResourceCleanUpStatic(cls.client.delete_volume,
+                                     VolumeId=volume_id)
+        cls.get_volume_waiter().wait_available(volume_id)
+
+        cls.ids = list()
+        for dummy in xrange(0, cls.SNAPSHOTS_COUNT):
+            data = cls.client.create_snapshot(VolumeId=volume_id)
+            snapshot_id = data['SnapshotId']
+            cls.addResourceCleanUpStatic(cls.client.delete_snapshot,
+                                         SnapshotId=snapshot_id)
+            cls.ids.append(snapshot_id)
+        for snapshot_id in cls.ids:
+            cls.get_snapshot_waiter().wait_available(snapshot_id,
+                                                     final_set=('completed'))
+
+    def test_simple_snapshots_paging_with_many_results(self):
+        data = self.client.describe_snapshots(MaxResults=500)
+        self.assertNotIn('NextToken', data)
+        self.assertNotEmpty(data['Snapshots'])
+        self.assertLessEqual(self.SNAPSHOTS_COUNT, len(data['Snapshots']))
+
+    def test_simple_snapshots_paging_with_min_results(self):
+        data = self.client.describe_snapshots(MaxResults=5)
+        self.assertIn('NextToken', data)
+        self.assertNotEmpty(data['Snapshots'])
+
+    def test_snapshots_paging_second_page(self):
+        data = self.client.describe_snapshots(MaxResults=5)
+        self.assertIn('NextToken', data)
+        self.assertNotEmpty(data['Snapshots'])
+        data = self.client.describe_snapshots(
+            MaxResults=5, NextToken=data['NextToken'])
+        self.assertNotIn('NextToken', data)
+        self.assertNotEmpty(data['Snapshots'])
+
+    def test_invalid_paging(self):
+        self.assertRaises('InvalidParameterValue',
+            self.client.describe_snapshots, MaxResults=4)
+
+        self.assertRaises('InvalidParameterCombination',
+            self.client.describe_snapshots,
+            MaxResults=5, SnapshotIds=[self.ids[0]])
+
+
+class InstancePagingTest(scenario_base.BaseScenarioTest):
+
+    RESERVATIONS_COUNT = 2
+    INSTANCES_IN_RESERVATIONS_COUNT = 3
+
+    @classmethod
+    @base.safe_setup
+    def setUpClass(cls):
+        super(InstancePagingTest, cls).setUpClass()
+        if not CONF.aws.image_id:
+            raise cls.skipException('aws image_id does not provided')
+
+        cls.ids = list()
+        kwargs = {
+            'ImageId': CONF.aws.image_id,
+            'InstanceType': CONF.aws.instance_type,
+            'Placement': {'AvailabilityZone': CONF.aws.aws_zone},
+            'MinCount': cls.INSTANCES_IN_RESERVATIONS_COUNT,
+            'MaxCount': cls.INSTANCES_IN_RESERVATIONS_COUNT
+        }
+        for dummy in xrange(0, cls.RESERVATIONS_COUNT):
+            data = cls.client.run_instances(*[], **kwargs)
+            for instance in data['Instances']:
+                cls.ids.append(instance['InstanceId'])
+
+        cls.addResourceCleanUpStatic(cls.client.terminate_instances,
+                                     InstanceIds=cls.ids)
+        for instance_id in cls.ids:
+            cls.get_instance_waiter().wait_available(instance_id,
+                                                     final_set=('running'))
+
+    def test_simple_instances_paging_with_many_results(self):
+        data = self.client.describe_instances(MaxResults=500)
+        self.assertNotIn('NextToken', data)
+        self.assertNotEmpty(data['Reservations'])
+        self.assertEqual(self.RESERVATIONS_COUNT, len(data['Reservations']))
+        count = self.RESERVATIONS_COUNT * self.INSTANCES_IN_RESERVATIONS_COUNT
+        self.assertEqual(count, self._count_instances(data))
+
+    def test_simple_instances_paging_with_min_results(self):
+        max_results = 5
+        data = self.client.describe_instances(MaxResults=max_results)
+        self.assertIn('NextToken', data)
+        self.assertEqual(max_results, self._count_instances(data))
+
+    def test_instances_paging_second_page(self):
+        max_results = 5
+        data = self.client.describe_instances(MaxResults=max_results)
+        self.assertIn('NextToken', data)
+        self.assertEqual(max_results, self._count_instances(data))
+        data = self.client.describe_instances(
+            MaxResults=max_results, NextToken=data['NextToken'])
+        self.assertNotIn('NextToken', data)
+        self.assertLess(0, self._count_instances(data))
+
+    def test_invalid_paging(self):
+        self.assertRaises('InvalidParameterValue',
+            self.client.describe_instances, MaxResults=4)
+
+        self.assertRaises('InvalidParameterCombination',
+            self.client.describe_instances,
+            MaxResults=5, InstanceIds=[self.ids[0]])
+
+    def _count_instances(self, data):
+        count = 0
+        for reservation in data['Reservations']:
+            count += len(reservation['Instances'])
+        return count

rally-scenarios/plugins/context_plugin_ec2_objects.py

@@ -130,7 +130,7 @@ class EC2Objects(base.Context):
         try:
             data = client.associate_address(*[], **kwargs)
         except Exception:
-            LOG.exception()
+            LOG.exception('')
             if is_vpc:
                 data = client.release_address(AllocationId=alloc_id)
             else:
@@ -183,13 +183,13 @@ class EC2Objects(base.Context):
                     data = client.detach_internet_gateway(
                         VpcId=vpc_id, InternetGatewayId=gw_id)
                 except Exception:
-                    LOG.exception()
+                    LOG.exception('')
                 time.sleep(1)
                 try:
                     data = client.delete_internet_gateway(
                         InternetGatewayId=gw_id)
                 except Exception:
-                    LOG.exception()
+                    LOG.exception('')
                 time.sleep(1)
             ni_ids = network.get("ni_ids")
             if ni_ids:
@@ -198,20 +198,20 @@ class EC2Objects(base.Context):
                         data = client.delete_network_interface(
                             NetworkInterfaceId=ni_id)
                     except Exception:
-                        LOG.exception()
+                        LOG.exception('')
                 time.sleep(1)
             subnet_id = network.get("subnet_id")
             if subnet_id:
                 try:
                     data = client.delete_subnet(SubnetId=subnet_id)
                 except Exception:
-                    LOG.exception()
+                    LOG.exception('')
                 time.sleep(1)
             if vpc_id:
                 try:
                     data = client.delete_vpc(VpcId=vpc_id)
                 except Exception:
-                    LOG.exception()
+                    LOG.exception('')
 
 
 @base.context(name="ec2_networks", order=451)