Support new v2 API image filters

Provide support in Glance API for querying the image list for
created_at and updated_at times using guidance from the API Working
Group recommendations for filtering. Filtering is applied at the DB
layer.

DocImpact
ApiImpact
MitakaPriority
Change-Id: Ie94295bb82779ec17ab773928c71ae4a9ee8fbcc
Implements bp: v2-additional-filtering

glance/common/exception.py

@@ -176,6 +176,10 @@ class InvalidSwiftStoreConfiguration(Invalid):
     message = _("Invalid configuration in glance-swift conf file.")
 
 
+class InvalidFilterOperatorValue(Invalid):
+    message = _("Unable to filter using the specified operator.")
+
+
 class InvalidFilterRangeValue(Invalid):
     message = _("Unable to filter using the specified range.")
 

glance/common/utils.py

@@ -648,3 +648,54 @@ def stash_conf_values():
     }
 
     return conf
+
+
+def split_filter_op(expression):
+    """Split operator from threshold in an expression.
+    Designed for use on a comparative-filtering query field.
+    When no operator is found, default to an equality comparison.
+
+    :param expression: the expression to parse
+
+    :returns a tuple (operator, threshold) parsed from expression
+    """
+    left, sep, right = expression.partition(':')
+    if sep:
+        op = left
+        threshold = right
+    else:
+        op = 'eq'  # default operator
+        threshold = left
+
+    # NOTE stevelle decoding escaped values may be needed later
+    return op, threshold
+
+
+def evaluate_filter_op(value, operator, threshold):
+    """Evaluate a comparison operator.
+    Designed for use on a comparative-filtering query field.
+
+    :param value: evaluated against the operator, as left side of expression
+    :param operator: any supported filter operation
+    :param threshold: to compare value against, as right side of expression
+
+    :raises InvalidFilterOperatorValue if an unknown operator is provided
+
+    :returns boolean result of applied comparison
+
+    """
+    if operator == 'gt':
+        return value > threshold
+    elif operator == 'gte':
+        return value >= threshold
+    elif operator == 'lt':
+        return value < threshold
+    elif operator == 'lte':
+        return value <= threshold
+    elif operator == 'neq':
+        return value != threshold
+    elif operator == 'eq':
+        return value == threshold
+
+    msg = _("Unable to filter on a unknown operator.")
+    raise exception.InvalidFilterOperatorValue(msg)

glance/db/simple/api.py

@@ -298,6 +298,13 @@ def _filter_images(images, filters, context,
                 to_add = image.get(key) >= value
             elif k.endswith('_max'):
                 to_add = image.get(key) <= value
+            elif k in ['created_at', 'updated_at']:
+                attr_value = image.get(key)
+                operator, isotime = utils.split_filter_op(value)
+                parsed_time = timeutils.parse_isotime(isotime)
+                threshold = timeutils.normalize_time(parsed_time)
+                to_add = utils.evaluate_filter_op(attr_value, operator,
+                                                  threshold)
             elif k != 'is_public' and image.get(k) is not None:
                 to_add = image.get(key) == value
             elif k == 'tags':

glance/db/sqlalchemy/api.py

@@ -473,6 +473,14 @@ def _make_conditions_from_filters(filters, is_public=None):
                 image_conditions.append(getattr(models.Image, key) >= v)
             if k.endswith('_max'):
                 image_conditions.append(getattr(models.Image, key) <= v)
+        elif k in ['created_at', 'updated_at']:
+            attr_value = getattr(models.Image, key)
+            operator, isotime = utils.split_filter_op(filters.pop(k))
+            parsed_time = timeutils.parse_isotime(isotime)
+            threshold = timeutils.normalize_time(parsed_time)
+            comparison = utils.evaluate_filter_op(attr_value, operator,
+                                                  threshold)
+            image_conditions.append(comparison)
 
     for (k, value) in filters.items():
         if hasattr(models.Image, k):

glance/db/sqlalchemy/migrate_repo/versions/043_add_image_created_updated_idx.py

@@ -0,0 +1,29 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from sqlalchemy import MetaData, Table, Index
+
+CREATED_AT_INDEX = 'created_at_image_idx'
+UPDATED_AT_INDEX = 'updated_at_image_idx'
+
+
+def upgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+    images = Table('images', meta, autoload=True)
+
+    created_index = Index(CREATED_AT_INDEX, images.c.created_at)
+    created_index.create(migrate_engine)
+    updated_index = Index(UPDATED_AT_INDEX, images.c.updated_at)
+    updated_index.create(migrate_engine)

glance/db/sqlalchemy/models.py

@@ -121,7 +121,9 @@ class Image(BASE, GlanceBase):
     __table_args__ = (Index('checksum_image_idx', 'checksum'),
                       Index('ix_images_is_public', 'is_public'),
                       Index('ix_images_deleted', 'deleted'),
-                      Index('owner_image_idx', 'owner'),)
+                      Index('owner_image_idx', 'owner'),
+                      Index('created_at_image_idx', 'created_at'),
+                      Index('updated_at_image_idx', 'updated_at'))
 
     id = Column(String(36), primary_key=True,
                 default=lambda: str(uuid.uuid4()))

glance/tests/functional/db/base.py

@@ -488,6 +488,22 @@ class DriverTests(object):
                                            filters={'poo': 'bear'})
         self.assertEqual(0, len(images))
 
+    def test_image_get_all_with_filter_comparative_created_at(self):
+        anchor = timeutils.isotime(self.fixtures[0]['created_at'])
+        time_expr = 'lt:' + anchor
+
+        images = self.db_api.image_get_all(self.context,
+                                           filters={'created_at': time_expr})
+        self.assertEqual(0, len(images))
+
+    def test_image_get_all_with_filter_comparative_updated_at(self):
+        anchor = timeutils.isotime(self.fixtures[0]['updated_at'])
+        time_expr = 'lt:' + anchor
+
+        images = self.db_api.image_get_all(self.context,
+                                           filters={'updated_at': time_expr})
+        self.assertEqual(0, len(images))
+
     def test_image_get_all_size_min_max(self):
         images = self.db_api.image_get_all(self.context,
                                            filters={

glance/tests/functional/v2/test_images.py

@@ -22,6 +22,7 @@ import requests
 import six
 # NOTE(jokke): simplified transition to py3, behaves like py2 xrange
 from six.moves import range
+from six.moves import urllib
 
 from glance.tests import functional
 from glance.tests import utils as test_utils
@@ -2452,6 +2453,26 @@ class TestImages(functional.FunctionalTest):
         self.assertEqual('/v2/images', body['first'])
         self.assertNotIn('next', jsonutils.loads(response.text))
 
+        # Image list filters by created_at time
+        url_template = '/v2/images?created_at=lt:%s'
+        path = self._url(url_template % images[0]['created_at'])
+        response = requests.get(path, headers=self._headers())
+        self.assertEqual(200, response.status_code)
+        body = jsonutils.loads(response.text)
+        self.assertEqual(0, len(body['images']))
+        self.assertEqual(url_template % images[0]['created_at'],
+                         urllib.parse.unquote(body['first']))
+
+        # Image list filters by updated_at time
+        url_template = '/v2/images?updated_at=lt:%s'
+        path = self._url(url_template % images[2]['updated_at'])
+        response = requests.get(path, headers=self._headers())
+        self.assertEqual(200, response.status_code)
+        body = jsonutils.loads(response.text)
+        self.assertGreaterEqual(3, len(body['images']))
+        self.assertEqual(url_template % images[2]['updated_at'],
+                         urllib.parse.unquote(body['first']))
+
         # Begin pagination after the first image
         template_url = ('/v2/images?limit=2&sort_dir=asc&sort_key=name'
                         '&marker=%s&type=kernel&ping=pong')

glance/tests/unit/common/test_utils.py

@@ -407,3 +407,78 @@ class TestUtils(test_utils.BaseTestCase):
             self.assertRaises(ValueError,
                               utils.parse_valid_host_port,
                               pair)
+
+
+class SplitFilterOpTestCase(test_utils.BaseTestCase):
+
+    def test_less_than_operator(self):
+        expr = 'lt:bar'
+        returned = utils.split_filter_op(expr)
+        self.assertEqual(('lt', 'bar'), returned)
+
+    def test_less_than_equal_operator(self):
+        expr = 'lte:bar'
+        returned = utils.split_filter_op(expr)
+        self.assertEqual(('lte', 'bar'), returned)
+
+    def test_greater_than_operator(self):
+        expr = 'gt:bar'
+        returned = utils.split_filter_op(expr)
+        self.assertEqual(('gt', 'bar'), returned)
+
+    def test_greater_than_equal_operator(self):
+        expr = 'gte:bar'
+        returned = utils.split_filter_op(expr)
+        self.assertEqual(('gte', 'bar'), returned)
+
+    def test_not_equal_operator(self):
+        expr = 'neq:bar'
+        returned = utils.split_filter_op(expr)
+        self.assertEqual(('neq', 'bar'), returned)
+
+    def test_equal_operator(self):
+        expr = 'eq:bar'
+        returned = utils.split_filter_op(expr)
+        self.assertEqual(('eq', 'bar'), returned)
+
+    def test_default_operator(self):
+        expr = 'bar'
+        returned = utils.split_filter_op(expr)
+        self.assertEqual(('eq', expr), returned)
+
+
+class EvaluateFilterOpTestCase(test_utils.BaseTestCase):
+
+    def test_less_than_operator(self):
+        self.assertTrue(utils.evaluate_filter_op(9, 'lt', 10))
+        self.assertFalse(utils.evaluate_filter_op(10, 'lt', 10))
+        self.assertFalse(utils.evaluate_filter_op(11, 'lt', 10))
+
+    def test_less_than_equal_operator(self):
+        self.assertTrue(utils.evaluate_filter_op(9, 'lte', 10))
+        self.assertTrue(utils.evaluate_filter_op(10, 'lte', 10))
+        self.assertFalse(utils.evaluate_filter_op(11, 'lte', 10))
+
+    def test_greater_than_operator(self):
+        self.assertFalse(utils.evaluate_filter_op(9, 'gt', 10))
+        self.assertFalse(utils.evaluate_filter_op(10, 'gt', 10))
+        self.assertTrue(utils.evaluate_filter_op(11, 'gt', 10))
+
+    def test_greater_than_equal_operator(self):
+        self.assertFalse(utils.evaluate_filter_op(9, 'gte', 10))
+        self.assertTrue(utils.evaluate_filter_op(10, 'gte', 10))
+        self.assertTrue(utils.evaluate_filter_op(11, 'gte', 10))
+
+    def test_not_equal_operator(self):
+        self.assertTrue(utils.evaluate_filter_op(9, 'neq', 10))
+        self.assertFalse(utils.evaluate_filter_op(10, 'neq', 10))
+        self.assertTrue(utils.evaluate_filter_op(11, 'neq', 10))
+
+    def test_equal_operator(self):
+        self.assertFalse(utils.evaluate_filter_op(9, 'eq', 10))
+        self.assertTrue(utils.evaluate_filter_op(10, 'eq', 10))
+        self.assertFalse(utils.evaluate_filter_op(11, 'eq', 10))
+
+    def test_invalid_operator(self):
+        self.assertRaises(exception.InvalidFilterOperatorValue,
+                          utils.evaluate_filter_op, '10', 'bar', '8')