Add a key-word-arg aware cache-key generator

Add in a key-word-arg cache key generator that may be optionally
used. This generator will generate a key that mirrors closely to
the original key generator. The key difference is that it will
flatten all arguments down to just the values ordered alphabetically
based on the argument name. This order will be used to ensure that
regardless of the order that the keys are passed (positional,
key-word, or out-of-order key-word) the cache key ends up being
the same.

This was not made the default keygenerator to avoid reverse
incompatibilities with developers that are relying on stable
cache-key generation (e.g. storing data via the memcache-like
interface to a non-volitile backend).

Fixes: #43
Co-authored-by: Mike Bayer <mike_mp@zzzcomputing.com>
Change-Id: I86c9d5e9c611090d5a84d8a746486a0b6c80039a
Pull-request: https://bitbucket.org/zzzeek/dogpile.cache/pull-requests/46

docs/build/api.rst

@@ -9,8 +9,6 @@ Region
 .. automodule:: dogpile.cache.region
     :members:
 
-.. autofunction:: dogpile.cache.util.function_key_generator
-
 Backend API
 =============
 
@@ -56,6 +54,8 @@ Utilities
 
 .. autofunction:: function_key_generator
 
+.. autofunction:: kwarg_function_key_generator
+
 .. autofunction:: sha1_mangle_key
 
 .. autofunction:: length_conditional_mangler

docs/build/changelog.rst

@@ -4,6 +4,14 @@ Changelog
 .. changelog::
     :version: 0.6.2
 
+    .. change::
+      :tags: feature
+      :tickets: 43
+
+       Added a new cache key generator :func:`.kwarg_function_key_generator`,
+       which takes keyword arguments as well as positional arguments into
+       account when forming the cache key.
+
 .. changelog::
     :version: 0.6.1
     :released: Mon Jun 6 2016

dogpile/cache/region.py

@@ -79,6 +79,13 @@ class CacheRegion(object):
         def my_function(a, b, **kw):
             return my_data()
 
+     .. seealso::
+
+        :func:`.function_key_generator` - default key generator
+
+        :func:`.kwarg_function_key_generator` - optional gen that also
+        uses keyword arguments
+
     :param function_multi_key_generator: Optional.
      Similar to ``function_key_generator`` parameter, but it's used in
      :meth:`.CacheRegion.cache_multi_on_arguments`. Generated function

dogpile/cache/util.py

@@ -11,8 +11,14 @@ def function_key_generator(namespace, fn, to_str=compat.string_type):
     This is used by :meth:`.CacheRegion.cache_on_arguments`
     to generate a cache key from a decorated function.
 
-    It can be replaced using the ``function_key_generator``
-    argument passed to :func:`.make_region`.
+    An alternate function may be used by specifying
+    the :paramref:`.CacheRegion.function_key_generator` argument
+    for :class:`.CacheRegion`.
+
+    .. seealso::
+
+        :func:`.kwarg_function_key_generator` - similar function that also
+        takes keyword arguments into account
 
     """
 
@@ -57,6 +63,60 @@ def function_multi_key_generator(namespace, fn, to_str=compat.string_type):
     return generate_keys
 
 
+def kwarg_function_key_generator(namespace, fn, to_str=compat.string_type):
+    """Return a function that generates a string
+    key, based on a given function as well as
+    arguments to the returned function itself.
+
+    For kwargs passed in, we will build a dict of
+    all argname (key) argvalue (values) including
+    default args from the argspec and then
+    alphabetize the list before generating the
+    key.
+
+    .. versionadded:: 0.6.2
+
+    .. seealso::
+
+        :func:`.function_key_generator` - default key generation function
+
+    """
+
+    if namespace is None:
+        namespace = '%s:%s' % (fn.__module__, fn.__name__)
+    else:
+        namespace = '%s:%s|%s' % (fn.__module__, fn.__name__, namespace)
+
+    argspec = inspect.getargspec(fn)
+    default_list = list(argspec.defaults or [])
+    # Reverse the list, as we want to compare the argspec by negative index,
+    # meaning default_list[0] should be args[-1], which works well with
+    # enumerate()
+    default_list.reverse()
+    # use idx*-1 to create the correct right-lookup index.
+    args_with_defaults = dict((argspec.args[(idx*-1)], default)
+                              for idx, default in enumerate(default_list, 1))
+    if argspec.args and argspec.args[0] in ('self', 'cls'):
+        arg_index_start = 1
+    else:
+        arg_index_start = 0
+
+    def generate_key(*args, **kwargs):
+        as_kwargs = dict(
+            [(argspec.args[idx], arg)
+             for idx, arg in enumerate(args[arg_index_start:],
+                                       arg_index_start)])
+        as_kwargs.update(kwargs)
+        for arg, val in args_with_defaults.items():
+            if arg not in as_kwargs:
+                as_kwargs[arg] = val
+
+        argument_values = [as_kwargs[key]
+                           for key in sorted(as_kwargs.keys())]
+        return namespace + '|' + " ".join(map(to_str, argument_values))
+    return generate_key
+
+
 def sha1_mangle_key(key):
     """a SHA1 key mangler."""
 

tests/cache/test_decorator.py

@@ -199,6 +199,75 @@ class KeyGenerationTest(TestCase):
             return fn
         return decorate, canary
 
+    def _kwarg_keygen_decorator(self, namespace=None, **kw):
+        canary = []
+
+        def decorate(fn):
+            canary.append(
+                util.kwarg_function_key_generator(namespace, fn, **kw))
+            return fn
+        return decorate, canary
+
+    def test_default_keygen_kwargs_raises_value_error(self):
+        decorate, canary = self._keygen_decorator()
+
+        @decorate
+        def one(a, b):
+            pass
+
+        gen = canary[0]
+        self.assertRaises(ValueError, gen, 1, b=2)
+
+    def test_kwarg_kegen_keygen_fn(self):
+        decorate, canary = self._kwarg_keygen_decorator()
+
+        @decorate
+        def one(a, b):
+            pass
+        gen = canary[0]
+
+        result_key = "tests.cache.test_decorator:one|1 2"
+
+        eq_(gen(1, 2), result_key)
+        eq_(gen(1, b=2), result_key)
+        eq_(gen(a=1, b=2), result_key)
+        eq_(gen(b=2, a=1), result_key)
+
+    def test_kwarg_kegen_keygen_fn_with_defaults_and_positional(self):
+        decorate, canary = self._kwarg_keygen_decorator()
+
+        @decorate
+        def one(a, b=None):
+            pass
+        gen = canary[0]
+
+        result_key = "tests.cache.test_decorator:one|1 2"
+
+        eq_(gen(1, 2), result_key)
+        eq_(gen(1, b=2), result_key)
+        eq_(gen(a=1, b=2), result_key)
+        eq_(gen(b=2, a=1), result_key)
+        eq_(gen(a=1), "tests.cache.test_decorator:one|1 None")
+
+    def test_kwarg_kegen_keygen_fn_all_defaults(self):
+        decorate, canary = self._kwarg_keygen_decorator()
+
+        @decorate
+        def one(a=True, b=None):
+            pass
+        gen = canary[0]
+
+        result_key = "tests.cache.test_decorator:one|1 2"
+
+        eq_(gen(1, 2), result_key)
+        eq_(gen(1, b=2), result_key)
+        eq_(gen(a=1, b=2), result_key)
+        eq_(gen(b=2, a=1), result_key)
+        eq_(gen(a=1), "tests.cache.test_decorator:one|1 None")
+        eq_(gen(1), "tests.cache.test_decorator:one|1 None")
+        eq_(gen(),  "tests.cache.test_decorator:one|True None")
+        eq_(gen(b=2),  "tests.cache.test_decorator:one|True 2")
+
     def test_keygen_fn(self):
         decorate, canary = self._keygen_decorator()
 
@@ -234,6 +303,17 @@ class KeyGenerationTest(TestCase):
         eq_(gen(1, 2), "tests.cache.test_decorator:one|mynamespace|1 2")
         eq_(gen(None, 5), "tests.cache.test_decorator:one|mynamespace|None 5")
 
+    def test_kwarg_keygen_fn_namespace(self):
+        decorate, canary = self._kwarg_keygen_decorator("mynamespace")
+
+        @decorate
+        def one(a, b):
+            pass
+        gen = canary[0]
+
+        eq_(gen(1, 2), "tests.cache.test_decorator:one|mynamespace|1 2")
+        eq_(gen(None, 5), "tests.cache.test_decorator:one|mynamespace|None 5")
+
     def test_key_isnt_unicode_bydefault(self):
         decorate, canary = self._keygen_decorator("mynamespace")
 
@@ -244,6 +324,17 @@ class KeyGenerationTest(TestCase):
 
         assert isinstance(gen('foo'), str)
 
+    def test_kwarg_kwgen_key_isnt_unicode_bydefault(self):
+        decorate, canary = self._kwarg_keygen_decorator("mynamespace")
+
+        @decorate
+        def one(a, b):
+            pass
+        gen = canary[0]
+
+        assert isinstance(gen('foo'), str)
+
+
     def test_unicode_key(self):
         decorate, canary = self._keygen_decorator("mynamespace",
                                                   to_str=compat.text_type)
@@ -257,6 +348,20 @@ class KeyGenerationTest(TestCase):
             compat.ue("tests.cache.test_decorator:"
                       "one|mynamespace|m\xe9il dr\xf4le"))
 
+    def test_unicode_key_kwarg_generator(self):
+        decorate, canary = self._kwarg_keygen_decorator(
+            "mynamespace",
+            to_str=compat.text_type)
+
+        @decorate
+        def one(a, b):
+            pass
+        gen = canary[0]
+
+        eq_(gen(compat.u('méil'), compat.u('drôle')),
+            compat.ue("tests.cache.test_decorator:"
+                      "one|mynamespace|m\xe9il dr\xf4le"))
+
     def test_unicode_key_multi(self):
         decorate, canary = self._multi_keygen_decorator(
             "mynamespace",
@@ -291,6 +396,23 @@ class KeyGenerationTest(TestCase):
             "tests.cache.test_decorator:"
             "one|mynamespace|m\xe9il dr\xf4le")
 
+    @requires_py3k
+    def test_unicode_key_by_default_kwarg_generator(self):
+        decorate, canary = self._kwarg_keygen_decorator(
+            "mynamespace",
+            to_str=compat.text_type)
+
+        @decorate
+        def one(a, b):
+            pass
+        gen = canary[0]
+
+        assert isinstance(gen('méil'), str)
+
+        eq_(gen('méil', 'drôle'),
+            "tests.cache.test_decorator:"
+            "one|mynamespace|m\xe9il dr\xf4le")
+
 
 class CacheDecoratorTest(_GenericBackendFixture, TestCase):
     backend = "mock"