Merge branch 'feature/keystone-initial-import'
Change-Id: I01b8a7ff6202cf8e1ed65bb5e5583b67b4aff7b9
This commit is contained in:
Davanum Srinivas
commit
ce06e75008
|
|
@ -0,0 +1,35 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""oslo.i18n integration module.
|
||||
|
||||
See http://docs.openstack.org/developer/oslo.i18n/usage.html
|
||||
|
||||
"""
|
||||
|
||||
import oslo_i18n
|
||||
|
||||
|
||||
_translators = oslo_i18n.TranslatorFactory(domain='oslo.versionedobjects')
|
||||
|
||||
# The primary translation function using the well-known name "_"
|
||||
_ = _translators.primary
|
||||
|
||||
# Translators for log levels.
|
||||
#
|
||||
# The abbreviated names are meant to reflect the usual use of a short
|
||||
# name like '_'. The "L" is for "log" and the other letter comes from
|
||||
# the level.
|
||||
_LI = _translators.log_info
|
||||
_LW = _translators.log_warning
|
||||
_LE = _translators.log_error
|
||||
_LC = _translators.log_critical
|
||||
|
|
@ -0,0 +1,244 @@
|
|||
# Copyright 2014 Mirantis Inc
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Thread-safe connection pool for python-memcached."""
|
||||
|
||||
# NOTE(yorik-sar): this file is copied between keystone and keystonemiddleware
|
||||
# and should be kept in sync until we can use external library for this.
|
||||
|
||||
import collections
|
||||
import contextlib
|
||||
import itertools
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
|
||||
import memcache
|
||||
from oslo_log import log
|
||||
from six.moves import queue, zip
|
||||
|
||||
from oslo_cache._i18n import _
|
||||
from oslo_cache import exception
|
||||
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
class _MemcacheClient(memcache.Client):
|
||||
"""Thread global memcache client
|
||||
|
||||
As client is inherited from threading.local we have to restore object
|
||||
methods overloaded by threading.local so we can reuse clients in
|
||||
different threads
|
||||
"""
|
||||
__delattr__ = object.__delattr__
|
||||
__getattribute__ = object.__getattribute__
|
||||
__new__ = object.__new__
|
||||
__setattr__ = object.__setattr__
|
||||
|
||||
def __del__(self):
|
||||
pass
|
||||
|
||||
|
||||
_PoolItem = collections.namedtuple('_PoolItem', ['ttl', 'connection'])
|
||||
|
||||
|
||||
class ConnectionPool(queue.Queue):
|
||||
"""Base connection pool class
|
||||
|
||||
This class implements the basic connection pool logic as an abstract base
|
||||
class.
|
||||
"""
|
||||
def __init__(self, maxsize, unused_timeout, conn_get_timeout=None):
|
||||
"""Initialize the connection pool.
|
||||
|
||||
:param maxsize: maximum number of client connections for the pool
|
||||
:type maxsize: int
|
||||
:param unused_timeout: idle time to live for unused clients (in
|
||||
seconds). If a client connection object has been
|
||||
in the pool and idle for longer than the
|
||||
unused_timeout, it will be reaped. This is to
|
||||
ensure resources are released as utilization
|
||||
goes down.
|
||||
:type unused_timeout: int
|
||||
:param conn_get_timeout: maximum time in seconds to wait for a
|
||||
connection. If set to `None` timeout is
|
||||
indefinite.
|
||||
:type conn_get_timeout: int
|
||||
"""
|
||||
# super() cannot be used here because Queue in stdlib is an
|
||||
# old-style class
|
||||
queue.Queue.__init__(self, maxsize)
|
||||
self._unused_timeout = unused_timeout
|
||||
self._connection_get_timeout = conn_get_timeout
|
||||
self._acquired = 0
|
||||
|
||||
def _create_connection(self):
|
||||
"""Returns a connection instance.
|
||||
|
||||
This is called when the pool needs another instance created.
|
||||
|
||||
:returns: a new connection instance
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def _destroy_connection(self, conn):
|
||||
"""Destroy and cleanup a connection instance.
|
||||
|
||||
This is called when the pool wishes to get rid of an existing
|
||||
connection. This is the opportunity for a subclass to free up
|
||||
resources and cleaup after itself.
|
||||
|
||||
:param conn: the connection object to destroy
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def _debug_logger(self, msg, *args, **kwargs):
|
||||
if LOG.isEnabledFor(logging.DEBUG):
|
||||
thread_id = threading.current_thread().ident
|
||||
args = (id(self), thread_id) + args
|
||||
prefix = 'Memcached pool %s, thread %s: '
|
||||
LOG.debug(prefix + msg, *args, **kwargs)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def acquire(self):
|
||||
self._debug_logger('Acquiring connection')
|
||||
try:
|
||||
conn = self.get(timeout=self._connection_get_timeout)
|
||||
except queue.Empty:
|
||||
raise exception.UnexpectedError(
|
||||
_('Unable to get a connection from pool id %(id)s after '
|
||||
'%(seconds)s seconds.') %
|
||||
{'id': id(self), 'seconds': self._connection_get_timeout})
|
||||
self._debug_logger('Acquired connection %s', id(conn))
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
self._debug_logger('Releasing connection %s', id(conn))
|
||||
self._drop_expired_connections()
|
||||
try:
|
||||
# super() cannot be used here because Queue in stdlib is an
|
||||
# old-style class
|
||||
queue.Queue.put(self, conn, block=False)
|
||||
except queue.Full:
|
||||
self._debug_logger('Reaping exceeding connection %s', id(conn))
|
||||
self._destroy_connection(conn)
|
||||
|
||||
def _qsize(self):
|
||||
if self.maxsize:
|
||||
return self.maxsize - self._acquired
|
||||
else:
|
||||
# A value indicating there is always a free connection
|
||||
# if maxsize is None or 0
|
||||
return 1
|
||||
|
||||
# NOTE(dstanek): stdlib and eventlet Queue implementations
|
||||
# have different names for the qsize method. This ensures
|
||||
# that we override both of them.
|
||||
if not hasattr(queue.Queue, '_qsize'):
|
||||
qsize = _qsize
|
||||
|
||||
def _get(self):
|
||||
if self.queue:
|
||||
conn = self.queue.pop().connection
|
||||
else:
|
||||
conn = self._create_connection()
|
||||
self._acquired += 1
|
||||
return conn
|
||||
|
||||
def _drop_expired_connections(self):
|
||||
"""Drop all expired connections from the right end of the queue."""
|
||||
now = time.time()
|
||||
while self.queue and self.queue[0].ttl < now:
|
||||
conn = self.queue.popleft().connection
|
||||
self._debug_logger('Reaping connection %s', id(conn))
|
||||
self._destroy_connection(conn)
|
||||
|
||||
def _put(self, conn):
|
||||
self.queue.append(_PoolItem(
|
||||
ttl=time.time() + self._unused_timeout,
|
||||
connection=conn,
|
||||
))
|
||||
self._acquired -= 1
|
||||
|
||||
|
||||
class MemcacheClientPool(ConnectionPool):
|
||||
def __init__(self, urls, arguments, **kwargs):
|
||||
# super() cannot be used here because Queue in stdlib is an
|
||||
# old-style class
|
||||
ConnectionPool.__init__(self, **kwargs)
|
||||
self.urls = urls
|
||||
self._arguments = arguments
|
||||
# NOTE(morganfainberg): The host objects expect an int for the
|
||||
# deaduntil value. Initialize this at 0 for each host with 0 indicating
|
||||
# the host is not dead.
|
||||
self._hosts_deaduntil = [0] * len(urls)
|
||||
|
||||
def _create_connection(self):
|
||||
return _MemcacheClient(self.urls, **self._arguments)
|
||||
|
||||
def _destroy_connection(self, conn):
|
||||
conn.disconnect_all()
|
||||
|
||||
def _get(self):
|
||||
# super() cannot be used here because Queue in stdlib is an
|
||||
# old-style class
|
||||
conn = ConnectionPool._get(self)
|
||||
try:
|
||||
# Propagate host state known to us to this client's list
|
||||
now = time.time()
|
||||
for deaduntil, host in zip(self._hosts_deaduntil, conn.servers):
|
||||
if deaduntil > now and host.deaduntil <= now:
|
||||
host.mark_dead('propagating death mark from the pool')
|
||||
host.deaduntil = deaduntil
|
||||
except Exception:
|
||||
# We need to be sure that connection doesn't leak from the pool.
|
||||
# This code runs before we enter context manager's try-finally
|
||||
# block, so we need to explicitly release it here.
|
||||
# super() cannot be used here because Queue in stdlib is an
|
||||
# old-style class
|
||||
ConnectionPool._put(self, conn)
|
||||
raise
|
||||
return conn
|
||||
|
||||
def _put(self, conn):
|
||||
try:
|
||||
# If this client found that one of the hosts is dead, mark it as
|
||||
# such in our internal list
|
||||
now = time.time()
|
||||
for i, host in zip(itertools.count(), conn.servers):
|
||||
deaduntil = self._hosts_deaduntil[i]
|
||||
# Do nothing if we already know this host is dead
|
||||
if deaduntil <= now:
|
||||
if host.deaduntil > now:
|
||||
self._hosts_deaduntil[i] = host.deaduntil
|
||||
self._debug_logger(
|
||||
'Marked host %s dead until %s',
|
||||
self.urls[i], host.deaduntil)
|
||||
else:
|
||||
self._hosts_deaduntil[i] = 0
|
||||
# If all hosts are dead we should forget that they're dead. This
|
||||
# way we won't get completely shut off until dead_retry seconds
|
||||
# pass, but will be checking servers as frequent as we can (over
|
||||
# way smaller socket_timeout)
|
||||
if all(deaduntil > now for deaduntil in self._hosts_deaduntil):
|
||||
self._debug_logger('All hosts are dead. Marking them as live.')
|
||||
self._hosts_deaduntil[:] = [0] * len(self._hosts_deaduntil)
|
||||
finally:
|
||||
# super() cannot be used here because Queue in stdlib is an
|
||||
# old-style class
|
||||
ConnectionPool._put(self, conn)
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
# Copyright 2012 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from oslo_config import cfg
|
||||
|
||||
|
||||
FILE_OPTIONS = {
|
||||
'cache': [
|
||||
cfg.StrOpt('config_prefix', default='cache.keystone',
|
||||
help='Prefix for building the configuration dictionary '
|
||||
'for the cache region. This should not need to be '
|
||||
'changed unless there is another dogpile.cache '
|
||||
'region with the same configuration name.'),
|
||||
cfg.IntOpt('expiration_time', default=600,
|
||||
help='Default TTL, in seconds, for any cached item in '
|
||||
'the dogpile.cache region. This applies to any '
|
||||
'cached method that doesn\'t have an explicit '
|
||||
'cache expiration time defined for it.'),
|
||||
# NOTE(morganfainberg): the dogpile.cache.memory acceptable in devstack
|
||||
# and other such single-process/thread deployments. Running
|
||||
# dogpile.cache.memory in any other configuration has the same pitfalls
|
||||
# as the KVS token backend. It is recommended that either Redis or
|
||||
# Memcached are used as the dogpile backend for real workloads. To
|
||||
# prevent issues with the memory cache ending up in "production"
|
||||
# unintentionally, we register a no-op as the keystone default caching
|
||||
# backend.
|
||||
cfg.StrOpt('backend', default='oslo_cache.noop',
|
||||
help='Dogpile.cache backend module. It is recommended '
|
||||
'that Memcache with pooling '
|
||||
'(keystone.cache.memcache_pool) or Redis '
|
||||
'(dogpile.cache.redis) be used in production '
|
||||
'deployments. Small workloads (single process) '
|
||||
'like devstack can use the dogpile.cache.memory '
|
||||
'backend.'),
|
||||
cfg.MultiStrOpt('backend_argument', default=[], secret=True,
|
||||
help='Arguments supplied to the backend module. '
|
||||
'Specify this option once per argument to be '
|
||||
'passed to the dogpile.cache backend. Example '
|
||||
'format: "<argname>:<value>".'),
|
||||
cfg.ListOpt('proxies', default=[],
|
||||
help='Proxy classes to import that will affect the way '
|
||||
'the dogpile.cache backend functions. See the '
|
||||
'dogpile.cache documentation on '
|
||||
'changing-backend-behavior.'),
|
||||
cfg.BoolOpt('enabled', default=False,
|
||||
help='Global toggle for all caching using the '
|
||||
'should_cache_fn mechanism.'),
|
||||
cfg.BoolOpt('debug_cache_backend', default=False,
|
||||
help='Extra debugging from the cache backend (cache '
|
||||
'keys, get/set/delete/etc calls). This is only '
|
||||
'really useful if you need to see the specific '
|
||||
'cache-backend get/set/delete calls with the '
|
||||
'keys/values. Typically this should be left set '
|
||||
'to false.'),
|
||||
cfg.ListOpt('memcache_servers', default=['localhost:11211'],
|
||||
help='Memcache servers in the format of "host:port".'
|
||||
' (dogpile.cache.memcache and keystone.cache.memcache_pool'
|
||||
' backends only).'),
|
||||
cfg.IntOpt('memcache_dead_retry',
|
||||
default=5 * 60,
|
||||
help='Number of seconds memcached server is considered dead'
|
||||
' before it is tried again. (dogpile.cache.memcache and'
|
||||
' keystone.cache.memcache_pool backends only).'),
|
||||
cfg.IntOpt('memcache_socket_timeout',
|
||||
default=3,
|
||||
help='Timeout in seconds for every call to a server.'
|
||||
' (dogpile.cache.memcache and keystone.cache.memcache_pool'
|
||||
' backends only).'),
|
||||
cfg.IntOpt('memcache_pool_maxsize',
|
||||
default=10,
|
||||
help='Max total number of open connections to every'
|
||||
' memcached server. (keystone.cache.memcache_pool backend'
|
||||
' only).'),
|
||||
cfg.IntOpt('memcache_pool_unused_timeout',
|
||||
default=60,
|
||||
help='Number of seconds a connection to memcached is held'
|
||||
' unused in the pool before it is closed.'
|
||||
' (keystone.cache.memcache_pool backend only).'),
|
||||
cfg.IntOpt('memcache_pool_connection_get_timeout',
|
||||
default=10,
|
||||
help='Number of seconds that an operation will wait to get '
|
||||
'a memcache client connection.'),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
def configure(conf=None):
|
||||
if conf is None:
|
||||
conf = CONF
|
||||
|
||||
for section in FILE_OPTIONS:
|
||||
for option in FILE_OPTIONS[section]:
|
||||
conf.register_opt(option, group=section)
|
||||
|
||||
|
||||
def list_opts():
|
||||
"""Return a list of oslo_config options available in Keystone.
|
||||
|
||||
The returned list includes all oslo_config options which are registered as
|
||||
the "FILE_OPTIONS" in keystone.common.config. This list will not include
|
||||
the options from the oslo-incubator library or any options registered
|
||||
dynamically at run time.
|
||||
|
||||
Each object in the list is a two element tuple. The first element of
|
||||
each tuple is the name of the group under which the list of options in the
|
||||
second element will be registered. A group name of None corresponds to the
|
||||
[DEFAULT] group in config files.
|
||||
|
||||
This function is also discoverable via the 'oslo_config.opts' entry point
|
||||
under the 'keystone.config.opts' namespace.
|
||||
|
||||
The purpose of this is to allow tools like the Oslo sample config file
|
||||
generator to discover the options exposed to users by this library.
|
||||
|
||||
:returns: a list of (group_name, opts) tuples
|
||||
"""
|
||||
return list(FILE_OPTIONS.items())
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
# Copyright 2014 Mirantis Inc
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""dogpile.cache backend that uses Memcached connection pool"""
|
||||
|
||||
import functools
|
||||
import logging
|
||||
|
||||
from dogpile.cache.backends import memcached as memcached_backend
|
||||
|
||||
from oslo_cache import _memcache_pool
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Helper to ease backend refactoring
|
||||
class ClientProxy(object):
|
||||
def __init__(self, client_pool):
|
||||
self.client_pool = client_pool
|
||||
|
||||
def _run_method(self, __name, *args, **kwargs):
|
||||
with self.client_pool.acquire() as client:
|
||||
return getattr(client, __name)(*args, **kwargs)
|
||||
|
||||
def __getattr__(self, name):
|
||||
return functools.partial(self._run_method, name)
|
||||
|
||||
|
||||
class PooledMemcachedBackend(memcached_backend.MemcachedBackend):
|
||||
# Composed from GenericMemcachedBackend's and MemcacheArgs's __init__
|
||||
def __init__(self, arguments):
|
||||
super(PooledMemcachedBackend, self).__init__(arguments)
|
||||
self.client_pool = _memcache_pool.MemcacheClientPool(
|
||||
self.url,
|
||||
arguments={
|
||||
'dead_retry': arguments.get('dead_retry', 5 * 60),
|
||||
'socket_timeout': arguments.get('socket_timeout', 3),
|
||||
},
|
||||
maxsize=arguments.get('pool_maxsize', 10),
|
||||
unused_timeout=arguments.get('pool_unused_timeout', 60),
|
||||
conn_get_timeout=arguments.get('pool_connection_get_timeout', 10),
|
||||
)
|
||||
|
||||
# Since all methods in backend just call one of methods of client, this
|
||||
# lets us avoid need to hack it too much
|
||||
@property
|
||||
def client(self):
|
||||
return ClientProxy(self.client_pool)
|
||||
|
|
@ -0,0 +1,561 @@
|
|||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import abc
|
||||
import datetime
|
||||
|
||||
from dogpile.cache import api
|
||||
from dogpile.cache import util as dp_util
|
||||
from oslo_log import log
|
||||
from oslo_utils import importutils
|
||||
from oslo_utils import timeutils
|
||||
import six
|
||||
|
||||
from oslo_cache import exception
|
||||
from oslo_cache._i18n import _, _LW
|
||||
|
||||
|
||||
NO_VALUE = api.NO_VALUE
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
|
||||
class MongoCacheBackend(api.CacheBackend):
|
||||
"""A MongoDB based caching backend implementing dogpile backend APIs.
|
||||
|
||||
Arguments accepted in the arguments dictionary:
|
||||
|
||||
:param db_hosts: string (required), hostname or IP address of the
|
||||
MongoDB server instance. This can be a single MongoDB connection URI,
|
||||
or a list of MongoDB connection URIs.
|
||||
|
||||
:param db_name: string (required), the name of the database to be used.
|
||||
|
||||
:param cache_collection: string (required), the name of collection to store
|
||||
cached data.
|
||||
*Note:* Different collection name can be provided if there is need to
|
||||
create separate container (i.e. collection) for cache data. So region
|
||||
configuration is done per collection.
|
||||
|
||||
Following are optional parameters for MongoDB backend configuration,
|
||||
|
||||
:param username: string, the name of the user to authenticate.
|
||||
|
||||
:param password: string, the password of the user to authenticate.
|
||||
|
||||
:param max_pool_size: integer, the maximum number of connections that the
|
||||
pool will open simultaneously. By default the pool size is 10.
|
||||
|
||||
:param w: integer, write acknowledgement for MongoDB client
|
||||
|
||||
If not provided, then no default is set on MongoDB and then write
|
||||
acknowledgement behavior occurs as per MongoDB default. This parameter
|
||||
name is same as what is used in MongoDB docs. This value is specified
|
||||
at collection level so its applicable to `cache_collection` db write
|
||||
operations.
|
||||
|
||||
If this is a replica set, write operations will block until they have
|
||||
been replicated to the specified number or tagged set of servers.
|
||||
Setting w=0 disables write acknowledgement and all other write concern
|
||||
options.
|
||||
|
||||
:param read_preference: string, the read preference mode for MongoDB client
|
||||
Expected value is ``primary``, ``primaryPreferred``, ``secondary``,
|
||||
``secondaryPreferred``, or ``nearest``. This read_preference is
|
||||
specified at collection level so its applicable to `cache_collection`
|
||||
db read operations.
|
||||
|
||||
:param use_replica: boolean, flag to indicate if replica client to be
|
||||
used. Default is `False`. `replicaset_name` value is required if
|
||||
`True`.
|
||||
|
||||
:param replicaset_name: string, name of replica set.
|
||||
Becomes required if `use_replica` is `True`
|
||||
|
||||
:param son_manipulator: string, name of class with module name which
|
||||
implements MongoDB SONManipulator.
|
||||
Default manipulator used is :class:`.BaseTransform`.
|
||||
|
||||
This manipulator is added per database. In multiple cache
|
||||
configurations, the manipulator name should be same if same
|
||||
database name ``db_name`` is used in those configurations.
|
||||
|
||||
SONManipulator is used to manipulate custom data types as they are
|
||||
saved or retrieved from MongoDB. Custom impl is only needed if cached
|
||||
data is custom class and needs transformations when saving or reading
|
||||
from db. If dogpile cached value contains built-in data types, then
|
||||
BaseTransform class is sufficient as it already handles dogpile
|
||||
CachedValue class transformation.
|
||||
|
||||
:param mongo_ttl_seconds: integer, interval in seconds to indicate maximum
|
||||
time-to-live value.
|
||||
If value is greater than 0, then its assumed that cache_collection
|
||||
needs to be TTL type (has index at 'doc_date' field).
|
||||
By default, the value is -1 and its disabled.
|
||||
Reference: <http://docs.mongodb.org/manual/tutorial/expire-data/>
|
||||
|
||||
.. NOTE::
|
||||
|
||||
This parameter is different from Dogpile own
|
||||
expiration_time, which is the number of seconds after which Dogpile
|
||||
will consider the value to be expired. When Dogpile considers a
|
||||
value to be expired, it continues to use the value until generation
|
||||
of a new value is complete, when using CacheRegion.get_or_create().
|
||||
Therefore, if you are setting `mongo_ttl_seconds`, you will want to
|
||||
make sure it is greater than expiration_time by at least enough
|
||||
seconds for new values to be generated, else the value would not
|
||||
be available during a regeneration, forcing all threads to wait for
|
||||
a regeneration each time a value expires.
|
||||
|
||||
:param ssl: boolean, If True, create the connection to the server
|
||||
using SSL. Default is `False`. Client SSL connection parameters depends
|
||||
on server side SSL setup. For further reference on SSL configuration:
|
||||
<http://docs.mongodb.org/manual/tutorial/configure-ssl/>
|
||||
|
||||
:param ssl_keyfile: string, the private keyfile used to identify the
|
||||
local connection against mongod. If included with the certfile then
|
||||
only the `ssl_certfile` is needed. Used only when `ssl` is `True`.
|
||||
|
||||
:param ssl_certfile: string, the certificate file used to identify the
|
||||
local connection against mongod. Used only when `ssl` is `True`.
|
||||
|
||||
:param ssl_ca_certs: string, the ca_certs file contains a set of
|
||||
concatenated 'certification authority' certificates, which are used to
|
||||
validate certificates passed from the other end of the connection.
|
||||
Used only when `ssl` is `True`.
|
||||
|
||||
:param ssl_cert_reqs: string, the parameter cert_reqs specifies whether
|
||||
a certificate is required from the other side of the connection, and
|
||||
whether it will be validated if provided. It must be one of the three
|
||||
values ``ssl.CERT_NONE`` (certificates ignored), ``ssl.CERT_OPTIONAL``
|
||||
(not required, but validated if provided), or
|
||||
``ssl.CERT_REQUIRED`` (required and validated). If the value of this
|
||||
parameter is not ``ssl.CERT_NONE``, then the ssl_ca_certs parameter
|
||||
must point to a file of CA certificates. Used only when `ssl`
|
||||
is `True`.
|
||||
|
||||
Rest of arguments are passed to mongo calls for read, write and remove.
|
||||
So related options can be specified to pass to these operations.
|
||||
|
||||
Further details of various supported arguments can be referred from
|
||||
<http://api.mongodb.org/python/current/api/pymongo/>
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, arguments):
|
||||
self.api = MongoApi(arguments)
|
||||
|
||||
@dp_util.memoized_property
|
||||
def client(self):
|
||||
"""Initializes MongoDB connection and collection defaults.
|
||||
|
||||
This initialization is done only once and performed as part of lazy
|
||||
inclusion of MongoDB dependency i.e. add imports only if related
|
||||
backend is used.
|
||||
|
||||
:return: :class:`.MongoApi` instance
|
||||
"""
|
||||
self.api.get_cache_collection()
|
||||
return self.api
|
||||
|
||||
def get(self, key):
|
||||
value = self.client.get(key)
|
||||
if value is None:
|
||||
return NO_VALUE
|
||||
else:
|
||||
return value
|
||||
|
||||
def get_multi(self, keys):
|
||||
values = self.client.get_multi(keys)
|
||||
return [
|
||||
NO_VALUE if key not in values
|
||||
else values[key] for key in keys
|
||||
]
|
||||
|
||||
def set(self, key, value):
|
||||
self.client.set(key, value)
|
||||
|
||||
def set_multi(self, mapping):
|
||||
self.client.set_multi(mapping)
|
||||
|
||||
def delete(self, key):
|
||||
self.client.delete(key)
|
||||
|
||||
def delete_multi(self, keys):
|
||||
self.client.delete_multi(keys)
|
||||
|
||||
|
||||
class MongoApi(object):
|
||||
"""Class handling MongoDB specific functionality.
|
||||
|
||||
This class uses PyMongo APIs internally to create database connection
|
||||
with configured pool size, ensures unique index on key, does database
|
||||
authentication and ensure TTL collection index if configured so.
|
||||
This class also serves as handle to cache collection for dogpile cache
|
||||
APIs.
|
||||
|
||||
In a single deployment, multiple cache configuration can be defined. In
|
||||
that case of multiple cache collections usage, db client connection pool
|
||||
is shared when cache collections are within same database.
|
||||
"""
|
||||
|
||||
# class level attributes for re-use of db client connection and collection
|
||||
_DB = {} # dict of db_name: db connection reference
|
||||
_MONGO_COLLS = {} # dict of cache_collection : db collection reference
|
||||
|
||||
def __init__(self, arguments):
|
||||
self._init_args(arguments)
|
||||
self._data_manipulator = None
|
||||
|
||||
def _init_args(self, arguments):
|
||||
"""Helper logic for collecting and parsing MongoDB specific arguments.
|
||||
|
||||
The arguments passed in are separated out in connection specific
|
||||
setting and rest of arguments are passed to create/update/delete
|
||||
db operations.
|
||||
"""
|
||||
self.conn_kwargs = {} # connection specific arguments
|
||||
|
||||
self.hosts = arguments.pop('db_hosts', None)
|
||||
if self.hosts is None:
|
||||
msg = _('db_hosts value is required')
|
||||
raise exception.ValidationError(message=msg)
|
||||
|
||||
self.db_name = arguments.pop('db_name', None)
|
||||
if self.db_name is None:
|
||||
msg = _('database db_name is required')
|
||||
raise exception.ValidationError(message=msg)
|
||||
|
||||
self.cache_collection = arguments.pop('cache_collection', None)
|
||||
if self.cache_collection is None:
|
||||
msg = _('cache_collection name is required')
|
||||
raise exception.ValidationError(message=msg)
|
||||
|
||||
self.username = arguments.pop('username', None)
|
||||
self.password = arguments.pop('password', None)
|
||||
self.max_pool_size = arguments.pop('max_pool_size', 10)
|
||||
|
||||
self.w = arguments.pop('w', -1)
|
||||
try:
|
||||
self.w = int(self.w)
|
||||
except ValueError:
|
||||
msg = _('integer value expected for w (write concern attribute)')
|
||||
raise exception.ValidationError(message=msg)
|
||||
|
||||
self.read_preference = arguments.pop('read_preference', None)
|
||||
|
||||
self.use_replica = arguments.pop('use_replica', False)
|
||||
if self.use_replica:
|
||||
if arguments.get('replicaset_name') is None:
|
||||
msg = _('replicaset_name required when use_replica is True')
|
||||
raise exception.ValidationError(message=msg)
|
||||
self.replicaset_name = arguments.get('replicaset_name')
|
||||
|
||||
self.son_manipulator = arguments.pop('son_manipulator', None)
|
||||
|
||||
# set if mongo collection needs to be TTL type.
|
||||
# This needs to be max ttl for any cache entry.
|
||||
# By default, -1 means don't use TTL collection.
|
||||
# With ttl set, it creates related index and have doc_date field with
|
||||
# needed expiration interval
|
||||
self.ttl_seconds = arguments.pop('mongo_ttl_seconds', -1)
|
||||
try:
|
||||
self.ttl_seconds = int(self.ttl_seconds)
|
||||
except ValueError:
|
||||
msg = _('integer value expected for mongo_ttl_seconds')
|
||||
raise exception.ValidationError(message=msg)
|
||||
|
||||
self.conn_kwargs['ssl'] = arguments.pop('ssl', False)
|
||||
if self.conn_kwargs['ssl']:
|
||||
ssl_keyfile = arguments.pop('ssl_keyfile', None)
|
||||
ssl_certfile = arguments.pop('ssl_certfile', None)
|
||||
ssl_ca_certs = arguments.pop('ssl_ca_certs', None)
|
||||
ssl_cert_reqs = arguments.pop('ssl_cert_reqs', None)
|
||||
if ssl_keyfile:
|
||||
self.conn_kwargs['ssl_keyfile'] = ssl_keyfile
|
||||
if ssl_certfile:
|
||||
self.conn_kwargs['ssl_certfile'] = ssl_certfile
|
||||
if ssl_ca_certs:
|
||||
self.conn_kwargs['ssl_ca_certs'] = ssl_ca_certs
|
||||
if ssl_cert_reqs:
|
||||
self.conn_kwargs['ssl_cert_reqs'] = (
|
||||
self._ssl_cert_req_type(ssl_cert_reqs))
|
||||
|
||||
# rest of arguments are passed to mongo crud calls
|
||||
self.meth_kwargs = arguments
|
||||
|
||||
def _ssl_cert_req_type(self, req_type):
|
||||
try:
|
||||
import ssl
|
||||
except ImportError:
|
||||
raise exception.ValidationError(_('no ssl support available'))
|
||||
req_type = req_type.upper()
|
||||
try:
|
||||
return {
|
||||
'NONE': ssl.CERT_NONE,
|
||||
'OPTIONAL': ssl.CERT_OPTIONAL,
|
||||
'REQUIRED': ssl.CERT_REQUIRED
|
||||
}[req_type]
|
||||
except KeyError:
|
||||
msg = _('Invalid ssl_cert_reqs value of %s, must be one of '
|
||||
'"NONE", "OPTIONAL", "REQUIRED"') % (req_type)
|
||||
raise exception.ValidationError(message=msg)
|
||||
|
||||
def _get_db(self):
|
||||
# defer imports until backend is used
|
||||
global pymongo
|
||||
import pymongo
|
||||
if self.use_replica:
|
||||
connection = pymongo.MongoReplicaSetClient(
|
||||
host=self.hosts, replicaSet=self.replicaset_name,
|
||||
max_pool_size=self.max_pool_size, **self.conn_kwargs)
|
||||
else: # used for standalone node or mongos in sharded setup
|
||||
connection = pymongo.MongoClient(
|
||||
host=self.hosts, max_pool_size=self.max_pool_size,
|
||||
**self.conn_kwargs)
|
||||
|
||||
database = getattr(connection, self.db_name)
|
||||
|
||||
self._assign_data_mainpulator()
|
||||
database.add_son_manipulator(self._data_manipulator)
|
||||
if self.username and self.password:
|
||||
database.authenticate(self.username, self.password)
|
||||
return database
|
||||
|
||||
def _assign_data_mainpulator(self):
|
||||
if self._data_manipulator is None:
|
||||
if self.son_manipulator:
|
||||
self._data_manipulator = importutils.import_object(
|
||||
self.son_manipulator)
|
||||
else:
|
||||
self._data_manipulator = BaseTransform()
|
||||
|
||||
def _get_doc_date(self):
|
||||
if self.ttl_seconds > 0:
|
||||
expire_delta = datetime.timedelta(seconds=self.ttl_seconds)
|
||||
doc_date = timeutils.utcnow() + expire_delta
|
||||
else:
|
||||
doc_date = timeutils.utcnow()
|
||||
return doc_date
|
||||
|
||||
def get_cache_collection(self):
|
||||
if self.cache_collection not in self._MONGO_COLLS:
|
||||
global pymongo
|
||||
import pymongo
|
||||
# re-use db client connection if already defined as part of
|
||||
# earlier dogpile cache configuration
|
||||
if self.db_name not in self._DB:
|
||||
self._DB[self.db_name] = self._get_db()
|
||||
coll = getattr(self._DB[self.db_name], self.cache_collection)
|
||||
|
||||
self._assign_data_mainpulator()
|
||||
if self.read_preference:
|
||||
# pymongo 3.0 renamed mongos_enum to read_pref_mode_from_name
|
||||
f = getattr(pymongo.read_preferences,
|
||||
'read_pref_mode_from_name', None)
|
||||
if not f:
|
||||
f = pymongo.read_preferences.mongos_enum
|
||||
self.read_preference = f(self.read_preference)
|
||||
coll.read_preference = self.read_preference
|
||||
if self.w > -1:
|
||||
coll.write_concern['w'] = self.w
|
||||
if self.ttl_seconds > 0:
|
||||
kwargs = {'expireAfterSeconds': self.ttl_seconds}
|
||||
coll.ensure_index('doc_date', cache_for=5, **kwargs)
|
||||
else:
|
||||
self._validate_ttl_index(coll, self.cache_collection,
|
||||
self.ttl_seconds)
|
||||
self._MONGO_COLLS[self.cache_collection] = coll
|
||||
|
||||
return self._MONGO_COLLS[self.cache_collection]
|
||||
|
||||
def _get_cache_entry(self, key, value, meta, doc_date):
|
||||
"""MongoDB cache data representation.
|
||||
|
||||
Storing cache key as ``_id`` field as MongoDB by default creates
|
||||
unique index on this field. So no need to create separate field and
|
||||
index for storing cache key. Cache data has additional ``doc_date``
|
||||
field for MongoDB TTL collection support.
|
||||
"""
|
||||
return dict(_id=key, value=value, meta=meta, doc_date=doc_date)
|
||||
|
||||
def _validate_ttl_index(self, collection, coll_name, ttl_seconds):
|
||||
"""Checks if existing TTL index is removed on a collection.
|
||||
|
||||
This logs warning when existing collection has TTL index defined and
|
||||
new cache configuration tries to disable index with
|
||||
``mongo_ttl_seconds < 0``. In that case, existing index needs
|
||||
to be addressed first to make new configuration effective.
|
||||
Refer to MongoDB documentation around TTL index for further details.
|
||||
"""
|
||||
indexes = collection.index_information()
|
||||
for indx_name, index_data in six.iteritems(indexes):
|
||||
if all(k in index_data for k in ('key', 'expireAfterSeconds')):
|
||||
existing_value = index_data['expireAfterSeconds']
|
||||
fld_present = 'doc_date' in index_data['key'][0]
|
||||
if fld_present and existing_value > -1 and ttl_seconds < 1:
|
||||
msg = _LW('TTL index already exists on db collection '
|
||||
'<%(c_name)s>, remove index <%(indx_name)s> '
|
||||
'first to make updated mongo_ttl_seconds value '
|
||||
'to be effective')
|
||||
LOG.warn(msg, {'c_name': coll_name,
|
||||
'indx_name': indx_name})
|
||||
|
||||
def get(self, key):
|
||||
critieria = {'_id': key}
|
||||
result = self.get_cache_collection().find_one(spec_or_id=critieria,
|
||||
**self.meth_kwargs)
|
||||
if result:
|
||||
return result['value']
|
||||
else:
|
||||
return None
|
||||
|
||||
def get_multi(self, keys):
|
||||
db_results = self._get_results_as_dict(keys)
|
||||
return {doc['_id']: doc['value'] for doc in six.itervalues(db_results)}
|
||||
|
||||
def _get_results_as_dict(self, keys):
|
||||
critieria = {'_id': {'$in': keys}}
|
||||
db_results = self.get_cache_collection().find(spec=critieria,
|
||||
**self.meth_kwargs)
|
||||
return {doc['_id']: doc for doc in db_results}
|
||||
|
||||
def set(self, key, value):
|
||||
doc_date = self._get_doc_date()
|
||||
ref = self._get_cache_entry(key, value.payload, value.metadata,
|
||||
doc_date)
|
||||
spec = {'_id': key}
|
||||
# find and modify does not have manipulator support
|
||||
# so need to do conversion as part of input document
|
||||
ref = self._data_manipulator.transform_incoming(ref, self)
|
||||
self.get_cache_collection().find_and_modify(spec, ref, upsert=True,
|
||||
**self.meth_kwargs)
|
||||
|
||||
def set_multi(self, mapping):
|
||||
"""Insert multiple documents specified as key, value pairs.
|
||||
|
||||
In this case, multiple documents can be added via insert provided they
|
||||
do not exist.
|
||||
Update of multiple existing documents is done one by one
|
||||
"""
|
||||
doc_date = self._get_doc_date()
|
||||
insert_refs = []
|
||||
update_refs = []
|
||||
existing_docs = self._get_results_as_dict(list(mapping.keys()))
|
||||
for key, value in mapping.items():
|
||||
ref = self._get_cache_entry(key, value.payload, value.metadata,
|
||||
doc_date)
|
||||
if key in existing_docs:
|
||||
ref['_id'] = existing_docs[key]['_id']
|
||||
update_refs.append(ref)
|
||||
else:
|
||||
insert_refs.append(ref)
|
||||
if insert_refs:
|
||||
self.get_cache_collection().insert(insert_refs, manipulate=True,
|
||||
**self.meth_kwargs)
|
||||
for upd_doc in update_refs:
|
||||
self.get_cache_collection().save(upd_doc, manipulate=True,
|
||||
**self.meth_kwargs)
|
||||
|
||||
def delete(self, key):
|
||||
critieria = {'_id': key}
|
||||
self.get_cache_collection().remove(spec_or_id=critieria,
|
||||
**self.meth_kwargs)
|
||||
|
||||
def delete_multi(self, keys):
|
||||
critieria = {'_id': {'$in': keys}}
|
||||
self.get_cache_collection().remove(spec_or_id=critieria,
|
||||
**self.meth_kwargs)
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class AbstractManipulator(object):
|
||||
"""Abstract class with methods which need to be implemented for custom
|
||||
manipulation.
|
||||
|
||||
Adding this as a base class for :class:`.BaseTransform` instead of adding
|
||||
import dependency of pymongo specific class i.e.
|
||||
`pymongo.son_manipulator.SONManipulator` and using that as base class.
|
||||
This is done to avoid pymongo dependency if MongoDB backend is not used.
|
||||
"""
|
||||
@abc.abstractmethod
|
||||
def transform_incoming(self, son, collection):
|
||||
"""Used while saving data to MongoDB.
|
||||
|
||||
:param son: the SON object to be inserted into the database
|
||||
:param collection: the collection the object is being inserted into
|
||||
|
||||
:returns: transformed SON object
|
||||
|
||||
"""
|
||||
raise exception.NotImplemented() # pragma: no cover
|
||||
|
||||
@abc.abstractmethod
|
||||
def transform_outgoing(self, son, collection):
|
||||
"""Used while reading data from MongoDB.
|
||||
|
||||
:param son: the SON object being retrieved from the database
|
||||
:param collection: the collection this object was stored in
|
||||
|
||||
:returns: transformed SON object
|
||||
"""
|
||||
raise exception.NotImplemented() # pragma: no cover
|
||||
|
||||
def will_copy(self):
|
||||
"""Will this SON manipulator make a copy of the incoming document?
|
||||
|
||||
Derived classes that do need to make a copy should override this
|
||||
method, returning `True` instead of `False`.
|
||||
|
||||
:returns: boolean
|
||||
"""
|
||||
return False
|
||||
|
||||
|
||||
class BaseTransform(AbstractManipulator):
|
||||
"""Base transformation class to store and read dogpile cached data
|
||||
from MongoDB.
|
||||
|
||||
This is needed as dogpile internally stores data as a custom class
|
||||
i.e. dogpile.cache.api.CachedValue
|
||||
|
||||
Note: Custom manipulator needs to always override ``transform_incoming``
|
||||
and ``transform_outgoing`` methods. MongoDB manipulator logic specifically
|
||||
checks that overridden method in instance and its super are different.
|
||||
"""
|
||||
|
||||
def transform_incoming(self, son, collection):
|
||||
"""Used while saving data to MongoDB."""
|
||||
for (key, value) in list(son.items()):
|
||||
if isinstance(value, api.CachedValue):
|
||||
son[key] = value.payload # key is 'value' field here
|
||||
son['meta'] = value.metadata
|
||||
elif isinstance(value, dict): # Make sure we recurse into sub-docs
|
||||
son[key] = self.transform_incoming(value, collection)
|
||||
return son
|
||||
|
||||
def transform_outgoing(self, son, collection):
|
||||
"""Used while reading data from MongoDB."""
|
||||
metadata = None
|
||||
# make sure its top level dictionary with all expected fields names
|
||||
# present
|
||||
if isinstance(son, dict) and all(k in son for k in
|
||||
('_id', 'value', 'meta', 'doc_date')):
|
||||
payload = son.pop('value', None)
|
||||
metadata = son.pop('meta', None)
|
||||
for (key, value) in list(son.items()):
|
||||
if isinstance(value, dict):
|
||||
son[key] = self.transform_outgoing(value, collection)
|
||||
if metadata is not None:
|
||||
son['value'] = api.CachedValue(payload, metadata)
|
||||
return son
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
# Copyright 2013 Metacloud
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from dogpile.cache import api
|
||||
|
||||
|
||||
NO_VALUE = api.NO_VALUE
|
||||
|
||||
|
||||
class NoopCacheBackend(api.CacheBackend):
|
||||
"""A no op backend as a default caching backend.
|
||||
|
||||
The no op backend is provided as the default caching backend for keystone
|
||||
to ensure that ``dogpile.cache.memory`` is not used in any real-world
|
||||
circumstances unintentionally. ``dogpile.cache.memory`` does not have a
|
||||
mechanism to cleanup it's internal dict and therefore could cause run-away
|
||||
memory utilization.
|
||||
"""
|
||||
def __init__(self, *args):
|
||||
return
|
||||
|
||||
def get(self, key):
|
||||
return NO_VALUE
|
||||
|
||||
def get_multi(self, keys):
|
||||
return [NO_VALUE for x in keys]
|
||||
|
||||
def set(self, key, value):
|
||||
return
|
||||
|
||||
def set_multi(self, mapping):
|
||||
return
|
||||
|
||||
def delete(self, key):
|
||||
return
|
||||
|
||||
def delete_multi(self, keys):
|
||||
return
|
||||
|
|
@ -0,0 +1,308 @@
|
|||
# Copyright 2013 Metacloud
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Keystone Caching Layer Implementation."""
|
||||
|
||||
import dogpile.cache
|
||||
from dogpile.cache import proxy
|
||||
from dogpile.cache import util
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log
|
||||
from oslo_utils import importutils
|
||||
|
||||
from oslo_cache import exception
|
||||
from oslo_cache._i18n import _, _LE
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
make_region = dogpile.cache.make_region
|
||||
|
||||
dogpile.cache.register_backend(
|
||||
'oslo_cache.noop',
|
||||
'oslo_cache.backends.noop',
|
||||
'NoopCacheBackend')
|
||||
|
||||
dogpile.cache.register_backend(
|
||||
'oslo_cache.mongo',
|
||||
'oslo_cache.backends.mongo',
|
||||
'MongoCacheBackend')
|
||||
|
||||
dogpile.cache.register_backend(
|
||||
'oslo_cache.memcache_pool',
|
||||
'oslo_cache.backends.memcache_pool',
|
||||
'PooledMemcachedBackend')
|
||||
|
||||
|
||||
class DebugProxy(proxy.ProxyBackend):
|
||||
"""Extra Logging ProxyBackend."""
|
||||
# NOTE(morganfainberg): Pass all key/values through repr to ensure we have
|
||||
# a clean description of the information. Without use of repr, it might
|
||||
# be possible to run into encode/decode error(s). For logging/debugging
|
||||
# purposes encode/decode is irrelevant and we should be looking at the
|
||||
# data exactly as it stands.
|
||||
|
||||
def get(self, key):
|
||||
value = self.proxied.get(key)
|
||||
LOG.debug('CACHE_GET: Key: "%(key)r" Value: "%(value)r"',
|
||||
{'key': key, 'value': value})
|
||||
return value
|
||||
|
||||
def get_multi(self, keys):
|
||||
values = self.proxied.get_multi(keys)
|
||||
LOG.debug('CACHE_GET_MULTI: "%(keys)r" Values: "%(values)r"',
|
||||
{'keys': keys, 'values': values})
|
||||
return values
|
||||
|
||||
def set(self, key, value):
|
||||
LOG.debug('CACHE_SET: Key: "%(key)r" Value: "%(value)r"',
|
||||
{'key': key, 'value': value})
|
||||
return self.proxied.set(key, value)
|
||||
|
||||
def set_multi(self, keys):
|
||||
LOG.debug('CACHE_SET_MULTI: "%r"', keys)
|
||||
self.proxied.set_multi(keys)
|
||||
|
||||
def delete(self, key):
|
||||
self.proxied.delete(key)
|
||||
LOG.debug('CACHE_DELETE: "%r"', key)
|
||||
|
||||
def delete_multi(self, keys):
|
||||
LOG.debug('CACHE_DELETE_MULTI: "%r"', keys)
|
||||
self.proxied.delete_multi(keys)
|
||||
|
||||
|
||||
def build_cache_config():
|
||||
"""Build the cache region dictionary configuration.
|
||||
|
||||
:returns: dict
|
||||
"""
|
||||
prefix = CONF.cache.config_prefix
|
||||
conf_dict = {}
|
||||
conf_dict['%s.backend' % prefix] = CONF.cache.backend
|
||||
conf_dict['%s.expiration_time' % prefix] = CONF.cache.expiration_time
|
||||
for argument in CONF.cache.backend_argument:
|
||||
try:
|
||||
(argname, argvalue) = argument.split(':', 1)
|
||||
except ValueError:
|
||||
msg = _LE('Unable to build cache config-key. Expected format '
|
||||
'"<argname>:<value>". Skipping unknown format: %s')
|
||||
LOG.error(msg, argument)
|
||||
continue
|
||||
|
||||
arg_key = '.'.join([prefix, 'arguments', argname])
|
||||
conf_dict[arg_key] = argvalue
|
||||
|
||||
LOG.debug('Oslo Cache Config: %s', conf_dict)
|
||||
# NOTE(yorik-sar): these arguments will be used for memcache-related
|
||||
# backends. Use setdefault for url to support old-style setting through
|
||||
# backend_argument=url:127.0.0.1:11211
|
||||
conf_dict.setdefault('%s.arguments.url' % prefix,
|
||||
CONF.cache.memcache_servers)
|
||||
for arg in ('dead_retry', 'socket_timeout', 'pool_maxsize',
|
||||
'pool_unused_timeout', 'pool_connection_get_timeout'):
|
||||
value = getattr(CONF.cache, 'memcache_' + arg)
|
||||
conf_dict['%s.arguments.%s' % (prefix, arg)] = value
|
||||
|
||||
return conf_dict
|
||||
|
||||
|
||||
def configure_cache_region(region):
|
||||
"""Configure a cache region.
|
||||
|
||||
:param region: optional CacheRegion object, if not provided a new region
|
||||
will be instantiated
|
||||
:raises: exception.ValidationError
|
||||
:returns: dogpile.cache.CacheRegion
|
||||
"""
|
||||
if not isinstance(region, dogpile.cache.CacheRegion):
|
||||
raise exception.ValidationError(
|
||||
_('region not type dogpile.cache.CacheRegion'))
|
||||
|
||||
if not region.is_configured:
|
||||
# NOTE(morganfainberg): this is how you tell if a region is configured.
|
||||
# There is a request logged with dogpile.cache upstream to make this
|
||||
# easier / less ugly.
|
||||
|
||||
config_dict = build_cache_config()
|
||||
region.configure_from_config(config_dict,
|
||||
'%s.' % CONF.cache.config_prefix)
|
||||
|
||||
if CONF.cache.debug_cache_backend:
|
||||
region.wrap(DebugProxy)
|
||||
|
||||
# NOTE(morganfainberg): if the backend requests the use of a
|
||||
# key_mangler, we should respect that key_mangler function. If a
|
||||
# key_mangler is not defined by the backend, use the sha1_mangle_key
|
||||
# mangler provided by dogpile.cache. This ensures we always use a fixed
|
||||
# size cache-key.
|
||||
if region.key_mangler is None:
|
||||
region.key_mangler = util.sha1_mangle_key
|
||||
|
||||
for class_path in CONF.cache.proxies:
|
||||
# NOTE(morganfainberg): if we have any proxy wrappers, we should
|
||||
# ensure they are added to the cache region's backend. Since
|
||||
# configure_from_config doesn't handle the wrap argument, we need
|
||||
# to manually add the Proxies. For information on how the
|
||||
# ProxyBackends work, see the dogpile.cache documents on
|
||||
# "changing-backend-behavior"
|
||||
cls = importutils.import_class(class_path)
|
||||
LOG.debug("Adding cache-proxy '%s' to backend.", class_path)
|
||||
region.wrap(cls)
|
||||
|
||||
return region
|
||||
|
||||
|
||||
def get_should_cache_fn(section):
|
||||
"""Build a function that returns a config section's caching status.
|
||||
|
||||
For any given driver in keystone that has caching capabilities, a boolean
|
||||
config option for that driver's section (e.g. ``token``) should exist and
|
||||
default to ``True``. This function will use that value to tell the caching
|
||||
decorator if caching for that driver is enabled. To properly use this
|
||||
with the decorator, pass this function the configuration section and assign
|
||||
the result to a variable. Pass the new variable to the caching decorator
|
||||
as the named argument ``should_cache_fn``. e.g.::
|
||||
|
||||
from keystone.common import cache
|
||||
|
||||
SHOULD_CACHE = cache.get_should_cache_fn('token')
|
||||
|
||||
@cache.on_arguments(should_cache_fn=SHOULD_CACHE)
|
||||
def function(arg1, arg2):
|
||||
...
|
||||
|
||||
:param section: name of the configuration section to examine
|
||||
:type section: string
|
||||
:returns: function reference
|
||||
"""
|
||||
def should_cache(value):
|
||||
if not CONF.cache.enabled:
|
||||
return False
|
||||
conf_group = getattr(CONF, section)
|
||||
return getattr(conf_group, 'caching', True)
|
||||
return should_cache
|
||||
|
||||
|
||||
def get_expiration_time_fn(section):
|
||||
"""Build a function that returns a config section's expiration time status.
|
||||
|
||||
For any given driver in keystone that has caching capabilities, an int
|
||||
config option called ``cache_time`` for that driver's section
|
||||
(e.g. ``token``) should exist and typically default to ``None``. This
|
||||
function will use that value to tell the caching decorator of the TTL
|
||||
override for caching the resulting objects. If the value of the config
|
||||
option is ``None`` the default value provided in the
|
||||
``[cache] expiration_time`` option will be used by the decorator. The
|
||||
default may be set to something other than ``None`` in cases where the
|
||||
caching TTL should not be tied to the global default(s) (e.g.
|
||||
revocation_list changes very infrequently and can be cached for >1h by
|
||||
default).
|
||||
|
||||
To properly use this with the decorator, pass this function the
|
||||
configuration section and assign the result to a variable. Pass the new
|
||||
variable to the caching decorator as the named argument
|
||||
``expiration_time``. e.g.::
|
||||
|
||||
from keystone.common import cache
|
||||
|
||||
EXPIRATION_TIME = cache.get_expiration_time_fn('token')
|
||||
|
||||
@cache.on_arguments(expiration_time=EXPIRATION_TIME)
|
||||
def function(arg1, arg2):
|
||||
...
|
||||
|
||||
:param section: name of the configuration section to examine
|
||||
:type section: string
|
||||
:rtype: function reference
|
||||
"""
|
||||
def get_expiration_time():
|
||||
conf_group = getattr(CONF, section)
|
||||
return getattr(conf_group, 'cache_time', None)
|
||||
return get_expiration_time
|
||||
|
||||
|
||||
def key_generate_to_str(s):
|
||||
# NOTE(morganfainberg): Since we need to stringify all arguments, attempt
|
||||
# to stringify and handle the Unicode error explicitly as needed.
|
||||
try:
|
||||
return str(s)
|
||||
except UnicodeEncodeError:
|
||||
return s.encode('utf-8')
|
||||
|
||||
|
||||
def function_key_generator(namespace, fn, to_str=key_generate_to_str):
|
||||
# NOTE(morganfainberg): This wraps dogpile.cache's default
|
||||
# function_key_generator to change the default to_str mechanism.
|
||||
return util.function_key_generator(namespace, fn, to_str=to_str)
|
||||
|
||||
|
||||
REGION = dogpile.cache.make_region(
|
||||
function_key_generator=function_key_generator)
|
||||
on_arguments = REGION.cache_on_arguments
|
||||
|
||||
|
||||
def get_memoization_decorator(section, expiration_section=None):
|
||||
"""Build a function based on the `on_arguments` decorator for the section.
|
||||
|
||||
For any given driver in Keystone that has caching capabilities, a
|
||||
pair of functions is required to properly determine the status of the
|
||||
caching capabilities (a toggle to indicate caching is enabled and any
|
||||
override of the default TTL for cached data). This function will return
|
||||
an object that has the memoization decorator ``on_arguments``
|
||||
pre-configured for the driver.
|
||||
|
||||
Example usage::
|
||||
|
||||
from keystone.common import cache
|
||||
|
||||
MEMOIZE = cache.get_memoization_decorator(section='token')
|
||||
|
||||
@MEMOIZE
|
||||
def function(arg1, arg2):
|
||||
...
|
||||
|
||||
|
||||
ALTERNATE_MEMOIZE = cache.get_memoization_decorator(
|
||||
section='token', expiration_section='revoke')
|
||||
|
||||
@ALTERNATE_MEMOIZE
|
||||
def function2(arg1, arg2):
|
||||
...
|
||||
|
||||
:param section: name of the configuration section to examine
|
||||
:type section: string
|
||||
:param expiration_section: name of the configuration section to examine
|
||||
for the expiration option. This will fall back
|
||||
to using ``section`` if the value is unspecified
|
||||
or ``None``
|
||||
:type expiration_section: string
|
||||
:rtype: function reference
|
||||
"""
|
||||
if expiration_section is None:
|
||||
expiration_section = section
|
||||
should_cache = get_should_cache_fn(section)
|
||||
expiration_time = get_expiration_time_fn(expiration_section)
|
||||
|
||||
memoize = REGION.cache_on_arguments(should_cache_fn=should_cache,
|
||||
expiration_time=expiration_time)
|
||||
|
||||
# Make sure the actual "should_cache" and "expiration_time" methods are
|
||||
# available. This is potentially interesting/useful to pre-seed cache
|
||||
# values.
|
||||
memoize.should_cache = should_cache
|
||||
memoize.get_expiration_time = expiration_time
|
||||
|
||||
return memoize
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
# Copyright 2012 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging as log
|
||||
|
||||
from oslo_config import cfg
|
||||
from oslo_utils import encodeutils
|
||||
import six
|
||||
|
||||
from oslo_cache._i18n import _, _LW
|
||||
|
||||
|
||||
CONF = cfg.CONF
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
# Tests use this to make exception message format errors fatal
|
||||
_FATAL_EXCEPTION_FORMAT_ERRORS = False
|
||||
|
||||
|
||||
class Error(Exception):
|
||||
"""Base error class.
|
||||
|
||||
Child classes should define an HTTP status code, title, and a
|
||||
message_format.
|
||||
|
||||
"""
|
||||
code = None
|
||||
title = None
|
||||
message_format = None
|
||||
|
||||
def __init__(self, message=None, **kwargs):
|
||||
try:
|
||||
message = self._build_message(message, **kwargs)
|
||||
except KeyError:
|
||||
# if you see this warning in your logs, please raise a bug report
|
||||
if _FATAL_EXCEPTION_FORMAT_ERRORS:
|
||||
raise
|
||||
else:
|
||||
LOG.warning(_LW('missing exception kwargs (programmer error)'))
|
||||
message = self.message_format
|
||||
|
||||
super(Error, self).__init__(message)
|
||||
|
||||
def _build_message(self, message, **kwargs):
|
||||
"""Builds and returns an exception message.
|
||||
|
||||
:raises: KeyError given insufficient kwargs
|
||||
|
||||
"""
|
||||
if not message:
|
||||
try:
|
||||
message = self.message_format % kwargs
|
||||
except UnicodeDecodeError:
|
||||
try:
|
||||
kwargs = {k: encodeutils.safe_decode(v)
|
||||
for k, v in six.iteritems(kwargs)}
|
||||
except UnicodeDecodeError:
|
||||
# NOTE(jamielennox): This is the complete failure case
|
||||
# at least by showing the template we have some idea
|
||||
# of where the error is coming from
|
||||
message = self.message_format
|
||||
else:
|
||||
message = self.message_format % kwargs
|
||||
|
||||
return message
|
||||
|
||||
|
||||
class ValidationError(Error):
|
||||
message_format = _("Expecting to find %(attribute)s in %(target)s -"
|
||||
" the server could not comply with the request"
|
||||
" since it is either malformed or otherwise"
|
||||
" incorrect. The client is assumed to be in error.")
|
||||
code = 400
|
||||
title = 'Bad Request'
|
||||
|
||||
|
||||
class NotImplemented(Error):
|
||||
message_format = _("The action you have requested has not"
|
||||
" been implemented.")
|
||||
code = 501
|
||||
title = 'Not Implemented'
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from oslo_cache import _opts
|
||||
|
||||
_opts.configure()
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2013 Metacloud
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
|
|
@ -12,17 +12,335 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
test_cache
|
||||
----------------------------------
|
||||
|
||||
Tests for `cache` module.
|
||||
"""
|
||||
import copy
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from dogpile.cache import api
|
||||
from dogpile.cache import proxy
|
||||
import mock
|
||||
from oslo_config import cfg
|
||||
from oslotest import base
|
||||
import six
|
||||
import testtools
|
||||
|
||||
from oslo_cache import core as cache
|
||||
from oslo_cache import exception
|
||||
from oslo_config import fixture as config_fixture
|
||||
|
||||
|
||||
class TestCache(base.BaseTestCase):
|
||||
CONF = cfg.CONF
|
||||
NO_VALUE = api.NO_VALUE
|
||||
|
||||
def test_something(self):
|
||||
pass
|
||||
|
||||
class BaseTestCase(base.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(BaseTestCase, self).setUp()
|
||||
|
||||
self.addCleanup(CONF.reset)
|
||||
|
||||
self.config_fixture = self.useFixture(config_fixture.Config(CONF))
|
||||
self.addCleanup(delattr, self, 'config_fixture')
|
||||
|
||||
|
||||
def _copy_value(value):
|
||||
if value is not NO_VALUE:
|
||||
value = copy.deepcopy(value)
|
||||
return value
|
||||
|
||||
|
||||
# NOTE(morganfainberg): WARNING - It is not recommended to use the Memory
|
||||
# backend for dogpile.cache in a real deployment under any circumstances. The
|
||||
# backend does no cleanup of expired values and therefore will leak memory. The
|
||||
# backend is not implemented in a way to share data across processes (e.g.
|
||||
# Keystone in HTTPD. This proxy is a hack to get around the lack of isolation
|
||||
# of values in memory. Currently it blindly stores and retrieves the values
|
||||
# from the cache, and modifications to dicts/lists/etc returned can result in
|
||||
# changes to the cached values. In short, do not use the dogpile.cache.memory
|
||||
# backend unless you are running tests or expecting odd/strange results.
|
||||
class CacheIsolatingProxy(proxy.ProxyBackend):
|
||||
"""Proxy that forces a memory copy of stored values.
|
||||
The default in-memory cache-region does not perform a copy on values it
|
||||
is meant to cache. Therefore if the value is modified after set or after
|
||||
get, the cached value also is modified. This proxy does a copy as the last
|
||||
thing before storing data.
|
||||
"""
|
||||
def get(self, key):
|
||||
return _copy_value(self.proxied.get(key))
|
||||
|
||||
def set(self, key, value):
|
||||
self.proxied.set(key, _copy_value(value))
|
||||
|
||||
|
||||
class TestProxy(proxy.ProxyBackend):
|
||||
def get(self, key):
|
||||
value = _copy_value(self.proxied.get(key))
|
||||
if value is not NO_VALUE:
|
||||
if isinstance(value[0], TestProxyValue):
|
||||
value[0].cached = True
|
||||
return value
|
||||
|
||||
|
||||
class TestProxyValue(object):
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
self.cached = False
|
||||
|
||||
|
||||
class CacheRegionTest(BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(CacheRegionTest, self).setUp()
|
||||
self.region = cache.make_region()
|
||||
cache.configure_cache_region(self.region)
|
||||
self.region.wrap(TestProxy)
|
||||
self.test_value = TestProxyValue('Decorator Test')
|
||||
|
||||
def _add_test_caching_option(self):
|
||||
self.config_fixture.register_opt(
|
||||
cfg.BoolOpt('caching', default=True), group='cache')
|
||||
|
||||
def _get_cacheable_function(self):
|
||||
with mock.patch.object(cache.REGION, 'cache_on_arguments',
|
||||
self.region.cache_on_arguments):
|
||||
memoize = cache.get_memoization_decorator(section='cache')
|
||||
|
||||
@memoize
|
||||
def cacheable_function(value):
|
||||
return value
|
||||
|
||||
return cacheable_function
|
||||
|
||||
# FIXME(dims) : Need to resurrect this test ported from keystone
|
||||
# def test_region_built_with_proxy_direct_cache_test(self):
|
||||
# # Verify cache regions are properly built with proxies.
|
||||
# test_value = TestProxyValue('Direct Cache Test')
|
||||
# self.region.set('cache_test', test_value)
|
||||
# cached_value = self.region.get('cache_test')
|
||||
# self.assertTrue(cached_value.cached)
|
||||
|
||||
def test_cache_region_no_error_multiple_config(self):
|
||||
# Verify configuring the CacheRegion again doesn't error.
|
||||
cache.configure_cache_region(self.region)
|
||||
cache.configure_cache_region(self.region)
|
||||
|
||||
def _get_cache_fallthrough_fn(self, cache_time):
|
||||
with mock.patch.object(cache.REGION, 'cache_on_arguments',
|
||||
self.region.cache_on_arguments):
|
||||
memoize = cache.get_memoization_decorator(
|
||||
section='cache',
|
||||
expiration_section='assignment')
|
||||
|
||||
class _test_obj(object):
|
||||
def __init__(self, value):
|
||||
self.test_value = value
|
||||
|
||||
@memoize
|
||||
def get_test_value(self):
|
||||
return self.test_value
|
||||
|
||||
def _do_test(value):
|
||||
|
||||
test_obj = _test_obj(value)
|
||||
|
||||
# Ensure the value has been cached
|
||||
test_obj.get_test_value()
|
||||
# Get the now cached value
|
||||
cached_value = test_obj.get_test_value()
|
||||
self.assertTrue(cached_value.cached)
|
||||
self.assertEqual(value.value, cached_value.value)
|
||||
self.assertEqual(cached_value.value, test_obj.test_value.value)
|
||||
# Change the underlying value on the test object.
|
||||
test_obj.test_value = TestProxyValue(uuid.uuid4().hex)
|
||||
self.assertEqual(cached_value.value,
|
||||
test_obj.get_test_value().value)
|
||||
# override the system time to ensure the non-cached new value
|
||||
# is returned
|
||||
new_time = time.time() + (cache_time * 2)
|
||||
with mock.patch.object(time, 'time',
|
||||
return_value=new_time):
|
||||
overriden_cache_value = test_obj.get_test_value()
|
||||
self.assertNotEqual(cached_value.value,
|
||||
overriden_cache_value.value)
|
||||
self.assertEqual(test_obj.test_value.value,
|
||||
overriden_cache_value.value)
|
||||
|
||||
return _do_test
|
||||
|
||||
# FIXME(dims) : Need to resurrect this test ported from keystone
|
||||
# def test_cache_no_fallthrough_expiration_time_fn(self):
|
||||
# # Since we do not re-configure the cache region, for ease of testing
|
||||
# # this value is set the same as the expiration_time default in the
|
||||
# # [cache] section
|
||||
# cache_time = 600
|
||||
# expiration_time = cache.get_expiration_time_fn('role')
|
||||
# do_test = self._get_cache_fallthrough_fn(cache_time)
|
||||
# # Run the test with the assignment cache_time value
|
||||
# self.config_fixture.config(cache_time=cache_time,
|
||||
# group='role')
|
||||
# test_value = TestProxyValue(uuid.uuid4().hex)
|
||||
# self.assertEqual(cache_time, expiration_time())
|
||||
# do_test(value=test_value)
|
||||
|
||||
# FIXME(dims) : Need to resurrect this test ported from keystone
|
||||
# def test_cache_fallthrough_expiration_time_fn(self):
|
||||
# # Since we do not re-configure the cache region, for ease of testing
|
||||
# # this value is set the same as the expiration_time default in the
|
||||
# # [cache] section
|
||||
# cache_time = 599
|
||||
# expiration_time = cache.get_expiration_time_fn('role')
|
||||
# do_test = self._get_cache_fallthrough_fn(cache_time)
|
||||
# # Run the test with the assignment cache_time value set to None and
|
||||
# # the global value set.
|
||||
# self.config_fixture.config(cache_time=None, group='role')
|
||||
# test_value = TestProxyValue(uuid.uuid4().hex)
|
||||
# self.assertIsNone(expiration_time())
|
||||
# do_test(value=test_value)
|
||||
|
||||
# FIXME(dims) : Need to resurrect this test ported from keystone
|
||||
# def test_should_cache_fn_global_cache_enabled(self):
|
||||
# # Verify should_cache_fn generates a sane function for subsystem and
|
||||
# # functions as expected with caching globally enabled.
|
||||
# cacheable_function = self._get_cacheable_function()
|
||||
#
|
||||
# self.config_fixture.config(group='cache', enabled=True)
|
||||
# cacheable_function(self.test_value)
|
||||
# cached_value = cacheable_function(self.test_value)
|
||||
# self.assertTrue(cached_value.cached)
|
||||
|
||||
@testtools.skipIf(six.PY3, 'FIXME: this test does not work python 3.x')
|
||||
def test_should_cache_fn_global_cache_disabled(self):
|
||||
# Verify should_cache_fn generates a sane function for subsystem and
|
||||
# functions as expected with caching globally disabled.
|
||||
cacheable_function = self._get_cacheable_function()
|
||||
|
||||
self.config_fixture.config(group='cache', enabled=False)
|
||||
cacheable_function(self.test_value)
|
||||
cached_value = cacheable_function(self.test_value)
|
||||
self.assertFalse(cached_value.cached)
|
||||
|
||||
@testtools.skipIf(six.PY3, 'FIXME: this test does not work python 3.x')
|
||||
def test_should_cache_fn_global_cache_disabled_section_cache_enabled(self):
|
||||
# Verify should_cache_fn generates a sane function for subsystem and
|
||||
# functions as expected with caching globally disabled and the specific
|
||||
# section caching enabled.
|
||||
cacheable_function = self._get_cacheable_function()
|
||||
|
||||
self._add_test_caching_option()
|
||||
self.config_fixture.config(group='cache', enabled=False)
|
||||
self.config_fixture.config(group='cache', caching=True)
|
||||
|
||||
cacheable_function(self.test_value)
|
||||
cached_value = cacheable_function(self.test_value)
|
||||
self.assertFalse(cached_value.cached)
|
||||
|
||||
@testtools.skipIf(six.PY3, 'FIXME: this test does not work python 3.x')
|
||||
def test_should_cache_fn_global_cache_enabled_section_cache_disabled(self):
|
||||
# Verify should_cache_fn generates a sane function for subsystem and
|
||||
# functions as expected with caching globally enabled and the specific
|
||||
# section caching disabled.
|
||||
cacheable_function = self._get_cacheable_function()
|
||||
|
||||
self._add_test_caching_option()
|
||||
self.config_fixture.config(group='cache', enabled=True)
|
||||
self.config_fixture.config(group='cache', caching=False)
|
||||
|
||||
cacheable_function(self.test_value)
|
||||
cached_value = cacheable_function(self.test_value)
|
||||
self.assertFalse(cached_value.cached)
|
||||
|
||||
# FIXME(dims) : Need to resurrect this test ported from keystone
|
||||
# def test_should_cache_fn_global_cache_enabled_section_cache_enabled(
|
||||
# self):
|
||||
# #Verify should_cache_fn generates a sane function for subsystem and
|
||||
# #functions as expected with caching globally enabled and the specific
|
||||
# #section caching enabled.
|
||||
# cacheable_function = self._get_cacheable_function()
|
||||
#
|
||||
# self._add_test_caching_option()
|
||||
# self.config_fixture.config(group='cache', enabled=True)
|
||||
# self.config_fixture.config(group='cache', caching=True)
|
||||
#
|
||||
# cacheable_function(self.test_value)
|
||||
# cached_value = cacheable_function(self.test_value)
|
||||
# self.assertTrue(cached_value.cached)
|
||||
|
||||
def test_cache_dictionary_config_builder(self):
|
||||
"""Validate we build a sane dogpile.cache dictionary config."""
|
||||
self.config_fixture.config(group='cache',
|
||||
config_prefix='test_prefix',
|
||||
backend='some_test_backend',
|
||||
expiration_time=86400,
|
||||
backend_argument=['arg1:test',
|
||||
'arg2:test:test',
|
||||
'arg3.invalid'])
|
||||
|
||||
config_dict = cache.build_cache_config()
|
||||
self.assertEqual(
|
||||
CONF.cache.backend, config_dict['test_prefix.backend'])
|
||||
self.assertEqual(
|
||||
CONF.cache.expiration_time,
|
||||
config_dict['test_prefix.expiration_time'])
|
||||
self.assertEqual('test', config_dict['test_prefix.arguments.arg1'])
|
||||
self.assertEqual('test:test',
|
||||
config_dict['test_prefix.arguments.arg2'])
|
||||
self.assertNotIn('test_prefix.arguments.arg3', config_dict)
|
||||
|
||||
# FIXME(dims) : Need to resurrect this test ported from keystone
|
||||
# def test_cache_debug_proxy(self):
|
||||
# single_value = 'Test Value'
|
||||
# single_key = 'testkey'
|
||||
# multi_values = {'key1': 1, 'key2': 2, 'key3': 3}
|
||||
#
|
||||
# self.region.set(single_key, single_value)
|
||||
# self.assertEqual(single_value, self.region.get(single_key))
|
||||
#
|
||||
# self.region.delete(single_key)
|
||||
# self.assertEqual(NO_VALUE, self.region.get(single_key))
|
||||
#
|
||||
# self.region.set_multi(multi_values)
|
||||
# cached_values = self.region.get_multi(multi_values.keys())
|
||||
# for value in multi_values.values():
|
||||
# self.assertIn(value, cached_values)
|
||||
# self.assertEqual(len(multi_values.values()), len(cached_values))
|
||||
#
|
||||
# self.region.delete_multi(multi_values.keys())
|
||||
# for value in self.region.get_multi(multi_values.keys()):
|
||||
# self.assertEqual(NO_VALUE, value)
|
||||
|
||||
def test_configure_non_region_object_raises_error(self):
|
||||
self.assertRaises(exception.ValidationError,
|
||||
cache.configure_cache_region,
|
||||
"bogus")
|
||||
|
||||
|
||||
class CacheNoopBackendTest(BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(CacheNoopBackendTest, self).setUp()
|
||||
self.region = cache.make_region()
|
||||
cache.configure_cache_region(self.region)
|
||||
|
||||
def config_overrides(self):
|
||||
super(CacheNoopBackendTest, self).config_overrides()
|
||||
self.config_fixture.config(group='cache',
|
||||
backend='oslo_cache.cache.noop')
|
||||
|
||||
@testtools.skipIf(six.PY3, 'FIXME: this test does not work python 3.x')
|
||||
def test_noop_backend(self):
|
||||
single_value = 'Test Value'
|
||||
single_key = 'testkey'
|
||||
multi_values = {'key1': 1, 'key2': 2, 'key3': 3}
|
||||
|
||||
self.region.set(single_key, single_value)
|
||||
self.assertEqual(NO_VALUE, self.region.get(single_key))
|
||||
|
||||
self.region.set_multi(multi_values)
|
||||
cached_values = self.region.get_multi(multi_values.keys())
|
||||
self.assertEqual(len(cached_values), len(multi_values.values()))
|
||||
for value in cached_values:
|
||||
self.assertEqual(NO_VALUE, value)
|
||||
|
||||
# Delete should not raise exceptions
|
||||
self.region.delete(single_key)
|
||||
self.region.delete_multi(multi_values.keys())
|
||||
|
|
|
|||
|
|
@ -0,0 +1,727 @@
|
|||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import collections
|
||||
import copy
|
||||
import functools
|
||||
import uuid
|
||||
|
||||
from dogpile.cache import api
|
||||
from dogpile.cache import region as dp_region
|
||||
import six
|
||||
from six.moves import range
|
||||
|
||||
from oslo_cache.backends import mongo
|
||||
from oslo_cache import exception
|
||||
from oslo_cache.tests import test_cache
|
||||
|
||||
# Mock database structure sample where 'ks_cache' is database and
|
||||
# 'cache' is collection. Dogpile CachedValue data is divided in two
|
||||
# fields `value` (CachedValue.payload) and `meta` (CachedValue.metadata)
|
||||
ks_cache = {
|
||||
"cache": [
|
||||
{
|
||||
"value": {
|
||||
"serviceType": "identity",
|
||||
"allVersionsUrl": "https://dummyUrl",
|
||||
"dateLastModified": "ISODDate(2014-02-08T18:39:13.237Z)",
|
||||
"serviceName": "Identity",
|
||||
"enabled": "True"
|
||||
},
|
||||
"meta": {
|
||||
"v": 1,
|
||||
"ct": 1392371422.015121
|
||||
},
|
||||
"doc_date": "ISODate('2014-02-14T09:50:22.015Z')",
|
||||
"_id": "8251dc95f63842719c077072f1047ddf"
|
||||
},
|
||||
{
|
||||
"value": "dummyValueX",
|
||||
"meta": {
|
||||
"v": 1,
|
||||
"ct": 1392371422.014058
|
||||
},
|
||||
"doc_date": "ISODate('2014-02-14T09:50:22.014Z')",
|
||||
"_id": "66730b9534d146f0804d23729ad35436"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
COLLECTIONS = {}
|
||||
SON_MANIPULATOR = None
|
||||
|
||||
|
||||
class MockCursor(object):
|
||||
|
||||
def __init__(self, collection, dataset_factory):
|
||||
super(MockCursor, self).__init__()
|
||||
self.collection = collection
|
||||
self._factory = dataset_factory
|
||||
self._dataset = self._factory()
|
||||
self._limit = None
|
||||
self._skip = None
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
if self._skip:
|
||||
for _ in range(self._skip):
|
||||
next(self._dataset)
|
||||
self._skip = None
|
||||
if self._limit is not None and self._limit <= 0:
|
||||
raise StopIteration()
|
||||
if self._limit is not None:
|
||||
self._limit -= 1
|
||||
return next(self._dataset)
|
||||
|
||||
next = __next__
|
||||
|
||||
def __getitem__(self, index):
|
||||
arr = [x for x in self._dataset]
|
||||
self._dataset = iter(arr)
|
||||
return arr[index]
|
||||
|
||||
|
||||
class MockCollection(object):
|
||||
|
||||
def __init__(self, db, name):
|
||||
super(MockCollection, self).__init__()
|
||||
self.name = name
|
||||
self._collection_database = db
|
||||
self._documents = {}
|
||||
self.write_concern = {}
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name == 'database':
|
||||
return self._collection_database
|
||||
|
||||
def ensure_index(self, key_or_list, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def index_information(self):
|
||||
return {}
|
||||
|
||||
def find_one(self, spec_or_id=None, *args, **kwargs):
|
||||
if spec_or_id is None:
|
||||
spec_or_id = {}
|
||||
if not isinstance(spec_or_id, collections.Mapping):
|
||||
spec_or_id = {'_id': spec_or_id}
|
||||
|
||||
try:
|
||||
return next(self.find(spec_or_id, *args, **kwargs))
|
||||
except StopIteration:
|
||||
return None
|
||||
|
||||
def find(self, spec=None, *args, **kwargs):
|
||||
return MockCursor(self, functools.partial(self._get_dataset, spec))
|
||||
|
||||
def _get_dataset(self, spec):
|
||||
dataset = (self._copy_doc(document, dict) for document in
|
||||
self._iter_documents(spec))
|
||||
return dataset
|
||||
|
||||
def _iter_documents(self, spec=None):
|
||||
return (SON_MANIPULATOR.transform_outgoing(document, self) for
|
||||
document in six.itervalues(self._documents)
|
||||
if self._apply_filter(document, spec))
|
||||
|
||||
def _apply_filter(self, document, query):
|
||||
for key, search in six.iteritems(query):
|
||||
doc_val = document.get(key)
|
||||
if isinstance(search, dict):
|
||||
op_dict = {'$in': lambda dv, sv: dv in sv}
|
||||
is_match = all(
|
||||
op_str in op_dict and op_dict[op_str](doc_val, search_val)
|
||||
for op_str, search_val in six.iteritems(search)
|
||||
)
|
||||
else:
|
||||
is_match = doc_val == search
|
||||
|
||||
return is_match
|
||||
|
||||
def _copy_doc(self, obj, container):
|
||||
if isinstance(obj, list):
|
||||
new = []
|
||||
for item in obj:
|
||||
new.append(self._copy_doc(item, container))
|
||||
return new
|
||||
if isinstance(obj, dict):
|
||||
new = container()
|
||||
for key, value in list(obj.items()):
|
||||
new[key] = self._copy_doc(value, container)
|
||||
return new
|
||||
else:
|
||||
return copy.copy(obj)
|
||||
|
||||
def insert(self, data, manipulate=True, **kwargs):
|
||||
if isinstance(data, list):
|
||||
return [self._insert(element) for element in data]
|
||||
return self._insert(data)
|
||||
|
||||
def save(self, data, manipulate=True, **kwargs):
|
||||
return self._insert(data)
|
||||
|
||||
def _insert(self, data):
|
||||
if '_id' not in data:
|
||||
data['_id'] = uuid.uuid4().hex
|
||||
object_id = data['_id']
|
||||
self._documents[object_id] = self._internalize_dict(data)
|
||||
return object_id
|
||||
|
||||
def find_and_modify(self, spec, document, upsert=False, **kwargs):
|
||||
self.update(spec, document, upsert, **kwargs)
|
||||
|
||||
def update(self, spec, document, upsert=False, **kwargs):
|
||||
|
||||
existing_docs = [doc for doc in six.itervalues(self._documents)
|
||||
if self._apply_filter(doc, spec)]
|
||||
if existing_docs:
|
||||
existing_doc = existing_docs[0] # should find only 1 match
|
||||
_id = existing_doc['_id']
|
||||
existing_doc.clear()
|
||||
existing_doc['_id'] = _id
|
||||
existing_doc.update(self._internalize_dict(document))
|
||||
elif upsert:
|
||||
existing_doc = self._documents[self._insert(document)]
|
||||
|
||||
def _internalize_dict(self, d):
|
||||
return {k: copy.deepcopy(v) for k, v in six.iteritems(d)}
|
||||
|
||||
def remove(self, spec_or_id=None, search_filter=None):
|
||||
"""Remove objects matching spec_or_id from the collection."""
|
||||
if spec_or_id is None:
|
||||
spec_or_id = search_filter if search_filter else {}
|
||||
if not isinstance(spec_or_id, dict):
|
||||
spec_or_id = {'_id': spec_or_id}
|
||||
to_delete = list(self.find(spec=spec_or_id))
|
||||
for doc in to_delete:
|
||||
doc_id = doc['_id']
|
||||
del self._documents[doc_id]
|
||||
|
||||
return {
|
||||
"connectionId": uuid.uuid4().hex,
|
||||
"n": len(to_delete),
|
||||
"ok": 1.0,
|
||||
"err": None,
|
||||
}
|
||||
|
||||
|
||||
class MockMongoDB(object):
|
||||
def __init__(self, dbname):
|
||||
self._dbname = dbname
|
||||
self.mainpulator = None
|
||||
|
||||
def authenticate(self, username, password):
|
||||
pass
|
||||
|
||||
def add_son_manipulator(self, manipulator):
|
||||
global SON_MANIPULATOR
|
||||
SON_MANIPULATOR = manipulator
|
||||
|
||||
def __getattr__(self, name):
|
||||
if name == 'authenticate':
|
||||
return self.authenticate
|
||||
elif name == 'name':
|
||||
return self._dbname
|
||||
elif name == 'add_son_manipulator':
|
||||
return self.add_son_manipulator
|
||||
else:
|
||||
return get_collection(self._dbname, name)
|
||||
|
||||
def __getitem__(self, name):
|
||||
return get_collection(self._dbname, name)
|
||||
|
||||
|
||||
class MockMongoClient(object):
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def __getattr__(self, dbname):
|
||||
return MockMongoDB(dbname)
|
||||
|
||||
|
||||
def get_collection(db_name, collection_name):
|
||||
mongo_collection = MockCollection(MockMongoDB(db_name), collection_name)
|
||||
return mongo_collection
|
||||
|
||||
|
||||
def pymongo_override():
|
||||
global pymongo
|
||||
import pymongo
|
||||
if pymongo.MongoClient is not MockMongoClient:
|
||||
pymongo.MongoClient = MockMongoClient
|
||||
if pymongo.MongoReplicaSetClient is not MockMongoClient:
|
||||
pymongo.MongoClient = MockMongoClient
|
||||
|
||||
|
||||
class MyTransformer(mongo.BaseTransform):
|
||||
"""Added here just to check manipulator logic is used correctly."""
|
||||
|
||||
def transform_incoming(self, son, collection):
|
||||
return super(MyTransformer, self).transform_incoming(son, collection)
|
||||
|
||||
def transform_outgoing(self, son, collection):
|
||||
return super(MyTransformer, self).transform_outgoing(son, collection)
|
||||
|
||||
|
||||
class MongoCache(test_cache.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(MongoCache, self).setUp()
|
||||
global COLLECTIONS
|
||||
COLLECTIONS = {}
|
||||
mongo.MongoApi._DB = {}
|
||||
mongo.MongoApi._MONGO_COLLS = {}
|
||||
pymongo_override()
|
||||
# using typical configuration
|
||||
self.arguments = {
|
||||
'db_hosts': 'localhost:27017',
|
||||
'db_name': 'ks_cache',
|
||||
'cache_collection': 'cache',
|
||||
'username': 'test_user',
|
||||
'password': 'test_password'
|
||||
}
|
||||
|
||||
def test_missing_db_hosts(self):
|
||||
self.arguments.pop('db_hosts')
|
||||
region = dp_region.make_region()
|
||||
self.assertRaises(exception.ValidationError, region.configure,
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments)
|
||||
|
||||
def test_missing_db_name(self):
|
||||
self.arguments.pop('db_name')
|
||||
region = dp_region.make_region()
|
||||
self.assertRaises(exception.ValidationError, region.configure,
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments)
|
||||
|
||||
def test_missing_cache_collection_name(self):
|
||||
self.arguments.pop('cache_collection')
|
||||
region = dp_region.make_region()
|
||||
self.assertRaises(exception.ValidationError, region.configure,
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments)
|
||||
|
||||
def test_incorrect_write_concern(self):
|
||||
self.arguments['w'] = 'one value'
|
||||
region = dp_region.make_region()
|
||||
self.assertRaises(exception.ValidationError, region.configure,
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments)
|
||||
|
||||
def test_correct_write_concern(self):
|
||||
self.arguments['w'] = 1
|
||||
region = dp_region.make_region().configure('oslo_cache.mongo',
|
||||
arguments=self.arguments)
|
||||
|
||||
random_key = uuid.uuid4().hex
|
||||
region.set(random_key, "dummyValue10")
|
||||
# There is no proxy so can access MongoCacheBackend directly
|
||||
self.assertEqual(1, region.backend.api.w)
|
||||
|
||||
def test_incorrect_read_preference(self):
|
||||
self.arguments['read_preference'] = 'inValidValue'
|
||||
region = dp_region.make_region().configure('oslo_cache.mongo',
|
||||
arguments=self.arguments)
|
||||
# As per delayed loading of pymongo, read_preference value should
|
||||
# still be string and NOT enum
|
||||
self.assertEqual('inValidValue', region.backend.api.read_preference)
|
||||
|
||||
random_key = uuid.uuid4().hex
|
||||
self.assertRaises(ValueError, region.set,
|
||||
random_key, "dummyValue10")
|
||||
|
||||
def test_correct_read_preference(self):
|
||||
self.arguments['read_preference'] = 'secondaryPreferred'
|
||||
region = dp_region.make_region().configure('oslo_cache.mongo',
|
||||
arguments=self.arguments)
|
||||
# As per delayed loading of pymongo, read_preference value should
|
||||
# still be string and NOT enum
|
||||
self.assertEqual('secondaryPreferred',
|
||||
region.backend.api.read_preference)
|
||||
|
||||
random_key = uuid.uuid4().hex
|
||||
region.set(random_key, "dummyValue10")
|
||||
|
||||
# Now as pymongo is loaded so expected read_preference value is enum.
|
||||
# There is no proxy so can access MongoCacheBackend directly
|
||||
self.assertEqual(3, region.backend.api.read_preference)
|
||||
|
||||
def test_missing_replica_set_name(self):
|
||||
self.arguments['use_replica'] = True
|
||||
region = dp_region.make_region()
|
||||
self.assertRaises(exception.ValidationError, region.configure,
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments)
|
||||
|
||||
def test_provided_replica_set_name(self):
|
||||
self.arguments['use_replica'] = True
|
||||
self.arguments['replicaset_name'] = 'my_replica'
|
||||
dp_region.make_region().configure('oslo_cache.mongo',
|
||||
arguments=self.arguments)
|
||||
self.assertTrue(True) # reached here means no initialization error
|
||||
|
||||
def test_incorrect_mongo_ttl_seconds(self):
|
||||
self.arguments['mongo_ttl_seconds'] = 'sixty'
|
||||
region = dp_region.make_region()
|
||||
self.assertRaises(exception.ValidationError, region.configure,
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments)
|
||||
|
||||
def test_cache_configuration_values_assertion(self):
|
||||
self.arguments['use_replica'] = True
|
||||
self.arguments['replicaset_name'] = 'my_replica'
|
||||
self.arguments['mongo_ttl_seconds'] = 60
|
||||
self.arguments['ssl'] = False
|
||||
region = dp_region.make_region().configure('oslo_cache.mongo',
|
||||
arguments=self.arguments)
|
||||
# There is no proxy so can access MongoCacheBackend directly
|
||||
self.assertEqual('localhost:27017', region.backend.api.hosts)
|
||||
self.assertEqual('ks_cache', region.backend.api.db_name)
|
||||
self.assertEqual('cache', region.backend.api.cache_collection)
|
||||
self.assertEqual('test_user', region.backend.api.username)
|
||||
self.assertEqual('test_password', region.backend.api.password)
|
||||
self.assertEqual(True, region.backend.api.use_replica)
|
||||
self.assertEqual('my_replica', region.backend.api.replicaset_name)
|
||||
self.assertEqual(False, region.backend.api.conn_kwargs['ssl'])
|
||||
self.assertEqual(60, region.backend.api.ttl_seconds)
|
||||
|
||||
def test_multiple_region_cache_configuration(self):
|
||||
arguments1 = copy.copy(self.arguments)
|
||||
arguments1['cache_collection'] = 'cache_region1'
|
||||
|
||||
region1 = dp_region.make_region().configure('oslo_cache.mongo',
|
||||
arguments=arguments1)
|
||||
# There is no proxy so can access MongoCacheBackend directly
|
||||
self.assertEqual('localhost:27017', region1.backend.api.hosts)
|
||||
self.assertEqual('ks_cache', region1.backend.api.db_name)
|
||||
self.assertEqual('cache_region1', region1.backend.api.cache_collection)
|
||||
self.assertEqual('test_user', region1.backend.api.username)
|
||||
self.assertEqual('test_password', region1.backend.api.password)
|
||||
# Should be None because of delayed initialization
|
||||
self.assertIsNone(region1.backend.api._data_manipulator)
|
||||
|
||||
random_key1 = uuid.uuid4().hex
|
||||
region1.set(random_key1, "dummyValue10")
|
||||
self.assertEqual("dummyValue10", region1.get(random_key1))
|
||||
# Now should have initialized
|
||||
self.assertIsInstance(region1.backend.api._data_manipulator,
|
||||
mongo.BaseTransform)
|
||||
|
||||
class_name = '%s.%s' % (MyTransformer.__module__, "MyTransformer")
|
||||
|
||||
arguments2 = copy.copy(self.arguments)
|
||||
arguments2['cache_collection'] = 'cache_region2'
|
||||
arguments2['son_manipulator'] = class_name
|
||||
|
||||
region2 = dp_region.make_region().configure('oslo_cache.mongo',
|
||||
arguments=arguments2)
|
||||
# There is no proxy so can access MongoCacheBackend directly
|
||||
self.assertEqual('localhost:27017', region2.backend.api.hosts)
|
||||
self.assertEqual('ks_cache', region2.backend.api.db_name)
|
||||
self.assertEqual('cache_region2', region2.backend.api.cache_collection)
|
||||
|
||||
# Should be None because of delayed initialization
|
||||
self.assertIsNone(region2.backend.api._data_manipulator)
|
||||
|
||||
random_key = uuid.uuid4().hex
|
||||
region2.set(random_key, "dummyValue20")
|
||||
self.assertEqual("dummyValue20", region2.get(random_key))
|
||||
# Now should have initialized
|
||||
self.assertIsInstance(region2.backend.api._data_manipulator,
|
||||
MyTransformer)
|
||||
|
||||
region1.set(random_key1, "dummyValue22")
|
||||
self.assertEqual("dummyValue22", region1.get(random_key1))
|
||||
|
||||
def test_typical_configuration(self):
|
||||
|
||||
dp_region.make_region().configure(
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments
|
||||
)
|
||||
self.assertTrue(True) # reached here means no initialization error
|
||||
|
||||
def test_backend_get_missing_data(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments
|
||||
)
|
||||
|
||||
random_key = uuid.uuid4().hex
|
||||
# should return NO_VALUE as key does not exist in cache
|
||||
self.assertEqual(api.NO_VALUE, region.get(random_key))
|
||||
|
||||
def test_backend_set_data(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments
|
||||
)
|
||||
|
||||
random_key = uuid.uuid4().hex
|
||||
region.set(random_key, "dummyValue")
|
||||
self.assertEqual("dummyValue", region.get(random_key))
|
||||
|
||||
def test_backend_set_data_with_string_as_valid_ttl(self):
|
||||
|
||||
self.arguments['mongo_ttl_seconds'] = '3600'
|
||||
region = dp_region.make_region().configure('oslo_cache.mongo',
|
||||
arguments=self.arguments)
|
||||
self.assertEqual(3600, region.backend.api.ttl_seconds)
|
||||
random_key = uuid.uuid4().hex
|
||||
region.set(random_key, "dummyValue")
|
||||
self.assertEqual("dummyValue", region.get(random_key))
|
||||
|
||||
def test_backend_set_data_with_int_as_valid_ttl(self):
|
||||
|
||||
self.arguments['mongo_ttl_seconds'] = 1800
|
||||
region = dp_region.make_region().configure('oslo_cache.mongo',
|
||||
arguments=self.arguments)
|
||||
self.assertEqual(1800, region.backend.api.ttl_seconds)
|
||||
random_key = uuid.uuid4().hex
|
||||
region.set(random_key, "dummyValue")
|
||||
self.assertEqual("dummyValue", region.get(random_key))
|
||||
|
||||
def test_backend_set_none_as_data(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments
|
||||
)
|
||||
|
||||
random_key = uuid.uuid4().hex
|
||||
region.set(random_key, None)
|
||||
self.assertIsNone(region.get(random_key))
|
||||
|
||||
def test_backend_set_blank_as_data(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments
|
||||
)
|
||||
|
||||
random_key = uuid.uuid4().hex
|
||||
region.set(random_key, "")
|
||||
self.assertEqual("", region.get(random_key))
|
||||
|
||||
def test_backend_set_same_key_multiple_times(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments
|
||||
)
|
||||
|
||||
random_key = uuid.uuid4().hex
|
||||
region.set(random_key, "dummyValue")
|
||||
self.assertEqual("dummyValue", region.get(random_key))
|
||||
|
||||
dict_value = {'key1': 'value1'}
|
||||
region.set(random_key, dict_value)
|
||||
self.assertEqual(dict_value, region.get(random_key))
|
||||
|
||||
region.set(random_key, "dummyValue2")
|
||||
self.assertEqual("dummyValue2", region.get(random_key))
|
||||
|
||||
def test_backend_multi_set_data(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments
|
||||
)
|
||||
random_key = uuid.uuid4().hex
|
||||
random_key1 = uuid.uuid4().hex
|
||||
random_key2 = uuid.uuid4().hex
|
||||
random_key3 = uuid.uuid4().hex
|
||||
mapping = {random_key1: 'dummyValue1',
|
||||
random_key2: 'dummyValue2',
|
||||
random_key3: 'dummyValue3'}
|
||||
region.set_multi(mapping)
|
||||
# should return NO_VALUE as key does not exist in cache
|
||||
self.assertEqual(api.NO_VALUE, region.get(random_key))
|
||||
self.assertFalse(region.get(random_key))
|
||||
self.assertEqual("dummyValue1", region.get(random_key1))
|
||||
self.assertEqual("dummyValue2", region.get(random_key2))
|
||||
self.assertEqual("dummyValue3", region.get(random_key3))
|
||||
|
||||
def test_backend_multi_get_data(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments
|
||||
)
|
||||
random_key = uuid.uuid4().hex
|
||||
random_key1 = uuid.uuid4().hex
|
||||
random_key2 = uuid.uuid4().hex
|
||||
random_key3 = uuid.uuid4().hex
|
||||
mapping = {random_key1: 'dummyValue1',
|
||||
random_key2: '',
|
||||
random_key3: 'dummyValue3'}
|
||||
region.set_multi(mapping)
|
||||
|
||||
keys = [random_key, random_key1, random_key2, random_key3]
|
||||
results = region.get_multi(keys)
|
||||
# should return NO_VALUE as key does not exist in cache
|
||||
self.assertEqual(api.NO_VALUE, results[0])
|
||||
self.assertEqual("dummyValue1", results[1])
|
||||
self.assertEqual("", results[2])
|
||||
self.assertEqual("dummyValue3", results[3])
|
||||
|
||||
def test_backend_multi_set_should_update_existing(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments
|
||||
)
|
||||
random_key = uuid.uuid4().hex
|
||||
random_key1 = uuid.uuid4().hex
|
||||
random_key2 = uuid.uuid4().hex
|
||||
random_key3 = uuid.uuid4().hex
|
||||
mapping = {random_key1: 'dummyValue1',
|
||||
random_key2: 'dummyValue2',
|
||||
random_key3: 'dummyValue3'}
|
||||
region.set_multi(mapping)
|
||||
# should return NO_VALUE as key does not exist in cache
|
||||
self.assertEqual(api.NO_VALUE, region.get(random_key))
|
||||
self.assertEqual("dummyValue1", region.get(random_key1))
|
||||
self.assertEqual("dummyValue2", region.get(random_key2))
|
||||
self.assertEqual("dummyValue3", region.get(random_key3))
|
||||
|
||||
mapping = {random_key1: 'dummyValue4',
|
||||
random_key2: 'dummyValue5'}
|
||||
region.set_multi(mapping)
|
||||
self.assertEqual(api.NO_VALUE, region.get(random_key))
|
||||
self.assertEqual("dummyValue4", region.get(random_key1))
|
||||
self.assertEqual("dummyValue5", region.get(random_key2))
|
||||
self.assertEqual("dummyValue3", region.get(random_key3))
|
||||
|
||||
def test_backend_multi_set_get_with_blanks_none(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments
|
||||
)
|
||||
random_key = uuid.uuid4().hex
|
||||
random_key1 = uuid.uuid4().hex
|
||||
random_key2 = uuid.uuid4().hex
|
||||
random_key3 = uuid.uuid4().hex
|
||||
random_key4 = uuid.uuid4().hex
|
||||
mapping = {random_key1: 'dummyValue1',
|
||||
random_key2: None,
|
||||
random_key3: '',
|
||||
random_key4: 'dummyValue4'}
|
||||
region.set_multi(mapping)
|
||||
# should return NO_VALUE as key does not exist in cache
|
||||
self.assertEqual(api.NO_VALUE, region.get(random_key))
|
||||
self.assertEqual("dummyValue1", region.get(random_key1))
|
||||
self.assertIsNone(region.get(random_key2))
|
||||
self.assertEqual("", region.get(random_key3))
|
||||
self.assertEqual("dummyValue4", region.get(random_key4))
|
||||
|
||||
keys = [random_key, random_key1, random_key2, random_key3, random_key4]
|
||||
results = region.get_multi(keys)
|
||||
|
||||
# should return NO_VALUE as key does not exist in cache
|
||||
self.assertEqual(api.NO_VALUE, results[0])
|
||||
self.assertEqual("dummyValue1", results[1])
|
||||
self.assertIsNone(results[2])
|
||||
self.assertEqual("", results[3])
|
||||
self.assertEqual("dummyValue4", results[4])
|
||||
|
||||
mapping = {random_key1: 'dummyValue5',
|
||||
random_key2: 'dummyValue6'}
|
||||
region.set_multi(mapping)
|
||||
self.assertEqual(api.NO_VALUE, region.get(random_key))
|
||||
self.assertEqual("dummyValue5", region.get(random_key1))
|
||||
self.assertEqual("dummyValue6", region.get(random_key2))
|
||||
self.assertEqual("", region.get(random_key3))
|
||||
|
||||
def test_backend_delete_data(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments
|
||||
)
|
||||
|
||||
random_key = uuid.uuid4().hex
|
||||
region.set(random_key, "dummyValue")
|
||||
self.assertEqual("dummyValue", region.get(random_key))
|
||||
|
||||
region.delete(random_key)
|
||||
# should return NO_VALUE as key no longer exists in cache
|
||||
self.assertEqual(api.NO_VALUE, region.get(random_key))
|
||||
|
||||
def test_backend_multi_delete_data(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments
|
||||
)
|
||||
random_key = uuid.uuid4().hex
|
||||
random_key1 = uuid.uuid4().hex
|
||||
random_key2 = uuid.uuid4().hex
|
||||
random_key3 = uuid.uuid4().hex
|
||||
mapping = {random_key1: 'dummyValue1',
|
||||
random_key2: 'dummyValue2',
|
||||
random_key3: 'dummyValue3'}
|
||||
region.set_multi(mapping)
|
||||
# should return NO_VALUE as key does not exist in cache
|
||||
self.assertEqual(api.NO_VALUE, region.get(random_key))
|
||||
self.assertEqual("dummyValue1", region.get(random_key1))
|
||||
self.assertEqual("dummyValue2", region.get(random_key2))
|
||||
self.assertEqual("dummyValue3", region.get(random_key3))
|
||||
self.assertEqual(api.NO_VALUE, region.get("InvalidKey"))
|
||||
|
||||
keys = mapping.keys()
|
||||
|
||||
region.delete_multi(keys)
|
||||
|
||||
self.assertEqual(api.NO_VALUE, region.get("InvalidKey"))
|
||||
# should return NO_VALUE as keys no longer exist in cache
|
||||
self.assertEqual(api.NO_VALUE, region.get(random_key1))
|
||||
self.assertEqual(api.NO_VALUE, region.get(random_key2))
|
||||
self.assertEqual(api.NO_VALUE, region.get(random_key3))
|
||||
|
||||
def test_additional_crud_method_arguments_support(self):
|
||||
"""Additional arguments should works across find/insert/update."""
|
||||
|
||||
self.arguments['wtimeout'] = 30000
|
||||
self.arguments['j'] = True
|
||||
self.arguments['continue_on_error'] = True
|
||||
self.arguments['secondary_acceptable_latency_ms'] = 60
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.mongo',
|
||||
arguments=self.arguments
|
||||
)
|
||||
|
||||
# There is no proxy so can access MongoCacheBackend directly
|
||||
api_methargs = region.backend.api.meth_kwargs
|
||||
self.assertEqual(30000, api_methargs['wtimeout'])
|
||||
self.assertEqual(True, api_methargs['j'])
|
||||
self.assertEqual(True, api_methargs['continue_on_error'])
|
||||
self.assertEqual(60, api_methargs['secondary_acceptable_latency_ms'])
|
||||
|
||||
random_key = uuid.uuid4().hex
|
||||
region.set(random_key, "dummyValue1")
|
||||
self.assertEqual("dummyValue1", region.get(random_key))
|
||||
|
||||
region.set(random_key, "dummyValue2")
|
||||
self.assertEqual("dummyValue2", region.get(random_key))
|
||||
|
||||
random_key = uuid.uuid4().hex
|
||||
region.set(random_key, "dummyValue3")
|
||||
self.assertEqual("dummyValue3", region.get(random_key))
|
||||
|
|
@ -3,3 +3,8 @@
|
|||
# process, which may cause wedges in the gate later.
|
||||
|
||||
Babel>=1.3
|
||||
dogpile.cache>=0.5.3
|
||||
six>=1.9.0
|
||||
oslo.config>=1.11.0 # Apache-2.0
|
||||
oslo.log>=1.2.0 # Apache-2.0
|
||||
oslo.utils>=1.6.0 # Apache-2.0
|
||||
|
|
|
|||
|
|
@ -5,3 +5,7 @@ hacking<0.11,>=0.10.0
|
|||
oslotest>=1.5.1 # Apache-2.0
|
||||
oslosphinx>=2.5.0 # Apache-2.0
|
||||
sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3
|
||||
testtools>=1.4.0
|
||||
|
||||
# Optional dogpile backend: MongoDB
|
||||
pymongo>=3.0.2
|
||||
|
|
|
|||
2
tox.ini
2
tox.ini
|
|
@ -32,7 +32,7 @@ commands = python setup.py testr --coverage --testr-args='{posargs}'
|
|||
# E123, E125 skipped as they are invalid PEP-8.
|
||||
|
||||
show-source = True
|
||||
ignore = E123,E125
|
||||
ignore = E123,E125,H301,H405
|
||||
builtins = _
|
||||
exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue