Support Redis and Redis Sentinel Cache
currently, only Memecache is supported while the standard Openstack caching library oslo.cache supports Redis as caching backend. This patch adds support Redis and Redis Sentinel w/ optional TLS Implements: redis-cache Change-Id: I4ef161ca5db1df2f289e8508f5d37dba0657d7a2
This commit is contained in:
parent
b81c50fea9
commit
abf598d0ce
|
@ -641,7 +641,6 @@ class AuthProtocol(BaseAuthProtocol):
|
|||
|
||||
self._www_authenticate_uri = \
|
||||
self._identity_server.www_authenticate_uri
|
||||
|
||||
self._token_cache = self._token_cache_factory()
|
||||
|
||||
def process_request(self, request):
|
||||
|
@ -861,7 +860,8 @@ class AuthProtocol(BaseAuthProtocol):
|
|||
def _token_cache_factory(self):
|
||||
|
||||
security_strategy = self._conf.get('memcache_security_strategy')
|
||||
|
||||
ssl = self._conf.get('redis_tls_enable')
|
||||
cache_backend = self._conf.get('cache_backend').lower()
|
||||
cache_kwargs = dict(
|
||||
cache_time=int(self._conf.get('token_cache_time')),
|
||||
env_cache_name=self._conf.get('cache'),
|
||||
|
@ -872,7 +872,45 @@ class AuthProtocol(BaseAuthProtocol):
|
|||
unused_timeout=self._conf.get('memcache_pool_unused_timeout'),
|
||||
conn_get_timeout=self._conf.get('memcache_pool_conn_get_timeout'),
|
||||
socket_timeout=self._conf.get('memcache_pool_socket_timeout'),
|
||||
cache_backend=cache_backend
|
||||
)
|
||||
if cache_backend in ('redis', 'redis-sentinel'):
|
||||
redis_kwargs = {}
|
||||
login_credentials = dict(
|
||||
username=self._conf.get('redis_username'),
|
||||
password=self._conf.get('redis_password'),
|
||||
)
|
||||
if cache_backend == 'redis-sentinel':
|
||||
redis_kwargs.update(login_credentials)
|
||||
else:
|
||||
cache_kwargs.update(login_credentials)
|
||||
if ssl:
|
||||
redis_kwargs.update(dict(
|
||||
ssl=ssl,
|
||||
ssl_certfile=self._conf.get('redis_tls_cert_file'),
|
||||
ssl_keyfile=self._conf.get('redis_tls_key_file'),
|
||||
ssl_ca_certs=self._conf.get('redis_tls_ca_file')
|
||||
)
|
||||
)
|
||||
cache_kwargs.update(dict(
|
||||
connection_kwargs=redis_kwargs,
|
||||
db=self._conf.get('redis_db')
|
||||
)
|
||||
)
|
||||
if cache_backend == 'redis-sentinel':
|
||||
cache_kwargs.update(dict(
|
||||
service_name=self._conf.get('redis_sentinel_service_name'),
|
||||
redis_sentinel_servers=self._conf.get(
|
||||
'redis_sentinel_servers'),
|
||||
sentinel_kwargs=redis_kwargs
|
||||
)
|
||||
)
|
||||
else:
|
||||
cache_kwargs.update(dict(
|
||||
host=self._conf.get('redis_host'),
|
||||
port=self._conf.get('redis_port'),
|
||||
)
|
||||
)
|
||||
|
||||
if security_strategy.lower() != 'none':
|
||||
secret_key = self._conf.get('memcache_secret_key')
|
||||
|
|
|
@ -104,6 +104,44 @@ class _MemcacheClientPool(object):
|
|||
yield client
|
||||
|
||||
|
||||
class _RedisCachePool(list):
|
||||
|
||||
class _RedisCacheRegion(object):
|
||||
"""A helper class for compatibility of dogpile cache region."""
|
||||
|
||||
def __init__(self, region):
|
||||
self.region = region
|
||||
|
||||
def set(self, key, value, time=0):
|
||||
# NOTE(mjenca): TokenCache passes 3 arguments to set
|
||||
self.region.set(key, value)
|
||||
|
||||
def get(self, key):
|
||||
from dogpile.cache.api import NoValue
|
||||
ret = self.region.get(key)
|
||||
# NOTE: We must return None instead of NoValue
|
||||
if isinstance(ret, NoValue):
|
||||
return None
|
||||
return ret
|
||||
|
||||
def __init__(self, arguments):
|
||||
self.arguments = arguments
|
||||
|
||||
@contextlib.contextmanager
|
||||
def reserve(self):
|
||||
try:
|
||||
c = self.pop()
|
||||
except IndexError:
|
||||
from dogpile.cache import make_region
|
||||
c = self._RedisCacheRegion(make_region().configure(
|
||||
self.arguments['dp_backend'], arguments=self.arguments
|
||||
))
|
||||
try:
|
||||
yield c
|
||||
finally:
|
||||
self.append(c)
|
||||
|
||||
|
||||
class TokenCache(object):
|
||||
"""Encapsulates the auth_token token cache functionality.
|
||||
|
||||
|
@ -121,32 +159,59 @@ class TokenCache(object):
|
|||
|
||||
_CACHE_KEY_TEMPLATE = 'tokens/%s'
|
||||
|
||||
def __init__(self, log, cache_time=None,
|
||||
env_cache_name=None, memcached_servers=None,
|
||||
def __init__(self, log, cache_time=None, cache_backend='memcache',
|
||||
env_cache_name=None, memcacached_servers=None,
|
||||
use_advanced_pool=True, dead_retry=None, socket_timeout=None,
|
||||
redis_sentinel_servers=None,
|
||||
**kwargs):
|
||||
self._LOG = log
|
||||
self._cache_time = cache_time
|
||||
self._env_cache_name = env_cache_name
|
||||
self._memcached_servers = memcached_servers
|
||||
self._memcached_servers = memcacached_servers
|
||||
self._cache_backend = cache_backend
|
||||
self._redis_sentinel_servers = redis_sentinel_servers
|
||||
self._use_advanced_pool = use_advanced_pool
|
||||
self._arguments = {
|
||||
'dead_retry': dead_retry,
|
||||
'socket_timeout': socket_timeout
|
||||
}
|
||||
self._memcache_pool_options = kwargs
|
||||
|
||||
self._cache_options = kwargs
|
||||
self._cache_pool = None
|
||||
self._initialized = False
|
||||
|
||||
def _get_cache_pool(self, cache):
|
||||
try:
|
||||
host = self._cache_options['host']
|
||||
except KeyError:
|
||||
host = None
|
||||
if cache:
|
||||
return _EnvCachePool(cache)
|
||||
|
||||
elif self._use_advanced_pool and self._memcached_servers:
|
||||
elif (
|
||||
self._use_advanced_pool and
|
||||
self._memcached_servers and
|
||||
self._cache_backend == 'memcache'
|
||||
):
|
||||
return _MemcacheClientPool(self._memcached_servers,
|
||||
self._arguments,
|
||||
**self._memcache_pool_options)
|
||||
**self._cache_options)
|
||||
elif (
|
||||
self._redis_sentinel_servers and
|
||||
self._cache_backend == 'redis-sentinel'
|
||||
):
|
||||
sentinels = [
|
||||
(server.split(":")[0], int(server.split(":")[1]))
|
||||
for server in self._redis_sentinel_servers.split(',')
|
||||
]
|
||||
dp_backend = 'dogpile.cache.redis_sentinel'
|
||||
self._cache_options.update(
|
||||
sentinels=sentinels,
|
||||
dp_backend=dp_backend
|
||||
)
|
||||
return _RedisCachePool(self._cache_options)
|
||||
elif host and self._cache_backend == 'redis':
|
||||
dp_backend = 'dogpile.cache.redis'
|
||||
self._cache_options.update(dp_backend=dp_backend)
|
||||
return _RedisCachePool(self._cache_options)
|
||||
|
||||
else:
|
||||
if not self._use_advanced_pool:
|
||||
|
@ -228,17 +293,14 @@ class TokenCache(object):
|
|||
return
|
||||
|
||||
key, context = self._get_cache_key(token_id)
|
||||
|
||||
with self._cache_pool.reserve() as cache:
|
||||
serialized = cache.get(key)
|
||||
|
||||
if serialized is None:
|
||||
return None
|
||||
|
||||
if isinstance(serialized, str):
|
||||
serialized = serialized.encode('utf8')
|
||||
data = self._deserialize(serialized, context)
|
||||
|
||||
if data is None:
|
||||
# In case decryption fails, e.g. data corrupted in memcached.
|
||||
return None
|
||||
|
@ -256,7 +318,6 @@ class TokenCache(object):
|
|||
|
||||
cache_key, context = self._get_cache_key(token_id)
|
||||
data_to_store = self._serialize(data, context)
|
||||
|
||||
with self._cache_pool.reserve() as cache:
|
||||
cache.set(cache_key, data_to_store, time=self._cache_time)
|
||||
|
||||
|
|
|
@ -181,6 +181,52 @@ _OPTS = [
|
|||
help='The name or type of the service as it appears in the'
|
||||
' service catalog. This is used to validate tokens that have'
|
||||
' restricted access rules.'),
|
||||
cfg.StrOpt('redis_host',
|
||||
help='Redis host. Used only with "redis backend"'
|
||||
' not "redis-sentinel".'),
|
||||
cfg.IntOpt('redis_port',
|
||||
help='Redis port. Used only with "redis backend"'
|
||||
' not "redis-sentinel". Default: 6379',
|
||||
default=6379),
|
||||
cfg.StrOpt('redis_sentinel_servers',
|
||||
default='',
|
||||
help='A comma-separated list of Redis Sentinel server addresses'
|
||||
' in the format "host:port".'
|
||||
' For example, "localhost:6379,localhost:6380".'),
|
||||
cfg.IntOpt('redis_db',
|
||||
default=0,
|
||||
help='The index of the Redis database to use. Default: 0'),
|
||||
cfg.StrOpt('redis_username',
|
||||
default='default',
|
||||
help='The username for Redis authentication. If not provided, '
|
||||
' authentication will not be attempted.'),
|
||||
cfg.StrOpt('redis_password',
|
||||
default='default',
|
||||
help='The password for Redis authentication. If not provided, '
|
||||
' authentication will not be attempted.'),
|
||||
cfg.StrOpt('redis_sentinel_service_name',
|
||||
default='mymaster',
|
||||
help='The name of the Redis service for Redis Sentinel. If not '
|
||||
'using Sentinel, this value is ignored.'),
|
||||
cfg.BoolOpt('redis_tls_enable',
|
||||
default=False,
|
||||
help='Enable TLS encryption for communication with Redis.'),
|
||||
cfg.StrOpt('redis_tls_cert_file',
|
||||
default='redis-cert.pem',
|
||||
help='The path to the certificate file for'
|
||||
' Redis TLS encryption.'),
|
||||
cfg.StrOpt('redis_tls_key_file',
|
||||
default='redis-key.pem',
|
||||
help='The path to the private key file for'
|
||||
' Redis TLS encryption.'),
|
||||
cfg.StrOpt('redis_tls_ca_file',
|
||||
default='ca-cert.pem',
|
||||
help='The path to the CA file for Redis TLS encryption.'),
|
||||
cfg.StrOpt('cache_backend',
|
||||
choices=('none', 'redis', 'redis-sentinel', 'memcache'),
|
||||
default='memcache',
|
||||
help='Caching backend, options are "memcache", "redis", "none"')
|
||||
|
||||
]
|
||||
|
||||
|
||||
|
|
|
@ -180,7 +180,53 @@ _EXTERNAL_AUTH2_OPTS = [
|
|||
cfg.BoolOpt('memcache_use_advanced_pool',
|
||||
default=True,
|
||||
help='(Optional) Use the advanced (eventlet safe) memcached '
|
||||
'client pool.')
|
||||
'client pool.'),
|
||||
cfg.StrOpt('redis_host',
|
||||
help='Redis host. Used only with "redis backend"'
|
||||
' not "redis-sentinel".'),
|
||||
cfg.IntOpt('redis_port',
|
||||
help='Redis port. Used only with "redis backend"'
|
||||
' not "redis-sentinel". Default: 6379',
|
||||
default=6379),
|
||||
cfg.StrOpt('redis_sentinel_servers',
|
||||
default='',
|
||||
help='A comma-separated list of Redis Sentinel server addresses'
|
||||
' in the format "host:port".'
|
||||
' For example, "localhost:6379,localhost:6380".'),
|
||||
cfg.IntOpt('redis_db',
|
||||
default=0,
|
||||
help='The index of the Redis database to use. Default: 0'),
|
||||
cfg.StrOpt('redis_username',
|
||||
default='default',
|
||||
help='The username for Redis authentication. If not provided, '
|
||||
' authentication will not be attempted.'),
|
||||
cfg.StrOpt('redis_password',
|
||||
default='default',
|
||||
help='The password for Redis authentication. If not provided, '
|
||||
' authentication will not be attempted.'),
|
||||
cfg.StrOpt('redis_sentinel_service_name',
|
||||
default='mymaster',
|
||||
help='The name of the Redis service for Redis Sentinel. If not '
|
||||
'using Sentinel, this value is ignored.'),
|
||||
cfg.BoolOpt('redis_tls_enable',
|
||||
default=False,
|
||||
help='Enable TLS encryption for communication with Redis.'),
|
||||
cfg.StrOpt('redis_tls_cert_file',
|
||||
default='redis-cert.pem',
|
||||
help='The path to the certificate file for'
|
||||
' Redis TLS encryption.'),
|
||||
cfg.StrOpt('redis_tls_key_file',
|
||||
default='redis-key.pem',
|
||||
help='The path to the private key file for'
|
||||
' Redis TLS encryption.'),
|
||||
cfg.StrOpt('redis_tls_ca_file',
|
||||
default='ca-cert.pem',
|
||||
help='The path to the CA file for Redis TLS encryption.'),
|
||||
cfg.StrOpt('cache_backend',
|
||||
choices=('none', 'redis', 'redis-sentinel', 'memcache'),
|
||||
default='memcache',
|
||||
help='Caching backend, options are "memcache", "redis", "none"')
|
||||
|
||||
]
|
||||
|
||||
cfg.CONF.register_opts(_EXTERNAL_AUTH2_OPTS,
|
||||
|
@ -496,7 +542,6 @@ class ExternalAuth2Protocol(object):
|
|||
all_opts,
|
||||
conf)
|
||||
self._token_cache = self._token_cache_factory()
|
||||
|
||||
self._session = self._create_session()
|
||||
self._audience = self._get_config_option('audience', is_required=True)
|
||||
self._introspect_endpoint = self._get_config_option(
|
||||
|
@ -512,27 +557,66 @@ class ExternalAuth2Protocol(object):
|
|||
|
||||
def _token_cache_factory(self):
|
||||
security_strategy = self._conf.get('memcache_security_strategy')
|
||||
ssl = self._conf.get('redis_tls_enable')
|
||||
cache_backend = self._conf.get('cache_backend').lower()
|
||||
cache_kwargs = dict(
|
||||
cache_time=int(self._conf.get('token_cache_time')),
|
||||
env_cache_name=self._conf.get('cache'),
|
||||
memcached_servers=self._conf.get('memcached_servers'),
|
||||
use_advanced_pool=self._conf.get(
|
||||
'memcache_use_advanced_pool'),
|
||||
use_advanced_pool=self._conf.get('memcache_use_advanced_pool'),
|
||||
dead_retry=self._conf.get('memcache_pool_dead_retry'),
|
||||
maxsize=self._conf.get('memcache_pool_maxsize'),
|
||||
unused_timeout=self._conf.get(
|
||||
'memcache_pool_unused_timeout'),
|
||||
conn_get_timeout=self._conf.get(
|
||||
'memcache_pool_conn_get_timeout'),
|
||||
socket_timeout=self._conf.get(
|
||||
'memcache_pool_socket_timeout'),
|
||||
unused_timeout=self._conf.get('memcache_pool_unused_timeout'),
|
||||
conn_get_timeout=self._conf.get('memcache_pool_conn_get_timeout'),
|
||||
socket_timeout=self._conf.get('memcache_pool_socket_timeout'),
|
||||
cache_backend=cache_backend
|
||||
)
|
||||
if cache_backend in ('redis', 'redis-sentinel'):
|
||||
redis_kwargs = {}
|
||||
login_credentials = dict(
|
||||
username=self._conf.get('redis_username'),
|
||||
password=self._conf.get('redis_password'),
|
||||
)
|
||||
if cache_backend == 'redis-sentinel':
|
||||
redis_kwargs.update(login_credentials)
|
||||
else:
|
||||
cache_kwargs.update(login_credentials)
|
||||
if ssl:
|
||||
redis_kwargs.update(dict(
|
||||
ssl=ssl,
|
||||
ssl_certfile=self._conf.get('redis_tls_cert_file'),
|
||||
ssl_keyfile=self._conf.get('redis_tls_key_file'),
|
||||
ssl_ca_certs=self._conf.get('redis_tls_ca_file')
|
||||
)
|
||||
)
|
||||
cache_kwargs.update(dict(
|
||||
connection_kwargs=redis_kwargs,
|
||||
db=self._conf.get('redis_db')
|
||||
)
|
||||
)
|
||||
if cache_backend == 'redis-sentinel':
|
||||
cache_kwargs.update(dict(
|
||||
service_name=self._conf.get('redis_sentinel_service_name'),
|
||||
redis_sentinel_servers=self._conf.get(
|
||||
'redis_sentinel_servers'),
|
||||
sentinel_kwargs=redis_kwargs
|
||||
)
|
||||
)
|
||||
else:
|
||||
cache_kwargs.update(dict(
|
||||
host=self._conf.get('redis_host'),
|
||||
port=self._conf.get('redis_port'),
|
||||
)
|
||||
)
|
||||
|
||||
if security_strategy.lower() != 'none':
|
||||
secret_key = self._conf.get('memcache_secret_key')
|
||||
return _cache.SecureTokenCache(self._log,
|
||||
return _cache.SecureTokenCache(self.log,
|
||||
security_strategy,
|
||||
secret_key,
|
||||
**cache_kwargs)
|
||||
return _cache.TokenCache(self._log, **cache_kwargs)
|
||||
else:
|
||||
return _cache.TokenCache(self.log, **cache_kwargs)
|
||||
|
||||
@webob.dec.wsgify()
|
||||
def __call__(self, req):
|
||||
|
|
|
@ -0,0 +1,90 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
from keystonemiddleware.tests.unit.auth_token import base
|
||||
import uuid
|
||||
|
||||
REDIS_AVAILABLE = False
|
||||
REDIS_TLS = False
|
||||
# BACKEND can be redis or redis-sentinel in this case
|
||||
BACKEND = "redis"
|
||||
REDIS_USERNAME = "default"
|
||||
REDIS_PASSWORD = "pass"
|
||||
REDIS_HOST = "127.0.0.1"
|
||||
REDIS_PORT = 6379
|
||||
REDIS_SENTINEL_SERVERS = "127.0.0.1:26379"
|
||||
REDIS_SERVICE_NAME = "mymaster"
|
||||
REDIS_CA = "/path/to/redis/certs/ca.crt"
|
||||
REDIS_CRT = "/path/to/redis/certs/redis.crt"
|
||||
REDIS_KEY = "redis-certs/redis-key.pem"
|
||||
|
||||
|
||||
class RedisTest(base.BaseAuthTokenTestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
||||
super(RedisTest, self).setUp()
|
||||
self.common_config = {
|
||||
"redis_username": REDIS_USERNAME,
|
||||
"redis_password": REDIS_PASSWORD,
|
||||
"redis_sentinel_servers": REDIS_SENTINEL_SERVERS,
|
||||
"redis_sentinel_service_name": REDIS_SERVICE_NAME,
|
||||
"redis_host": REDIS_HOST,
|
||||
"redis_port": REDIS_PORT,
|
||||
"redis_tls_enable": REDIS_TLS,
|
||||
"redis_tls_cert_file": REDIS_CRT,
|
||||
"redis_tls_key_file": REDIS_KEY,
|
||||
"redis_tls_ca_file": REDIS_CA,
|
||||
"cache_backend": BACKEND,
|
||||
"redis_db": 0
|
||||
}
|
||||
|
||||
if not REDIS_AVAILABLE:
|
||||
self.skipTest("Redis server is not available")
|
||||
|
||||
def _test_redis_helper(self, extra_config={}):
|
||||
def _merge(d1, d2):
|
||||
res = d1.copy()
|
||||
res.update(d2)
|
||||
return res
|
||||
token = uuid.uuid4().hex.encode()
|
||||
data = uuid.uuid4().hex
|
||||
cfg = _merge(self.common_config, extra_config)
|
||||
token_cache = self.create_simple_middleware(conf=cfg)._token_cache
|
||||
token_cache.initialize({})
|
||||
token_cache.set(token, data)
|
||||
self.assertEqual(token_cache.get(token), data)
|
||||
|
||||
def test_redis(self):
|
||||
self._test_redis_helper()
|
||||
|
||||
def test_redis_encrypt_data(self):
|
||||
conf = {
|
||||
'memcache_security_strategy': 'encrypt',
|
||||
'memcache_secret_key': 'mysecret'
|
||||
}
|
||||
self._test_redis_helper(conf)
|
||||
|
||||
def test_redis_mac_data(self):
|
||||
conf = {
|
||||
'memcache_security_strategy': 'mac',
|
||||
'memcache_secret_key': 'mysecret'
|
||||
}
|
||||
self._test_redis_helper(conf)
|
||||
|
||||
def test_no_value(self):
|
||||
token = uuid.uuid4().hex.encode()
|
||||
token_cache = self.create_simple_middleware(
|
||||
conf=self.common_config)._token_cache
|
||||
token_cache.initialize({})
|
||||
self.assertIsNone(token_cache.get(token))
|
Loading…
Reference in New Issue