Support Redis and Redis Sentinel Cache
currently, only Memecache is supported while the standard Openstack caching library oslo.cache supports Redis as caching backend. This patch adds support Redis and Redis Sentinel w/ optional TLS Implements: redis-cache Change-Id: I4ef161ca5db1df2f289e8508f5d37dba0657d7a2
This commit is contained in:
parent
b81c50fea9
commit
d49c093357
|
@ -641,7 +641,6 @@ class AuthProtocol(BaseAuthProtocol):
|
|||
|
||||
self._www_authenticate_uri = \
|
||||
self._identity_server.www_authenticate_uri
|
||||
|
||||
self._token_cache = self._token_cache_factory()
|
||||
|
||||
def process_request(self, request):
|
||||
|
@ -861,7 +860,8 @@ class AuthProtocol(BaseAuthProtocol):
|
|||
def _token_cache_factory(self):
|
||||
|
||||
security_strategy = self._conf.get('memcache_security_strategy')
|
||||
|
||||
ssl = self._conf.get('redis_tls_enable')
|
||||
cache_backend = self._conf.get('cache_backend').lower()
|
||||
cache_kwargs = dict(
|
||||
cache_time=int(self._conf.get('token_cache_time')),
|
||||
env_cache_name=self._conf.get('cache'),
|
||||
|
@ -872,7 +872,45 @@ class AuthProtocol(BaseAuthProtocol):
|
|||
unused_timeout=self._conf.get('memcache_pool_unused_timeout'),
|
||||
conn_get_timeout=self._conf.get('memcache_pool_conn_get_timeout'),
|
||||
socket_timeout=self._conf.get('memcache_pool_socket_timeout'),
|
||||
cache_backend=cache_backend
|
||||
)
|
||||
if cache_backend in ('redis', 'redis-sentinel'):
|
||||
redis_kwargs = {}
|
||||
login_credentials = dict(
|
||||
username=self._conf.get('redis_username'),
|
||||
password=self._conf.get('redis_password'),
|
||||
)
|
||||
if cache_backend == 'redis-sentinel':
|
||||
redis_kwargs.update(login_credentials)
|
||||
else:
|
||||
cache_kwargs.update(login_credentials)
|
||||
if ssl:
|
||||
redis_kwargs.update(dict(
|
||||
ssl=ssl,
|
||||
ssl_certfile=self._conf.get('redis_tls_cert_file'),
|
||||
ssl_keyfile=self._conf.get('redis_tls_key_file'),
|
||||
ssl_ca_certs=self._conf.get('redis_tls_ca_file')
|
||||
)
|
||||
)
|
||||
cache_kwargs.update(dict(
|
||||
connection_kwargs=redis_kwargs,
|
||||
db=self._conf.get('redis_db')
|
||||
)
|
||||
)
|
||||
if cache_backend == 'redis-sentinel':
|
||||
cache_kwargs.update(dict(
|
||||
service_name=self._conf.get('redis_sentinel_service_name'),
|
||||
redis_sentinel_servers=self._conf.get(
|
||||
'redis_sentinel_servers'),
|
||||
sentinel_kwargs=redis_kwargs
|
||||
)
|
||||
)
|
||||
else:
|
||||
cache_kwargs.update(dict(
|
||||
host=self._conf.get('redis_host'),
|
||||
port=self._conf.get('redis_port'),
|
||||
)
|
||||
)
|
||||
|
||||
if security_strategy.lower() != 'none':
|
||||
secret_key = self._conf.get('memcache_secret_key')
|
||||
|
|
|
@ -104,6 +104,44 @@ class _MemcacheClientPool(object):
|
|||
yield client
|
||||
|
||||
|
||||
class _RedisCachePool(list):
|
||||
|
||||
class _RedisCacheRegion(object):
|
||||
"""A helper class for compatibility of dogpile cache region."""
|
||||
|
||||
def __init__(self, region):
|
||||
self.region = region
|
||||
|
||||
def set(self, key, value, time=0):
|
||||
# NOTE(mjenca): TokenCache passes 3 arguments to set
|
||||
self.region.set(key, value)
|
||||
|
||||
def get(self, key):
|
||||
from dogpile.cache.api import NoValue
|
||||
ret = self.region.get(key)
|
||||
# NOTE: We must return None instead of NoValue
|
||||
if type(ret) == NoValue:
|
||||
return None
|
||||
return ret
|
||||
|
||||
def __init__(self, arguments):
|
||||
self.arguments = arguments
|
||||
|
||||
@contextlib.contextmanager
|
||||
def reserve(self):
|
||||
try:
|
||||
c = self.pop()
|
||||
except IndexError:
|
||||
from dogpile.cache import make_region
|
||||
c = self._RedisCacheRegion(make_region().configure(
|
||||
self.arguments['dp_backend'], arguments=self.arguments
|
||||
))
|
||||
try:
|
||||
yield c
|
||||
finally:
|
||||
self.append(c)
|
||||
|
||||
|
||||
class TokenCache(object):
|
||||
"""Encapsulates the auth_token token cache functionality.
|
||||
|
||||
|
@ -121,32 +159,59 @@ class TokenCache(object):
|
|||
|
||||
_CACHE_KEY_TEMPLATE = 'tokens/%s'
|
||||
|
||||
def __init__(self, log, cache_time=None,
|
||||
env_cache_name=None, memcached_servers=None,
|
||||
def __init__(self, log, cache_time=None, cache_backend='memcache',
|
||||
env_cache_name=None, memcacached_servers=None,
|
||||
use_advanced_pool=True, dead_retry=None, socket_timeout=None,
|
||||
redis_sentinel_servers=None,
|
||||
**kwargs):
|
||||
self._LOG = log
|
||||
self._cache_time = cache_time
|
||||
self._env_cache_name = env_cache_name
|
||||
self._memcached_servers = memcached_servers
|
||||
self._memcached_servers = memcacached_servers
|
||||
self._cache_backend = cache_backend
|
||||
self._redis_sentinel_servers = redis_sentinel_servers
|
||||
self._use_advanced_pool = use_advanced_pool
|
||||
self._arguments = {
|
||||
'dead_retry': dead_retry,
|
||||
'socket_timeout': socket_timeout
|
||||
}
|
||||
self._memcache_pool_options = kwargs
|
||||
|
||||
self._cache_options = kwargs
|
||||
self._cache_pool = None
|
||||
self._initialized = False
|
||||
|
||||
def _get_cache_pool(self, cache):
|
||||
try:
|
||||
host = self._cache_options['host']
|
||||
except KeyError:
|
||||
host = None
|
||||
if cache:
|
||||
return _EnvCachePool(cache)
|
||||
|
||||
elif self._use_advanced_pool and self._memcached_servers:
|
||||
elif (
|
||||
self._use_advanced_pool and
|
||||
self._memcached_servers and
|
||||
self._cache_backend == 'memcache'
|
||||
):
|
||||
return _MemcacheClientPool(self._memcached_servers,
|
||||
self._arguments,
|
||||
**self._memcache_pool_options)
|
||||
**self._cache_options)
|
||||
elif (
|
||||
self._redis_sentinel_servers and
|
||||
self._cache_backend == 'redis-sentinel'
|
||||
):
|
||||
sentinels = [
|
||||
(server.split(":")[0], int(server.split(":")[1]))
|
||||
for server in self._redis_sentinel_servers.split(',')
|
||||
]
|
||||
dp_backend = 'dogpile.cache.redis_sentinel'
|
||||
self._cache_options.update(
|
||||
sentinels=sentinels,
|
||||
dp_backend=dp_backend
|
||||
)
|
||||
return _RedisCachePool(self._cache_options)
|
||||
elif host and self._cache_backend == 'redis':
|
||||
dp_backend = 'dogpile.cache.redis'
|
||||
self._cache_options.update(dp_backend=dp_backend)
|
||||
return _RedisCachePool(self._cache_options)
|
||||
|
||||
else:
|
||||
if not self._use_advanced_pool:
|
||||
|
@ -228,17 +293,14 @@ class TokenCache(object):
|
|||
return
|
||||
|
||||
key, context = self._get_cache_key(token_id)
|
||||
|
||||
with self._cache_pool.reserve() as cache:
|
||||
serialized = cache.get(key)
|
||||
|
||||
if serialized is None:
|
||||
return None
|
||||
|
||||
if isinstance(serialized, str):
|
||||
serialized = serialized.encode('utf8')
|
||||
data = self._deserialize(serialized, context)
|
||||
|
||||
if data is None:
|
||||
# In case decryption fails, e.g. data corrupted in memcached.
|
||||
return None
|
||||
|
@ -256,7 +318,6 @@ class TokenCache(object):
|
|||
|
||||
cache_key, context = self._get_cache_key(token_id)
|
||||
data_to_store = self._serialize(data, context)
|
||||
|
||||
with self._cache_pool.reserve() as cache:
|
||||
cache.set(cache_key, data_to_store, time=self._cache_time)
|
||||
|
||||
|
|
|
@ -181,6 +181,52 @@ _OPTS = [
|
|||
help='The name or type of the service as it appears in the'
|
||||
' service catalog. This is used to validate tokens that have'
|
||||
' restricted access rules.'),
|
||||
cfg.StrOpt('redis_host',
|
||||
help='Redis host. Used only with "redis backend"'
|
||||
' not "redis-sentinel".'),
|
||||
cfg.IntOpt('redis_port',
|
||||
help='Redis port. Used only with "redis backend"'
|
||||
' not "redis-sentinel". Default: 6379',
|
||||
default=6379),
|
||||
cfg.StrOpt('redis_sentinel_servers',
|
||||
default='',
|
||||
help='A comma-separated list of Redis Sentinel server addresses'
|
||||
' in the format "host:port".'
|
||||
' For example, "localhost:6379,localhost:6380".'),
|
||||
cfg.IntOpt('redis_db',
|
||||
default=0,
|
||||
help='The index of the Redis database to use. Default: 0'),
|
||||
cfg.StrOpt('redis_username',
|
||||
default='default',
|
||||
help='The username for Redis authentication. If not provided, '
|
||||
' authentication will not be attempted.'),
|
||||
cfg.StrOpt('redis_password',
|
||||
default='default',
|
||||
help='The password for Redis authentication. If not provided, '
|
||||
' authentication will not be attempted.'),
|
||||
cfg.StrOpt('redis_sentinel_service_name',
|
||||
default='mymaster',
|
||||
help='The name of the Redis service for Redis Sentinel. If not '
|
||||
'using Sentinel, this value is ignored.'),
|
||||
cfg.BoolOpt('redis_tls_enable',
|
||||
default=False,
|
||||
help='Enable TLS encryption for communication with Redis.'),
|
||||
cfg.StrOpt('redis_tls_cert_file',
|
||||
default='redis-cert.pem',
|
||||
help='The path to the certificate file for'
|
||||
' Redis TLS encryption.'),
|
||||
cfg.StrOpt('redis_tls_key_file',
|
||||
default='redis-key.pem',
|
||||
help='The path to the private key file for'
|
||||
' Redis TLS encryption.'),
|
||||
cfg.StrOpt('redis_tls_ca_file',
|
||||
default='ca-cert.pem',
|
||||
help='The path to the CA file for Redis TLS encryption.'),
|
||||
cfg.StrOpt('cache_backend',
|
||||
choices=('none', 'redis', 'redis-sentinel', 'memcache'),
|
||||
default='memcache',
|
||||
help='Caching backend, options are "memcache", "redis", "none"')
|
||||
|
||||
]
|
||||
|
||||
|
||||
|
|
|
@ -36,7 +36,8 @@ from keystonemiddleware.auth_token import _cache
|
|||
from keystonemiddleware.exceptions import ConfigurationError
|
||||
from keystonemiddleware.exceptions import KeystoneMiddlewareException
|
||||
from keystonemiddleware.i18n import _
|
||||
|
||||
from oslo_log import log as logging
|
||||
_LOG = logging.getLogger(__name__)
|
||||
oslo_cache.configure(cfg.CONF)
|
||||
_EXT_AUTH_CONFIG_GROUP_NAME = 'ext_oauth2_auth'
|
||||
_EXTERNAL_AUTH2_OPTS = [
|
||||
|
@ -180,7 +181,53 @@ _EXTERNAL_AUTH2_OPTS = [
|
|||
cfg.BoolOpt('memcache_use_advanced_pool',
|
||||
default=True,
|
||||
help='(Optional) Use the advanced (eventlet safe) memcached '
|
||||
'client pool.')
|
||||
'client pool.'),
|
||||
cfg.StrOpt('redis_host',
|
||||
help='Redis host. Used only with "redis backend"'
|
||||
' not "redis-sentinel".'),
|
||||
cfg.IntOpt('redis_port',
|
||||
help='Redis port. Used only with "redis backend"'
|
||||
' not "redis-sentinel". Default: 6379',
|
||||
default=6379),
|
||||
cfg.StrOpt('redis_sentinel_servers',
|
||||
default='',
|
||||
help='A comma-separated list of Redis Sentinel server addresses'
|
||||
' in the format "host:port".'
|
||||
' For example, "localhost:6379,localhost:6380".'),
|
||||
cfg.IntOpt('redis_db',
|
||||
default=0,
|
||||
help='The index of the Redis database to use. Default: 0'),
|
||||
cfg.StrOpt('redis_username',
|
||||
default='default',
|
||||
help='The username for Redis authentication. If not provided, '
|
||||
' authentication will not be attempted.'),
|
||||
cfg.StrOpt('redis_password',
|
||||
default='default',
|
||||
help='The password for Redis authentication. If not provided, '
|
||||
' authentication will not be attempted.'),
|
||||
cfg.StrOpt('redis_sentinel_service_name',
|
||||
default='mymaster',
|
||||
help='The name of the Redis service for Redis Sentinel. If not '
|
||||
'using Sentinel, this value is ignored.'),
|
||||
cfg.BoolOpt('redis_tls_enable',
|
||||
default=False,
|
||||
help='Enable TLS encryption for communication with Redis.'),
|
||||
cfg.StrOpt('redis_tls_cert_file',
|
||||
default='redis-cert.pem',
|
||||
help='The path to the certificate file for'
|
||||
' Redis TLS encryption.'),
|
||||
cfg.StrOpt('redis_tls_key_file',
|
||||
default='redis-key.pem',
|
||||
help='The path to the private key file for'
|
||||
' Redis TLS encryption.'),
|
||||
cfg.StrOpt('redis_tls_ca_file',
|
||||
default='ca-cert.pem',
|
||||
help='The path to the CA file for Redis TLS encryption.'),
|
||||
cfg.StrOpt('cache_backend',
|
||||
choices=('none', 'redis', 'redis-sentinel', 'memcache'),
|
||||
default='memcache',
|
||||
help='Caching backend, options are "memcache", "redis", "none"')
|
||||
|
||||
]
|
||||
|
||||
cfg.CONF.register_opts(_EXTERNAL_AUTH2_OPTS,
|
||||
|
@ -496,7 +543,6 @@ class ExternalAuth2Protocol(object):
|
|||
all_opts,
|
||||
conf)
|
||||
self._token_cache = self._token_cache_factory()
|
||||
|
||||
self._session = self._create_session()
|
||||
self._audience = self._get_config_option('audience', is_required=True)
|
||||
self._introspect_endpoint = self._get_config_option(
|
||||
|
@ -512,27 +558,66 @@ class ExternalAuth2Protocol(object):
|
|||
|
||||
def _token_cache_factory(self):
|
||||
security_strategy = self._conf.get('memcache_security_strategy')
|
||||
ssl = self._conf.get('redis_tls_enable')
|
||||
cache_backend = self._conf.get('cache_backend').lower()
|
||||
cache_kwargs = dict(
|
||||
cache_time=int(self._conf.get('token_cache_time')),
|
||||
env_cache_name=self._conf.get('cache'),
|
||||
memcached_servers=self._conf.get('memcached_servers'),
|
||||
use_advanced_pool=self._conf.get(
|
||||
'memcache_use_advanced_pool'),
|
||||
use_advanced_pool=self._conf.get('memcache_use_advanced_pool'),
|
||||
dead_retry=self._conf.get('memcache_pool_dead_retry'),
|
||||
maxsize=self._conf.get('memcache_pool_maxsize'),
|
||||
unused_timeout=self._conf.get(
|
||||
'memcache_pool_unused_timeout'),
|
||||
conn_get_timeout=self._conf.get(
|
||||
'memcache_pool_conn_get_timeout'),
|
||||
socket_timeout=self._conf.get(
|
||||
'memcache_pool_socket_timeout'),
|
||||
unused_timeout=self._conf.get('memcache_pool_unused_timeout'),
|
||||
conn_get_timeout=self._conf.get('memcache_pool_conn_get_timeout'),
|
||||
socket_timeout=self._conf.get('memcache_pool_socket_timeout'),
|
||||
cache_backend=cache_backend
|
||||
)
|
||||
if cache_backend in ('redis', 'redis-sentinel'):
|
||||
redis_kwargs = {}
|
||||
login_credentials = dict(
|
||||
username=self._conf.get('redis_username'),
|
||||
password=self._conf.get('redis_password'),
|
||||
)
|
||||
if cache_backend == 'redis-sentinel':
|
||||
redis_kwargs.update(login_credentials)
|
||||
else:
|
||||
cache_kwargs.update(login_credentials)
|
||||
if ssl:
|
||||
redis_kwargs.update(dict(
|
||||
ssl=ssl,
|
||||
ssl_certfile=self._conf.get('redis_tls_cert_file'),
|
||||
ssl_keyfile=self._conf.get('redis_tls_key_file'),
|
||||
ssl_ca_certs=self._conf.get('redis_tls_ca_file')
|
||||
)
|
||||
)
|
||||
cache_kwargs.update(dict(
|
||||
connection_kwargs=redis_kwargs,
|
||||
db=self._conf.get('redis_db')
|
||||
)
|
||||
)
|
||||
if cache_backend == 'redis-sentinel':
|
||||
cache_kwargs.update(dict(
|
||||
service_name=self._conf.get('redis_sentinel_service_name'),
|
||||
redis_sentinel_servers=self._conf.get(
|
||||
'redis_sentinel_servers'),
|
||||
sentinel_kwargs=redis_kwargs
|
||||
)
|
||||
)
|
||||
else:
|
||||
cache_kwargs.update(dict(
|
||||
host=self._conf.get('redis_host'),
|
||||
port=self._conf.get('redis_port'),
|
||||
)
|
||||
)
|
||||
|
||||
if security_strategy.lower() != 'none':
|
||||
secret_key = self._conf.get('memcache_secret_key')
|
||||
return _cache.SecureTokenCache(self._log,
|
||||
return _cache.SecureTokenCache(self.log,
|
||||
security_strategy,
|
||||
secret_key,
|
||||
**cache_kwargs)
|
||||
return _cache.TokenCache(self._log, **cache_kwargs)
|
||||
else:
|
||||
return _cache.TokenCache(self.log, **cache_kwargs)
|
||||
|
||||
@webob.dec.wsgify()
|
||||
def __call__(self, req):
|
||||
|
|
|
@ -0,0 +1,83 @@
|
|||
|
||||
import uuid
|
||||
import redis
|
||||
from keystonemiddleware.auth_token import _cache as cache
|
||||
from keystonemiddleware.tests.unit.auth_token import base
|
||||
from keystonemiddleware.tests.unit import utils
|
||||
from keystonemiddleware.tests.unit.auth_token import test_auth_token_middleware
|
||||
|
||||
|
||||
REDIS_AVAILABLE = False
|
||||
REDIS_TLS = False
|
||||
# BACKEND can be redis or redis-sentinel in this case
|
||||
BACKEND = "redis"
|
||||
REDIS_USERNAME = "default"
|
||||
REDIS_PASSWORD = "pass"
|
||||
REDIS_HOST = "127.0.0.1"
|
||||
REDIS_PORT = 6379
|
||||
REDIS_SENTINEL_SERVERS = "127.0.0.1:26379"
|
||||
REDIS_SERVICE_NAME = "mymaster"
|
||||
REDIS_CA = "/path/to/redis/certs/ca.crt"
|
||||
REDIS_CRT = "/path/to/redis/certs/redis.crt"
|
||||
REDIS_KEY = "redis-certs/redis-key.pem"
|
||||
|
||||
|
||||
class RedisTest(base.BaseAuthTokenTestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
||||
super(RedisTest, self).setUp()
|
||||
self.common_config = {
|
||||
"redis_username": REDIS_USERNAME,
|
||||
"redis_password": REDIS_PASSWORD,
|
||||
"redis_sentinel_servers": REDIS_SENTINEL_SERVERS,
|
||||
"redis_sentinel_service_name": REDIS_SERVICE_NAME,
|
||||
"redis_host": REDIS_HOST,
|
||||
"redis_port": REDIS_PORT,
|
||||
"redis_tls_enable": REDIS_TLS,
|
||||
"redis_tls_cert_file": REDIS_CRT,
|
||||
"redis_tls_key_file": REDIS_KEY,
|
||||
"redis_tls_ca_file": REDIS_CA,
|
||||
"cache_backend": BACKEND,
|
||||
"redis_db": 0
|
||||
}
|
||||
|
||||
if not REDIS_AVAILABLE:
|
||||
self.skipTest("Redis server is not available")
|
||||
|
||||
def _test_redis_helper(self, extra_config={}):
|
||||
def _merge(d1, d2):
|
||||
res = d1.copy()
|
||||
res.update(d2)
|
||||
return res
|
||||
token = uuid.uuid4().hex.encode()
|
||||
data = uuid.uuid4().hex
|
||||
cfg = _merge(self.common_config, extra_config)
|
||||
token_cache = self.create_simple_middleware(conf=cfg)._token_cache
|
||||
token_cache.initialize({})
|
||||
token_cache.set(token, data)
|
||||
self.assertEqual(token_cache.get(token), data)
|
||||
|
||||
def test_redis(self):
|
||||
self._test_redis_helper()
|
||||
|
||||
def test_redis_encrypt_data(self):
|
||||
conf = {
|
||||
'memcache_security_strategy': 'encrypt',
|
||||
'memcache_secret_key': 'mysecret'
|
||||
}
|
||||
self._test_redis_helper(conf)
|
||||
|
||||
def test_redis_mac_data(self):
|
||||
conf = {
|
||||
'memcache_security_strategy': 'mac',
|
||||
'memcache_secret_key': 'mysecret'
|
||||
}
|
||||
self._test_redis_helper(conf)
|
||||
|
||||
def test_no_value(self):
|
||||
token = uuid.uuid4().hex.encode()
|
||||
token_cache = self.create_simple_middleware(
|
||||
conf=self.common_config)._token_cache
|
||||
token_cache.initialize({})
|
||||
self.assertIsNone(token_cache.get(token))
|
Loading…
Reference in New Issue