openstack
/
keystonemiddleware
Code Issues Proposed changes

Support Redis and Redis Sentinel Cache 

currently, only Memecache is supported while the
standard Openstack caching library oslo.cache
supports Redis as caching backend.

This patch adds support Redis and Redis Sentinel w/ optional TLS

Implements: redis-cache
Change-Id: I4ef161ca5db1df2f289e8508f5d37dba0657d7a2
This commit is contained in:
Matus Jenca 2024-04-15 15:14:30 +02:00
parent b81c50fea9
commit abf598d0ce
5 changed files with 345 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
keystonemiddleware/auth_token/__init__.py
View File

@ -641,7 +641,6 @@ class AuthProtocol(BaseAuthProtocol):
self._www_authenticate_uri = \
self._identity_server.www_authenticate_uri
self._token_cache = self._token_cache_factory()
def process_request(self, request):
@ -861,7 +860,8 @@ class AuthProtocol(BaseAuthProtocol):
def _token_cache_factory(self):
security_strategy = self._conf.get('memcache_security_strategy')
ssl = self._conf.get('redis_tls_enable')
cache_backend = self._conf.get('cache_backend').lower()
cache_kwargs = dict(
cache_time=int(self._conf.get('token_cache_time')),
env_cache_name=self._conf.get('cache'),
@ -872,7 +872,45 @@ class AuthProtocol(BaseAuthProtocol):
unused_timeout=self._conf.get('memcache_pool_unused_timeout'),
conn_get_timeout=self._conf.get('memcache_pool_conn_get_timeout'),
socket_timeout=self._conf.get('memcache_pool_socket_timeout'),
cache_backend=cache_backend
)
if cache_backend in ('redis', 'redis-sentinel'):
redis_kwargs = {}
login_credentials = dict(
username=self._conf.get('redis_username'),
password=self._conf.get('redis_password'),
)
if cache_backend == 'redis-sentinel':
redis_kwargs.update(login_credentials)
else:
cache_kwargs.update(login_credentials)
if ssl:
redis_kwargs.update(dict(
ssl=ssl,
ssl_certfile=self._conf.get('redis_tls_cert_file'),
ssl_keyfile=self._conf.get('redis_tls_key_file'),
ssl_ca_certs=self._conf.get('redis_tls_ca_file')
)
)
cache_kwargs.update(dict(
connection_kwargs=redis_kwargs,
db=self._conf.get('redis_db')
)
)
if cache_backend == 'redis-sentinel':
cache_kwargs.update(dict(
service_name=self._conf.get('redis_sentinel_service_name'),
redis_sentinel_servers=self._conf.get(
'redis_sentinel_servers'),
sentinel_kwargs=redis_kwargs
)
)
else:
cache_kwargs.update(dict(
host=self._conf.get('redis_host'),
port=self._conf.get('redis_port'),
)
)
if security_strategy.lower() != 'none':
secret_key = self._conf.get('memcache_secret_key')

85
keystonemiddleware/auth_token/_cache.py
View File

@ -104,6 +104,44 @@ class _MemcacheClientPool(object):
yield client
class _RedisCachePool(list):
class _RedisCacheRegion(object):
"""A helper class for compatibility of dogpile cache region."""
def __init__(self, region):
self.region = region
def set(self, key, value, time=0):
# NOTE(mjenca): TokenCache passes 3 arguments to set
self.region.set(key, value)
def get(self, key):
from dogpile.cache.api import NoValue
ret = self.region.get(key)
# NOTE: We must return None instead of NoValue
if isinstance(ret, NoValue):
return None
return ret
def __init__(self, arguments):
self.arguments = arguments
@contextlib.contextmanager
def reserve(self):
try:
c = self.pop()
except IndexError:
from dogpile.cache import make_region
c = self._RedisCacheRegion(make_region().configure(
self.arguments['dp_backend'], arguments=self.arguments
))
try:
yield c
finally:
self.append(c)
class TokenCache(object):
"""Encapsulates the auth_token token cache functionality.
@ -121,32 +159,59 @@ class TokenCache(object):
_CACHE_KEY_TEMPLATE = 'tokens/%s'
def __init__(self, log, cache_time=None,
env_cache_name=None, memcached_servers=None,
def __init__(self, log, cache_time=None, cache_backend='memcache',
env_cache_name=None, memcacached_servers=None,
use_advanced_pool=True, dead_retry=None, socket_timeout=None,
redis_sentinel_servers=None,
**kwargs):
self._LOG = log
self._cache_time = cache_time
self._env_cache_name = env_cache_name
self._memcached_servers = memcached_servers
self._memcached_servers = memcacached_servers
self._cache_backend = cache_backend
self._redis_sentinel_servers = redis_sentinel_servers
self._use_advanced_pool = use_advanced_pool
self._arguments = {
'dead_retry': dead_retry,
'socket_timeout': socket_timeout
}
self._memcache_pool_options = kwargs
self._cache_options = kwargs
self._cache_pool = None
self._initialized = False
def _get_cache_pool(self, cache):
try:
host = self._cache_options['host']
except KeyError:
host = None
if cache:
return _EnvCachePool(cache)
elif self._use_advanced_pool and self._memcached_servers:
elif (
self._use_advanced_pool and
self._memcached_servers and
self._cache_backend == 'memcache'
):
return _MemcacheClientPool(self._memcached_servers,
self._arguments,
**self._memcache_pool_options)
**self._cache_options)
elif (
self._redis_sentinel_servers and
self._cache_backend == 'redis-sentinel'
):
sentinels = [
(server.split(":")[0], int(server.split(":")[1]))
for server in self._redis_sentinel_servers.split(',')
]
dp_backend = 'dogpile.cache.redis_sentinel'
self._cache_options.update(
sentinels=sentinels,
dp_backend=dp_backend
)
return _RedisCachePool(self._cache_options)
elif host and self._cache_backend == 'redis':
dp_backend = 'dogpile.cache.redis'
self._cache_options.update(dp_backend=dp_backend)
return _RedisCachePool(self._cache_options)
else:
if not self._use_advanced_pool:
@ -228,17 +293,14 @@ class TokenCache(object):
return
key, context = self._get_cache_key(token_id)
with self._cache_pool.reserve() as cache:
serialized = cache.get(key)
if serialized is None:
return None
if isinstance(serialized, str):
serialized = serialized.encode('utf8')
data = self._deserialize(serialized, context)
if data is None:
# In case decryption fails, e.g. data corrupted in memcached.
return None
@ -256,7 +318,6 @@ class TokenCache(object):
cache_key, context = self._get_cache_key(token_id)
data_to_store = self._serialize(data, context)
with self._cache_pool.reserve() as cache:
cache.set(cache_key, data_to_store, time=self._cache_time)

46
keystonemiddleware/auth_token/_opts.py
View File

@ -181,6 +181,52 @@ _OPTS = [
help='The name or type of the service as it appears in the'
' service catalog. This is used to validate tokens that have'
' restricted access rules.'),
cfg.StrOpt('redis_host',
help='Redis host. Used only with "redis backend"'
' not "redis-sentinel".'),
cfg.IntOpt('redis_port',
help='Redis port. Used only with "redis backend"'
' not "redis-sentinel". Default: 6379',
default=6379),
cfg.StrOpt('redis_sentinel_servers',
default='',
help='A comma-separated list of Redis Sentinel server addresses'
' in the format "host:port".'
' For example, "localhost:6379,localhost:6380".'),
cfg.IntOpt('redis_db',
default=0,
help='The index of the Redis database to use. Default: 0'),
cfg.StrOpt('redis_username',
default='default',
help='The username for Redis authentication. If not provided, '
' authentication will not be attempted.'),
cfg.StrOpt('redis_password',
default='default',
help='The password for Redis authentication. If not provided, '
' authentication will not be attempted.'),
cfg.StrOpt('redis_sentinel_service_name',
default='mymaster',
help='The name of the Redis service for Redis Sentinel. If not '
'using Sentinel, this value is ignored.'),
cfg.BoolOpt('redis_tls_enable',
default=False,
help='Enable TLS encryption for communication with Redis.'),
cfg.StrOpt('redis_tls_cert_file',
default='redis-cert.pem',
help='The path to the certificate file for'
' Redis TLS encryption.'),
cfg.StrOpt('redis_tls_key_file',
default='redis-key.pem',
help='The path to the private key file for'
' Redis TLS encryption.'),
cfg.StrOpt('redis_tls_ca_file',
default='ca-cert.pem',
help='The path to the CA file for Redis TLS encryption.'),
cfg.StrOpt('cache_backend',
choices=('none', 'redis', 'redis-sentinel', 'memcache'),
default='memcache',
help='Caching backend, options are "memcache", "redis", "none"')
]

108
keystonemiddleware/external_oauth2_token.py
View File

@ -180,7 +180,53 @@ _EXTERNAL_AUTH2_OPTS = [
cfg.BoolOpt('memcache_use_advanced_pool',
default=True,
help='(Optional) Use the advanced (eventlet safe) memcached '
'client pool.')
'client pool.'),
cfg.StrOpt('redis_host',
help='Redis host. Used only with "redis backend"'
' not "redis-sentinel".'),
cfg.IntOpt('redis_port',
help='Redis port. Used only with "redis backend"'
' not "redis-sentinel". Default: 6379',
default=6379),
cfg.StrOpt('redis_sentinel_servers',
default='',
help='A comma-separated list of Redis Sentinel server addresses'
' in the format "host:port".'
' For example, "localhost:6379,localhost:6380".'),
cfg.IntOpt('redis_db',
default=0,
help='The index of the Redis database to use. Default: 0'),
cfg.StrOpt('redis_username',
default='default',
help='The username for Redis authentication. If not provided, '
' authentication will not be attempted.'),
cfg.StrOpt('redis_password',
default='default',
help='The password for Redis authentication. If not provided, '
' authentication will not be attempted.'),
cfg.StrOpt('redis_sentinel_service_name',
default='mymaster',
help='The name of the Redis service for Redis Sentinel. If not '
'using Sentinel, this value is ignored.'),
cfg.BoolOpt('redis_tls_enable',
default=False,
help='Enable TLS encryption for communication with Redis.'),
cfg.StrOpt('redis_tls_cert_file',
default='redis-cert.pem',
help='The path to the certificate file for'
' Redis TLS encryption.'),
cfg.StrOpt('redis_tls_key_file',
default='redis-key.pem',
help='The path to the private key file for'
' Redis TLS encryption.'),
cfg.StrOpt('redis_tls_ca_file',
default='ca-cert.pem',
help='The path to the CA file for Redis TLS encryption.'),
cfg.StrOpt('cache_backend',
choices=('none', 'redis', 'redis-sentinel', 'memcache'),
default='memcache',
help='Caching backend, options are "memcache", "redis", "none"')
]
cfg.CONF.register_opts(_EXTERNAL_AUTH2_OPTS,
@ -496,7 +542,6 @@ class ExternalAuth2Protocol(object):
all_opts,
conf)
self._token_cache = self._token_cache_factory()
self._session = self._create_session()
self._audience = self._get_config_option('audience', is_required=True)
self._introspect_endpoint = self._get_config_option(
@ -512,27 +557,66 @@ class ExternalAuth2Protocol(object):
def _token_cache_factory(self):
security_strategy = self._conf.get('memcache_security_strategy')
ssl = self._conf.get('redis_tls_enable')
cache_backend = self._conf.get('cache_backend').lower()
cache_kwargs = dict(
cache_time=int(self._conf.get('token_cache_time')),
env_cache_name=self._conf.get('cache'),
memcached_servers=self._conf.get('memcached_servers'),
use_advanced_pool=self._conf.get(
'memcache_use_advanced_pool'),
use_advanced_pool=self._conf.get('memcache_use_advanced_pool'),
dead_retry=self._conf.get('memcache_pool_dead_retry'),
maxsize=self._conf.get('memcache_pool_maxsize'),
unused_timeout=self._conf.get(
'memcache_pool_unused_timeout'),
conn_get_timeout=self._conf.get(
'memcache_pool_conn_get_timeout'),
socket_timeout=self._conf.get(
'memcache_pool_socket_timeout'),
unused_timeout=self._conf.get('memcache_pool_unused_timeout'),
conn_get_timeout=self._conf.get('memcache_pool_conn_get_timeout'),
socket_timeout=self._conf.get('memcache_pool_socket_timeout'),
cache_backend=cache_backend
)
if cache_backend in ('redis', 'redis-sentinel'):
redis_kwargs = {}
login_credentials = dict(
username=self._conf.get('redis_username'),
password=self._conf.get('redis_password'),
)
if cache_backend == 'redis-sentinel':
redis_kwargs.update(login_credentials)
else:
cache_kwargs.update(login_credentials)
if ssl:
redis_kwargs.update(dict(
ssl=ssl,
ssl_certfile=self._conf.get('redis_tls_cert_file'),
ssl_keyfile=self._conf.get('redis_tls_key_file'),
ssl_ca_certs=self._conf.get('redis_tls_ca_file')
)
)
cache_kwargs.update(dict(
connection_kwargs=redis_kwargs,
db=self._conf.get('redis_db')
)
)
if cache_backend == 'redis-sentinel':
cache_kwargs.update(dict(
service_name=self._conf.get('redis_sentinel_service_name'),
redis_sentinel_servers=self._conf.get(
'redis_sentinel_servers'),
sentinel_kwargs=redis_kwargs
)
)
else:
cache_kwargs.update(dict(
host=self._conf.get('redis_host'),
port=self._conf.get('redis_port'),
)
)
if security_strategy.lower() != 'none':
secret_key = self._conf.get('memcache_secret_key')
return _cache.SecureTokenCache(self._log,
return _cache.SecureTokenCache(self.log,
security_strategy,
secret_key,
**cache_kwargs)
return _cache.TokenCache(self._log, **cache_kwargs)
else:
return _cache.TokenCache(self.log, **cache_kwargs)
@webob.dec.wsgify()
def __call__(self, req):

90
keystonemiddleware/tests/unit/auth_token/test_cache_redis.py Normal file
View File

@ -0,0 +1,90 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystonemiddleware.tests.unit.auth_token import base
import uuid
REDIS_AVAILABLE = False
REDIS_TLS = False
# BACKEND can be redis or redis-sentinel in this case
BACKEND = "redis"
REDIS_USERNAME = "default"
REDIS_PASSWORD = "pass"
REDIS_HOST = "127.0.0.1"
REDIS_PORT = 6379
REDIS_SENTINEL_SERVERS = "127.0.0.1:26379"
REDIS_SERVICE_NAME = "mymaster"
REDIS_CA = "/path/to/redis/certs/ca.crt"
REDIS_CRT = "/path/to/redis/certs/redis.crt"
REDIS_KEY = "redis-certs/redis-key.pem"
class RedisTest(base.BaseAuthTokenTestCase):
def setUp(self):
super(RedisTest, self).setUp()
self.common_config = {
"redis_username": REDIS_USERNAME,
"redis_password": REDIS_PASSWORD,
"redis_sentinel_servers": REDIS_SENTINEL_SERVERS,
"redis_sentinel_service_name": REDIS_SERVICE_NAME,
"redis_host": REDIS_HOST,
"redis_port": REDIS_PORT,
"redis_tls_enable": REDIS_TLS,
"redis_tls_cert_file": REDIS_CRT,
"redis_tls_key_file": REDIS_KEY,
"redis_tls_ca_file": REDIS_CA,
"cache_backend": BACKEND,
"redis_db": 0
}
if not REDIS_AVAILABLE:
self.skipTest("Redis server is not available")
def _test_redis_helper(self, extra_config={}):
def _merge(d1, d2):
res = d1.copy()
res.update(d2)
return res
token = uuid.uuid4().hex.encode()
data = uuid.uuid4().hex
cfg = _merge(self.common_config, extra_config)
token_cache = self.create_simple_middleware(conf=cfg)._token_cache
token_cache.initialize({})
token_cache.set(token, data)
self.assertEqual(token_cache.get(token), data)
def test_redis(self):
self._test_redis_helper()
def test_redis_encrypt_data(self):
conf = {
'memcache_security_strategy': 'encrypt',
'memcache_secret_key': 'mysecret'
}
self._test_redis_helper(conf)
def test_redis_mac_data(self):
conf = {
'memcache_security_strategy': 'mac',
'memcache_secret_key': 'mysecret'
}
self._test_redis_helper(conf)
def test_no_value(self):
token = uuid.uuid4().hex.encode()
token_cache = self.create_simple_middleware(
conf=self.common_config)._token_cache
token_cache.initialize({})
self.assertIsNone(token_cache.get(token))