From 28411250dadaf73f0263b55e1d07337fc79b202b Mon Sep 17 00:00:00 2001 From: Takashi Kajinami Date: Sat, 3 Feb 2024 21:10:28 +0900 Subject: [PATCH] Add native options for redis backend This introduces a few new redis_* options to simplify the settings required to use the redis backend. The main aim of this change is to replace the requirement to inject url by backend_argument. [cache] backend=dogpile.cache.redis redis_server=127.0.0.1:6379 redis_username=default redis_password=a_big_secret redis_socket_timeout=1.0 Closes-Bug: #2052351 Change-Id: Id72878f9cddaa99146eab5fb4ee76c8e6a633809 --- oslo_cache/_opts.py | 12 +++ oslo_cache/core.py | 89 ++++++++++++------- oslo_cache/tests/unit/test_cache_basics.py | 37 +++++++- .../redis-backend-opts-27915f2b672512c9.yaml | 16 ++++ 4 files changed, 122 insertions(+), 32 deletions(-) create mode 100644 releasenotes/notes/redis-backend-opts-27915f2b672512c9.yaml diff --git a/oslo_cache/_opts.py b/oslo_cache/_opts.py index 0787959f..569a2401 100644 --- a/oslo_cache/_opts.py +++ b/oslo_cache/_opts.py @@ -128,6 +128,18 @@ FILE_OPTIONS = { default='', secret=True, help='the password for the memcached which SASL enabled'), + cfg.StrOpt('redis_server', + default='localhost:6379', + help='Redis server in the format of "host:port"'), + cfg.StrOpt('redis_username', + help='the user name for redis'), + cfg.StrOpt('redis_password', + secret=True, + help='the password for redis'), + cfg.FloatOpt('redis_socket_timeout', + default=1.0, + help='Timeout in seconds for every call to a server.' + ' (dogpile.cache.redis backend only).'), cfg.BoolOpt('tls_enabled', default=False, help='Global toggle for TLS usage when communicating with' diff --git a/oslo_cache/core.py b/oslo_cache/core.py index 2ae6e087..b5a8ce07 100644 --- a/oslo_cache/core.py +++ b/oslo_cache/core.py @@ -35,6 +35,7 @@ The library has special public value for nonexistent or expired keys called NO_VALUE = core.NO_VALUE """ import ssl +import urllib.parse import dogpile.cache from dogpile.cache import api @@ -136,37 +137,63 @@ def _build_cache_config(conf): conf_dict[arg_key] = argvalue _LOG.debug('Oslo Cache Config: %s', conf_dict) - # NOTE(yorik-sar): these arguments will be used for memcache-related - # backends. Use setdefault for url to support old-style setting through - # backend_argument=url:127.0.0.1:11211 - # - # NOTE(morgan): If requested by config, 'flush_on_reconnect' will be set - # for pooled connections. This can ensure that stale data is never - # consumed from a server that pops in/out due to a network partition - # or disconnect. - # - # See the help from python-memcached: - # - # param flush_on_reconnect: optional flag which prevents a - # scenario that can cause stale data to be read: If there's more - # than one memcached server and the connection to one is - # interrupted, keys that mapped to that server will get - # reassigned to another. If the first server comes back, those - # keys will map to it again. If it still has its data, get()s - # can read stale data that was overwritten on another - # server. This flag is off by default for backwards - # compatibility. - # - # The normal non-pooled clients connect explicitly on each use and - # does not need the explicit flush_on_reconnect - conf_dict.setdefault('%s.arguments.url' % prefix, - conf.cache.memcache_servers) - for arg in ('dead_retry', 'socket_timeout', 'pool_maxsize', - 'pool_unused_timeout', 'pool_connection_get_timeout', - 'pool_flush_on_reconnect', 'sasl_enabled', 'username', - 'password'): - value = getattr(conf.cache, 'memcache_' + arg) - conf_dict['%s.arguments.%s' % (prefix, arg)] = value + + if conf.cache.backend == 'dogpile.cache.redis': + if conf.cache.redis_password is None: + netloc = conf.cache.redis_server + else: + if conf.cache.redis_username: + netloc = '%s:%s@%s' % (conf.cache.redis_username, + conf.cache.redis_password, + conf.cache.redis_server) + else: + netloc = ':%s@%s' % (conf.cache.redis_password, + conf.cache.redis_server) + + parts = urllib.parse.ParseResult( + scheme=('rediss' if conf.cache.tls_enabled else 'redis'), + netloc=netloc, path='', params='', query='', fragment='') + + conf_dict.setdefault( + '%s.arguments.url' % prefix, + urllib.parse.urlunparse(parts) + ) + for arg in ('socket_timeout',): + value = getattr(conf.cache, 'redis_' + arg) + conf_dict['%s.arguments.%s' % (prefix, arg)] = value + else: + # NOTE(yorik-sar): these arguments will be used for memcache-related + # backends. Use setdefault for url to support old-style setting through + # backend_argument=url:127.0.0.1:11211 + # + # NOTE(morgan): If requested by config, 'flush_on_reconnect' will be + # set for pooled connections. This can ensure that stale data is never + # consumed from a server that pops in/out due to a network partition + # or disconnect. + # + # See the help from python-memcached: + # + # param flush_on_reconnect: optional flag which prevents a + # scenario that can cause stale data to be read: If there's more + # than one memcached server and the connection to one is + # interrupted, keys that mapped to that server will get + # reassigned to another. If the first server comes back, those + # keys will map to it again. If it still has its data, get()s + # can read stale data that was overwritten on another + # server. This flag is off by default for backwards + # compatibility. + # + # The normal non-pooled clients connect explicitly on each use and + # does not need the explicit flush_on_reconnect + conf_dict.setdefault('%s.arguments.url' % prefix, + conf.cache.memcache_servers) + + for arg in ('dead_retry', 'socket_timeout', 'pool_maxsize', + 'pool_unused_timeout', 'pool_connection_get_timeout', + 'pool_flush_on_reconnect', 'sasl_enabled', 'username', + 'password'): + value = getattr(conf.cache, 'memcache_' + arg) + conf_dict['%s.arguments.%s' % (prefix, arg)] = value if conf.cache.tls_enabled: if conf.cache.backend in ('dogpile.cache.bmemcache', diff --git a/oslo_cache/tests/unit/test_cache_basics.py b/oslo_cache/tests/unit/test_cache_basics.py index eb2181a3..8791df38 100644 --- a/oslo_cache/tests/unit/test_cache_basics.py +++ b/oslo_cache/tests/unit/test_cache_basics.py @@ -306,7 +306,9 @@ class CacheRegionTest(test_cache.BaseTestCase): tls_allowed_ciphers='allowed_ciphers') config_dict = cache._build_cache_config(self.config_fixture.conf) - + self.assertEqual( + 'redis://localhost:6379', + config_dict['test_prefix.arguments.url']) self.assertFalse(self.config_fixture.conf.cache.tls_enabled) self.assertNotIn('test_prefix.arguments.connection_kwargs', config_dict) @@ -351,6 +353,9 @@ class CacheRegionTest(test_cache.BaseTestCase): self.assertTrue(self.config_fixture.conf.cache.tls_enabled) self.assertIn('test_prefix.arguments.connection_kwargs', config_dict) + self.assertEqual( + 'rediss://localhost:6379', + config_dict['test_prefix.arguments.url']) self.assertEqual( { 'ssl_ca_certs': 'path_to_ca_file', @@ -694,6 +699,36 @@ class CacheRegionTest(test_cache.BaseTestCase): self.assertFalse(config_dict['test_prefix.arguments' '.pool_flush_on_reconnect']) + def test_cache_dictionary_config_builder_redis(self): + """Validate the backend is reset to default if caching is disabled.""" + self.config_fixture.config(group='cache', + config_prefix='test_prefix', + backend='dogpile.cache.redis', + redis_server='[::1]:6379', + redis_username='user', + redis_password='secrete') + + config_dict = cache._build_cache_config(self.config_fixture.conf) + self.assertEqual( + 'redis://user:secrete@[::1]:6379', + config_dict['test_prefix.arguments.url']) + self.assertEqual( + 1.0, config_dict['test_prefix.arguments.socket_timeout']) + + def test_cache_dictionary_config_builder_redis_with_auth(self): + """Validate the backend is reset to default if caching is disabled.""" + self.config_fixture.config(group='cache', + config_prefix='test_prefix', + backend='dogpile.cache.redis', + redis_server='[::1]:6379', + redis_username='user', + redis_password='secrete') + + config_dict = cache._build_cache_config(self.config_fixture.conf) + self.assertEqual( + 'redis://user:secrete@[::1]:6379', + config_dict['test_prefix.arguments.url']) + def test_cache_debug_proxy(self): single_value = 'Test Value' single_key = 'testkey' diff --git a/releasenotes/notes/redis-backend-opts-27915f2b672512c9.yaml b/releasenotes/notes/redis-backend-opts-27915f2b672512c9.yaml new file mode 100644 index 00000000..6b3d3994 --- /dev/null +++ b/releasenotes/notes/redis-backend-opts-27915f2b672512c9.yaml @@ -0,0 +1,16 @@ +--- +features: + - | + The following new options are added. These options are used to customize + connections in the ``dogpile.cache.redis`` backend. + + - ``redis_server`` + - ``redis_username`` + - ``redis_password`` + - ``redis_socket_timeout`` + +upgrade: + - | + The ``[cache] memcache_socket_timeout`` option no longer takes affect in + when the ``dogpile.cache.redis`` backend, which is the documented behavior. + Use the ``[cache] redis_socket_timeout`` option instead.