From 56efc8ac76415070041d125a6e30523caf9b3cbd Mon Sep 17 00:00:00 2001 From: Fang Zhen Date: Wed, 9 Mar 2016 14:46:12 +0800 Subject: [PATCH] Switch to oslo.cache Oslo incubator is about to stop for cache module. We could use oslo.cache instead. The legacy memory backend is replaced by oslo_cache.dict. Closes-Bug: #1517883 Change-Id: I108242ca9f27c9ec47959ce7615bc7d84cae014b --- etc/oslo-config-generator/metadata_agent.ini | 1 + neutron/agent/metadata/agent.py | 12 +- neutron/agent/metadata_agent.py | 2 +- neutron/common/cache_utils.py | 150 +++++++++++ neutron/common/utils.py | 63 ----- neutron/openstack/common/__init__.py | 0 neutron/openstack/common/cache/__init__.py | 0 .../common/cache/_backends/__init__.py | 0 .../common/cache/_backends/memory.py | 166 ------------ neutron/openstack/common/cache/backends.py | 250 ------------------ neutron/openstack/common/cache/cache.py | 92 ------- neutron/opts.py | 4 +- .../tests/unit/agent/metadata/test_agent.py | 52 ++-- neutron/tests/unit/common/test_cache_utils.py | 130 +++++++++ neutron/tests/unit/common/test_utils.py | 58 ---- ...cache-url-deprecated-16cd3d335c5962eb.yaml | 9 + requirements.txt | 1 + 17 files changed, 331 insertions(+), 659 deletions(-) create mode 100644 neutron/common/cache_utils.py delete mode 100644 neutron/openstack/common/__init__.py delete mode 100644 neutron/openstack/common/cache/__init__.py delete mode 100644 neutron/openstack/common/cache/_backends/__init__.py delete mode 100644 neutron/openstack/common/cache/_backends/memory.py delete mode 100644 neutron/openstack/common/cache/backends.py delete mode 100644 neutron/openstack/common/cache/cache.py create mode 100644 neutron/tests/unit/common/test_cache_utils.py create mode 100644 releasenotes/notes/oslo-cache-cache-url-deprecated-16cd3d335c5962eb.yaml diff --git a/etc/oslo-config-generator/metadata_agent.ini b/etc/oslo-config-generator/metadata_agent.ini index 5c27309a6ca..a154af4f906 100644 --- a/etc/oslo-config-generator/metadata_agent.ini +++ b/etc/oslo-config-generator/metadata_agent.ini @@ -4,3 +4,4 @@ wrap_width = 79 namespace = neutron.metadata.agent namespace = oslo.log +namespace = oslo.cache \ No newline at end of file diff --git a/neutron/agent/metadata/agent.py b/neutron/agent/metadata/agent.py index c772fb6ea39..9069ea6b540 100644 --- a/neutron/agent/metadata/agent.py +++ b/neutron/agent/metadata/agent.py @@ -30,12 +30,11 @@ from neutron._i18n import _, _LE, _LW from neutron.agent.linux import utils as agent_utils from neutron.agent.metadata import config from neutron.agent import rpc as agent_rpc +from neutron.common import cache_utils as cache from neutron.common import constants as n_const from neutron.common import rpc as n_rpc from neutron.common import topics -from neutron.common import utils from neutron import context -from neutron.openstack.common.cache import cache LOG = logging.getLogger(__name__) @@ -76,10 +75,7 @@ class MetadataProxyHandler(object): def __init__(self, conf): self.conf = conf - if self.conf.cache_url: - self._cache = cache.get_cache(self.conf.cache_url) - else: - self._cache = False + self._cache = cache.get_cache(self.conf) self.plugin_rpc = MetadataPluginAPI(topics.PLUGIN) self.context = context.get_admin_context_without_session() @@ -121,13 +117,13 @@ class MetadataProxyHandler(object): return filters - @utils.cache_method_results + @cache.cache_method_results def _get_router_networks(self, router_id): """Find all networks connected to given router.""" internal_ports = self._get_ports_from_server(router_id=router_id) return tuple(p['network_id'] for p in internal_ports) - @utils.cache_method_results + @cache.cache_method_results def _get_ports_for_remote_address(self, remote_address, networks): """Get list of ports that has given ip address and are part of given networks. diff --git a/neutron/agent/metadata_agent.py b/neutron/agent/metadata_agent.py index b392ed1f3ce..aa75ddb56a4 100644 --- a/neutron/agent/metadata_agent.py +++ b/neutron/agent/metadata_agent.py @@ -20,9 +20,9 @@ from oslo_log import log as logging from neutron.agent.common import config as agent_conf from neutron.agent.metadata import agent from neutron.agent.metadata import config as metadata_conf +from neutron.common import cache_utils as cache from neutron.common import config from neutron.common import utils -from neutron.openstack.common.cache import cache LOG = logging.getLogger(__name__) diff --git a/neutron/common/cache_utils.py b/neutron/common/cache_utils.py new file mode 100644 index 00000000000..97c46110b1a --- /dev/null +++ b/neutron/common/cache_utils.py @@ -0,0 +1,150 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools + +from oslo_cache import core as cache +from oslo_config import cfg +from oslo_log import log as logging +from oslo_utils import reflection +from six.moves.urllib import parse + +from neutron._i18n import _ +from neutron.common import utils + + +cache_opts = [ + cfg.StrOpt('cache_url', default='', + deprecated_for_removal=True, + help=_('URL to connect to the cache back end. ' + 'This option is deprecated in the Newton release and ' + 'will be removed. Please add a [cache] group for ' + 'oslo.cache in your neutron.conf and add "enable" and ' + '"backend" options in this section.')), +] + +LOG = logging.getLogger(__name__) + + +def register_oslo_configs(conf): + conf.register_opts(cache_opts) + cache.configure(conf) + + +def get_cache(conf): + """Used to get cache client""" + # cache_url is still used, we just respect it. Memory backend is the only + # backend supported before and default_ttl is the only options of Memory + # backend. We use dict backend of oslo.cache for this. + if conf.cache_url: + return _get_cache_region_for_legacy(conf.cache_url) + elif conf.cache.enabled: + return _get_cache_region(conf) + else: + return False + + +def _get_cache_region(conf): + region = cache.create_region() + cache.configure_cache_region(conf, region) + return region + + +def _get_cache_region_for_legacy(url): + parsed = parse.urlparse(url) + backend = parsed.scheme + + if backend == 'memory': + backend = 'oslo_cache.dict' + query = parsed.query + # NOTE(fangzhen): The following NOTE and code is from legacy + # oslo-incubator cache module. Previously reside in neutron at + # neutron/openstack/common/cache/cache.py:78 + # NOTE(flaper87): We need the following hack + # for python versions < 2.7.5. Previous versions + # of python parsed query params just for 'known' + # schemes. This was changed in this patch: + # http://hg.python.org/cpython/rev/79e6ff3d9afd + if not query and '?' in parsed.path: + query = parsed.path.split('?', 1)[-1] + parameters = parse.parse_qs(query) + expiration_time = int(parameters.get('default_ttl', [0])[0]) + + region = cache.create_region() + region.configure(backend, expiration_time=expiration_time) + return region + else: + raise RuntimeError(_('Old style configuration can use only memory ' + '(dict) backend')) + + +class cache_method_results(object): + """This decorator is intended for object methods only.""" + + def __init__(self, func): + self.func = func + functools.update_wrapper(self, func) + self._first_call = True + self._not_cached = cache.NO_VALUE + + def _get_from_cache(self, target_self, *args, **kwargs): + target_self_cls_name = reflection.get_class_name(target_self, + fully_qualified=False) + func_name = "%(module)s.%(class)s.%(func_name)s" % { + 'module': target_self.__module__, + 'class': target_self_cls_name, + 'func_name': self.func.__name__, + } + key = (func_name,) + args + if kwargs: + key += utils.dict2tuple(kwargs) + try: + item = target_self._cache.get(key) + except TypeError: + LOG.debug("Method %(func_name)s cannot be cached due to " + "unhashable parameters: args: %(args)s, kwargs: " + "%(kwargs)s", + {'func_name': func_name, + 'args': args, + 'kwargs': kwargs}) + return self.func(target_self, *args, **kwargs) + + if item is self._not_cached: + item = self.func(target_self, *args, **kwargs) + target_self._cache.set(key, item) + + return item + + def __call__(self, target_self, *args, **kwargs): + target_self_cls_name = reflection.get_class_name(target_self, + fully_qualified=False) + if not hasattr(target_self, '_cache'): + raise NotImplementedError( + _("Instance of class %(module)s.%(class)s must contain _cache " + "attribute") % { + 'module': target_self.__module__, + 'class': target_self_cls_name}) + if not target_self._cache: + if self._first_call: + LOG.debug("Instance of class %(module)s.%(class)s doesn't " + "contain attribute _cache therefore results " + "cannot be cached for %(func_name)s.", + {'module': target_self.__module__, + 'class': target_self_cls_name, + 'func_name': self.func.__name__}) + self._first_call = False + return self.func(target_self, *args, **kwargs) + return self._get_from_cache(target_self, *args, **kwargs) + + def __get__(self, obj, objtype): + return functools.partial(self.__call__, obj) diff --git a/neutron/common/utils.py b/neutron/common/utils.py index 9744c6a3c15..46aa899d43f 100644 --- a/neutron/common/utils.py +++ b/neutron/common/utils.py @@ -42,7 +42,6 @@ from oslo_db import exception as db_exc from oslo_log import log as logging from oslo_utils import excutils from oslo_utils import importutils -from oslo_utils import reflection import six from stevedore import driver @@ -58,68 +57,6 @@ MAX_UINT16 = 0xffff synchronized = lockutils.synchronized_with_prefix(SYNCHRONIZED_PREFIX) -class cache_method_results(object): - """This decorator is intended for object methods only.""" - - def __init__(self, func): - self.func = func - functools.update_wrapper(self, func) - self._first_call = True - self._not_cached = object() - - def _get_from_cache(self, target_self, *args, **kwargs): - target_self_cls_name = reflection.get_class_name(target_self, - fully_qualified=False) - func_name = "%(module)s.%(class)s.%(func_name)s" % { - 'module': target_self.__module__, - 'class': target_self_cls_name, - 'func_name': self.func.__name__, - } - key = (func_name,) + args - if kwargs: - key += dict2tuple(kwargs) - try: - item = target_self._cache.get(key, self._not_cached) - except TypeError: - LOG.debug("Method %(func_name)s cannot be cached due to " - "unhashable parameters: args: %(args)s, kwargs: " - "%(kwargs)s", - {'func_name': func_name, - 'args': args, - 'kwargs': kwargs}) - return self.func(target_self, *args, **kwargs) - - if item is self._not_cached: - item = self.func(target_self, *args, **kwargs) - target_self._cache.set(key, item, None) - - return item - - def __call__(self, target_self, *args, **kwargs): - target_self_cls_name = reflection.get_class_name(target_self, - fully_qualified=False) - if not hasattr(target_self, '_cache'): - raise NotImplementedError( - _("Instance of class %(module)s.%(class)s must contain _cache " - "attribute") % { - 'module': target_self.__module__, - 'class': target_self_cls_name}) - if not target_self._cache: - if self._first_call: - LOG.debug("Instance of class %(module)s.%(class)s doesn't " - "contain attribute _cache therefore results " - "cannot be cached for %(func_name)s.", - {'module': target_self.__module__, - 'class': target_self_cls_name, - 'func_name': self.func.__name__}) - self._first_call = False - return self.func(target_self, *args, **kwargs) - return self._get_from_cache(target_self, *args, **kwargs) - - def __get__(self, obj, objtype): - return functools.partial(self.__call__, obj) - - def ensure_dir(dir_path): """Ensure a directory with 755 permissions mode.""" try: diff --git a/neutron/openstack/common/__init__.py b/neutron/openstack/common/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/neutron/openstack/common/cache/__init__.py b/neutron/openstack/common/cache/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/neutron/openstack/common/cache/_backends/__init__.py b/neutron/openstack/common/cache/_backends/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/neutron/openstack/common/cache/_backends/memory.py b/neutron/openstack/common/cache/_backends/memory.py deleted file mode 100644 index 8d14de6eb0d..00000000000 --- a/neutron/openstack/common/cache/_backends/memory.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import collections - -from oslo_concurrency import lockutils -from oslo_utils import timeutils - -from neutron.openstack.common.cache import backends - - -class MemoryBackend(backends.BaseCache): - - def __init__(self, parsed_url, options=None): - super(MemoryBackend, self).__init__(parsed_url, options) - self._clear() - - def _set_unlocked(self, key, value, ttl=0): - expires_at = 0 - if ttl != 0: - expires_at = timeutils.utcnow_ts() + ttl - - self._cache[key] = (expires_at, value) - - if expires_at: - self._keys_expires[expires_at].add(key) - - def _set(self, key, value, ttl=0, not_exists=False): - with lockutils.lock(key): - - # NOTE(flaper87): This is needed just in `set` - # calls, hence it's not in `_set_unlocked` - if not_exists and self._exists_unlocked(key): - return False - - self._set_unlocked(key, value, ttl) - return True - - def _get_unlocked(self, key, default=None): - now = timeutils.utcnow_ts() - - try: - timeout, value = self._cache[key] - except KeyError: - return (0, default) - - if timeout and now >= timeout: - - # NOTE(flaper87): Record expired, - # remove it from the cache but catch - # KeyError and ValueError in case - # _purge_expired removed this key already. - try: - del self._cache[key] - except KeyError: - pass - - try: - # NOTE(flaper87): Keys with ttl == 0 - # don't exist in the _keys_expires dict - self._keys_expires[timeout].remove(key) - except (KeyError, ValueError): - pass - - return (0, default) - - return (timeout, value) - - def _get(self, key, default=None): - with lockutils.lock(key): - return self._get_unlocked(key, default)[1] - - def _exists_unlocked(self, key): - now = timeutils.utcnow_ts() - try: - timeout = self._cache[key][0] - return not timeout or now <= timeout - except KeyError: - return False - - def __contains__(self, key): - with lockutils.lock(key): - return self._exists_unlocked(key) - - def _incr_append(self, key, other): - with lockutils.lock(key): - timeout, value = self._get_unlocked(key) - - if value is None: - return None - - ttl = timeutils.utcnow_ts() - timeout - new_value = value + other - self._set_unlocked(key, new_value, ttl) - return new_value - - def _incr(self, key, delta): - if not isinstance(delta, int): - raise TypeError('delta must be an int instance') - - return self._incr_append(key, delta) - - def _append_tail(self, key, tail): - return self._incr_append(key, tail) - - def _purge_expired(self): - """Removes expired keys from the cache.""" - - now = timeutils.utcnow_ts() - for timeout in sorted(self._keys_expires.keys()): - - # NOTE(flaper87): If timeout is greater - # than `now`, stop the iteration, remaining - # keys have not expired. - if now < timeout: - break - - # NOTE(flaper87): Unset every key in - # this set from the cache if its timeout - # is equal to `timeout`. (The key might - # have been updated) - for subkey in self._keys_expires.pop(timeout): - try: - if self._cache[subkey][0] == timeout: - del self._cache[subkey] - except KeyError: - continue - - def __delitem__(self, key): - self._purge_expired() - - # NOTE(flaper87): Delete the key. Using pop - # since it could have been deleted already - value = self._cache.pop(key, None) - - if value: - try: - # NOTE(flaper87): Keys with ttl == 0 - # don't exist in the _keys_expires dict - self._keys_expires[value[0]].remove(key) - except (KeyError, ValueError): - pass - - def _clear(self): - self._cache = {} - self._keys_expires = collections.defaultdict(set) - - def _get_many(self, keys, default): - return super(MemoryBackend, self)._get_many(keys, default) - - def _set_many(self, data, ttl=0): - return super(MemoryBackend, self)._set_many(data, ttl) - - def _unset_many(self, keys): - return super(MemoryBackend, self)._unset_many(keys) diff --git a/neutron/openstack/common/cache/backends.py b/neutron/openstack/common/cache/backends.py deleted file mode 100644 index 1bea8912a00..00000000000 --- a/neutron/openstack/common/cache/backends.py +++ /dev/null @@ -1,250 +0,0 @@ -# Copyright 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc - -import six - - -NOTSET = object() - - -@six.add_metaclass(abc.ABCMeta) -class BaseCache(object): - """Base Cache Abstraction - - :params parsed_url: Parsed url object. - :params options: A dictionary with configuration parameters - for the cache. For example: - - - default_ttl: An integer defining the default ttl for keys. - """ - - def __init__(self, parsed_url, options=None): - self._parsed_url = parsed_url - self._options = options or {} - self._default_ttl = int(self._options.get('default_ttl', 0)) - - @abc.abstractmethod - def _set(self, key, value, ttl, not_exists=False): - """Implementations of this class have to override this method.""" - - def set(self, key, value, ttl, not_exists=False): - """Sets or updates a cache entry - - .. note:: Thread-safety is required and has to be guaranteed by the - backend implementation. - - :params key: Item key as string. - :type key: `unicode string` - :params value: Value to assign to the key. This can be anything that - is handled by the current backend. - :params ttl: Key's timeout in seconds. 0 means no timeout. - :type ttl: int - :params not_exists: If True, the key will be set if it doesn't exist. - Otherwise, it'll always be set. - :type not_exists: bool - - :returns: True if the operation succeeds, False otherwise. - """ - if ttl is None: - ttl = self._default_ttl - - return self._set(key, value, ttl, not_exists) - - def __setitem__(self, key, value): - self.set(key, value, self._default_ttl) - - def setdefault(self, key, value): - """Sets the key value to `value` if it doesn't exist - - :params key: Item key as string. - :type key: `unicode string` - :params value: Value to assign to the key. This can be anything that - is handled by the current backend. - """ - try: - return self[key] - except KeyError: - self[key] = value - return value - - @abc.abstractmethod - def _get(self, key, default): - """Implementations of this class have to override this method.""" - - def get(self, key, default=None): - """Gets one item from the cache - - .. note:: Thread-safety is required and it has to be guaranteed - by the backend implementation. - - :params key: Key for the item to retrieve from the cache. - :params default: The default value to return. - - :returns: `key`'s value in the cache if it exists, otherwise - `default` should be returned. - """ - return self._get(key, default) - - def __getitem__(self, key): - value = self.get(key, NOTSET) - - if value is NOTSET: - raise KeyError - - return value - - @abc.abstractmethod - def __delitem__(self, key): - """Removes an item from cache. - - .. note:: Thread-safety is required and it has to be guaranteed by - the backend implementation. - - :params key: The key to remove. - - :returns: The key value if there's one - """ - - @abc.abstractmethod - def _clear(self): - """Implementations of this class have to override this method.""" - - def clear(self): - """Removes all items from the cache. - - .. note:: Thread-safety is required and it has to be guaranteed by - the backend implementation. - """ - return self._clear() - - @abc.abstractmethod - def _incr(self, key, delta): - """Implementations of this class have to override this method.""" - - def incr(self, key, delta=1): - """Increments the value for a key - - :params key: The key for the value to be incremented - :params delta: Number of units by which to increment the value. - Pass a negative number to decrement the value. - - :returns: The new value - """ - return self._incr(key, delta) - - @abc.abstractmethod - def _append_tail(self, key, tail): - """Implementations of this class have to override this method.""" - - def append_tail(self, key, tail): - """Appends `tail` to `key`'s value. - - :params key: The key of the value to which `tail` should be appended. - :params tail: The list of values to append to the original. - - :returns: The new value - """ - - if not hasattr(tail, "__iter__"): - raise TypeError('Tail must be an iterable') - - if not isinstance(tail, list): - # NOTE(flaper87): Make sure we pass a list - # down to the implementation. Not all drivers - # have support for generators, sets or other - # iterables. - tail = list(tail) - - return self._append_tail(key, tail) - - def append(self, key, value): - """Appends `value` to `key`'s value. - - :params key: The key of the value to which `tail` should be appended. - :params value: The value to append to the original. - - :returns: The new value - """ - return self.append_tail(key, [value]) - - @abc.abstractmethod - def __contains__(self, key): - """Verifies that a key exists. - - :params key: The key to verify. - - :returns: True if the key exists, otherwise False. - """ - - @abc.abstractmethod - def _get_many(self, keys, default): - """Implementations of this class have to override this method.""" - return ((k, self.get(k, default=default)) for k in keys) - - def get_many(self, keys, default=NOTSET): - """Gets keys' value from cache - - :params keys: List of keys to retrieve. - :params default: The default value to return for each key that is not - in the cache. - - :returns: A generator of (key, value) - """ - return self._get_many(keys, default) - - @abc.abstractmethod - def _set_many(self, data, ttl): - """Implementations of this class have to override this method.""" - - for key, value in data.items(): - self.set(key, value, ttl=ttl) - - def set_many(self, data, ttl=None): - """Puts several items into the cache at once - - Depending on the backend, this operation may or may not be efficient. - The default implementation calls set for each (key, value) pair - passed, other backends support set_many operations as part of their - protocols. - - :params data: A dictionary like {key: val} to store in the cache. - :params ttl: Key's timeout in seconds. - """ - - if ttl is None: - ttl = self._default_ttl - - self._set_many(data, ttl) - - def update(self, **kwargs): - """Sets several (key, value) paris. - - Refer to the `set_many` docstring. - """ - self.set_many(kwargs, ttl=self._default_ttl) - - @abc.abstractmethod - def _unset_many(self, keys): - """Implementations of this class have to override this method.""" - for key in keys: - del self[key] - - def unset_many(self, keys): - """Removes several keys from the cache at once - - :params keys: List of keys to unset. - """ - self._unset_many(keys) diff --git a/neutron/openstack/common/cache/cache.py b/neutron/openstack/common/cache/cache.py deleted file mode 100644 index 5b77b6b44a7..00000000000 --- a/neutron/openstack/common/cache/cache.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Cache library. - -Supported configuration options: - -`default_backend`: Name of the cache backend to use. -`key_namespace`: Namespace under which keys will be created. -""" - -######################################################################## -# -# THIS MODULE IS DEPRECATED -# -# Please refer to -# https://etherpad.openstack.org/p/kilo-neutron-library-proposals for -# the discussion leading to this deprecation. -# -# We recommend helping with the new oslo.cache library being created -# as a wrapper for dogpile. -# -######################################################################## - - -from six.moves.urllib import parse -from stevedore import driver - - -def _get_oslo_configs(): - """Returns the oslo config options to register.""" - # NOTE(flaper87): Oslo config should be - # optional. Instead of doing try / except - # at the top of this file, lets import cfg - # here and assume that the caller of this - # function already took care of this dependency. - from oslo_config import cfg - - return [ - cfg.StrOpt('cache_url', default='memory://', - help='URL to connect to the cache back end.') - ] - - -def register_oslo_configs(conf): - """Registers a cache configuration options - - :params conf: Config object. - :type conf: `cfg.ConfigOptions` - """ - conf.register_opts(_get_oslo_configs()) - - -def get_cache(url='memory://'): - """Loads the cache backend - - This function loads the cache backend - specified in the given configuration. - - :param conf: Configuration instance to use - """ - - parsed = parse.urlparse(url) - backend = parsed.scheme - - query = parsed.query - # NOTE(flaper87): We need the following hack - # for python versions < 2.7.5. Previous versions - # of python parsed query params just for 'known' - # schemes. This was changed in this patch: - # http://hg.python.org/cpython/rev/79e6ff3d9afd - if not query and '?' in parsed.path: - query = parsed.path.split('?', 1)[-1] - parameters = parse.parse_qsl(query) - kwargs = {'options': dict(parameters)} - - mgr = driver.DriverManager('neutron.openstack.common.cache.backends', backend, - invoke_on_load=True, - invoke_args=[parsed], - invoke_kwds=kwargs) - return mgr.driver diff --git a/neutron/opts.py b/neutron/opts.py index 8a7e7d4e525..bb8949fbb25 100644 --- a/neutron/opts.py +++ b/neutron/opts.py @@ -29,6 +29,7 @@ import neutron.agent.linux.ra import neutron.agent.metadata.config import neutron.agent.ovsdb.api import neutron.agent.securitygroups_rpc +import neutron.common.cache_utils import neutron.conf.quota import neutron.conf.service import neutron.db.agents_db @@ -43,7 +44,6 @@ import neutron.db.migration.cli import neutron.extensions.allowedaddresspairs import neutron.extensions.l3 import neutron.extensions.securitygroup -import neutron.openstack.common.cache.cache import neutron.plugins.ml2.config import neutron.plugins.ml2.drivers.agent.config import neutron.plugins.ml2.drivers.linuxbridge.agent.common.config @@ -218,7 +218,7 @@ def list_metadata_agent_opts(): neutron.agent.metadata.config.SHARED_OPTS, neutron.agent.metadata.config.METADATA_PROXY_HANDLER_OPTS, neutron.agent.metadata.config.UNIX_DOMAIN_METADATA_PROXY_OPTS, - neutron.openstack.common.cache.cache._get_oslo_configs()) + neutron.common.cache_utils.cache_opts) ), ('AGENT', neutron.agent.common.config.AGENT_STATE_OPTS) ] diff --git a/neutron/tests/unit/agent/metadata/test_agent.py b/neutron/tests/unit/agent/metadata/test_agent.py index c9fc5704f3b..7e872b9154f 100644 --- a/neutron/tests/unit/agent/metadata/test_agent.py +++ b/neutron/tests/unit/agent/metadata/test_agent.py @@ -17,35 +17,47 @@ from neutron_lib import constants as n_const import testtools import webob +from oslo_config import cfg +from oslo_config import fixture as config_fixture + from neutron.agent.linux import utils as agent_utils from neutron.agent.metadata import agent from neutron.agent.metadata import config from neutron.agent import metadata_agent +from neutron.common import cache_utils as cache from neutron.common import utils from neutron.tests import base -class FakeConf(object): - auth_ca_cert = None - nova_metadata_ip = '9.9.9.9' - nova_metadata_port = 8775 - metadata_proxy_shared_secret = 'secret' - nova_metadata_protocol = 'http' - nova_metadata_insecure = True - nova_client_cert = 'nova_cert' - nova_client_priv_key = 'nova_priv_key' - cache_url = '' +class ConfFixture(config_fixture.Config): + def setUp(self): + super(ConfFixture, self).setUp() + self.conf.register_opts(config.METADATA_PROXY_HANDLER_OPTS) + self.config(auth_ca_cert=None, + nova_metadata_ip='9.9.9.9', + nova_metadata_port=8775, + metadata_proxy_shared_secret='secret', + nova_metadata_protocol='http', + nova_metadata_insecure=True, + nova_client_cert='nova_cert', + nova_client_priv_key='nova_priv_key') + cache.register_oslo_configs(self.conf) + self.config(cache_url='') -class FakeConfCache(FakeConf): - cache_url = 'memory://?default_ttl=5' +class CacheConfFixture(ConfFixture): + def setUp(self): + super(CacheConfFixture, self).setUp() + self.config(cache_url='memory://?default_ttl=5') class TestMetadataProxyHandlerBase(base.BaseTestCase): - fake_conf = FakeConf + fake_conf = cfg.CONF + fake_conf_fixture = ConfFixture(fake_conf) def setUp(self): super(TestMetadataProxyHandlerBase, self).setUp() + self.useFixture(self.fake_conf_fixture) self.log_p = mock.patch.object(agent, 'LOG') self.log = self.log_p.start() self.handler = agent.MetadataProxyHandler(self.fake_conf) @@ -85,7 +97,8 @@ class TestMetadataProxyHandlerRpc(TestMetadataProxyHandlerBase): class TestMetadataProxyHandlerCache(TestMetadataProxyHandlerBase): - fake_conf = FakeConfCache + fake_conf = cfg.CONF + fake_conf_fixture = CacheConfFixture(fake_conf) def test_call(self): req = mock.Mock() @@ -337,10 +350,10 @@ class TestMetadataProxyHandlerCache(TestMetadataProxyHandlerBase): ca_certs=None, disable_ssl_certificate_validation=True) mock_http.assert_has_calls([ mock.call().add_certificate( - FakeConf.nova_client_priv_key, - FakeConf.nova_client_cert, - "%s:%s" % (FakeConf.nova_metadata_ip, - FakeConf.nova_metadata_port) + self.fake_conf.nova_client_priv_key, + self.fake_conf.nova_client_cert, + "%s:%s" % (self.fake_conf.nova_metadata_ip, + self.fake_conf.nova_metadata_port) ), mock.call().request( 'http://9.9.9.9:8775/the_path', @@ -399,7 +412,8 @@ class TestMetadataProxyHandlerCache(TestMetadataProxyHandlerBase): class TestMetadataProxyHandlerNoCache(TestMetadataProxyHandlerCache): - fake_conf = FakeConf + fake_conf = cfg.CONF + fake_conf_fixture = ConfFixture(fake_conf) def test_get_router_networks_twice(self): self._test_get_router_networks_twice_helper() diff --git a/neutron/tests/unit/common/test_cache_utils.py b/neutron/tests/unit/common/test_cache_utils.py new file mode 100644 index 00000000000..d1e20dc8c88 --- /dev/null +++ b/neutron/tests/unit/common/test_cache_utils.py @@ -0,0 +1,130 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from oslo_config import cfg +from oslo_config import fixture as config_fixture + +from neutron.common import cache_utils as cache +from neutron.tests import base + + +class CacheConfFixture(config_fixture.Config): + def setUp(self): + super(CacheConfFixture, self).setUp() + cache.register_oslo_configs(self.conf) + self.config(enabled=True, group='cache') + + +class TestOsloCache(base.BaseTestCase): + def setUp(self): + super(TestOsloCache, self).setUp() + self.memory_conf = cfg.ConfigOpts() + memory_conf_fixture = CacheConfFixture(self.memory_conf) + self.useFixture(memory_conf_fixture) + memory_conf_fixture.config(cache_url='memory://?default_ttl=5') + + self.dict_conf = cfg.ConfigOpts() + dict_conf_fixture = CacheConfFixture(self.dict_conf) + self.useFixture(dict_conf_fixture) + dict_conf_fixture.config(expiration_time=60, + backend='oslo_cache.dict', group='cache') + + self.null_cache_conf = cfg.ConfigOpts() + null_conf_fixture = CacheConfFixture(self.null_cache_conf) + self.useFixture(null_conf_fixture) + null_conf_fixture.config(expiration_time=600, + backend='dogpile.cache.null', group='cache') + + def _test_get_cache_region_helper(self, conf): + region = cache._get_cache_region(conf) + self.assertIsNotNone(region) + + def test_get_cache_region(self): + self._test_get_cache_region_helper(self.dict_conf) + self._test_get_cache_region_helper(self.null_cache_conf) + + def test_get_cache_region_for_legacy(self): + region = cache._get_cache_region_for_legacy("memory://?default_ttl=10") + self.assertIsNotNone(region) + self.assertEqual(10, region.expiration_time) + + @mock.patch('neutron.common.cache_utils._get_cache_region_for_legacy') + @mock.patch('neutron.common.cache_utils._get_cache_region') + def test_get_cache(self, mock_get_cache_region, + mock_get_cache_region_for_legacy): + self.assertIsNotNone(cache.get_cache(self.memory_conf)) + self.assertIsNotNone(cache.get_cache(self.dict_conf)) + self.assertIsNotNone(cache.get_cache(self.null_cache_conf)) + mock_get_cache_region.assert_has_calls( + [mock.call(self.dict_conf), + mock.call(self.null_cache_conf)] + ) + mock_get_cache_region_for_legacy.assert_has_calls( + [mock.call('memory://?default_ttl=5')] + ) + + +class _CachingDecorator(object): + def __init__(self): + self.func_retval = 'bar' + self._cache = mock.Mock() + + @cache.cache_method_results + def func(self, *args, **kwargs): + return self.func_retval + + +class TestCachingDecorator(base.BaseTestCase): + def setUp(self): + super(TestCachingDecorator, self).setUp() + self.decor = _CachingDecorator() + self.func_name = '%(module)s._CachingDecorator.func' % { + 'module': self.__module__ + } + self.not_cached = self.decor.func.func.__self__._not_cached + + def test_cache_miss(self): + expected_key = (self.func_name, 1, 2, ('foo', 'bar')) + args = (1, 2) + kwargs = {'foo': 'bar'} + self.decor._cache.get.return_value = self.not_cached + retval = self.decor.func(*args, **kwargs) + self.decor._cache.set.assert_called_once_with( + expected_key, self.decor.func_retval) + self.assertEqual(self.decor.func_retval, retval) + + def test_cache_hit(self): + expected_key = (self.func_name, 1, 2, ('foo', 'bar')) + args = (1, 2) + kwargs = {'foo': 'bar'} + retval = self.decor.func(*args, **kwargs) + self.assertFalse(self.decor._cache.set.called) + self.assertEqual(self.decor._cache.get.return_value, retval) + self.decor._cache.get.assert_called_once_with(expected_key) + + def test_get_unhashable(self): + expected_key = (self.func_name, [1], 2) + self.decor._cache.get.side_effect = TypeError + retval = self.decor.func([1], 2) + self.assertFalse(self.decor._cache.set.called) + self.assertEqual(self.decor.func_retval, retval) + self.decor._cache.get.assert_called_once_with(expected_key) + + def test_missing_cache(self): + delattr(self.decor, '_cache') + self.assertRaises(NotImplementedError, self.decor.func, (1, 2)) + + def test_no_cache(self): + self.decor._cache = False + retval = self.decor.func((1, 2)) + self.assertEqual(self.decor.func_retval, retval) diff --git a/neutron/tests/unit/common/test_utils.py b/neutron/tests/unit/common/test_utils.py index 33e1d0368c7..4b758acbba0 100644 --- a/neutron/tests/unit/common/test_utils.py +++ b/neutron/tests/unit/common/test_utils.py @@ -414,64 +414,6 @@ class TestDictUtils(base.BaseTestCase): self.assertEqual(removed, [dict(key3="value3")]) -class _CachingDecorator(object): - def __init__(self): - self.func_retval = 'bar' - self._cache = mock.Mock() - - @utils.cache_method_results - def func(self, *args, **kwargs): - return self.func_retval - - -class TestCachingDecorator(base.BaseTestCase): - def setUp(self): - super(TestCachingDecorator, self).setUp() - self.decor = _CachingDecorator() - self.func_name = '%(module)s._CachingDecorator.func' % { - 'module': self.__module__ - } - self.not_cached = self.decor.func.func.__self__._not_cached - - def test_cache_miss(self): - expected_key = (self.func_name, 1, 2, ('foo', 'bar')) - args = (1, 2) - kwargs = {'foo': 'bar'} - self.decor._cache.get.return_value = self.not_cached - retval = self.decor.func(*args, **kwargs) - self.decor._cache.set.assert_called_once_with( - expected_key, self.decor.func_retval, None) - self.assertEqual(self.decor.func_retval, retval) - - def test_cache_hit(self): - expected_key = (self.func_name, 1, 2, ('foo', 'bar')) - args = (1, 2) - kwargs = {'foo': 'bar'} - retval = self.decor.func(*args, **kwargs) - self.assertFalse(self.decor._cache.set.called) - self.assertEqual(self.decor._cache.get.return_value, retval) - self.decor._cache.get.assert_called_once_with(expected_key, - self.not_cached) - - def test_get_unhashable(self): - expected_key = (self.func_name, [1], 2) - self.decor._cache.get.side_effect = TypeError - retval = self.decor.func([1], 2) - self.assertFalse(self.decor._cache.set.called) - self.assertEqual(self.decor.func_retval, retval) - self.decor._cache.get.assert_called_once_with(expected_key, - self.not_cached) - - def test_missing_cache(self): - delattr(self.decor, '_cache') - self.assertRaises(NotImplementedError, self.decor.func, (1, 2)) - - def test_no_cache(self): - self.decor._cache = False - retval = self.decor.func((1, 2)) - self.assertEqual(self.decor.func_retval, retval) - - class TestDict2Tuples(base.BaseTestCase): def test_dict(self): input_dict = {'foo': 'bar', '42': 'baz', 'aaa': 'zzz'} diff --git a/releasenotes/notes/oslo-cache-cache-url-deprecated-16cd3d335c5962eb.yaml b/releasenotes/notes/oslo-cache-cache-url-deprecated-16cd3d335c5962eb.yaml new file mode 100644 index 00000000000..e17a2b6abf0 --- /dev/null +++ b/releasenotes/notes/oslo-cache-cache-url-deprecated-16cd3d335c5962eb.yaml @@ -0,0 +1,9 @@ +--- +features: + - Neutron switched to using oslo.cache library to cache port state in + metadata agent. With it, more caching backends are now available, including + Memcached and Mongo. More details in oslo.cache documentation. +deprecations: + - The cache_url configuration option is deprecated as of Newton, and will be + removed in Ocata. Please configure metadata cache using [cache] group, + setting enable = True and configuring your backend. diff --git a/requirements.txt b/requirements.txt index a7e5052a960..4953411a1aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,7 @@ keystoneauth1>=2.1.0 # Apache-2.0 alembic>=0.8.4 # MIT six>=1.9.0 # MIT stevedore>=1.10.0 # Apache-2.0 +oslo.cache>=1.5.0 # Apache-2.0 oslo.concurrency>=3.8.0 # Apache-2.0 oslo.config>=3.9.0 # Apache-2.0 oslo.context>=2.2.0 # Apache-2.0