diff --git a/oslo_cache/backends/etcd3gw.py b/oslo_cache/backends/etcd3gw.py new file mode 100644 index 00000000..d3cb986b --- /dev/null +++ b/oslo_cache/backends/etcd3gw.py @@ -0,0 +1,75 @@ +# Copyright 2015 Mirantis Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""dogpile.cache backend that uses etcd 3.x for storage""" + +from __future__ import absolute_import +import ast +from dogpile.cache import api +import etcd3gw + +from oslo_cache import core + +__all__ = [ + 'Etcd3gwCacheBackend' +] + +_NO_VALUE = core.NO_VALUE + + +class Etcd3gwCacheBackend(api.CacheBackend): + #: Default socket/lock/member/leader timeout used when none is provided. + DEFAULT_TIMEOUT = 30 + + #: Default hostname used when none is provided. + DEFAULT_HOST = "localhost" + + #: Default port used if none provided (4001 or 2379 are the common ones). + DEFAULT_PORT = 2379 + + def __init__(self, arguments): + self.host = arguments.get('host', self.DEFAULT_HOST) + self.port = arguments.get('port', self.DEFAULT_PORT) + self.timeout = int(arguments.get('timeout', self.DEFAULT_TIMEOUT)) + self._client = etcd3gw.client(host=self.host, + port=self.port, + timeout=self.timeout) + + def get(self, key): + values = self._client.get(key, False) + if not values: + return core.NO_VALUE + (value, metadata) = ast.literal_eval(values[0]) + return api.CachedValue(value, metadata) + + def get_multi(self, keys): + """Retrieves the value for a list of keys.""" + return [self.get(key) for key in keys] + + def set(self, key, value): + self.set_multi({key: value}) + + def set_multi(self, mapping): + lease = None + if self.timeout: + lease = self._client.lease(ttl=self.timeout) + for key, value in mapping.items(): + self._client.put(key, value, lease) + + def delete(self, key): + self._client.delete(key) + + def delete_multi(self, keys): + for key in keys: + self._client.delete(key) diff --git a/oslo_cache/tests/functional/__init__.py b/oslo_cache/tests/functional/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/oslo_cache/tests/functional/test_cache_backend_etcd3gw.py b/oslo_cache/tests/functional/test_cache_backend_etcd3gw.py new file mode 100644 index 00000000..d8fe5440 --- /dev/null +++ b/oslo_cache/tests/functional/test_cache_backend_etcd3gw.py @@ -0,0 +1,269 @@ +# Copyright 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from dogpile.cache import region as dp_region +from oslo_utils import uuidutils +import urllib3 + +from oslo_cache import core +from oslo_cache.tests import test_cache + + +NO_VALUE = core.NO_VALUE + + +class Etcd3gwCache(test_cache.BaseTestCase): + arguments = { + 'host': '127.0.0.1', + 'port': 2379, + } + + def setUp(self): + test_cache.BaseTestCase.setUp(self) + try: + urllib3.PoolManager().request( + 'GET', + '%s:%d' % (self.arguments['host'], self.arguments['port']) + ) + return True + except urllib3.exceptions.HTTPError: + self.skipTest("skipping this test") + + def test_typical_configuration(self): + + dp_region.make_region().configure( + 'oslo_cache.etcd3gw', + arguments=self.arguments + ) + self.assertTrue(True) # reached here means no initialization error + + def test_backend_get_missing_data(self): + + region = dp_region.make_region().configure( + 'oslo_cache.etcd3gw', + arguments=self.arguments + ) + + random_key = uuidutils.generate_uuid(dashed=False) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(NO_VALUE, region.get(random_key)) + + def test_backend_set_data(self): + region = dp_region.make_region().configure( + 'oslo_cache.etcd3gw', + arguments=self.arguments + ) + + random_key = uuidutils.generate_uuid(dashed=False) + region.set(random_key, "dummyValue") + self.assertEqual("dummyValue", region.get(random_key)) + + def test_backend_set_none_as_data(self): + + region = dp_region.make_region().configure( + 'oslo_cache.etcd3gw', + arguments=self.arguments + ) + + random_key = uuidutils.generate_uuid(dashed=False) + region.set(random_key, None) + self.assertIsNone(region.get(random_key)) + + def test_backend_set_blank_as_data(self): + + region = dp_region.make_region().configure( + 'oslo_cache.etcd3gw', + arguments=self.arguments + ) + + random_key = uuidutils.generate_uuid(dashed=False) + region.set(random_key, "") + self.assertEqual("", region.get(random_key)) + + def test_backend_set_same_key_multiple_times(self): + + region = dp_region.make_region().configure( + 'oslo_cache.etcd3gw', + arguments=self.arguments + ) + + random_key = uuidutils.generate_uuid(dashed=False) + region.set(random_key, "dummyValue") + self.assertEqual("dummyValue", region.get(random_key)) + + dict_value = {'key1': 'value1'} + region.set(random_key, dict_value) + self.assertEqual(dict_value, region.get(random_key)) + + region.set(random_key, "dummyValue2") + self.assertEqual("dummyValue2", region.get(random_key)) + + def test_backend_multi_set_data(self): + + region = dp_region.make_region().configure( + 'oslo_cache.etcd3gw', + arguments=self.arguments + ) + random_key = uuidutils.generate_uuid(dashed=False) + random_key1 = uuidutils.generate_uuid(dashed=False) + random_key2 = uuidutils.generate_uuid(dashed=False) + random_key3 = uuidutils.generate_uuid(dashed=False) + mapping = {random_key1: 'dummyValue1', + random_key2: 'dummyValue2', + random_key3: 'dummyValue3'} + region.set_multi(mapping) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(NO_VALUE, region.get(random_key)) + self.assertFalse(region.get(random_key)) + self.assertEqual("dummyValue1", region.get(random_key1)) + self.assertEqual("dummyValue2", region.get(random_key2)) + self.assertEqual("dummyValue3", region.get(random_key3)) + + def test_backend_multi_get_data(self): + + region = dp_region.make_region().configure( + 'oslo_cache.etcd3gw', + arguments=self.arguments + ) + random_key = uuidutils.generate_uuid(dashed=False) + random_key1 = uuidutils.generate_uuid(dashed=False) + random_key2 = uuidutils.generate_uuid(dashed=False) + random_key3 = uuidutils.generate_uuid(dashed=False) + mapping = {random_key1: 'dummyValue1', + random_key2: '', + random_key3: 'dummyValue3'} + region.set_multi(mapping) + + keys = [random_key, random_key1, random_key2, random_key3] + results = region.get_multi(keys) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(NO_VALUE, results[0]) + self.assertEqual("dummyValue1", results[1]) + self.assertEqual("", results[2]) + self.assertEqual("dummyValue3", results[3]) + + def test_backend_multi_set_should_update_existing(self): + + region = dp_region.make_region().configure( + 'oslo_cache.etcd3gw', + arguments=self.arguments + ) + random_key = uuidutils.generate_uuid(dashed=False) + random_key1 = uuidutils.generate_uuid(dashed=False) + random_key2 = uuidutils.generate_uuid(dashed=False) + random_key3 = uuidutils.generate_uuid(dashed=False) + mapping = {random_key1: 'dummyValue1', + random_key2: 'dummyValue2', + random_key3: 'dummyValue3'} + region.set_multi(mapping) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(NO_VALUE, region.get(random_key)) + self.assertEqual("dummyValue1", region.get(random_key1)) + self.assertEqual("dummyValue2", region.get(random_key2)) + self.assertEqual("dummyValue3", region.get(random_key3)) + + mapping = {random_key1: 'dummyValue4', + random_key2: 'dummyValue5'} + region.set_multi(mapping) + self.assertEqual(NO_VALUE, region.get(random_key)) + self.assertEqual("dummyValue4", region.get(random_key1)) + self.assertEqual("dummyValue5", region.get(random_key2)) + self.assertEqual("dummyValue3", region.get(random_key3)) + + def test_backend_multi_set_get_with_blanks_none(self): + + region = dp_region.make_region().configure( + 'oslo_cache.etcd3gw', + arguments=self.arguments + ) + random_key = uuidutils.generate_uuid(dashed=False) + random_key1 = uuidutils.generate_uuid(dashed=False) + random_key2 = uuidutils.generate_uuid(dashed=False) + random_key3 = uuidutils.generate_uuid(dashed=False) + random_key4 = uuidutils.generate_uuid(dashed=False) + mapping = {random_key1: 'dummyValue1', + random_key2: None, + random_key3: '', + random_key4: 'dummyValue4'} + region.set_multi(mapping) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(NO_VALUE, region.get(random_key)) + self.assertEqual("dummyValue1", region.get(random_key1)) + self.assertIsNone(region.get(random_key2)) + self.assertEqual("", region.get(random_key3)) + self.assertEqual("dummyValue4", region.get(random_key4)) + + keys = [random_key, random_key1, random_key2, random_key3, random_key4] + results = region.get_multi(keys) + + # should return NO_VALUE as key does not exist in cache + self.assertEqual(NO_VALUE, results[0]) + self.assertEqual("dummyValue1", results[1]) + self.assertIsNone(results[2]) + self.assertEqual("", results[3]) + self.assertEqual("dummyValue4", results[4]) + + mapping = {random_key1: 'dummyValue5', + random_key2: 'dummyValue6'} + region.set_multi(mapping) + self.assertEqual(NO_VALUE, region.get(random_key)) + self.assertEqual("dummyValue5", region.get(random_key1)) + self.assertEqual("dummyValue6", region.get(random_key2)) + self.assertEqual("", region.get(random_key3)) + + def test_backend_delete_data(self): + + region = dp_region.make_region().configure( + 'oslo_cache.etcd3gw', + arguments=self.arguments + ) + + random_key = uuidutils.generate_uuid(dashed=False) + region.set(random_key, "dummyValue") + self.assertEqual("dummyValue", region.get(random_key)) + + region.delete(random_key) + # should return NO_VALUE as key no longer exists in cache + self.assertEqual(NO_VALUE, region.get(random_key)) + + def test_backend_multi_delete_data(self): + + region = dp_region.make_region().configure( + 'oslo_cache.etcd3gw', + arguments=self.arguments + ) + random_key = uuidutils.generate_uuid(dashed=False) + random_key1 = uuidutils.generate_uuid(dashed=False) + random_key2 = uuidutils.generate_uuid(dashed=False) + random_key3 = uuidutils.generate_uuid(dashed=False) + mapping = {random_key1: 'dummyValue1', + random_key2: 'dummyValue2', + random_key3: 'dummyValue3'} + region.set_multi(mapping) + # should return NO_VALUE as key does not exist in cache + self.assertEqual(NO_VALUE, region.get(random_key)) + self.assertEqual("dummyValue1", region.get(random_key1)) + self.assertEqual("dummyValue2", region.get(random_key2)) + self.assertEqual("dummyValue3", region.get(random_key3)) + self.assertEqual(NO_VALUE, region.get("InvalidKey")) + + keys = mapping.keys() + + region.delete_multi(keys) + + self.assertEqual(NO_VALUE, region.get("InvalidKey")) + # should return NO_VALUE as keys no longer exist in cache + self.assertEqual(NO_VALUE, region.get(random_key1)) + self.assertEqual(NO_VALUE, region.get(random_key2)) + self.assertEqual(NO_VALUE, region.get(random_key3)) diff --git a/releasenotes/notes/etcd3gw_driver-8ba4511ae9553a91.yaml b/releasenotes/notes/etcd3gw_driver-8ba4511ae9553a91.yaml new file mode 100644 index 00000000..73f12b35 --- /dev/null +++ b/releasenotes/notes/etcd3gw_driver-8ba4511ae9553a91.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Added a new etcd3gw driver that uses the etcd 3.x grpc-gateway + HTTP "/v3alpha" API to cache key/value pairs. \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index ca75ed97..8dcc37db 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,12 +30,15 @@ dogpile.cache = oslo_cache.mongo = oslo_cache.backends.mongo:MongoCacheBackend oslo_cache.memcache_pool = oslo_cache.backends.memcache_pool:PooledMemcachedBackend oslo_cache.dict = oslo_cache.backends.dictionary:DictCacheBackend + oslo_cache.etcd3gw = oslo_cache.backends.etcd3gw:Etcd3gwCacheBackend [extras] dogpile = python-memcached>=1.56 # PSF mongo = pymongo!=3.1,>=3.0.2 # Apache-2.0 +etcd3gw = + etcd3gw>=0.1.0 # Apache-2.0 [pbr] warnerrors = true diff --git a/test-requirements.txt b/test-requirements.txt index 5cdedae1..03ac7417 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,5 +5,6 @@ hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0 mock>=2.0 # BSD oslotest>=1.10.0 # Apache-2.0 oslosphinx>=4.7.0 # Apache-2.0 +pifpaf>=0.10.0 # Apache-2.0 sphinx!=1.6.1,>=1.5.1 # BSD reno>=1.8.0 # Apache-2.0 diff --git a/tools/setup-etcd-env.sh b/tools/setup-etcd-env.sh new file mode 100755 index 00000000..cfdbaac5 --- /dev/null +++ b/tools/setup-etcd-env.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -eux +if [ -z "$(which etcd)" ]; then + ETCD_VERSION=3.1.3 + case `uname -s` in + Darwin) + OS=darwin + SUFFIX=zip + ;; + Linux) + OS=linux + SUFFIX=tar.gz + ;; + *) + echo "Unsupported OS" + exit 1 + esac + case `uname -m` in + x86_64) + MACHINE=amd64 + ;; + *) + echo "Unsupported machine" + exit 1 + esac + TARBALL_NAME=etcd-v${ETCD_VERSION}-$OS-$MACHINE + test ! -d "$TARBALL_NAME" && curl -L https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/${TARBALL_NAME}.${SUFFIX} | tar xz + export PATH=$PATH:$TARBALL_NAME +fi + +$* diff --git a/tox.ini b/tox.ini index 80292da5..daa326fc 100644 --- a/tox.ini +++ b/tox.ini @@ -10,11 +10,22 @@ setenv = install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages} deps = .[dogpile] .[mongo] + .[etcd3gw] -r{toxinidir}/test-requirements.txt commands = find . -type f -name "*.pyc" -delete python setup.py testr --slowest --testr-args='{posargs}' +[testenv:py27-functional-etcd3gw] +commands = + find . -type f -name "*.pyc" -delete + {toxinidir}/tools/setup-etcd-env.sh pifpaf -e OSLO_CACHE_TEST run etcd -- python setup.py testr --slowest --testr-args='functional.*' + +[testenv:py35-functional-etcd3gw] +commands = + find . -type f -name "*.pyc" -delete + {toxinidir}/tools/setup-etcd-env.sh pifpaf -e OSLO_CACHE_TEST run etcd -- python setup.py testr --slowest --testr-args='functional.*' + [testenv:pep8] commands = flake8