etcd3 driver for oslo.cache
Support for oslo.cache to use the etcd3 grpc gateway based HTTP API (/v3alpha) using the python-etd3gw library Change-Id: I41ed9f6ea72641dc1db5fec45920dc41de5088bf
This commit is contained in:
Davanum Srinivas
parent
d584504bda
commit
e79d1ab159
|
|
@ -0,0 +1,75 @@
|
|||
# Copyright 2015 Mirantis Inc
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""dogpile.cache backend that uses etcd 3.x for storage"""
|
||||
|
||||
from __future__ import absolute_import
|
||||
import ast
|
||||
from dogpile.cache import api
|
||||
import etcd3gw
|
||||
|
||||
from oslo_cache import core
|
||||
|
||||
__all__ = [
|
||||
'Etcd3gwCacheBackend'
|
||||
]
|
||||
|
||||
_NO_VALUE = core.NO_VALUE
|
||||
|
||||
|
||||
class Etcd3gwCacheBackend(api.CacheBackend):
|
||||
#: Default socket/lock/member/leader timeout used when none is provided.
|
||||
DEFAULT_TIMEOUT = 30
|
||||
|
||||
#: Default hostname used when none is provided.
|
||||
DEFAULT_HOST = "localhost"
|
||||
|
||||
#: Default port used if none provided (4001 or 2379 are the common ones).
|
||||
DEFAULT_PORT = 2379
|
||||
|
||||
def __init__(self, arguments):
|
||||
self.host = arguments.get('host', self.DEFAULT_HOST)
|
||||
self.port = arguments.get('port', self.DEFAULT_PORT)
|
||||
self.timeout = int(arguments.get('timeout', self.DEFAULT_TIMEOUT))
|
||||
self._client = etcd3gw.client(host=self.host,
|
||||
port=self.port,
|
||||
timeout=self.timeout)
|
||||
|
||||
def get(self, key):
|
||||
values = self._client.get(key, False)
|
||||
if not values:
|
||||
return core.NO_VALUE
|
||||
(value, metadata) = ast.literal_eval(values[0])
|
||||
return api.CachedValue(value, metadata)
|
||||
|
||||
def get_multi(self, keys):
|
||||
"""Retrieves the value for a list of keys."""
|
||||
return [self.get(key) for key in keys]
|
||||
|
||||
def set(self, key, value):
|
||||
self.set_multi({key: value})
|
||||
|
||||
def set_multi(self, mapping):
|
||||
lease = None
|
||||
if self.timeout:
|
||||
lease = self._client.lease(ttl=self.timeout)
|
||||
for key, value in mapping.items():
|
||||
self._client.put(key, value, lease)
|
||||
|
||||
def delete(self, key):
|
||||
self._client.delete(key)
|
||||
|
||||
def delete_multi(self, keys):
|
||||
for key in keys:
|
||||
self._client.delete(key)
|
||||
|
|
@ -0,0 +1,269 @@
|
|||
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from dogpile.cache import region as dp_region
|
||||
from oslo_utils import uuidutils
|
||||
import urllib3
|
||||
|
||||
from oslo_cache import core
|
||||
from oslo_cache.tests import test_cache
|
||||
|
||||
|
||||
NO_VALUE = core.NO_VALUE
|
||||
|
||||
|
||||
class Etcd3gwCache(test_cache.BaseTestCase):
|
||||
arguments = {
|
||||
'host': '127.0.0.1',
|
||||
'port': 2379,
|
||||
}
|
||||
|
||||
def setUp(self):
|
||||
test_cache.BaseTestCase.setUp(self)
|
||||
try:
|
||||
urllib3.PoolManager().request(
|
||||
'GET',
|
||||
'%s:%d' % (self.arguments['host'], self.arguments['port'])
|
||||
)
|
||||
return True
|
||||
except urllib3.exceptions.HTTPError:
|
||||
self.skipTest("skipping this test")
|
||||
|
||||
def test_typical_configuration(self):
|
||||
|
||||
dp_region.make_region().configure(
|
||||
'oslo_cache.etcd3gw',
|
||||
arguments=self.arguments
|
||||
)
|
||||
self.assertTrue(True) # reached here means no initialization error
|
||||
|
||||
def test_backend_get_missing_data(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.etcd3gw',
|
||||
arguments=self.arguments
|
||||
)
|
||||
|
||||
random_key = uuidutils.generate_uuid(dashed=False)
|
||||
# should return NO_VALUE as key does not exist in cache
|
||||
self.assertEqual(NO_VALUE, region.get(random_key))
|
||||
|
||||
def test_backend_set_data(self):
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.etcd3gw',
|
||||
arguments=self.arguments
|
||||
)
|
||||
|
||||
random_key = uuidutils.generate_uuid(dashed=False)
|
||||
region.set(random_key, "dummyValue")
|
||||
self.assertEqual("dummyValue", region.get(random_key))
|
||||
|
||||
def test_backend_set_none_as_data(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.etcd3gw',
|
||||
arguments=self.arguments
|
||||
)
|
||||
|
||||
random_key = uuidutils.generate_uuid(dashed=False)
|
||||
region.set(random_key, None)
|
||||
self.assertIsNone(region.get(random_key))
|
||||
|
||||
def test_backend_set_blank_as_data(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.etcd3gw',
|
||||
arguments=self.arguments
|
||||
)
|
||||
|
||||
random_key = uuidutils.generate_uuid(dashed=False)
|
||||
region.set(random_key, "")
|
||||
self.assertEqual("", region.get(random_key))
|
||||
|
||||
def test_backend_set_same_key_multiple_times(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.etcd3gw',
|
||||
arguments=self.arguments
|
||||
)
|
||||
|
||||
random_key = uuidutils.generate_uuid(dashed=False)
|
||||
region.set(random_key, "dummyValue")
|
||||
self.assertEqual("dummyValue", region.get(random_key))
|
||||
|
||||
dict_value = {'key1': 'value1'}
|
||||
region.set(random_key, dict_value)
|
||||
self.assertEqual(dict_value, region.get(random_key))
|
||||
|
||||
region.set(random_key, "dummyValue2")
|
||||
self.assertEqual("dummyValue2", region.get(random_key))
|
||||
|
||||
def test_backend_multi_set_data(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.etcd3gw',
|
||||
arguments=self.arguments
|
||||
)
|
||||
random_key = uuidutils.generate_uuid(dashed=False)
|
||||
random_key1 = uuidutils.generate_uuid(dashed=False)
|
||||
random_key2 = uuidutils.generate_uuid(dashed=False)
|
||||
random_key3 = uuidutils.generate_uuid(dashed=False)
|
||||
mapping = {random_key1: 'dummyValue1',
|
||||
random_key2: 'dummyValue2',
|
||||
random_key3: 'dummyValue3'}
|
||||
region.set_multi(mapping)
|
||||
# should return NO_VALUE as key does not exist in cache
|
||||
self.assertEqual(NO_VALUE, region.get(random_key))
|
||||
self.assertFalse(region.get(random_key))
|
||||
self.assertEqual("dummyValue1", region.get(random_key1))
|
||||
self.assertEqual("dummyValue2", region.get(random_key2))
|
||||
self.assertEqual("dummyValue3", region.get(random_key3))
|
||||
|
||||
def test_backend_multi_get_data(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.etcd3gw',
|
||||
arguments=self.arguments
|
||||
)
|
||||
random_key = uuidutils.generate_uuid(dashed=False)
|
||||
random_key1 = uuidutils.generate_uuid(dashed=False)
|
||||
random_key2 = uuidutils.generate_uuid(dashed=False)
|
||||
random_key3 = uuidutils.generate_uuid(dashed=False)
|
||||
mapping = {random_key1: 'dummyValue1',
|
||||
random_key2: '',
|
||||
random_key3: 'dummyValue3'}
|
||||
region.set_multi(mapping)
|
||||
|
||||
keys = [random_key, random_key1, random_key2, random_key3]
|
||||
results = region.get_multi(keys)
|
||||
# should return NO_VALUE as key does not exist in cache
|
||||
self.assertEqual(NO_VALUE, results[0])
|
||||
self.assertEqual("dummyValue1", results[1])
|
||||
self.assertEqual("", results[2])
|
||||
self.assertEqual("dummyValue3", results[3])
|
||||
|
||||
def test_backend_multi_set_should_update_existing(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.etcd3gw',
|
||||
arguments=self.arguments
|
||||
)
|
||||
random_key = uuidutils.generate_uuid(dashed=False)
|
||||
random_key1 = uuidutils.generate_uuid(dashed=False)
|
||||
random_key2 = uuidutils.generate_uuid(dashed=False)
|
||||
random_key3 = uuidutils.generate_uuid(dashed=False)
|
||||
mapping = {random_key1: 'dummyValue1',
|
||||
random_key2: 'dummyValue2',
|
||||
random_key3: 'dummyValue3'}
|
||||
region.set_multi(mapping)
|
||||
# should return NO_VALUE as key does not exist in cache
|
||||
self.assertEqual(NO_VALUE, region.get(random_key))
|
||||
self.assertEqual("dummyValue1", region.get(random_key1))
|
||||
self.assertEqual("dummyValue2", region.get(random_key2))
|
||||
self.assertEqual("dummyValue3", region.get(random_key3))
|
||||
|
||||
mapping = {random_key1: 'dummyValue4',
|
||||
random_key2: 'dummyValue5'}
|
||||
region.set_multi(mapping)
|
||||
self.assertEqual(NO_VALUE, region.get(random_key))
|
||||
self.assertEqual("dummyValue4", region.get(random_key1))
|
||||
self.assertEqual("dummyValue5", region.get(random_key2))
|
||||
self.assertEqual("dummyValue3", region.get(random_key3))
|
||||
|
||||
def test_backend_multi_set_get_with_blanks_none(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.etcd3gw',
|
||||
arguments=self.arguments
|
||||
)
|
||||
random_key = uuidutils.generate_uuid(dashed=False)
|
||||
random_key1 = uuidutils.generate_uuid(dashed=False)
|
||||
random_key2 = uuidutils.generate_uuid(dashed=False)
|
||||
random_key3 = uuidutils.generate_uuid(dashed=False)
|
||||
random_key4 = uuidutils.generate_uuid(dashed=False)
|
||||
mapping = {random_key1: 'dummyValue1',
|
||||
random_key2: None,
|
||||
random_key3: '',
|
||||
random_key4: 'dummyValue4'}
|
||||
region.set_multi(mapping)
|
||||
# should return NO_VALUE as key does not exist in cache
|
||||
self.assertEqual(NO_VALUE, region.get(random_key))
|
||||
self.assertEqual("dummyValue1", region.get(random_key1))
|
||||
self.assertIsNone(region.get(random_key2))
|
||||
self.assertEqual("", region.get(random_key3))
|
||||
self.assertEqual("dummyValue4", region.get(random_key4))
|
||||
|
||||
keys = [random_key, random_key1, random_key2, random_key3, random_key4]
|
||||
results = region.get_multi(keys)
|
||||
|
||||
# should return NO_VALUE as key does not exist in cache
|
||||
self.assertEqual(NO_VALUE, results[0])
|
||||
self.assertEqual("dummyValue1", results[1])
|
||||
self.assertIsNone(results[2])
|
||||
self.assertEqual("", results[3])
|
||||
self.assertEqual("dummyValue4", results[4])
|
||||
|
||||
mapping = {random_key1: 'dummyValue5',
|
||||
random_key2: 'dummyValue6'}
|
||||
region.set_multi(mapping)
|
||||
self.assertEqual(NO_VALUE, region.get(random_key))
|
||||
self.assertEqual("dummyValue5", region.get(random_key1))
|
||||
self.assertEqual("dummyValue6", region.get(random_key2))
|
||||
self.assertEqual("", region.get(random_key3))
|
||||
|
||||
def test_backend_delete_data(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.etcd3gw',
|
||||
arguments=self.arguments
|
||||
)
|
||||
|
||||
random_key = uuidutils.generate_uuid(dashed=False)
|
||||
region.set(random_key, "dummyValue")
|
||||
self.assertEqual("dummyValue", region.get(random_key))
|
||||
|
||||
region.delete(random_key)
|
||||
# should return NO_VALUE as key no longer exists in cache
|
||||
self.assertEqual(NO_VALUE, region.get(random_key))
|
||||
|
||||
def test_backend_multi_delete_data(self):
|
||||
|
||||
region = dp_region.make_region().configure(
|
||||
'oslo_cache.etcd3gw',
|
||||
arguments=self.arguments
|
||||
)
|
||||
random_key = uuidutils.generate_uuid(dashed=False)
|
||||
random_key1 = uuidutils.generate_uuid(dashed=False)
|
||||
random_key2 = uuidutils.generate_uuid(dashed=False)
|
||||
random_key3 = uuidutils.generate_uuid(dashed=False)
|
||||
mapping = {random_key1: 'dummyValue1',
|
||||
random_key2: 'dummyValue2',
|
||||
random_key3: 'dummyValue3'}
|
||||
region.set_multi(mapping)
|
||||
# should return NO_VALUE as key does not exist in cache
|
||||
self.assertEqual(NO_VALUE, region.get(random_key))
|
||||
self.assertEqual("dummyValue1", region.get(random_key1))
|
||||
self.assertEqual("dummyValue2", region.get(random_key2))
|
||||
self.assertEqual("dummyValue3", region.get(random_key3))
|
||||
self.assertEqual(NO_VALUE, region.get("InvalidKey"))
|
||||
|
||||
keys = mapping.keys()
|
||||
|
||||
region.delete_multi(keys)
|
||||
|
||||
self.assertEqual(NO_VALUE, region.get("InvalidKey"))
|
||||
# should return NO_VALUE as keys no longer exist in cache
|
||||
self.assertEqual(NO_VALUE, region.get(random_key1))
|
||||
self.assertEqual(NO_VALUE, region.get(random_key2))
|
||||
self.assertEqual(NO_VALUE, region.get(random_key3))
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
features:
|
||||
- |
|
||||
Added a new etcd3gw driver that uses the etcd 3.x grpc-gateway
|
||||
HTTP "/v3alpha" API to cache key/value pairs.
|
||||
|
|
@ -30,12 +30,15 @@ dogpile.cache =
|
|||
oslo_cache.mongo = oslo_cache.backends.mongo:MongoCacheBackend
|
||||
oslo_cache.memcache_pool = oslo_cache.backends.memcache_pool:PooledMemcachedBackend
|
||||
oslo_cache.dict = oslo_cache.backends.dictionary:DictCacheBackend
|
||||
oslo_cache.etcd3gw = oslo_cache.backends.etcd3gw:Etcd3gwCacheBackend
|
||||
|
||||
[extras]
|
||||
dogpile =
|
||||
python-memcached>=1.56 # PSF
|
||||
mongo =
|
||||
pymongo!=3.1,>=3.0.2 # Apache-2.0
|
||||
etcd3gw =
|
||||
etcd3gw>=0.1.0 # Apache-2.0
|
||||
|
||||
[pbr]
|
||||
warnerrors = true
|
||||
|
|
|
|||
|
|
@ -5,5 +5,6 @@ hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
|
|||
mock>=2.0 # BSD
|
||||
oslotest>=1.10.0 # Apache-2.0
|
||||
oslosphinx>=4.7.0 # Apache-2.0
|
||||
pifpaf>=0.10.0 # Apache-2.0
|
||||
sphinx!=1.6.1,>=1.5.1 # BSD
|
||||
reno>=1.8.0 # Apache-2.0
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
#!/bin/bash
|
||||
set -eux
|
||||
if [ -z "$(which etcd)" ]; then
|
||||
ETCD_VERSION=3.1.3
|
||||
case `uname -s` in
|
||||
Darwin)
|
||||
OS=darwin
|
||||
SUFFIX=zip
|
||||
;;
|
||||
Linux)
|
||||
OS=linux
|
||||
SUFFIX=tar.gz
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported OS"
|
||||
exit 1
|
||||
esac
|
||||
case `uname -m` in
|
||||
x86_64)
|
||||
MACHINE=amd64
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported machine"
|
||||
exit 1
|
||||
esac
|
||||
TARBALL_NAME=etcd-v${ETCD_VERSION}-$OS-$MACHINE
|
||||
test ! -d "$TARBALL_NAME" && curl -L https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/${TARBALL_NAME}.${SUFFIX} | tar xz
|
||||
export PATH=$PATH:$TARBALL_NAME
|
||||
fi
|
||||
|
||||
$*
|
||||
11
tox.ini
11
tox.ini
|
|
@ -10,11 +10,22 @@ setenv =
|
|||
install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
|
||||
deps = .[dogpile]
|
||||
.[mongo]
|
||||
.[etcd3gw]
|
||||
-r{toxinidir}/test-requirements.txt
|
||||
commands =
|
||||
find . -type f -name "*.pyc" -delete
|
||||
python setup.py testr --slowest --testr-args='{posargs}'
|
||||
|
||||
[testenv:py27-functional-etcd3gw]
|
||||
commands =
|
||||
find . -type f -name "*.pyc" -delete
|
||||
{toxinidir}/tools/setup-etcd-env.sh pifpaf -e OSLO_CACHE_TEST run etcd -- python setup.py testr --slowest --testr-args='functional.*'
|
||||
|
||||
[testenv:py35-functional-etcd3gw]
|
||||
commands =
|
||||
find . -type f -name "*.pyc" -delete
|
||||
{toxinidir}/tools/setup-etcd-env.sh pifpaf -e OSLO_CACHE_TEST run etcd -- python setup.py testr --slowest --testr-args='functional.*'
|
||||
|
||||
[testenv:pep8]
|
||||
commands = flake8
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue