summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMorgan Fainberg <m@metacloud.com>2013-12-09 05:28:33 (GMT)
committerGerrit Code Review <review@openstack.org>2014-02-19 23:55:17 (GMT)
commit813d1254eb4f7a7d40009b23bbadbc4c5cc5daac (patch)
tree85dab7cb87dc7783b57495206a1fa2cb28c454b7
parent57d02590f97a4692e6e7c339dfa61847a58e28a8 (diff)
Convert Token Memcache backend to new KeyValueStore Implrefs/changes/43/60743/25
This patchset converts the memcache backend for tokens to the new KeyValueStore implementation. This implementation provides basic configuration choices for utilizing memcache as a token backend. The test_backend_memcache test suite has been removed, as there is no extra functionality added on top of the KeyValueStore base besides the basic configuration choices. The test_backend_memcache test suite was only used for testing token memcache backend. The important unicode tests have been moved into the base token test suite and should now validate against all token backends. Closes-Bug: #1260080 bp: dogpile-kvs-backends Change-Id: I8f0bca409a6393176e77f7a27ba15959756da06f
Notes
Notes (review): Verified+2: Jenkins Approved+1: Dolph Mathews <dolph.mathews@gmail.com> Code-Review+2: Dolph Mathews <dolph.mathews@gmail.com> Code-Review+2: ayoung <ayoung@redhat.com> Code-Review+1: Fabio Giannetti <fabio.giannetti@hp.com> Submitted-by: Jenkins Submitted-at: Thu, 20 Feb 2014 04:46:39 +0000 Reviewed-on: https://review.openstack.org/60743 Project: openstack/keystone Branch: refs/heads/master
-rw-r--r--keystone/tests/test_backend_memcache.py257
-rw-r--r--keystone/token/backends/memcache.py263
2 files changed, 12 insertions, 508 deletions
diff --git a/keystone/tests/test_backend_memcache.py b/keystone/tests/test_backend_memcache.py
deleted file mode 100644
index e95ed2e..0000000
--- a/keystone/tests/test_backend_memcache.py
+++ /dev/null
@@ -1,257 +0,0 @@
1# Copyright 2012 OpenStack Foundation
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import copy
16import datetime
17import uuid
18
19import memcache
20import six
21
22from keystone.common import utils
23from keystone import config
24from keystone import exception
25from keystone.openstack.common import jsonutils
26from keystone.openstack.common import timeutils
27from keystone import tests
28from keystone.tests import test_backend
29from keystone.tests import test_utils
30from keystone import token
31from keystone.token.backends import memcache as token_memcache
32
33CONF = config.CONF
34
35
36class MemcacheClient(object):
37 """Replicates a tiny subset of memcached client interface."""
38
39 def __init__(self, *args, **kwargs):
40 """Ignores the passed in args."""
41 self.cache = {}
42 self.reject_cas = False
43
44 def add(self, key, value):
45 if self.get(key):
46 return False
47 return self.set(key, value)
48
49 def append(self, key, value):
50 existing_value = self.get(key)
51 if existing_value:
52 self.set(key, existing_value + value)
53 return True
54 return False
55
56 def check_key(self, key):
57 if not isinstance(key, str):
58 raise memcache.Client.MemcachedStringEncodingError()
59
60 def gets(self, key):
61 #Call self.get() since we don't really do 'cas' here.
62 return self.get(key)
63
64 def get(self, key):
65 """Retrieves the value for a key or None."""
66 self.check_key(key)
67 obj = self.cache.get(key)
68 now = utils.unixtime(timeutils.utcnow())
69 if obj and (obj[1] == 0 or obj[1] > now):
70 # NOTE(morganfainberg): This behaves more like memcache
71 # actually does and prevents modification of the passed in
72 # reference from affecting the cached back-end data. This makes
73 # tests a little easier to write.
74 #
75 # The back-end store should only change with an explicit
76 # set/delete/append/etc
77 data_copy = copy.deepcopy(obj[0])
78 return data_copy
79
80 def set(self, key, value, time=0):
81 """Sets the value for a key."""
82 self.check_key(key)
83 # NOTE(morganfainberg): This behaves more like memcache
84 # actually does and prevents modification of the passed in
85 # reference from affecting the cached back-end data. This makes
86 # tests a little easier to write.
87 #
88 # The back-end store should only change with an explicit
89 # set/delete/append/etc
90 data_copy = copy.deepcopy(value)
91 self.cache[key] = (data_copy, time)
92 return True
93
94 def cas(self, key, value, time=0, min_compress_len=0):
95 # Call self.set() since we don't really do 'cas' here.
96 if self.reject_cas:
97 return False
98 return self.set(key, value, time=time)
99
100 def reset_cas(self):
101 #This is a stub for the memcache client reset_cas function.
102 pass
103
104 def delete(self, key):
105 self.check_key(key)
106 try:
107 del self.cache[key]
108 except KeyError:
109 #NOTE(bcwaldon): python-memcached always returns the same value
110 pass
111
112
113class MemcacheToken(tests.TestCase, test_backend.TokenTests):
114 def setUp(self):
115 super(MemcacheToken, self).setUp()
116 # Use the memcache backend for the token driver.
117 self.opt_in_group('token',
118 driver='keystone.token.backends.memcache.Token')
119 self.load_backends()
120 # Override the memcache client with the "dummy" client.
121 fake_client = MemcacheClient()
122 self.token_man = token.Manager()
123 self.token_man.driver = token_memcache.Token(client=fake_client)
124 self.token_api = self.token_man
125
126 def test_create_unicode_token_id(self):
127 token_id = six.text_type(self._create_token_id())
128 data = {'id': token_id, 'a': 'b',
129 'user': {'id': 'testuserid'}}
130 self.token_api.create_token(token_id, data)
131 self.token_api.get_token(token_id)
132
133 def test_create_unicode_user_id(self):
134 token_id = self._create_token_id()
135 user_id = six.text_type(uuid.uuid4().hex)
136 data = {'id': token_id, 'a': 'b',
137 'user': {'id': user_id}}
138 self.token_api.create_token(token_id, data)
139 self.token_api.get_token(token_id)
140
141 def test_list_tokens_unicode_user_id(self):
142 user_id = six.text_type(uuid.uuid4().hex)
143 self.token_api.list_tokens(user_id)
144
145 def test_flush_expired_token(self):
146 self.assertRaises(exception.NotImplemented,
147 self.token_api.flush_expired_tokens)
148
149 def test_cleanup_user_index_on_create(self):
150 valid_token_id = uuid.uuid4().hex
151 second_valid_token_id = uuid.uuid4().hex
152 expired_token_id = uuid.uuid4().hex
153 user_id = six.text_type(uuid.uuid4().hex)
154
155 expire_delta = datetime.timedelta(seconds=CONF.token.expiration)
156
157 valid_data = {'id': valid_token_id, 'a': 'b',
158 'user': {'id': user_id}}
159 second_valid_data = {'id': second_valid_token_id, 'a': 'b',
160 'user': {'id': user_id}}
161 expired_data = {'id': expired_token_id, 'a': 'b',
162 'user': {'id': user_id}}
163 self.token_api.create_token(valid_token_id, valid_data)
164 self.token_api.create_token(expired_token_id, expired_data)
165 # NOTE(morganfainberg): Directly access the data cache since we need to
166 # get expired tokens as well as valid tokens. token_api._list_tokens()
167 # will not return any expired tokens in the list.
168 user_key = self.token_api.driver._prefix_user_id(user_id)
169 user_token_list = self.token_api.driver.client.get(user_key)
170 self.assertEqual(len(user_token_list), 2)
171 # user_token_list is a list of (token, expiry) tuples
172 expired_idx = [i[0] for i in user_token_list].index(expired_token_id)
173 # set the token as expired.
174 user_token_list[expired_idx] = (user_token_list[expired_idx][0],
175 timeutils.utcnow() - expire_delta)
176 self.token_api.driver.client.set(user_key, user_token_list)
177
178 self.token_api.create_token(second_valid_token_id, second_valid_data)
179 user_token_list = self.token_api.driver.client.get(user_key)
180 self.assertEqual(len(user_token_list), 2)
181
182 def test_convert_token_list_from_json(self):
183 token_list = ','.join(['"%s"' % uuid.uuid4().hex for x in xrange(5)])
184 token_list_loaded = jsonutils.loads('[%s]' % token_list)
185 converted_list = self.token_api.driver._convert_user_index_from_json(
186 token_list, 'test-key')
187 for idx, item in enumerate(converted_list):
188 token_id, expiry = item
189 self.assertEqual(token_id, token_list_loaded[idx])
190 self.assertIsInstance(expiry, datetime.datetime)
191
192 def test_convert_token_list_from_json_non_string(self):
193 token_list = self.token_api.driver._convert_user_index_from_json(
194 None, 'test-key')
195 self.assertEqual([], token_list)
196
197 def test_convert_token_list_from_json_invalid_json(self):
198 token_list = self.token_api.driver._convert_user_index_from_json(
199 'invalid_json_list', 'test-key')
200 self.assertEqual([], token_list)
201
202 def test_cas_failure(self):
203 expire_delta = datetime.timedelta(seconds=86400)
204 self.token_api.driver.client.reject_cas = True
205 token_id = uuid.uuid4().hex
206 user_id = six.text_type(uuid.uuid4().hex)
207 token_data = {'expires': timeutils.utcnow() + expire_delta,
208 'id': token_id}
209 user_key = self.token_api.driver._prefix_user_id(user_id)
210 self.assertRaises(
211 exception.UnexpectedError,
212 self.token_api.driver._update_user_list_with_cas,
213 user_key, token_id, token_data)
214
215 def test_token_expire_timezone(self):
216
217 @test_utils.timezone
218 def _create_token(expire_time):
219 token_id = uuid.uuid4().hex
220 user_id = six.text_type(uuid.uuid4().hex)
221 data = {'id': token_id, 'a': 'b', 'user': {'id': user_id},
222 'expires': expire_time
223 }
224 self.token_api.create_token(token_id, data)
225 return data
226
227 for d in ['+0', '-11', '-8', '-5', '+5', '+8', '+14']:
228 test_utils.TZ = 'UTC' + d
229 expire_time = timeutils.utcnow() + \
230 datetime.timedelta(minutes=1)
231 data_in = _create_token(expire_time)
232 data_get = None
233 data_get = self.token_api.get_token(data_in['id'])
234
235 self.assertIsNotNone(data_get, "TZ=%s" % test_utils.TZ)
236 self.assertEqual(data_in['id'], data_get['id'],
237 "TZ=%s" % test_utils.TZ)
238
239 expire_time_expired = timeutils.utcnow() + \
240 datetime.timedelta(minutes=-1)
241 data_in = _create_token(expire_time_expired)
242 self.assertRaises(exception.TokenNotFound,
243 self.token_api.get_token, data_in['id'])
244
245
246class MemcacheTokenCacheInvalidation(tests.TestCase,
247 test_backend.TokenCacheInvalidation):
248 def setUp(self):
249 super(MemcacheTokenCacheInvalidation, self).setUp()
250 CONF.token.driver = 'keystone.token.backends.memcache.Token'
251 self.load_backends()
252 fake_client = MemcacheClient()
253 self.token_man = token.Manager()
254 self.token_man.driver = token_memcache.Token(client=fake_client)
255 self.token_api = self.token_man
256 self.token_provider_api.driver.token_api = self.token_api
257 self._create_test_data()
diff --git a/keystone/token/backends/memcache.py b/keystone/token/backends/memcache.py
index 2f77d4e..a08e78d 100644
--- a/keystone/token/backends/memcache.py
+++ b/keystone/token/backends/memcache.py
@@ -1,3 +1,6 @@
1# -*- coding: utf-8 -*-
2
3# Copyright 2013 Metacloud, Inc.
1# Copyright 2012 OpenStack Foundation 4# Copyright 2012 OpenStack Foundation
2# 5#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may 6# Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -12,260 +15,18 @@
12# License for the specific language governing permissions and limitations 15# License for the specific language governing permissions and limitations
13# under the License. 16# under the License.
14 17
15from __future__ import absolute_import 18from keystone.common import config
16import copy 19from keystone.token.backends import kvs
17import datetime
18
19import memcache
20
21from keystone.common import utils
22from keystone import config
23from keystone import exception
24from keystone.openstack.common import jsonutils
25from keystone.openstack.common import log
26from keystone.openstack.common import timeutils
27from keystone import token
28 20
29 21
30CONF = config.CONF 22CONF = config.CONF
31 23
32LOG = log.getLogger(__name__)
33
34
35class Token(token.Driver):
36 revocation_key = 'revocation-list'
37
38 def __init__(self, client=None):
39 self._memcache_client = client
40
41 @property
42 def client(self):
43 return self._memcache_client or self._get_memcache_client()
44
45 def _get_memcache_client(self):
46 memcache_servers = CONF.memcache.servers
47 # NOTE(morganfainberg): The memcache client library for python is NOT
48 # thread safe and should not be passed between threads. This is highly
49 # specific to the cas() (compare and set) methods and the caching of
50 # the previous value(s). It appears greenthread should ensure there is
51 # a single data structure per spawned greenthread.
52 self._memcache_client = memcache.Client(memcache_servers, debug=0,
53 cache_cas=True)
54 return self._memcache_client
55
56 def _prefix_token_id(self, token_id):
57 return 'token-%s' % token_id.encode('utf-8')
58
59 def _prefix_user_id(self, user_id):
60 return 'usertokens-%s' % user_id.encode('utf-8')
61
62 def get_token(self, token_id):
63 if token_id is None:
64 raise exception.TokenNotFound(token_id='')
65 ptk = self._prefix_token_id(token_id)
66 token_ref = self.client.get(ptk)
67 if token_ref is None:
68 raise exception.TokenNotFound(token_id=token_id)
69
70 return token_ref
71
72 def create_token(self, token_id, data):
73 data_copy = copy.deepcopy(data)
74 ptk = self._prefix_token_id(token_id)
75 if not data_copy.get('expires'):
76 data_copy['expires'] = token.default_expire_time()
77 if not data_copy.get('user_id'):
78 data_copy['user_id'] = data_copy['user']['id']
79 kwargs = {}
80 if data_copy['expires'] is not None:
81 expires_ts = utils.unixtime(data_copy['expires'])
82 kwargs['time'] = expires_ts
83 self.client.set(ptk, data_copy, **kwargs)
84 if 'id' in data['user']:
85 user_id = data['user']['id']
86 user_key = self._prefix_user_id(user_id)
87 # Append the new token_id to the token-index-list stored in the
88 # user-key within memcache.
89 self._update_user_list_with_cas(user_key, token_id, data_copy)
90 return copy.deepcopy(data_copy)
91
92 def _convert_user_index_from_json(self, token_list, user_key):
93 try:
94 # NOTE(morganfainberg): Try loading in the old format
95 # of the list.
96 token_list = jsonutils.loads('[%s]' % token_list)
97
98 # NOTE(morganfainberg): Build a delta based upon the
99 # token TTL configured. Since we are using the old
100 # format index-list, we will create a "fake" expiration
101 # that should be further in the future than the actual
102 # expiry. To avoid locking up keystone trying to
103 # communicate to memcached, it is better to use a fake
104 # value. The logic that utilizes this list already
105 # knows how to handle the case of tokens that are
106 # no longer valid being included.
107 delta = datetime.timedelta(
108 seconds=CONF.token.expiration)
109 new_expiry = timeutils.normalize_time(
110 timeutils.utcnow()) + delta
111
112 for idx, token_id in enumerate(token_list):
113 token_list[idx] = (token_id, new_expiry)
114
115 except Exception:
116 # NOTE(morganfainberg): Catch any errors thrown here. There is
117 # nothing the admin or operator needs to do in this case, but
118 # it should be logged that there was an error and some action was
119 # taken to correct it
120 LOG.exception(_('Error converting user-token-index to new format; '
121 'clearing user token index record "%s".'),
122 user_key)
123 token_list = []
124 return token_list
125
126 def _update_user_list_with_cas(self, user_key, token_id, token_data):
127 cas_retry = 0
128 max_cas_retry = CONF.memcache.max_compare_and_set_retry
129 current_time = timeutils.normalize_time(timeutils.utcnow())
130
131 self.client.reset_cas()
132
133 while cas_retry <= max_cas_retry:
134 # NOTE(morganfainberg): cas or "compare and set" is a function of
135 # memcache. It will return false if the value has changed since the
136 # last call to client.gets(). This is the memcache supported method
137 # of avoiding race conditions on set(). Memcache is already atomic
138 # on the back-end and serializes operations.
139 #
140 # cas_retry is for tracking our iterations before we give up (in
141 # case memcache is down or something horrible happens we don't
142 # iterate forever trying to compare and set the new value.
143 cas_retry += 1
144 token_list = self.client.gets(user_key)
145 filtered_list = []
146
147 if token_list is not None:
148 if not isinstance(token_list, list):
149 token_list = self._convert_user_index_from_json(token_list,
150 user_key)
151 for token_i, expiry in token_list:
152 expires_at = timeutils.normalize_time(expiry)
153 if expires_at < current_time:
154 # skip tokens that are expired.
155 continue
156
157 # Add the still valid token_id to the list.
158 filtered_list.append((token_i, expiry))
159 # Add the new token_id and expiry.
160 filtered_list.append(
161 (token_id, timeutils.normalize_time(token_data['expires'])))
162
163 # Use compare-and-set (cas) to set the new value for the
164 # token-index-list for the user-key. Cas is used to prevent race
165 # conditions from causing the loss of valid token ids from this
166 # list.
167 if self.client.cas(user_key, filtered_list):
168 msg = _('Successful set of token-index-list for user-key '
169 '"%(user_key)s", #%(count)d records')
170 LOG.debug(msg, {'user_key': user_key,
171 'count': len(filtered_list)})
172 return filtered_list
173
174 # The cas function will return true if it succeeded or false if it
175 # failed for any reason, including memcache server being down, cas
176 # id changed since gets() called (the data changed between when
177 # this loop started and this point, etc.
178 error_msg = _('Failed to set token-index-list for user-key '
179 '"%(user_key)s". Attempt %(cas_retry)d of '
180 '%(cas_retry_max)d')
181 LOG.debug(error_msg,
182 {'user_key': user_key,
183 'cas_retry': cas_retry,
184 'cas_retry_max': max_cas_retry})
185
186 # Exceeded the maximum retry attempts.
187 error_msg = _('Unable to add token user list')
188 raise exception.UnexpectedError(error_msg)
189
190 def _add_to_revocation_list(self, data):
191 data_json = jsonutils.dumps(data)
192 if not self.client.append(self.revocation_key, ',%s' % data_json):
193 if not self.client.add(self.revocation_key, data_json):
194 if not self.client.append(self.revocation_key,
195 ',%s' % data_json):
196 msg = _('Unable to add token to revocation list.')
197 raise exception.UnexpectedError(msg)
198
199 def delete_token(self, token_id):
200 # Test for existence
201 data = self.get_token(token_id)
202 ptk = self._prefix_token_id(token_id)
203 result = self.client.delete(ptk)
204 self._add_to_revocation_list(data)
205 return result
206
207 def delete_tokens(self, user_id, tenant_id=None, trust_id=None,
208 consumer_id=None):
209 return super(Token, self).delete_tokens(
210 user_id=user_id,
211 tenant_id=tenant_id,
212 trust_id=trust_id,
213 consumer_id=consumer_id,
214 )
215
216 def _list_tokens(self, user_id, tenant_id=None, trust_id=None,
217 consumer_id=None):
218 tokens = []
219 user_key = self._prefix_user_id(user_id)
220 current_time = timeutils.normalize_time(timeutils.utcnow())
221 token_list = self.client.get(user_key) or []
222 if not isinstance(token_list, list):
223 # NOTE(morganfainberg): This is for compatibility for old-format
224 # token-lists that were a JSON string of just token_ids. This code
225 # will reference the underlying expires directly from the
226 # token_ref vs in this list, so setting to none just ensures the
227 # loop works as expected.
228 token_list = [(i, None) for i in
229 jsonutils.loads('[%s]' % token_list)]
230 for token_id, expiry in token_list:
231 ptk = self._prefix_token_id(token_id)
232 token_ref = self.client.get(ptk)
233 if token_ref:
234 if tenant_id is not None:
235 tenant = token_ref.get('tenant')
236 if not tenant:
237 continue
238 if tenant.get('id') != tenant_id:
239 continue
240 if trust_id is not None:
241 trust = token_ref.get('trust_id')
242 if not trust:
243 continue
244 if trust != trust_id:
245 continue
246 if consumer_id is not None:
247 try:
248 oauth = token_ref['token_data']['token']['OS-OAUTH1']
249 if oauth.get('consumer_id') != consumer_id:
250 continue
251 except KeyError:
252 continue
253
254 if (timeutils.normalize_time(token_ref['expires']) <
255 current_time):
256 # Skip expired tokens.
257 continue
258
259 tokens.append(token_id)
260 return tokens
261 24
262 def list_revoked_tokens(self): 25class Token(kvs.Token):
263 list_json = self.client.get(self.revocation_key) 26 kvs_backend = 'openstack.kvs.Memcached'
264 if list_json:
265 return jsonutils.loads('[%s]' % list_json)
266 return []
267 27
268 def flush_expired_tokens(self): 28 def __init__(self, *args, **kwargs):
269 """Archive or delete tokens that have expired. 29 kwargs['no_expiry_keys'] = [self.revocation_key]
270 """ 30 kwargs['memcached_expire_time'] = CONF.token.expiration
271 raise exception.NotImplemented() 31 kwargs['url'] = CONF.memcache.servers
32 super(Token, self).__init__(*args, **kwargs)