openstack
/
keystone
Code Issues Proposed changes

Added cache for id mapping manager 

When using a identity driver without providing uuid, and using default
sql id mapping driver, if there are lots of users, then it may take
minutes to list users. Adding cache to the id mapping manager can
improve the performance.

After adding the cache, when listing 12000 users though the keystone
api, and the time is reduced from about 75 seconds to 20 seconds.

Closes-Bug: #1582585
Change-Id: I72eeb88926d8babb09a61e99f6f594371987f393
This commit is contained in:
liuhongjiang 2016-06-13 08:11:16 +08:00
parent 38aa653366
commit 53bb53a814
3 changed files with 126 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
keystone/identity/core.py
View File

@ -19,6 +19,7 @@ import os
import threading
import uuid
from oslo_cache import core as oslo_cache
from oslo_config import cfg
from oslo_log import log
from oslo_log import versionutils
@ -45,6 +46,10 @@ LOG = log.getLogger(__name__)
MEMOIZE = cache.get_memoization_decorator(group='identity')
ID_MAPPING_REGION = oslo_cache.create_region()
MEMOIZE_ID_MAPPING = cache.get_memoization_decorator(group='identity',
region=ID_MAPPING_REGION)
DOMAIN_CONF_FHEAD = 'keystone.'
DOMAIN_CONF_FTAIL = '.conf'
@ -1286,6 +1291,48 @@ class MappingManager(manager.Manager):
def __init__(self):
super(MappingManager, self).__init__(CONF.identity_mapping.driver)
@MEMOIZE_ID_MAPPING
def _get_public_id(self, domain_id, local_id, entity_type):
return self.driver.get_public_id({'domain_id': domain_id,
'local_id': local_id,
'entity_type': entity_type})
def get_public_id(self, local_entity):
return self._get_public_id(local_entity['domain_id'],
local_entity['local_id'],
local_entity['entity_type'])
@MEMOIZE_ID_MAPPING
def get_id_mapping(self, public_id):
return self.driver.get_id_mapping(public_id)
def create_id_mapping(self, local_entity, public_id=None):
public_id = self.driver.create_id_mapping(local_entity, public_id)
if MEMOIZE_ID_MAPPING.should_cache(public_id):
self._get_public_id.set(public_id, self,
local_entity['domain_id'],
local_entity['local_id'],
local_entity['entity_type'])
self.get_id_mapping.set(local_entity, self, public_id)
return public_id
def delete_id_mapping(self, public_id):
local_entity = self.get_id_mapping.get(self, public_id)
self.driver.delete_id_mapping(public_id)
# Delete the key of entity from cache
if local_entity:
self._get_public_id.invalidate(self, local_entity['domain_id'],
local_entity['local_id'],
local_entity['entity_type'])
self.get_id_mapping.invalidate(self, public_id)
def purge_mappings(self, purge_filter):
# Purge mapping is rarely used and only used by the command client,
# it's quite complex to invalidate part of the cache based on the purge
# filters, so here invalidate the whole cache when purging mappings.
self.driver.purge_mappings(purge_filter)
ID_MAPPING_REGION.invalidate()
@versionutils.deprecated(
versionutils.deprecated.NEWTON,

3
keystone/server/backends.py
View File

@ -42,6 +42,9 @@ def load_backends():
cache.apply_invalidation_patch(region=revoke.REVOKE_REGION,
region_name=revoke.REVOKE_REGION.name)
cache.configure_cache(region=token.provider.TOKENS_REGION)
cache.configure_cache(region=identity.ID_MAPPING_REGION)
cache.apply_invalidation_patch(region=identity.ID_MAPPING_REGION,
region_name=identity.ID_MAPPING_REGION.name)
# Ensure that the identity driver is created before the assignment manager
# and that the assignment driver is created before the resource manager.

76
keystone/tests/unit/test_backend_id_mapping_sql.py
View File

@ -214,3 +214,79 @@ class SqlIDMapping(test_backend_sql.SqlTests):
public_id3 = self.id_mapping_api.create_id_mapping(
local_entity, public_id=uuid.uuid4().hex)
self.assertEqual(public_id1, public_id3)
@unit.skip_if_cache_disabled('identity')
def test_cache_when_id_mapping_crud(self):
local_id = uuid.uuid4().hex
local_entity = {'domain_id': self.domainA['id'],
'local_id': local_id,
'entity_type': mapping.EntityType.USER}
# Check no mappings for the new local entity
self.assertIsNone(self.id_mapping_api.get_public_id(local_entity))
# Create new mappings, and it should be in the cache after created
public_id = self.id_mapping_api.create_id_mapping(local_entity)
self.assertEqual(
public_id, self.id_mapping_api.get_public_id(local_entity))
local_id_ref = self.id_mapping_api.get_id_mapping(public_id)
self.assertEqual(self.domainA['id'], local_id_ref['domain_id'])
self.assertEqual(local_id, local_id_ref['local_id'])
self.assertEqual(mapping.EntityType.USER, local_id_ref['entity_type'])
# After delete the mapping, should be deleted from cache too
self.id_mapping_api.delete_id_mapping(public_id)
self.assertIsNone(self.id_mapping_api.get_public_id(local_entity))
self.assertIsNone(self.id_mapping_api.get_id_mapping(public_id))
@unit.skip_if_cache_disabled('identity')
def test_invalidate_cache_when_purge_mappings(self):
local_id1 = uuid.uuid4().hex
local_id2 = uuid.uuid4().hex
local_id3 = uuid.uuid4().hex
local_id4 = uuid.uuid4().hex
local_id5 = uuid.uuid4().hex
# Create five mappings,two in domainA, three in domainB
local_entity1 = {'domain_id': self.domainA['id'],
'local_id': local_id1,
'entity_type': mapping.EntityType.USER}
local_entity2 = {'domain_id': self.domainA['id'],
'local_id': local_id2,
'entity_type': mapping.EntityType.USER}
local_entity3 = {'domain_id': self.domainB['id'],
'local_id': local_id3,
'entity_type': mapping.EntityType.GROUP}
local_entity4 = {'domain_id': self.domainB['id'],
'local_id': local_id4,
'entity_type': mapping.EntityType.USER}
local_entity5 = {'domain_id': self.domainB['id'],
'local_id': local_id5,
'entity_type': mapping.EntityType.USER}
self.id_mapping_api.create_id_mapping(local_entity1)
self.id_mapping_api.create_id_mapping(local_entity2)
self.id_mapping_api.create_id_mapping(local_entity3)
self.id_mapping_api.create_id_mapping(local_entity4)
self.id_mapping_api.create_id_mapping(local_entity5)
# Purge mappings for domainA, should be left with those in B
self.id_mapping_api.purge_mappings(
{'domain_id': self.domainA['id']})
self.assertIsNone(self.id_mapping_api.get_public_id(local_entity1))
self.assertIsNone(self.id_mapping_api.get_public_id(local_entity2))
# Purge mappings for type Group, should purge one more
self.id_mapping_api.purge_mappings(
{'entity_type': mapping.EntityType.GROUP})
self.assertIsNone(self.id_mapping_api.get_public_id(local_entity3))
# Purge mapping for a specific local identifier
self.id_mapping_api.purge_mappings(
{'domain_id': self.domainB['id'], 'local_id': local_id4,
'entity_type': mapping.EntityType.USER})
self.assertIsNone(self.id_mapping_api.get_public_id(local_entity4))
# Purge mappings the remaining mappings
self.id_mapping_api.purge_mappings({})
self.assertIsNone(self.id_mapping_api.get_public_id(local_entity5))