openstack
/
keystone
Code Issues Proposed changes

list_revoked_tokens sql speedup for havana 

This consists of the following 3 patches:

    Narrow columns used in list_revoked_tokens sql

    Currently the SQL backend lists revoked tokens by selecting all of the
    columns, including the massive "extra" column. This places a significant
    burden on the client library and wastes resources. We only need the
    id/expired columns to satisfy the API call.

    In tests this query was several orders of magnitude faster with just two
    thousand un-expired revoked tokens.
    (cherry picked from commit ab7221246a)

    Add index to cover revoked token list

    The individual expires and valid indexes do not fully cover the most
    common query, which is the one that lists revoked tokens.

    Because valid is only ever used in conjunction with expires, we do not
    need it to have its own index now that there is a covering compound
    index for expires and valid.

    Note that he expires index is still useful alone for purging old tokens
    as we do not filter for valid in that case.
    (cherry picked from commit dd2c80c566)

    Remove unused token.valid index

    Because valid is only ever used in conjunction with expires, we do not
    need it to have its own index now that there is a covering compound
    index for expires and valid.

    Note that he expires index is still useful alone for purging old tokens
    as we do not filter for valid in that case.
    (cherry picked from commit 5d8a1a4142)

Change-Id: I04d62b98d5d760a3fbc3c8db61530f7ebccb0a48
Closes-Bug: #1253755
This commit is contained in:
Clint Byrum 2013-11-22 08:50:39 -08:00 committed by Dirk Mueller
parent 9983fbeaf3
commit 2414bab51d
5 changed files with 104 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
keystone/common/sql/migrate_repo/versions/035_add_compound_revoked_token_index.py Normal file
View File

@ -0,0 +1,33 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sql
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
token = sql.Table('token', meta, autoload=True)
idx = sql.Index('ix_token_expires_valid', token.c.expires, token.c.valid)
idx.create(migrate_engine)
def downgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
token = sql.Table('token', meta, autoload=True)
idx = sql.Index('ix_token_expires_valid', token.c.expires, token.c.valid)
idx.drop(migrate_engine)

33
keystone/common/sql/migrate_repo/versions/036_token_drop_valid_index.py Normal file
View File

@ -0,0 +1,33 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy as sql
def upgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
token = sql.Table('token', meta, autoload=True)
idx = sql.Index('ix_token_valid', token.c.valid)
idx.drop(migrate_engine)
def downgrade(migrate_engine):
meta = sql.MetaData()
meta.bind = migrate_engine
token = sql.Table('token', meta, autoload=True)
idx = sql.Index('ix_token_valid', token.c.valid)
idx.create(migrate_engine)

20
keystone/tests/test_backend_sql.py
View File

@ -22,9 +22,11 @@ from keystone.common import sql
from keystone import config
from keystone import exception
from keystone.identity.backends import sql as identity_sql
from keystone.openstack.common.fixture import moxstubout
from keystone import tests
from keystone.tests import default_fixtures
from keystone.tests import test_backend
from keystone.token.backends import sql as token_sql
CONF = config.CONF
@ -352,7 +354,23 @@ class SqlTrust(SqlTests, test_backend.TrustTests):
class SqlToken(SqlTests, test_backend.TokenTests):
pass
def test_token_revocation_list_uses_right_columns(self):
# This query used to be heavy with too many columns. We want
# to make sure it is only running with the minimum columns
# necessary.
fixture = self.useFixture(moxstubout.MoxStubout())
self.mox = fixture.mox
tok = token_sql.Token()
session = tok.get_session()
q = session.query(token_sql.TokenModel.id,
token_sql.TokenModel.expires)
self.mox.StubOutWithMock(session, 'query')
session.query(token_sql.TokenModel.id,
token_sql.TokenModel.expires).AndReturn(q)
self.mox.StubOutWithMock(tok, 'get_session')
tok.get_session().AndReturn(session)
self.mox.ReplayAll()
tok.list_revoked_tokens()
class SqlCatalog(SqlTests, test_backend.CatalogTests):

15
keystone/tests/test_sql_upgrade.py
View File

@ -1333,6 +1333,21 @@ class SqlUpgradeTests(SqlMigrateBase):
else:
self.assertEqual(len(index_data), 0)
def test_revoked_token_index(self):
self.upgrade(35)
table = sqlalchemy.Table('token', self.metadata, autoload=True)
index_data = [(idx.name, idx.columns.keys())
for idx in table.indexes]
self.assertIn(('ix_token_expires_valid', ['expires', 'valid']),
index_data)
def test_dropped_valid_index(self):
self.upgrade(36)
table = sqlalchemy.Table('token', self.metadata, autoload=True)
index_data = [(idx.name, idx.columns.keys())
for idx in table.indexes]
self.assertNotIn(('ix_token_valid', ['valid']), index_data)
def test_migrate_ec2_credential(self):
user = {
'id': 'foo',

8
keystone/token/backends/sql.py
View File

@ -33,7 +33,7 @@ class TokenModel(sql.ModelBase, sql.DictBase):
trust_id = sql.Column(sql.String(64))
__table_args__ = (
sql.Index('ix_token_expires', 'expires'),
sql.Index('ix_token_valid', 'valid')
sql.Index('ix_token_expires_valid', 'expires', 'valid')
)
@ -181,13 +181,13 @@ class Token(sql.Base, token.Driver):
session = self.get_session()
tokens = []
now = timeutils.utcnow()
query = session.query(TokenModel)
query = session.query(TokenModel.id, TokenModel.expires)
query = query.filter(TokenModel.expires > now)
token_references = query.filter_by(valid=False)
for token_ref in token_references:
record = {
'id': token_ref['id'],
'expires': token_ref['expires'],
'id': token_ref[0],
'expires': token_ref[1],
}
tokens.append(record)
return tokens