Add database layer for the quota part

Added a db abstraction for quota sync methods.
Implementation currently is based on sqlalchemy.
Added database migration enablement.
Added the db unit tests.
Further enhancement to the db API will be added
in a separate comment.

Change-Id: I56284919ad54d2359fa03d476cc89a4acdc68122
This commit is contained in:
Dimitri Mazmanov 2015-11-23 15:04:37 +01:00
parent de686cd200
commit 0613521f19
19 changed files with 771 additions and 8 deletions

View File

@ -1,4 +1,5 @@
# Copyright 2015 Huawei Technologies Co., Ltd.
# Copyright 2015 Ericsson AB.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -16,9 +17,9 @@
"""
Kingbird base exception handling.
"""
import six
from oslo_utils import excutils
import six
from kingbird.common.i18n import _
@ -82,3 +83,7 @@ class InUse(KingbirdException):
class InvalidConfigurationOption(KingbirdException):
message = _("An invalid value was provided for %(opt_name)s: "
"%(opt_value)s")
class ProjectQuotaNotFound(NotFound):
message = _("Quota for project %(project_id) doesn't exist.")

81
kingbird/db/api.py Normal file
View File

@ -0,0 +1,81 @@
# Copyright (c) 2015 Ericsson AB.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
'''
Interface for database access.
SQLAlchemy is currently the only supported backend.
'''
from oslo_config import cfg
from oslo_db import api
CONF = cfg.CONF
_BACKEND_MAPPING = {'sqlalchemy': 'kingbird.db.sqlalchemy.api'}
IMPL = api.DBAPI.from_config(CONF, backend_mapping=_BACKEND_MAPPING)
def get_engine():
return IMPL.get_engine()
def get_session():
return IMPL.get_session()
# quota usage db methods
###################
def quota_create(context, project_id, resource, limit):
"""Create a quota for the given project and resource."""
return IMPL.quota_create(context, project_id, resource, limit)
def quota_get(context, project_id, resource):
"""Retrieve a quota or raise if it does not exist."""
return IMPL.quota_get(context, project_id, resource)
def quota_get_all_by_project(context, project_id):
"""Retrieve all quotas associated with a given project."""
return IMPL.quota_get_all_by_project(context, project_id)
def quota_update(context, project_id, resource, limit):
"""Update a quota or raise if it does not exist."""
return IMPL.quota_update(context, project_id, resource, limit)
def quota_destroy(context, project_id, resource):
"""Destroy the quota or raise if it does not exist."""
return IMPL.quota_destroy(context, project_id, resource)
def quota_destroy_all(context, project_id):
"""Destroy the quota or raise if it does not exist."""
return IMPL.quota_destroy(context, project_id)
def db_sync(engine, version=None):
"""Migrate the database to `version` or the most recent version."""
return IMPL.db_sync(engine, version=version)
def db_version(engine):
"""Display the current database version."""
return IMPL.db_version(engine)

View File

View File

@ -0,0 +1,199 @@
# Copyright (c) 2015 Ericsson AB.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
'''
Implementation of SQLAlchemy backend.
'''
import sys
from oslo_config import cfg
from oslo_db.sqlalchemy import session as db_session
from oslo_log import log as logging
from kingbird.common import exceptions as exception
from kingbird.common.i18n import _
from kingbird.db.sqlalchemy import migration
from kingbird.db.sqlalchemy import models
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
_facade = None
def get_facade():
global _facade
if not _facade:
_facade = db_session.EngineFacade.from_config(CONF)
return _facade
get_engine = lambda: get_facade().get_engine()
get_session = lambda: get_facade().get_session()
def get_backend():
"""The backend is this module itself."""
return sys.modules[__name__]
def model_query(context, *args):
session = _session(context)
query = session.query(*args)
return query
def _session(context):
return get_session()
def is_admin_context(context):
"""Indicates if the request context is an administrator."""
if not context:
LOG.warn(_('Use of empty request context is deprecated'),
DeprecationWarning)
raise Exception('die')
return context.is_admin
def is_user_context(context):
"""Indicates if the request context is a normal user."""
if not context:
return False
if context.is_admin:
return False
if not context.user_id or not context.project_id:
return False
return True
def require_admin_context(f):
"""Decorator to require admin request context.
The first argument to the wrapped function must be the context.
"""
def wrapper(*args, **kwargs):
if not is_admin_context(args[0]):
raise exception.AdminRequired()
return f(*args, **kwargs)
return wrapper
def require_context(f):
"""Decorator to require *any* user or admin context.
This does no authorization for user or project access matching, see
:py:func:`authorize_project_context` and
:py:func:`authorize_user_context`.
The first argument to the wrapped function must be the context.
"""
def wrapper(*args, **kwargs):
if not is_admin_context(args[0]) and not is_user_context(args[0]):
raise exception.Forbidden()
return f(*args, **kwargs)
return wrapper
###################
@require_context
def _quota_get(context, project_id, resource, session=None):
result = model_query(context, models.Quota). \
filter_by(project_id=project_id). \
filter_by(resource=resource). \
first()
if not result:
raise exception.ProjectQuotaNotFound(project_id=project_id)
return result
@require_context
def quota_get(context, project_id, resource):
return _quota_get(context, project_id, resource)
@require_context
def quota_get_all_by_project(context, project_id):
rows = model_query(context, models.Quota). \
filter_by(project_id=project_id). \
all()
result = {'project_id': project_id}
for row in rows:
result[row.resource] = row.hard_limit
return result
@require_admin_context
def quota_create(context, project_id, resource, limit):
quota_ref = models.Quota()
quota_ref.project_id = project_id
quota_ref.resource = resource
quota_ref.hard_limit = limit
session = _session(context)
with session.begin():
quota_ref.save(session)
return quota_ref
@require_admin_context
def quota_update(context, project_id, resource, limit):
session = _session(context)
with session.begin():
quota_ref = _quota_get(context, project_id, resource, session=session)
quota_ref.hard_limit = limit
quota_ref.save(_session(context))
return quota_ref
@require_admin_context
def quota_destroy(context, project_id, resource):
session = _session(context)
quota_ref = _quota_get(context, project_id, resource, session=session)
quota_ref.delete(session=session)
@require_admin_context
def quota_destroy_all(context, project_id):
session = _session(context)
quotas = model_query(context, models.Quota). \
filter_by(project_id=project_id). \
all()
for quota_ref in quotas:
quota_ref.delete(session=session)
def db_sync(engine, version=None):
"""Migrate the database to `version` or the most recent version."""
return migration.db_sync(engine, version=version)
def db_version(engine):
"""Display the current database version."""
return migration.db_version(engine)

View File

@ -0,0 +1,4 @@
This is a database migration repository.
More information at
http://code.google.com/p/sqlalchemy-migrate/

View File

@ -0,0 +1,5 @@
#!/usr/bin/env python
from migrate.versioning.shell import main
if __name__ == '__main__':
main(debug='False')

View File

@ -0,0 +1,25 @@
[db_settings]
# Used to identify which repository this database is versioned under.
# You can use the name of your project.
repository_id=kingbird
# The name of the database table used to track the schema version.
# This name shouldn't already be used by your project.
# If this is changed once a database is under version control, you'll need to
# change the table name in each database too.
version_table=migrate_version
# When committing a change script, Migrate will attempt to generate the
# sql for all supported databases; normally, if one of them fails - probably
# because you don't have that database installed - it is ignored and the
# commit continues, perhaps ending successfully.
# Databases in this list MUST compile successfully during a commit, or the
# entire commit will fail. List the databases your application will actually
# be using to ensure your updates to that database work properly.
# This must be a list; example: ['postgres','sqlite']
required_dbs=[]
# When creating new change scripts, Migrate will stamp the new script with
# a version number. By default this is latest_version + 1. You can set this
# to 'true' to tell Migrate to use the UTC timestamp instead.
use_timestamp_numbering=False

View File

@ -0,0 +1,54 @@
# Copyright (c) 2015 Ericsson AB.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy
def upgrade(migrate_engine):
meta = sqlalchemy.MetaData()
meta.bind = migrate_engine
quotas = sqlalchemy.Table(
'quotas', meta,
sqlalchemy.Column('id', sqlalchemy.Integer,
primary_key=True, nullable=False),
sqlalchemy.Column('project_id', sqlalchemy.String(36)),
sqlalchemy.Column('resource', sqlalchemy.String(255), nullable=False),
sqlalchemy.Column('hard_limit', sqlalchemy.Integer, nullable=False),
sqlalchemy.Column('created_at', sqlalchemy.DateTime),
sqlalchemy.Column('updated_at', sqlalchemy.DateTime),
sqlalchemy.Column('deleted_at', sqlalchemy.DateTime),
sqlalchemy.Column('deleted', sqlalchemy.Integer),
mysql_engine='InnoDB',
mysql_charset='utf8'
)
tables = (
quotas,
)
for index, table in enumerate(tables):
try:
table.create()
except Exception:
# If an error occurs, drop all tables created so far to return
# to the previously existing state.
meta.drop_all(tables=tables[:index])
raise
def downgrade(migrate_engine):
raise NotImplementedError('Database downgrade not supported - '
'would drop all tables')

View File

@ -0,0 +1,40 @@
# Copyright (c) 2015 Ericsson AB.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from oslo_db.sqlalchemy import migration as oslo_migration
INIT_VERSION = 0
def db_sync(engine, version=None):
path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'migrate_repo')
return oslo_migration.db_sync(engine, path, version,
init_version=INIT_VERSION)
def db_version(engine):
path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'migrate_repo')
return oslo_migration.db_version(engine, path, INIT_VERSION)
def db_version_control(engine, version=None):
path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'migrate_repo')
return oslo_migration.db_version_control(engine, path, version)

View File

@ -0,0 +1,77 @@
# Copyright (c) 2015 Ericsson AB
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
SQLAlchemy models for kingbird data.
"""
from oslo_config import cfg
from oslo_db.sqlalchemy import models
from sqlalchemy.orm import session as orm_session
from sqlalchemy import (Column, Integer, String)
from sqlalchemy.ext.declarative import declarative_base
CONF = cfg.CONF
BASE = declarative_base()
def get_session():
from kingbird.db.sqlalchemy import api as db_api
return db_api.get_session()
class KingbirdBase(models.ModelBase,
models.SoftDeleteMixin,
models.TimestampMixin):
"""Base class for Kingbird Models."""
__table_args__ = {'mysql_engine': 'InnoDB'}
def expire(self, session=None, attrs=None):
if not session:
session = orm_session.Session.object_session(self)
if not session:
session = get_session()
session.expire(self, attrs)
def refresh(self, session=None, attrs=None):
"""Refresh this object."""
if not session:
session = orm_session.Session.object_session(self)
if not session:
session = get_session()
session.refresh(self, attrs)
def delete(self, session=None):
"""Delete this object."""
if not session:
session = orm_session.Session.object_session(self)
if not session:
session = get_session()
session.begin()
session.delete(self)
session.commit()
class Quota(BASE, KingbirdBase):
__tablename__ = 'quotas'
id = Column(Integer, primary_key=True)
project_id = Column(String(36))
resource = Column(String(255), nullable=False)
hard_limit = Column(Integer, nullable=False)

48
kingbird/db/utils.py Normal file
View File

@ -0,0 +1,48 @@
# Copyright (c) 2015 Ericsson AB.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
class LazyPluggable(object):
"""A pluggable backend loaded lazily based on some value."""
def __init__(self, pivot, **backends):
self.__backends = backends
self.__pivot = pivot
self.__backend = None
def __get_backend(self):
if not self.__backend:
backend_name = 'sqlalchemy'
backend = self.__backends[backend_name]
if isinstance(backend, tuple):
name = backend[0]
fromlist = backend[1]
else:
name = backend
fromlist = backend
self.__backend = __import__(name, None, None, fromlist)
return self.__backend
def __getattr__(self, key):
backend = self.__get_backend()
return getattr(backend, key)
IMPL = LazyPluggable('backend', sqlalchemy='kingbird.db.sqlalchemy.api')
def purge_deleted(age, granularity='days'):
IMPL.purge_deleted(age, granularity)

View File

@ -1,13 +1,11 @@
# -*- coding: utf-8 -*-
# Copyright 2010-2011 OpenStack Foundation
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
# Copyright (c) 2015 Ericsson AB
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
@ -19,5 +17,4 @@ from oslotest import base
class TestCase(base.BaseTestCase):
"""Test case base class for all unit tests."""

View File

View File

@ -0,0 +1,136 @@
# Copyright (c) 2015 Ericsson AB
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sqlalchemy
from oslo_config import cfg
from oslo_db import options
from kingbird.common import exceptions
from kingbird.db import api as api
from kingbird.db.sqlalchemy import api as db_api
from kingbird.tests import base
from kingbird.tests import utils
get_engine = api.get_engine
UUID1 = utils.UUID1
UUID2 = utils.UUID2
UUID3 = utils.UUID3
class DBAPIQuotaTest(base.TestCase):
def setup_dummy_db(self):
options.cfg.set_defaults(options.database_opts,
sqlite_synchronous=False)
options.set_defaults(cfg.CONF, connection="sqlite://",
sqlite_db='kingbird.db')
engine = get_engine()
db_api.db_sync(engine)
engine.connect()
def reset_dummy_db(self):
engine = get_engine()
meta = sqlalchemy.MetaData()
meta.reflect(bind=engine)
for table in reversed(meta.sorted_tables):
if table.name == 'migrate_version':
continue
engine.execute(table.delete())
def create_quota_limit(self, ctxt, **kwargs):
values = {
'project_id': utils.UUID1,
'resource': "ram",
'limit': 10,
}
values.update(kwargs)
return db_api.quota_create(ctxt, **values)
def setUp(self):
super(DBAPIQuotaTest, self).setUp()
self.setup_dummy_db()
self.addCleanup(self.reset_dummy_db)
self.ctx = utils.dummy_context()
def test_create_quota_limit(self):
project_id = UUID2
resource = 'cores'
limit = self.create_quota_limit(self.ctx, project_id=project_id,
resource=resource, limit=15)
self.assertIsNotNone(limit)
cores_limit = db_api.quota_get(self.ctx, project_id, resource)
self.assertIsNotNone(cores_limit)
self.assertEqual(15, cores_limit.hard_limit)
def test_update_quota_limit(self):
project_id = UUID2
resource = 'cores'
limit = self.create_quota_limit(self.ctx, project_id=project_id,
resource=resource, limit=15)
self.assertIsNotNone(limit)
updated = db_api.quota_update(self.ctx, project_id, resource, 10)
self.assertIsNotNone(updated)
updated_limit = db_api.quota_get(self.ctx, project_id, resource)
self.assertEqual(10, updated_limit.hard_limit)
def test_delete_quota_limit(self):
project_id = UUID2
resource = 'cores'
limit = self.create_quota_limit(self.ctx, project_id=project_id,
resource=resource, limit=15)
self.assertIsNotNone(limit)
db_api.quota_destroy(self.ctx, project_id, resource)
self.assertRaises(exceptions.ProjectQuotaNotFound,
db_api.quota_get,
self.ctx, project_id, resource)
def test_delete_all_quota_limit(self):
project_id = UUID2
resources = [('cores', 2), ('ram', 2)]
for r in resources:
self.create_quota_limit(self.ctx,
project_id=project_id,
resource=r[0],
limit=r[1])
db_api.quota_destroy_all(self.ctx, project_id)
for r in resources:
self.assertRaises(exceptions.ProjectQuotaNotFound,
db_api.quota_get,
self.ctx, project_id, r[0])
def test_quota_get_by_project(self):
project_id = UUID2
resource = 'cores'
limit = self.create_quota_limit(self.ctx, project_id=project_id,
resource=resource, limit=15)
self.assertIsNotNone(limit)
by_project = db_api.quota_get_all_by_project(self.ctx, project_id)
self.assertIsNotNone(by_project)
self.assertEqual(project_id, by_project['project_id'])

View File

@ -23,6 +23,5 @@ from kingbird.tests import base
class TestKingbird(base.TestCase):
def test_something(self):
pass

91
kingbird/tests/utils.py Normal file
View File

@ -0,0 +1,91 @@
# Copyright (c) 2015 Ericsson AB
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import random
import sqlalchemy
import string
import uuid
from oslo_config import cfg
from oslo_db import options
from kingbird.common import context
from kingbird.db import api as db_api
get_engine = db_api.get_engine
class UUIDStub(object):
def __init__(self, value):
self.value = value
def __enter__(self):
self.uuid4 = uuid.uuid4
uuid_stub = lambda: self.value
uuid.uuid4 = uuid_stub
def __exit__(self, *exc_info):
uuid.uuid4 = self.uuid4
UUIDs = (UUID1, UUID2, UUID3) = sorted([str(uuid.uuid4())
for x in range(3)])
def random_name():
return ''.join(random.choice(string.ascii_uppercase)
for x in range(10))
def setup_dummy_db():
options.cfg.set_defaults(options.database_opts, sqlite_synchronous=False)
options.set_defaults(cfg.CONF, connection="sqlite://",
sqlite_db='kingbird.db')
engine = get_engine()
db_api.db_sync(engine)
engine.connect()
def reset_dummy_db():
engine = get_engine()
meta = sqlalchemy.MetaData()
meta.reflect(bind=engine)
for table in reversed(meta.sorted_tables):
if table.name == 'migrate_version':
continue
engine.execute(table.delete())
def create_quota_limit(ctxt, **kwargs):
values = {
'project_id': UUID1,
'resource': "ram",
'limit': 10,
}
values.update(kwargs)
return db_api.quota_create(ctxt, **values)
def dummy_context(user='test_username', tenant='test_project_id',
region_name=None):
return context.ContextBase.from_dict({
'auth_token': 'abcd1234',
'user': user,
'tenant': tenant,
'is_admin': True,
'region_name': region_name
})

View File

@ -39,3 +39,5 @@ oslo.serialization>=1.10.0 # Apache-2.0
oslo.service>=0.12.0 # Apache-2.0
oslo.utils!=2.6.0,>=2.4.0 # Apache-2.0
oslo.versionedobjects>=0.9.0
SQLAlchemy<1.1.0,>=0.9.9
sqlalchemy-migrate>=0.9.6