Add online migration to move quotas to API database

This moves quota limits and classes from the main database to the
API database in an online migration.

Part of blueprint cells-quota-api-db

Change-Id: I64b600b30f6e54db0ec9083c6c176e895c6d0cc2
This commit is contained in:
melanie witt 2016-12-14 18:18:17 +00:00 committed by Matt Riedemann
parent b31062803e
commit b291061cb1
4 changed files with 258 additions and 0 deletions

View File

@ -90,6 +90,7 @@ from nova.objects import host_mapping as host_mapping_obj
from nova.objects import instance as instance_obj
from nova.objects import instance_group as instance_group_obj
from nova.objects import keypair as keypair_obj
from nova.objects import quotas as quotas_obj
from nova.objects import request_spec
from nova import quota
from nova import rpc
@ -646,6 +647,10 @@ class DbCommands(object):
build_request_obj.delete_build_requests_with_no_instance_uuid,
# Added in Pike
db.service_uuids_online_data_migration,
# Added in Pike
quotas_obj.migrate_quota_limits_to_api_db,
# Added in Pike
quotas_obj.migrate_quota_classes_to_api_db,
)
def __init__(self):

View File

@ -19,6 +19,7 @@ from oslo_db import exception as db_exc
from nova import db
from nova.db.sqlalchemy import api as db_api
from nova.db.sqlalchemy import api_models
from nova.db.sqlalchemy import models as main_models
from nova import exception
from nova.objects import base
from nova.objects import fields
@ -515,3 +516,131 @@ class QuotasNoOp(Quotas):
def check_deltas(cls, context, deltas, *count_args, **count_kwargs):
pass
@db_api.require_context
@db_api.pick_context_manager_reader
def _get_main_per_project_limits(context, limit):
return context.session.query(main_models.Quota).\
filter_by(deleted=0).\
limit(limit).\
all()
@db_api.require_context
@db_api.pick_context_manager_reader
def _get_main_per_user_limits(context, limit):
return context.session.query(main_models.ProjectUserQuota).\
filter_by(deleted=0).\
limit(limit).\
all()
@db_api.require_context
@db_api.pick_context_manager_writer
def _destroy_main_per_project_limits(context, project_id, resource):
context.session.query(main_models.Quota).\
filter_by(deleted=0).\
filter_by(project_id=project_id).\
filter_by(resource=resource).\
soft_delete(synchronize_session=False)
@db_api.require_context
@db_api.pick_context_manager_writer
def _destroy_main_per_user_limits(context, project_id, resource, user_id):
context.session.query(main_models.ProjectUserQuota).\
filter_by(deleted=0).\
filter_by(project_id=project_id).\
filter_by(user_id=user_id).\
filter_by(resource=resource).\
soft_delete(synchronize_session=False)
@db_api.api_context_manager.writer
def _create_limits_in_api_db(context, db_limits, per_user=False):
for db_limit in db_limits:
user_id = db_limit.user_id if per_user else None
Quotas._create_limit_in_db(context, db_limit.project_id,
db_limit.resource, db_limit.hard_limit,
user_id=user_id)
def migrate_quota_limits_to_api_db(context, count):
# Migrate per project limits
main_per_project_limits = _get_main_per_project_limits(context, count)
done = 0
try:
# Create all the limits in a single transaction.
_create_limits_in_api_db(context, main_per_project_limits)
except exception.QuotaExists:
# NOTE(melwitt): This can happen if the migration is interrupted after
# limits were created in the api db but before they were deleted from
# the main db, and the migration is re-run.
pass
# Delete the limits separately.
for db_limit in main_per_project_limits:
_destroy_main_per_project_limits(context, db_limit.project_id,
db_limit.resource)
done += 1
if done == count:
return len(main_per_project_limits), done
# Migrate per user limits
count -= done
main_per_user_limits = _get_main_per_user_limits(context, count)
try:
# Create all the limits in a single transaction.
_create_limits_in_api_db(context, main_per_user_limits, per_user=True)
except exception.QuotaExists:
# NOTE(melwitt): This can happen if the migration is interrupted after
# limits were created in the api db but before they were deleted from
# the main db, and the migration is re-run.
pass
# Delete the limits separately.
for db_limit in main_per_user_limits:
_destroy_main_per_user_limits(context, db_limit.project_id,
db_limit.resource, db_limit.user_id)
done += 1
return len(main_per_project_limits) + len(main_per_user_limits), done
@db_api.require_context
@db_api.pick_context_manager_reader
def _get_main_quota_classes(context, limit):
return context.session.query(main_models.QuotaClass).\
filter_by(deleted=0).\
limit(limit).\
all()
@db_api.pick_context_manager_writer
def _destroy_main_quota_classes(context, db_classes):
for db_class in db_classes:
context.session.query(main_models.QuotaClass).\
filter_by(deleted=0).\
filter_by(id=db_class.id).\
soft_delete(synchronize_session=False)
@db_api.api_context_manager.writer
def _create_classes_in_api_db(context, db_classes):
for db_class in db_classes:
Quotas._create_class_in_db(context, db_class.class_name,
db_class.resource, db_class.hard_limit)
def migrate_quota_classes_to_api_db(context, count):
main_quota_classes = _get_main_quota_classes(context, count)
done = 0
try:
# Create all the classes in a single transaction.
_create_classes_in_api_db(context, main_quota_classes)
except exception.QuotaClassExists:
# NOTE(melwitt): This can happen if the migration is interrupted after
# classes were created in the api db but before they were deleted from
# the main db, and the migration is re-run.
pass
# Delete the classes in a single transaction.
_destroy_main_quota_classes(context, main_quota_classes)
found = done = len(main_quota_classes)
return found, done

View File

@ -11,6 +11,7 @@
# under the License.
from nova import context
from nova.db.sqlalchemy import api as db_api
from nova import exception
from nova.objects import quotas
from nova import test
@ -206,3 +207,113 @@ class QuotasObjectTestCase(test.TestCase,
self.assertEqual('foo', limits_dict['class_name'])
self.assertEqual(5, limits_dict['instances'])
self.assertEqual(10, limits_dict['cores'])
def test_migrate_quota_limits(self):
# Create a limit in api db
quotas.Quotas._create_limit_in_db(self.context, 'fake-project',
'instances', 5, user_id='fake-user')
# Create 4 limits in main db
db_api.quota_create(self.context, 'fake-project', 'cores', 10,
user_id='fake-user')
db_api.quota_create(self.context, 'fake-project', 'ram', 8192,
user_id='fake-user')
db_api.quota_create(self.context, 'fake-project', 'fixed_ips', 10)
db_api.quota_create(self.context, 'fake-project', 'floating_ips', 10)
# Migrate with a count/limit of 3
total, done = quotas.migrate_quota_limits_to_api_db(self.context, 3)
self.assertEqual(3, total)
self.assertEqual(3, done)
# This only fetches from the api db. There should now be 4 limits.
api_user_limits = quotas.Quotas._get_all_from_db(self.context,
'fake-project')
api_proj_limits_dict = quotas.Quotas._get_all_from_db_by_project(
self.context, 'fake-project')
api_proj_limits_dict.pop('project_id', None)
self.assertEqual(4,
len(api_user_limits) + len(api_proj_limits_dict))
# This only fetches from the main db. There should be one left.
main_user_limits = db_api.quota_get_all(self.context, 'fake-project')
main_proj_limits_dict = db_api.quota_get_all_by_project(self.context,
'fake-project')
main_proj_limits_dict.pop('project_id', None)
self.assertEqual(1, len(main_user_limits) + len(main_proj_limits_dict))
self.assertEqual((1, 1),
quotas.migrate_quota_limits_to_api_db(
self.context, 100))
self.assertEqual((0, 0),
quotas.migrate_quota_limits_to_api_db(
self.context, 100))
def test_migrate_quota_limits_skips_existing(self):
quotas.Quotas._create_limit_in_db(self.context, 'fake-project',
'instances', 5, user_id='fake-user')
db_api.quota_create(self.context, 'fake-project', 'instances', 5,
user_id='fake-user')
total, done = quotas.migrate_quota_limits_to_api_db(
self.context, 100)
self.assertEqual(1, total)
self.assertEqual(1, done)
total, done = quotas.migrate_quota_limits_to_api_db(
self.context, 100)
self.assertEqual(0, total)
self.assertEqual(0, done)
self.assertEqual(1, len(quotas.Quotas._get_all_from_db(
self.context, 'fake-project')))
def test_migrate_quota_classes(self):
# Create a class in api db
quotas.Quotas._create_class_in_db(self.context, 'foo', 'instances', 5)
# Create 3 classes in main db
db_api.quota_class_create(self.context, 'foo', 'cores', 10)
db_api.quota_class_create(self.context, db_api._DEFAULT_QUOTA_NAME,
'instances', 10)
db_api.quota_class_create(self.context, 'foo', 'ram', 8192)
total, done = quotas.migrate_quota_classes_to_api_db(self.context, 2)
self.assertEqual(2, total)
self.assertEqual(2, done)
# This only fetches from the api db
api_foo_dict = quotas.Quotas._get_all_class_from_db_by_name(
self.context, 'foo')
api_foo_dict.pop('class_name', None)
api_default_dict = quotas.Quotas._get_all_class_from_db_by_name(
self.context, db_api._DEFAULT_QUOTA_NAME)
api_default_dict.pop('class_name', None)
self.assertEqual(3,
len(api_foo_dict) + len(api_default_dict))
# This only fetches from the main db
main_foo_dict = db_api.quota_class_get_all_by_name(self.context, 'foo')
main_foo_dict.pop('class_name', None)
main_default_dict = db_api.quota_class_get_default(self.context)
main_default_dict.pop('class_name', None)
self.assertEqual(1, len(main_foo_dict) + len(main_default_dict))
self.assertEqual((1, 1),
quotas.migrate_quota_classes_to_api_db(
self.context, 100))
self.assertEqual((0, 0),
quotas.migrate_quota_classes_to_api_db(
self.context, 100))
def test_migrate_quota_classes_skips_existing(self):
quotas.Quotas._create_class_in_db(self.context, 'foo-class',
'instances', 5)
db_api.quota_class_create(self.context, 'foo-class', 'instances', 7)
total, done = quotas.migrate_quota_classes_to_api_db(
self.context, 100)
self.assertEqual(1, total)
self.assertEqual(1, done)
total, done = quotas.migrate_quota_classes_to_api_db(
self.context, 100)
self.assertEqual(0, total)
self.assertEqual(0, done)
# Existing class should not be overwritten in the result
db_class = quotas.Quotas._get_all_class_from_db_by_name(
self.context, 'foo-class')
self.assertEqual(5, db_class['instances'])

View File

@ -0,0 +1,13 @@
---
upgrade:
- |
Quota limits and classes are being moved to the API database for Cells v2.
In this release, the online data migrations will move any quota limits and
classes you have in your main database to the API database, retaining all
attributes.
.. note:: Quota limits and classes can no longer be soft-deleted as the API
database does not replicate the legacy soft-delete functionality from the
main database. As such, deleted quota limits and classes are not migrated
and the behavior users will experience will be the same as if a purge of
deleted records was performed.