Merge "Add online migration to move quotas to API database"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
19f78e2f5f
|
|
@ -90,6 +90,7 @@ from nova.objects import host_mapping as host_mapping_obj
|
|||
from nova.objects import instance as instance_obj
|
||||
from nova.objects import instance_group as instance_group_obj
|
||||
from nova.objects import keypair as keypair_obj
|
||||
from nova.objects import quotas as quotas_obj
|
||||
from nova.objects import request_spec
|
||||
from nova import quota
|
||||
from nova import rpc
|
||||
|
|
@ -646,6 +647,10 @@ class DbCommands(object):
|
|||
build_request_obj.delete_build_requests_with_no_instance_uuid,
|
||||
# Added in Pike
|
||||
db.service_uuids_online_data_migration,
|
||||
# Added in Pike
|
||||
quotas_obj.migrate_quota_limits_to_api_db,
|
||||
# Added in Pike
|
||||
quotas_obj.migrate_quota_classes_to_api_db,
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from oslo_db import exception as db_exc
|
|||
from nova import db
|
||||
from nova.db.sqlalchemy import api as db_api
|
||||
from nova.db.sqlalchemy import api_models
|
||||
from nova.db.sqlalchemy import models as main_models
|
||||
from nova import exception
|
||||
from nova.objects import base
|
||||
from nova.objects import fields
|
||||
|
|
@ -515,3 +516,131 @@ class QuotasNoOp(Quotas):
|
|||
|
||||
def check_deltas(cls, context, deltas, *count_args, **count_kwargs):
|
||||
pass
|
||||
|
||||
|
||||
@db_api.require_context
|
||||
@db_api.pick_context_manager_reader
|
||||
def _get_main_per_project_limits(context, limit):
|
||||
return context.session.query(main_models.Quota).\
|
||||
filter_by(deleted=0).\
|
||||
limit(limit).\
|
||||
all()
|
||||
|
||||
|
||||
@db_api.require_context
|
||||
@db_api.pick_context_manager_reader
|
||||
def _get_main_per_user_limits(context, limit):
|
||||
return context.session.query(main_models.ProjectUserQuota).\
|
||||
filter_by(deleted=0).\
|
||||
limit(limit).\
|
||||
all()
|
||||
|
||||
|
||||
@db_api.require_context
|
||||
@db_api.pick_context_manager_writer
|
||||
def _destroy_main_per_project_limits(context, project_id, resource):
|
||||
context.session.query(main_models.Quota).\
|
||||
filter_by(deleted=0).\
|
||||
filter_by(project_id=project_id).\
|
||||
filter_by(resource=resource).\
|
||||
soft_delete(synchronize_session=False)
|
||||
|
||||
|
||||
@db_api.require_context
|
||||
@db_api.pick_context_manager_writer
|
||||
def _destroy_main_per_user_limits(context, project_id, resource, user_id):
|
||||
context.session.query(main_models.ProjectUserQuota).\
|
||||
filter_by(deleted=0).\
|
||||
filter_by(project_id=project_id).\
|
||||
filter_by(user_id=user_id).\
|
||||
filter_by(resource=resource).\
|
||||
soft_delete(synchronize_session=False)
|
||||
|
||||
|
||||
@db_api.api_context_manager.writer
|
||||
def _create_limits_in_api_db(context, db_limits, per_user=False):
|
||||
for db_limit in db_limits:
|
||||
user_id = db_limit.user_id if per_user else None
|
||||
Quotas._create_limit_in_db(context, db_limit.project_id,
|
||||
db_limit.resource, db_limit.hard_limit,
|
||||
user_id=user_id)
|
||||
|
||||
|
||||
def migrate_quota_limits_to_api_db(context, count):
|
||||
# Migrate per project limits
|
||||
main_per_project_limits = _get_main_per_project_limits(context, count)
|
||||
done = 0
|
||||
try:
|
||||
# Create all the limits in a single transaction.
|
||||
_create_limits_in_api_db(context, main_per_project_limits)
|
||||
except exception.QuotaExists:
|
||||
# NOTE(melwitt): This can happen if the migration is interrupted after
|
||||
# limits were created in the api db but before they were deleted from
|
||||
# the main db, and the migration is re-run.
|
||||
pass
|
||||
# Delete the limits separately.
|
||||
for db_limit in main_per_project_limits:
|
||||
_destroy_main_per_project_limits(context, db_limit.project_id,
|
||||
db_limit.resource)
|
||||
done += 1
|
||||
if done == count:
|
||||
return len(main_per_project_limits), done
|
||||
# Migrate per user limits
|
||||
count -= done
|
||||
main_per_user_limits = _get_main_per_user_limits(context, count)
|
||||
try:
|
||||
# Create all the limits in a single transaction.
|
||||
_create_limits_in_api_db(context, main_per_user_limits, per_user=True)
|
||||
except exception.QuotaExists:
|
||||
# NOTE(melwitt): This can happen if the migration is interrupted after
|
||||
# limits were created in the api db but before they were deleted from
|
||||
# the main db, and the migration is re-run.
|
||||
pass
|
||||
# Delete the limits separately.
|
||||
for db_limit in main_per_user_limits:
|
||||
_destroy_main_per_user_limits(context, db_limit.project_id,
|
||||
db_limit.resource, db_limit.user_id)
|
||||
done += 1
|
||||
return len(main_per_project_limits) + len(main_per_user_limits), done
|
||||
|
||||
|
||||
@db_api.require_context
|
||||
@db_api.pick_context_manager_reader
|
||||
def _get_main_quota_classes(context, limit):
|
||||
return context.session.query(main_models.QuotaClass).\
|
||||
filter_by(deleted=0).\
|
||||
limit(limit).\
|
||||
all()
|
||||
|
||||
|
||||
@db_api.pick_context_manager_writer
|
||||
def _destroy_main_quota_classes(context, db_classes):
|
||||
for db_class in db_classes:
|
||||
context.session.query(main_models.QuotaClass).\
|
||||
filter_by(deleted=0).\
|
||||
filter_by(id=db_class.id).\
|
||||
soft_delete(synchronize_session=False)
|
||||
|
||||
|
||||
@db_api.api_context_manager.writer
|
||||
def _create_classes_in_api_db(context, db_classes):
|
||||
for db_class in db_classes:
|
||||
Quotas._create_class_in_db(context, db_class.class_name,
|
||||
db_class.resource, db_class.hard_limit)
|
||||
|
||||
|
||||
def migrate_quota_classes_to_api_db(context, count):
|
||||
main_quota_classes = _get_main_quota_classes(context, count)
|
||||
done = 0
|
||||
try:
|
||||
# Create all the classes in a single transaction.
|
||||
_create_classes_in_api_db(context, main_quota_classes)
|
||||
except exception.QuotaClassExists:
|
||||
# NOTE(melwitt): This can happen if the migration is interrupted after
|
||||
# classes were created in the api db but before they were deleted from
|
||||
# the main db, and the migration is re-run.
|
||||
pass
|
||||
# Delete the classes in a single transaction.
|
||||
_destroy_main_quota_classes(context, main_quota_classes)
|
||||
found = done = len(main_quota_classes)
|
||||
return found, done
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
# under the License.
|
||||
|
||||
from nova import context
|
||||
from nova.db.sqlalchemy import api as db_api
|
||||
from nova import exception
|
||||
from nova.objects import quotas
|
||||
from nova import test
|
||||
|
|
@ -206,3 +207,113 @@ class QuotasObjectTestCase(test.TestCase,
|
|||
self.assertEqual('foo', limits_dict['class_name'])
|
||||
self.assertEqual(5, limits_dict['instances'])
|
||||
self.assertEqual(10, limits_dict['cores'])
|
||||
|
||||
def test_migrate_quota_limits(self):
|
||||
# Create a limit in api db
|
||||
quotas.Quotas._create_limit_in_db(self.context, 'fake-project',
|
||||
'instances', 5, user_id='fake-user')
|
||||
# Create 4 limits in main db
|
||||
db_api.quota_create(self.context, 'fake-project', 'cores', 10,
|
||||
user_id='fake-user')
|
||||
db_api.quota_create(self.context, 'fake-project', 'ram', 8192,
|
||||
user_id='fake-user')
|
||||
db_api.quota_create(self.context, 'fake-project', 'fixed_ips', 10)
|
||||
db_api.quota_create(self.context, 'fake-project', 'floating_ips', 10)
|
||||
|
||||
# Migrate with a count/limit of 3
|
||||
total, done = quotas.migrate_quota_limits_to_api_db(self.context, 3)
|
||||
self.assertEqual(3, total)
|
||||
self.assertEqual(3, done)
|
||||
|
||||
# This only fetches from the api db. There should now be 4 limits.
|
||||
api_user_limits = quotas.Quotas._get_all_from_db(self.context,
|
||||
'fake-project')
|
||||
api_proj_limits_dict = quotas.Quotas._get_all_from_db_by_project(
|
||||
self.context, 'fake-project')
|
||||
api_proj_limits_dict.pop('project_id', None)
|
||||
self.assertEqual(4,
|
||||
len(api_user_limits) + len(api_proj_limits_dict))
|
||||
|
||||
# This only fetches from the main db. There should be one left.
|
||||
main_user_limits = db_api.quota_get_all(self.context, 'fake-project')
|
||||
main_proj_limits_dict = db_api.quota_get_all_by_project(self.context,
|
||||
'fake-project')
|
||||
main_proj_limits_dict.pop('project_id', None)
|
||||
self.assertEqual(1, len(main_user_limits) + len(main_proj_limits_dict))
|
||||
|
||||
self.assertEqual((1, 1),
|
||||
quotas.migrate_quota_limits_to_api_db(
|
||||
self.context, 100))
|
||||
self.assertEqual((0, 0),
|
||||
quotas.migrate_quota_limits_to_api_db(
|
||||
self.context, 100))
|
||||
|
||||
def test_migrate_quota_limits_skips_existing(self):
|
||||
quotas.Quotas._create_limit_in_db(self.context, 'fake-project',
|
||||
'instances', 5, user_id='fake-user')
|
||||
db_api.quota_create(self.context, 'fake-project', 'instances', 5,
|
||||
user_id='fake-user')
|
||||
total, done = quotas.migrate_quota_limits_to_api_db(
|
||||
self.context, 100)
|
||||
self.assertEqual(1, total)
|
||||
self.assertEqual(1, done)
|
||||
total, done = quotas.migrate_quota_limits_to_api_db(
|
||||
self.context, 100)
|
||||
self.assertEqual(0, total)
|
||||
self.assertEqual(0, done)
|
||||
self.assertEqual(1, len(quotas.Quotas._get_all_from_db(
|
||||
self.context, 'fake-project')))
|
||||
|
||||
def test_migrate_quota_classes(self):
|
||||
# Create a class in api db
|
||||
quotas.Quotas._create_class_in_db(self.context, 'foo', 'instances', 5)
|
||||
# Create 3 classes in main db
|
||||
db_api.quota_class_create(self.context, 'foo', 'cores', 10)
|
||||
db_api.quota_class_create(self.context, db_api._DEFAULT_QUOTA_NAME,
|
||||
'instances', 10)
|
||||
db_api.quota_class_create(self.context, 'foo', 'ram', 8192)
|
||||
|
||||
total, done = quotas.migrate_quota_classes_to_api_db(self.context, 2)
|
||||
self.assertEqual(2, total)
|
||||
self.assertEqual(2, done)
|
||||
|
||||
# This only fetches from the api db
|
||||
api_foo_dict = quotas.Quotas._get_all_class_from_db_by_name(
|
||||
self.context, 'foo')
|
||||
api_foo_dict.pop('class_name', None)
|
||||
api_default_dict = quotas.Quotas._get_all_class_from_db_by_name(
|
||||
self.context, db_api._DEFAULT_QUOTA_NAME)
|
||||
api_default_dict.pop('class_name', None)
|
||||
self.assertEqual(3,
|
||||
len(api_foo_dict) + len(api_default_dict))
|
||||
|
||||
# This only fetches from the main db
|
||||
main_foo_dict = db_api.quota_class_get_all_by_name(self.context, 'foo')
|
||||
main_foo_dict.pop('class_name', None)
|
||||
main_default_dict = db_api.quota_class_get_default(self.context)
|
||||
main_default_dict.pop('class_name', None)
|
||||
self.assertEqual(1, len(main_foo_dict) + len(main_default_dict))
|
||||
|
||||
self.assertEqual((1, 1),
|
||||
quotas.migrate_quota_classes_to_api_db(
|
||||
self.context, 100))
|
||||
self.assertEqual((0, 0),
|
||||
quotas.migrate_quota_classes_to_api_db(
|
||||
self.context, 100))
|
||||
|
||||
def test_migrate_quota_classes_skips_existing(self):
|
||||
quotas.Quotas._create_class_in_db(self.context, 'foo-class',
|
||||
'instances', 5)
|
||||
db_api.quota_class_create(self.context, 'foo-class', 'instances', 7)
|
||||
total, done = quotas.migrate_quota_classes_to_api_db(
|
||||
self.context, 100)
|
||||
self.assertEqual(1, total)
|
||||
self.assertEqual(1, done)
|
||||
total, done = quotas.migrate_quota_classes_to_api_db(
|
||||
self.context, 100)
|
||||
self.assertEqual(0, total)
|
||||
self.assertEqual(0, done)
|
||||
# Existing class should not be overwritten in the result
|
||||
db_class = quotas.Quotas._get_all_class_from_db_by_name(
|
||||
self.context, 'foo-class')
|
||||
self.assertEqual(5, db_class['instances'])
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
---
|
||||
upgrade:
|
||||
- |
|
||||
Quota limits and classes are being moved to the API database for Cells v2.
|
||||
In this release, the online data migrations will move any quota limits and
|
||||
classes you have in your main database to the API database, retaining all
|
||||
attributes.
|
||||
|
||||
.. note:: Quota limits and classes can no longer be soft-deleted as the API
|
||||
database does not replicate the legacy soft-delete functionality from the
|
||||
main database. As such, deleted quota limits and classes are not migrated
|
||||
and the behavior users will experience will be the same as if a purge of
|
||||
deleted records was performed.
|
||||
Loading…
Reference in New Issue