From ff03b157b930de23e5912802cbfbc86889c869c2 Mon Sep 17 00:00:00 2001 From: Jack Ding Date: Tue, 25 Sep 2018 13:20:25 -0400 Subject: [PATCH] Handle missing marker during online data migration During upgrade the instance used by the request spec marker could be deleted and purged between sessions. This would cause the database online data migration to fail as the marker instance couldn't be found. Fix by handling the MarkerNotFound exception and re-trying without the marker. This will go through all the instances and reset the marker when done. Closes-Bug: #1793419 Change-Id: If96e3d038346f16cc93209bccf3db028bacfe59b Signed-off-by: Jack Ding --- nova/objects/request_spec.py | 18 ++++---- nova/tests/functional/db/test_request_spec.py | 42 +++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/nova/objects/request_spec.py b/nova/objects/request_spec.py index d1a3c199f811..964248a2538d 100644 --- a/nova/objects/request_spec.py +++ b/nova/objects/request_spec.py @@ -697,13 +697,17 @@ def migrate_instances_add_request_spec(context, max_count): # Prevent lazy-load of those fields for every instance later. attrs = ['system_metadata', 'flavor', 'pci_requests', 'numa_topology', 'availability_zone'] - instances = objects.InstanceList.get_by_filters(context, - filters={'deleted': False}, - sort_key='created_at', - sort_dir='asc', - limit=max_count, - marker=marker, - expected_attrs=attrs) + try: + instances = objects.InstanceList.get_by_filters( + context, filters={'deleted': False}, sort_key='created_at', + sort_dir='asc', limit=max_count, marker=marker, + expected_attrs=attrs) + except exception.MarkerNotFound: + # Instance referenced by marker may have been purged. + # Try again but get all instances. + instances = objects.InstanceList.get_by_filters( + context, filters={'deleted': False}, sort_key='created_at', + sort_dir='asc', limit=max_count, expected_attrs=attrs) count_all = len(instances) count_hit = 0 for instance in instances: diff --git a/nova/tests/functional/db/test_request_spec.py b/nova/tests/functional/db/test_request_spec.py index dcfc3688b457..6cfe6274d656 100644 --- a/nova/tests/functional/db/test_request_spec.py +++ b/nova/tests/functional/db/test_request_spec.py @@ -166,3 +166,45 @@ class RequestSpecInstanceMigrationTestCase( self.context, 50) self.assertEqual(5, match) self.assertEqual(0, done) + + def test_migration_with_missing_marker(self): + self._create_instances(old=2, total=5) + + # Start with 2 old (without request_spec) and 3 new instances: + # [old, old, new, new, new] + match, done = request_spec.migrate_instances_add_request_spec( + self.context, 2) + # Instance list after session 1: + # [upgraded, upgraded, new, new, new] + self.assertEqual(2, match) + self.assertEqual(2, done) + + # Delete and remove the marker instance from api table while leaving + # the spec in request_specs table. This triggers MarkerNotFound + # exception in the latter session. + self.api.delete_server(self.instances[1].uuid) + db.archive_deleted_rows(max_rows=100) + # Instance list after deletion: [upgraded, new, new, new] + + # This session of migration hits MarkerNotFound exception and then + # starts from the beginning of the list + match, done = request_spec.migrate_instances_add_request_spec( + self.context, 50) + self.assertEqual(4, match) + self.assertEqual(0, done) + + # Make sure we ran over all the instances + match, done = request_spec.migrate_instances_add_request_spec( + self.context, 50) + self.assertEqual(0, match) + self.assertEqual(0, done) + + # Make sure all instances have now a related RequestSpec + for instance in self.instances: + uuid = instance.uuid + try: + spec = objects.RequestSpec.get_by_instance_uuid( + self.context, uuid) + self.assertEqual(instance.project_id, spec.project_id) + except exception.RequestSpecNotFound: + self.fail("RequestSpec not found for instance UUID :%s ", uuid)