Handle missing marker during online data migration

During upgrade the instance used by the request spec marker could be
deleted and purged between sessions. This would cause the database
online data migration to fail as the marker instance couldn't be found.

Fix by handling the MarkerNotFound exception and re-trying without the
marker. This will go through all the instances and reset the marker when
done.

Closes-Bug: #1793419
Change-Id: If96e3d038346f16cc93209bccf3db028bacfe59b
Signed-off-by: Jack Ding <jack.ding@windriver.com>
(cherry picked from commit ff03b157b9)
(cherry picked from commit 57b46754ff)

nova/objects/request_spec.py

@@ -645,13 +645,17 @@ def migrate_instances_add_request_spec(context, max_count):
     # Prevent lazy-load of those fields for every instance later.
     attrs = ['system_metadata', 'flavor', 'pci_requests', 'numa_topology',
              'availability_zone']
-    instances = objects.InstanceList.get_by_filters(context,
-                                                    filters={'deleted': False},
-                                                    sort_key='created_at',
-                                                    sort_dir='asc',
-                                                    limit=max_count,
-                                                    marker=marker,
-                                                    expected_attrs=attrs)
+    try:
+        instances = objects.InstanceList.get_by_filters(
+            context, filters={'deleted': False}, sort_key='created_at',
+            sort_dir='asc', limit=max_count, marker=marker,
+            expected_attrs=attrs)
+    except exception.MarkerNotFound:
+        # Instance referenced by marker may have been purged.
+        # Try again but get all instances.
+        instances = objects.InstanceList.get_by_filters(
+            context, filters={'deleted': False}, sort_key='created_at',
+            sort_dir='asc', limit=max_count, expected_attrs=attrs)
     count_all = len(instances)
     count_hit = 0
     for instance in instances:

nova/tests/functional/db/test_request_spec.py

@@ -167,3 +167,45 @@ class RequestSpecInstanceMigrationTestCase(
             self.context, 50)
         self.assertEqual(5, match)
         self.assertEqual(0, done)
+
+    def test_migration_with_missing_marker(self):
+        self._create_instances(old=2, total=5)
+
+        # Start with 2 old (without request_spec) and 3 new instances:
+        # [old, old, new, new, new]
+        match, done = request_spec.migrate_instances_add_request_spec(
+            self.context, 2)
+        # Instance list after session 1:
+        # [upgraded, upgraded<MARKER>, new, new, new]
+        self.assertEqual(2, match)
+        self.assertEqual(2, done)
+
+        # Delete and remove the marker instance from api table while leaving
+        # the spec in request_specs table. This triggers MarkerNotFound
+        # exception in the latter session.
+        self.api.delete_server(self.instances[1].uuid)
+        db.archive_deleted_rows(max_rows=100)
+        # Instance list after deletion: [upgraded, new, new, new]
+
+        # This session of migration hits MarkerNotFound exception and then
+        # starts from the beginning of the list
+        match, done = request_spec.migrate_instances_add_request_spec(
+            self.context, 50)
+        self.assertEqual(4, match)
+        self.assertEqual(0, done)
+
+        # Make sure we ran over all the instances
+        match, done = request_spec.migrate_instances_add_request_spec(
+            self.context, 50)
+        self.assertEqual(0, match)
+        self.assertEqual(0, done)
+
+        # Make sure all instances have now a related RequestSpec
+        for instance in self.instances:
+            uuid = instance.uuid
+            try:
+                spec = objects.RequestSpec.get_by_instance_uuid(
+                    self.context, uuid)
+                self.assertEqual(instance.project_id, spec.project_id)
+            except exception.RequestSpecNotFound:
+                self.fail("RequestSpec not found for instance UUID :%s ", uuid)