Use unique consumer_id when doing online data migration

If there are multiple consumers having allocations to the same
resource provider, with different classes, it will attempt
multiple INSERTs with the same consumer_id which is not allowed
because of the database constraints.

This patch adds a simple GROUP BY in order to ensure that the
database server only provides us with unique values to avoid
trying to INSERT duplicate values.

Conflicts:
      nova/tests/functional/api/openstack/placement/db/test_consumer.py

NOTE(mriedem): The conflict is due to not having change
I7f5f08691ca3f73073c66c29dddb996fb2c2b266 in Rocky.

Change-Id: I1acba5e65cd562472f29e354c6077f82844fa87d
Closes-Bug: #1798163
(cherry picked from commit 730936e535)
This commit is contained in:
Mohammed Naser 2018-10-16 19:54:40 +02:00 committed by Matt Riedemann
parent 83396b3254
commit 6bd7e8a958
3 changed files with 14 additions and 13 deletions

View File

@ -55,6 +55,10 @@ def create_incomplete_consumers(ctx, batch_size):
sel = sa.select(cols)
sel = sel.select_from(alloc_to_consumer)
sel = sel.where(CONSUMER_TBL.c.id.is_(None))
# NOTE(mnaser): It is possible to have multiple consumers having many
# allocations to the same resource provider, which would
# make the INSERT FROM SELECT fail due to duplicates.
sel = sel.group_by(_ALLOC_TBL.c.consumer_id)
sel = sel.limit(batch_size)
target_cols = ['uuid', 'project_id', 'user_id']
ins_stmt = CONSUMER_TBL.insert().from_select(target_cols, sel)

View File

@ -1917,6 +1917,10 @@ def _create_incomplete_consumers_for_provider(ctx, rp_id):
sa.and_(
_ALLOC_TBL.c.resource_provider_id == rp_id,
consumer_obj.CONSUMER_TBL.c.id.is_(None)))
# NOTE(mnaser): It is possible to have multiple consumers having many
# allocations to the same resource provider, which would
# make the INSERT FROM SELECT fail due to duplicates.
sel = sel.group_by(_ALLOC_TBL.c.consumer_id)
target_cols = ['uuid', 'project_id', 'user_id']
ins_stmt = consumer_obj.CONSUMER_TBL.insert().from_select(
target_cols, sel)

View File

@ -11,7 +11,6 @@
# under the License.
from oslo_config import cfg
from oslo_db import exception as db_exc
import sqlalchemy as sa
from nova.api.openstack.placement import db_api
@ -231,20 +230,14 @@ class CreateIncompleteConsumersTestCase(base.TestCase):
# Run the online data migration to migrate one consumer. The batch size
# needs to be large enough to hit more than one consumer for this test
# where each consumer has two allocations.
# FIXME(mriedem): This should not raise a UniqueConstraint error once
# bug 1798163 is fixed.
# res = consumer_obj.create_incomplete_consumers(self.ctx, 2)
# self.assertEqual((2, 2), res)
self.assertRaises(db_exc.DBDuplicateEntry,
consumer_obj.create_incomplete_consumers,
self.ctx, 2)
res = consumer_obj.create_incomplete_consumers(self.ctx, 2)
self.assertEqual((2, 2), res)
# Migrate the rest by listing allocations on the resource provider.
rp1 = rp_obj.ResourceProvider(self.ctx, id=1)
# FIXME(mriedem): This should not raise a UniqueConstraint error once
# bug 1798163 is fixed.
self.assertRaises(db_exc.DBDuplicateEntry,
rp_obj.AllocationList.get_all_by_resource_provider,
self.ctx, rp1)
rp_obj.AllocationList.get_all_by_resource_provider(self.ctx, rp1)
self._check_incomplete_consumers(self.ctx)
res = consumer_obj.create_incomplete_consumers(self.ctx, 10)
self.assertEqual((0, 0), res)
class DeleteConsumerIfNoAllocsTestCase(tb.PlacementDbBaseTestCase):