diff --git a/nova/api/openstack/placement/exception.py b/nova/api/openstack/placement/exception.py index 231c6e7a37ae..faf5673e55af 100644 --- a/nova/api/openstack/placement/exception.py +++ b/nova/api/openstack/placement/exception.py @@ -66,6 +66,10 @@ class NotFound(_BaseException): msg_fmt = _("Resource could not be found.") +class Exists(_BaseException): + msg_fmt = _("Resource already exists.") + + class InvalidInventory(_BaseException): msg_fmt = _("Inventory for '%(resource_class)s' on " "resource provider '%(resource_provider)s' invalid.") @@ -172,3 +176,27 @@ class TraitInUse(_BaseException): class TraitNotFound(NotFound): msg_fmt = _("No such trait(s): %(names)s.") + + +class ProjectNotFound(NotFound): + msg_fmt = _("No such project(s): %(external_id)s.") + + +class ProjectExists(Exists): + msg_fmt = _("The project %(external_id)s already exists.") + + +class UserNotFound(NotFound): + msg_fmt = _("No such user(s): %(external_id)s.") + + +class UserExists(Exists): + msg_fmt = _("The user %(external_id)s already exists.") + + +class ConsumerNotFound(NotFound): + msg_fmt = _("No such consumer(s): %(uuid)s.") + + +class ConsumerExists(Exists): + msg_fmt = _("The consumer %(uuid)s already exists.") diff --git a/nova/api/openstack/placement/handlers/allocation.py b/nova/api/openstack/placement/handlers/allocation.py index 3ec1273d619f..13c46558bbaf 100644 --- a/nova/api/openstack/placement/handlers/allocation.py +++ b/nova/api/openstack/placement/handlers/allocation.py @@ -217,6 +217,7 @@ def _new_allocations(context, resource_provider_uuid, consumer_uuid, _("Allocation for resource provider '%(rp_uuid)s' " "that does not exist.") % {'rp_uuid': resource_provider_uuid}) + util.ensure_consumer(context, consumer_uuid, project_id, user_id) for resource_class in resources: allocation = rp_obj.Allocation( resource_provider=resource_provider, diff --git a/nova/api/openstack/placement/objects/consumer.py b/nova/api/openstack/placement/objects/consumer.py new file mode 100644 index 000000000000..26eac2624fbc --- /dev/null +++ b/nova/api/openstack/placement/objects/consumer.py @@ -0,0 +1,153 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_db import exception as db_exc +from oslo_versionedobjects import base +from oslo_versionedobjects import fields +import sqlalchemy as sa + +from nova.api.openstack.placement import exception +from nova.api.openstack.placement.objects import project as project_obj +from nova.api.openstack.placement.objects import resource_provider as rp_obj +from nova.api.openstack.placement.objects import user as user_obj +from nova.db.sqlalchemy import api as db_api +from nova.db.sqlalchemy import api_models as models + +CONSUMER_TBL = models.Consumer.__table__ + + +@db_api.api_context_manager.writer +def create_incomplete_consumers(ctx, batch_size): + """Finds all the consumer records that are missing for allocations and + creates consumer records for them, using the "incomplete consumer" project + and user CONF options. + + Returns a tuple containing two identical elements with the number of + consumer records created, since this is the expected return format for data + migration routines. + """ + # Create a record in the projects table for our incomplete project + incomplete_proj_id = project_obj.ensure_incomplete_project(ctx) + + # Create a record in the users table for our incomplete user + incomplete_user_id = user_obj.ensure_incomplete_user(ctx) + + # Create a consumer table record for all consumers where + # allocations.consumer_id doesn't exist in the consumers table. Use the + # incomplete consumer project and user ID. + alloc_to_consumer = sa.outerjoin( + rp_obj._ALLOC_TBL, CONSUMER_TBL, + rp_obj._ALLOC_TBL.c.consumer_id == CONSUMER_TBL.c.uuid) + cols = [ + rp_obj._ALLOC_TBL.c.consumer_id, + incomplete_proj_id, + incomplete_user_id, + ] + sel = sa.select(cols) + sel = sel.select_from(alloc_to_consumer) + sel = sel.where(CONSUMER_TBL.c.id.is_(None)) + sel = sel.limit(batch_size) + target_cols = ['uuid', 'project_id', 'user_id'] + ins_stmt = CONSUMER_TBL.insert().from_select(target_cols, sel) + res = ctx.session.execute(ins_stmt) + return res.rowcount, res.rowcount + + +@db_api.api_context_manager.reader +def _get_consumer_by_uuid(ctx, uuid): + # The SQL for this looks like the following: + # SELECT + # c.id, c.uuid, + # p.id AS project_id, p.external_id AS project_external_id, + # u.id AS user_id, u.external_id AS user_external_id, + # c.updated_at, c.created_at + # FROM consumers c + # INNER JOIN projects p + # ON c.project_id = p.id + # INNER JOIN users u + # ON c.user_id = u.id + # WHERE c.uuid = $uuid + consumers = sa.alias(CONSUMER_TBL, name="c") + projects = sa.alias(project_obj.PROJECT_TBL, name="p") + users = sa.alias(user_obj.USER_TBL, name="u") + cols = [ + consumers.c.id, + consumers.c.uuid, + projects.c.id.label("project_id"), + projects.c.external_id.label("project_external_id"), + users.c.id.label("user_id"), + users.c.external_id.label("user_external_id"), + consumers.c.updated_at, + consumers.c.created_at + ] + c_to_p_join = sa.join( + consumers, projects, consumers.c.project_id == projects.c.id) + c_to_u_join = sa.join( + c_to_p_join, users, consumers.c.user_id == users.c.id) + sel = sa.select(cols).select_from(c_to_u_join) + sel = sel.where(consumers.c.uuid == uuid) + res = ctx.session.execute(sel).fetchone() + if not res: + raise exception.ConsumerNotFound(uuid=uuid) + + return dict(res) + + +@base.VersionedObjectRegistry.register_if(False) +class Consumer(base.VersionedObject, base.TimestampedObject): + + fields = { + 'id': fields.IntegerField(read_only=True), + 'uuid': fields.UUIDField(nullable=False), + 'project': fields.ObjectField('Project', nullable=False), + 'user': fields.ObjectField('User', nullable=False), + } + + @staticmethod + def _from_db_object(ctx, target, source): + target.id = source['id'] + target.uuid = source['uuid'] + target.created_at = source['created_at'] + target.updated_at = source['updated_at'] + + target.project = project_obj.Project( + ctx, id=source['project_id'], + external_id=source['project_external_id']) + target.user = user_obj.User( + ctx, id=source['user_id'], + external_id=source['user_external_id']) + + target._context = ctx + target.obj_reset_changes() + return target + + @classmethod + def get_by_uuid(cls, ctx, uuid): + res = _get_consumer_by_uuid(ctx, uuid) + return cls._from_db_object(ctx, cls(ctx), res) + + def create(self): + @db_api.api_context_manager.writer + def _create_in_db(ctx): + db_obj = models.Consumer( + uuid=self.uuid, project_id=self.project.id, + user_id=self.user.id) + try: + db_obj.save(ctx.session) + # NOTE(jaypipes): We don't do the normal _from_db_object() + # thing here because models.Consumer doesn't have a + # project_external_id or user_external_id attribute. + self.id = db_obj.id + except db_exc.DBDuplicateEntry: + raise exception.ConsumerExists(uuid=self.uuid) + _create_in_db(self._context) + self.obj_reset_changes() diff --git a/nova/api/openstack/placement/objects/project.py b/nova/api/openstack/placement/objects/project.py new file mode 100644 index 000000000000..eac3c7ff357d --- /dev/null +++ b/nova/api/openstack/placement/objects/project.py @@ -0,0 +1,92 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_db import exception as db_exc +from oslo_versionedobjects import base +from oslo_versionedobjects import fields +import sqlalchemy as sa + +from nova.api.openstack.placement import exception +from nova.db.sqlalchemy import api as db_api +from nova.db.sqlalchemy import api_models as models + +CONF = cfg.CONF +PROJECT_TBL = models.Project.__table__ + + +@db_api.api_context_manager.writer +def ensure_incomplete_project(ctx): + """Ensures that a project record is created for the "incomplete consumer + project". Returns the internal ID of that record. + """ + incomplete_id = CONF.placement.incomplete_consumer_project_id + sel = sa.select([PROJECT_TBL.c.id]).where( + PROJECT_TBL.c.external_id == incomplete_id) + res = ctx.session.execute(sel).fetchone() + if res: + return res[0] + ins = PROJECT_TBL.insert().values(external_id=incomplete_id) + res = ctx.session.execute(ins) + return res.inserted_primary_key[0] + + +@db_api.api_context_manager.reader +def _get_project_by_external_id(ctx, external_id): + projects = sa.alias(PROJECT_TBL, name="p") + cols = [ + projects.c.id, + projects.c.external_id, + projects.c.updated_at, + projects.c.created_at + ] + sel = sa.select(cols) + sel = sel.where(projects.c.external_id == external_id) + res = ctx.session.execute(sel).fetchone() + if not res: + raise exception.ProjectNotFound(external_id=external_id) + + return dict(res) + + +@base.VersionedObjectRegistry.register_if(False) +class Project(base.VersionedObject): + + fields = { + 'id': fields.IntegerField(read_only=True), + 'external_id': fields.StringField(nullable=False), + } + + @staticmethod + def _from_db_object(ctx, target, source): + for field in target.fields: + setattr(target, field, source[field]) + + target._context = ctx + target.obj_reset_changes() + return target + + @classmethod + def get_by_external_id(cls, ctx, external_id): + res = _get_project_by_external_id(ctx, external_id) + return cls._from_db_object(ctx, cls(ctx), res) + + def create(self): + @db_api.api_context_manager.writer + def _create_in_db(ctx): + db_obj = models.Project(external_id=self.external_id) + try: + db_obj.save(ctx.session) + except db_exc.DBDuplicateEntry: + raise exception.ProjectExists(external_id=self.external_id) + self._from_db_object(ctx, self, db_obj) + _create_in_db(self._context) diff --git a/nova/api/openstack/placement/objects/resource_provider.py b/nova/api/openstack/placement/objects/resource_provider.py index dffcdcedee1b..0c5af550c38a 100644 --- a/nova/api/openstack/placement/objects/resource_provider.py +++ b/nova/api/openstack/placement/objects/resource_provider.py @@ -1555,15 +1555,15 @@ class Allocation(base.VersionedObject, base.TimestampedObject): :param ctx: `nova.context.RequestContext` object that has the oslo.db Session object in it """ - # If project_id and user_id are not set then silently - # move on. This allows microversion <1.8 to continue to work. Since - # then the fields are required and the enforcement is at the HTTP - # API layer. - if not ('project_id' in self and - self.project_id is not None and - 'user_id' in self and - self.user_id is not None): - return + # If project_id and user_id are not set then create a consumer record + # pointing to the incomplete consumer project and user ID. + # This allows microversion <1.8 to continue to work. Since then the + # fields are required and the enforcement is at the HTTP API layer. + if 'project_id' not in self or self.project_id is None: + self.project_id = CONF.placement.incomplete_consumer_project_id + if 'user_id' not in self or self.user_id is None: + self.user_id = CONF.placement.incomplete_consumer_user_id + # Grab the project internal ID if it exists in the projects table pid = _ensure_project(ctx, self.project_id) # Grab the user internal ID if it exists in the users table diff --git a/nova/api/openstack/placement/objects/user.py b/nova/api/openstack/placement/objects/user.py new file mode 100644 index 000000000000..4a12ee2a35d8 --- /dev/null +++ b/nova/api/openstack/placement/objects/user.py @@ -0,0 +1,92 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_db import exception as db_exc +from oslo_versionedobjects import base +from oslo_versionedobjects import fields +import sqlalchemy as sa + +from nova.api.openstack.placement import exception +from nova.db.sqlalchemy import api as db_api +from nova.db.sqlalchemy import api_models as models + +CONF = cfg.CONF +USER_TBL = models.User.__table__ + + +@db_api.api_context_manager.writer +def ensure_incomplete_user(ctx): + """Ensures that a user record is created for the "incomplete consumer + user". Returns the internal ID of that record. + """ + incomplete_id = CONF.placement.incomplete_consumer_user_id + sel = sa.select([USER_TBL.c.id]).where( + USER_TBL.c.external_id == incomplete_id) + res = ctx.session.execute(sel).fetchone() + if res: + return res[0] + ins = USER_TBL.insert().values(external_id=incomplete_id) + res = ctx.session.execute(ins) + return res.inserted_primary_key[0] + + +@db_api.api_context_manager.reader +def _get_user_by_external_id(ctx, external_id): + users = sa.alias(USER_TBL, name="u") + cols = [ + users.c.id, + users.c.external_id, + users.c.updated_at, + users.c.created_at + ] + sel = sa.select(cols) + sel = sel.where(users.c.external_id == external_id) + res = ctx.session.execute(sel).fetchone() + if not res: + raise exception.UserNotFound(external_id=external_id) + + return dict(res) + + +@base.VersionedObjectRegistry.register_if(False) +class User(base.VersionedObject): + + fields = { + 'id': fields.IntegerField(read_only=True), + 'external_id': fields.StringField(nullable=False), + } + + @staticmethod + def _from_db_object(ctx, target, source): + for field in target.fields: + setattr(target, field, source[field]) + + target._context = ctx + target.obj_reset_changes() + return target + + @classmethod + def get_by_external_id(cls, ctx, external_id): + res = _get_user_by_external_id(ctx, external_id) + return cls._from_db_object(ctx, cls(ctx), res) + + def create(self): + @db_api.api_context_manager.writer + def _create_in_db(ctx): + db_obj = models.User(external_id=self.external_id) + try: + db_obj.save(ctx.session) + except db_exc.DBDuplicateEntry: + raise exception.UserExists(external_id=self.external_id) + self._from_db_object(ctx, self, db_obj) + _create_in_db(self._context) diff --git a/nova/api/openstack/placement/util.py b/nova/api/openstack/placement/util.py index e2f35a9c3161..6acf03fe2143 100644 --- a/nova/api/openstack/placement/util.py +++ b/nova/api/openstack/placement/util.py @@ -15,6 +15,7 @@ import functools import re import jsonschema +from oslo_config import cfg from oslo_middleware import request_id from oslo_serialization import jsonutils from oslo_utils import timeutils @@ -22,12 +23,18 @@ from oslo_utils import uuidutils import webob from nova.api.openstack.placement import errors +from nova.api.openstack.placement import exception from nova.api.openstack.placement import lib as placement_lib # NOTE(cdent): avoid cyclical import conflict between util and # microversion import nova.api.openstack.placement.microversion +from nova.api.openstack.placement.objects import consumer as consumer_obj +from nova.api.openstack.placement.objects import project as project_obj +from nova.api.openstack.placement.objects import user as user_obj from nova.i18n import _ +CONF = cfg.CONF + # Error code handling constants ENV_ERROR_CODE = 'placement.error_code' ERROR_CODE_MICROVERSION = (1, 23) @@ -568,3 +575,53 @@ def parse_qs_request_groups(req): raise webob.exc.HTTPBadRequest(msg % ', '.join(conflicting_traits)) return by_suffix + + +def ensure_consumer(ctx, consumer_uuid, project_id, user_id): + """Ensures there are records in the consumers, projects and users table for + the supplied external identifiers. + + Returns a populated Consumer object containing Project and User sub-objects + + :param ctx: The request context. + :param consumer_uuid: The uuid of the consumer of the resources. + :param project_id: The external ID of the project consuming the resources. + :param user_id: The external ID of the user consuming the resources. + """ + if project_id is None: + project_id = CONF.placement.incomplete_consumer_project_id + user_id = CONF.placement.incomplete_consumer_user_id + try: + proj = project_obj.Project.get_by_external_id(ctx, project_id) + except exception.NotFound: + # Auto-create the project if we found no record of it... + try: + proj = project_obj.Project(ctx, external_id=project_id) + proj.create() + except exception.ProjectExists: + # No worries, another thread created this project already + proj = project_obj.Project.get_by_external_id(ctx, project_id) + try: + user = user_obj.User.get_by_external_id(ctx, user_id) + except exception.NotFound: + # Auto-create the user if we found no record of it... + try: + user = user_obj.User(ctx, external_id=user_id) + user.create() + except exception.UserExists: + # No worries, another thread created this user already + user = user_obj.User.get_by_external_id(ctx, user_id) + + try: + consumer = consumer_obj.Consumer.get_by_uuid(ctx, consumer_uuid) + except exception.NotFound: + # No such consumer. This is common for new allocations. Create the + # consumer record + try: + consumer = consumer_obj.Consumer( + ctx, uuid=consumer_uuid, project=proj, user=user) + consumer.create() + except exception.ConsumerExists: + # No worries, another thread created this user already + consumer = consumer_obj.Consumer.get_by_uuid(ctx, consumer_uuid) + return consumer diff --git a/nova/cmd/manage.py b/nova/cmd/manage.py index cba54fbd6869..fca8a8245738 100644 --- a/nova/cmd/manage.py +++ b/nova/cmd/manage.py @@ -44,6 +44,7 @@ import six import six.moves.urllib.parse as urlparse from sqlalchemy.engine import url as sqla_url +from nova.api.openstack.placement.objects import consumer as consumer_obj from nova.cmd import common as cmd_common import nova.conf from nova import config @@ -400,6 +401,8 @@ class DbCommands(object): # Queens and Pike since instance.avz of instances before Pike # need to be populated if it was not specified during boot time. instance_obj.populate_missing_availability_zones, + # Added in Rocky + consumer_obj.create_incomplete_consumers, ) def __init__(self): diff --git a/nova/conf/placement.py b/nova/conf/placement.py index 7d03a72785e1..417a17ec78ac 100644 --- a/nova/conf/placement.py +++ b/nova/conf/placement.py @@ -17,6 +17,7 @@ from nova.conf import utils as confutils DEFAULT_SERVICE_TYPE = 'placement' +DEFAULT_CONSUMER_MISSING_ID = '00000000-0000-0000-0000-0000000000000' placement_group = cfg.OptGroup( 'placement', @@ -44,6 +45,26 @@ is determined. default='placement-policy.yaml', help='The file that defines placement policies. This can be an ' 'absolute path or relative to the configuration file.'), + cfg.StrOpt( + 'incomplete_consumer_project_id', + default=DEFAULT_CONSUMER_MISSING_ID, + help=""" +Early API microversions (<1.8) allowed creating allocations and not specifying +a project or user identifier for the consumer. In cleaning up the data +modeling, we no longer allow missing project and user information. if an older +client makes an allocation, we'll use this in place of the information it +doesn't provide. +"""), +cfg.StrOpt( + 'incomplete_consumer_user_id', + default=DEFAULT_CONSUMER_MISSING_ID, + help=""" +Early API microversions (<1.8) allowed creating allocations and not specifying +a project or user identifier for the consumer. In cleaning up the data +modeling, we no longer allow missing project and user information. if an older +client makes an allocation, we'll use this in place of the information it +doesn't provide. +"""), ] diff --git a/nova/tests/functional/api/openstack/placement/db/test_consumer.py b/nova/tests/functional/api/openstack/placement/db/test_consumer.py new file mode 100644 index 000000000000..db8b6cba4705 --- /dev/null +++ b/nova/tests/functional/api/openstack/placement/db/test_consumer.py @@ -0,0 +1,133 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +import sqlalchemy as sa + +from nova.api.openstack.placement import exception +from nova.api.openstack.placement.objects import consumer as consumer_obj +from nova.api.openstack.placement.objects import project as project_obj +from nova.api.openstack.placement.objects import resource_provider as rp_obj +from nova.api.openstack.placement.objects import user as user_obj +from nova.db.sqlalchemy import api as db_api +from nova.tests.functional.api.openstack.placement.db import test_base as tb +from nova.tests import uuidsentinel as uuids + +CONF = cfg.CONF +CONSUMER_TBL = consumer_obj.CONSUMER_TBL +PROJECT_TBL = project_obj.PROJECT_TBL +USER_TBL = user_obj.USER_TBL +ALLOC_TBL = rp_obj._ALLOC_TBL + + +class ConsumerTestCase(tb.PlacementDbBaseTestCase): + def test_non_existing_consumer(self): + self.assertRaises(exception.ConsumerNotFound, + consumer_obj.Consumer.get_by_uuid, self.ctx, + uuids.non_existing_consumer) + + def test_create_and_get(self): + u = user_obj.User(self.ctx, external_id='fake-user') + u.create() + p = project_obj.Project(self.ctx, external_id='fake-project') + p.create() + c = consumer_obj.Consumer( + self.ctx, uuid=uuids.consumer, user=u, project=p) + c.create() + c = consumer_obj.Consumer.get_by_uuid(self.ctx, uuids.consumer) + self.assertEqual(1, c.id) + self.assertEqual(1, c.project.id) + self.assertEqual(1, c.user.id) + self.assertRaises(exception.ConsumerExists, c.create) + + +class CreateIncompleteConsumersTestCase(tb.PlacementDbBaseTestCase): + @db_api.api_context_manager.writer + def _create_incomplete_allocations(self, ctx): + # Create some allocations with consumers that don't exist in the + # consumers table to represent old allocations that we expect to be + # "cleaned up" with consumers table records that point to the sentinel + # project/user records. + c1_missing_uuid = uuids.c1_missing + c2_missing_uuid = uuids.c2_missing + ins_stmt = ALLOC_TBL.insert().values( + resource_provider_id=1, resource_class_id=0, + consumer_id=c1_missing_uuid, used=1) + ctx.session.execute(ins_stmt) + ins_stmt = ALLOC_TBL.insert().values( + resource_provider_id=1, resource_class_id=0, + consumer_id=c2_missing_uuid, used=1) + ctx.session.execute(ins_stmt) + # Verify there are no records in the projects/users table + project_count = ctx.session.scalar( + sa.select([sa.func.count('*')]).select_from(PROJECT_TBL)) + self.assertEqual(0, project_count) + user_count = ctx.session.scalar( + sa.select([sa.func.count('*')]).select_from(USER_TBL)) + self.assertEqual(0, user_count) + # Verify there are no consumer records for the missing consumers + sel = CONSUMER_TBL.select( + CONSUMER_TBL.c.uuid.in_([c1_missing_uuid, c2_missing_uuid])) + res = ctx.session.execute(sel).fetchall() + self.assertEqual(0, len(res)) + + @db_api.api_context_manager.reader + def _check_incomplete_consumers(self, ctx): + incomplete_external_id = CONF.placement.incomplete_consumer_project_id + + # Verify we have a record in projects for the missing sentinel + sel = PROJECT_TBL.select( + PROJECT_TBL.c.external_id == incomplete_external_id) + rec = ctx.session.execute(sel).first() + self.assertEqual(incomplete_external_id, rec['external_id']) + incomplete_proj_id = rec['id'] + + # Verify we have a record in users for the missing sentinel + sel = user_obj.USER_TBL.select( + USER_TBL.c.external_id == incomplete_external_id) + rec = ctx.session.execute(sel).first() + self.assertEqual(incomplete_external_id, rec['external_id']) + incomplete_user_id = rec['id'] + + # Verify there are records in the consumers table for our old + # allocation records created in the pre-migration setup and that the + # projects and users referenced in those consumer records point to the + # incomplete project/user + sel = CONSUMER_TBL.select(CONSUMER_TBL.c.uuid == uuids.c1_missing) + missing_c1 = ctx.session.execute(sel).first() + self.assertEqual(incomplete_proj_id, missing_c1['project_id']) + self.assertEqual(incomplete_user_id, missing_c1['user_id']) + sel = CONSUMER_TBL.select(CONSUMER_TBL.c.uuid == uuids.c2_missing) + missing_c2 = ctx.session.execute(sel).first() + self.assertEqual(incomplete_proj_id, missing_c2['project_id']) + self.assertEqual(incomplete_user_id, missing_c2['user_id']) + + # Ensure there are no more allocations with incomplete consumers + alloc_to_consumer = sa.outerjoin( + ALLOC_TBL, CONSUMER_TBL, + ALLOC_TBL.c.consumer_id == CONSUMER_TBL.c.uuid) + sel = sa.select([ALLOC_TBL]) + sel = sel.select_from(alloc_to_consumer) + sel = sel.where(CONSUMER_TBL.c.id.is_(None)) + res = ctx.session.execute(sel).fetchall() + self.assertEqual(0, len(res)) + + def test_create_incomplete_consumers(self): + """Test the online data migration that creates incomplete consumer + records along with the incomplete consumer project/user records. + """ + self._create_incomplete_allocations(self.ctx) + res = consumer_obj.create_incomplete_consumers(self.ctx, 10) + self.assertEqual((2, 2), res) + self._check_incomplete_consumers(self.ctx) + res = consumer_obj.create_incomplete_consumers(self.ctx, 10) + self.assertEqual((0, 0), res) diff --git a/nova/tests/functional/api/openstack/placement/db/test_project.py b/nova/tests/functional/api/openstack/placement/db/test_project.py new file mode 100644 index 000000000000..8e640b6f961a --- /dev/null +++ b/nova/tests/functional/api/openstack/placement/db/test_project.py @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova.api.openstack.placement import exception +from nova.api.openstack.placement.objects import project as project_obj +from nova.tests.functional.api.openstack.placement.db import test_base as tb +from nova.tests import uuidsentinel as uuids + + +class ProjectTestCase(tb.PlacementDbBaseTestCase): + def test_non_existing_project(self): + self.assertRaises( + exception.ProjectNotFound, project_obj.Project.get_by_external_id, + self.ctx, uuids.non_existing_project) + + def test_create_and_get(self): + p = project_obj.Project(self.ctx, external_id='fake-project') + p.create() + p = project_obj.Project.get_by_external_id(self.ctx, 'fake-project') + self.assertEqual(1, p.id) + self.assertRaises(exception.ProjectExists, p.create) diff --git a/nova/tests/functional/api/openstack/placement/db/test_user.py b/nova/tests/functional/api/openstack/placement/db/test_user.py new file mode 100644 index 000000000000..1e35bfa13bbf --- /dev/null +++ b/nova/tests/functional/api/openstack/placement/db/test_user.py @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova.api.openstack.placement import exception +from nova.api.openstack.placement.objects import user as user_obj +from nova.tests.functional.api.openstack.placement.db import test_base as tb +from nova.tests import uuidsentinel as uuids + + +class UserTestCase(tb.PlacementDbBaseTestCase): + def test_non_existing_user(self): + self.assertRaises( + exception.UserNotFound, user_obj.User.get_by_external_id, + self.ctx, uuids.non_existing_user) + + def test_create_and_get(self): + u = user_obj.User(self.ctx, external_id='fake-user') + u.create() + u = user_obj.User.get_by_external_id(self.ctx, 'fake-user') + self.assertEqual(1, u.id) + self.assertRaises(exception.UserExists, u.create) diff --git a/nova/tests/functional/api/openstack/placement/gabbits/ensure-consumer.yaml b/nova/tests/functional/api/openstack/placement/gabbits/ensure-consumer.yaml new file mode 100644 index 000000000000..2c574df99b65 --- /dev/null +++ b/nova/tests/functional/api/openstack/placement/gabbits/ensure-consumer.yaml @@ -0,0 +1,41 @@ +# Tests of the ensure consumer behaviour for versions of the API from <1.7 to +# 1.8-1.26 and finally 1.27+ +fixtures: + - AllocationFixture + +defaults: + request_headers: + x-auth-token: admin + accept: application/json + openstack-api-version: placement 1.7 + +vars: +- &default_incomplete_id 00000000-0000-0000-0000-0000000000000 +- &consumer_id fbad1a87-c341-4ac0-be49-777b21ce1b7b +tests: + +- name: put an allocation without project/user (1.7) + PUT: /allocations/*consumer_id + request_headers: + content-type: application/json + openstack-api-version: placement 1.7 + data: + allocations: + - resource_provider: + uuid: $ENVIRON['RP_UUID'] + resources: + DISK_GB: 10 + status: 204 + +# We now ALWAYS create a consumer record, and if project or user isn't +# specified (as was the case in 1.7) we should get the project/user +# corresponding to the CONF option for incomplete consumers when asking for the +# allocation information at a microversion that shows project/user information +# (1.12+) +- name: get with 1.12 microversion and check project and user are filled + GET: /allocations/*consumer_id + request_headers: + openstack-api-version: placement 1.12 + response_json_paths: + $.project_id: *default_incomplete_id + $.user_id: *default_incomplete_id diff --git a/nova/tests/unit/api/openstack/placement/test_util.py b/nova/tests/unit/api/openstack/placement/test_util.py index cf932b6b1aef..5a823482d453 100644 --- a/nova/tests/unit/api/openstack/placement/test_util.py +++ b/nova/tests/unit/api/openstack/placement/test_util.py @@ -18,19 +18,25 @@ import datetime import fixtures import microversion_parse import mock +from oslo_config import cfg from oslo_middleware import request_id from oslo_utils import timeutils import webob import six +from nova.api.openstack.placement import exception from nova.api.openstack.placement import lib as pl from nova.api.openstack.placement import microversion +from nova.api.openstack.placement.objects import project as project_obj from nova.api.openstack.placement.objects import resource_provider as rp_obj +from nova.api.openstack.placement.objects import user as user_obj from nova.api.openstack.placement import util from nova import test from nova.tests import uuidsentinel +CONF = cfg.CONF + class TestCheckAccept(test.NoDBTestCase): """Confirm behavior of util.check_accept.""" @@ -894,3 +900,86 @@ class TestPickLastModified(test.NoDBTestCase): None, self.resource_provider) self.assertEqual(now, chosen_time) mock_utc.assert_called_once_with(with_timezone=True) + + +class TestEnsureConsumer(test.NoDBTestCase): + def setUp(self): + super(TestEnsureConsumer, self).setUp() + self.mock_project_get = self.useFixture(fixtures.MockPatch( + 'nova.api.openstack.placement.objects.project.' + 'Project.get_by_external_id')).mock + self.mock_user_get = self.useFixture(fixtures.MockPatch( + 'nova.api.openstack.placement.objects.user.' + 'User.get_by_external_id')).mock + self.mock_consumer_get = self.useFixture(fixtures.MockPatch( + 'nova.api.openstack.placement.objects.consumer.' + 'Consumer.get_by_uuid')).mock + self.mock_project_create = self.useFixture(fixtures.MockPatch( + 'nova.api.openstack.placement.objects.project.' + 'Project.create')).mock + self.mock_user_create = self.useFixture(fixtures.MockPatch( + 'nova.api.openstack.placement.objects.user.' + 'User.create')).mock + self.mock_consumer_create = self.useFixture(fixtures.MockPatch( + 'nova.api.openstack.placement.objects.consumer.' + 'Consumer.create')).mock + self.ctx = mock.sentinel.ctx + self.consumer_id = uuidsentinel.consumer + self.project_id = uuidsentinel.project + self.user_id = uuidsentinel.user + + def test_no_existing_project_user_consumer(self): + self.mock_project_get.side_effect = exception.NotFound + self.mock_user_get.side_effect = exception.NotFound + self.mock_consumer_get.side_effect = exception.NotFound + + util.ensure_consumer( + self.ctx, self.consumer_id, self.project_id, self.user_id) + + self.mock_project_get.assert_called_once_with( + self.ctx, self.project_id) + self.mock_user_get.assert_called_once_with( + self.ctx, self.user_id) + self.mock_consumer_get.assert_called_once_with( + self.ctx, self.consumer_id) + self.mock_project_create.assert_called_once() + self.mock_user_create.assert_called_once() + self.mock_consumer_create.assert_called_once() + + def test_no_existing_project_user_consumer_use_incomplete(self): + """Verify that if the project_id arg is None, that we fall back to the + CONF options for incomplete project and user ID. + """ + self.mock_project_get.side_effect = exception.NotFound + self.mock_user_get.side_effect = exception.NotFound + self.mock_consumer_get.side_effect = exception.NotFound + + util.ensure_consumer( + self.ctx, self.consumer_id, None, None) + + self.mock_project_get.assert_called_once_with( + self.ctx, CONF.placement.incomplete_consumer_project_id) + self.mock_user_get.assert_called_once_with( + self.ctx, CONF.placement.incomplete_consumer_user_id) + self.mock_consumer_get.assert_called_once_with( + self.ctx, self.consumer_id) + self.mock_project_create.assert_called_once() + self.mock_user_create.assert_called_once() + self.mock_consumer_create.assert_called_once() + + def test_existing_project_user_no_existing_consumer(self): + """Check that if we find an existing project and user, that we use + those found objects in creating the consumer. + """ + proj = project_obj.Project(self.ctx, id=1, external_id=self.project_id) + self.mock_project_get.return_value = proj + user = user_obj.User(self.ctx, id=1, external_id=self.user_id) + self.mock_user_get.return_value = user + self.mock_consumer_get.side_effect = exception.NotFound + + util.ensure_consumer( + self.ctx, self.consumer_id, self.project_id, self.user_id) + + self.mock_project_create.assert_not_called() + self.mock_user_create.assert_not_called() + self.mock_consumer_create.assert_called_once() diff --git a/releasenotes/notes/placement-incomplete-consumer-configuration-b775dac1bcd34f9d.yaml b/releasenotes/notes/placement-incomplete-consumer-configuration-b775dac1bcd34f9d.yaml new file mode 100644 index 000000000000..ec45ad2008df --- /dev/null +++ b/releasenotes/notes/placement-incomplete-consumer-configuration-b775dac1bcd34f9d.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + Prior to microversion 1.8 of the placement API, one could create + allocations and not supply a project or user ID for the consumer of the + allocated resources. While this is no longer allowed after placement API + 1.8, older allocations exist and we now ensure that a consumer record is + created for these older allocations. Use the two new CONF options + ``CONF.placement.incomplete_consumer_project_id`` and + ``CONF.placement.incomplete_consumer_user_id`` to control the project and + user identifiers that are written for these incomplete consumer records.