From 03d80cf0de85099bf9d306578cd065af5bed7994 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Thu, 10 May 2018 13:40:45 -0400
Subject: [PATCH] placement: always create consumer records

Adds objects for Consumer, Project, and User data models, in their own
files. They do not contain logic that comes from the API microversions
and are meant to be plain-old-data objects that represent the current
schema in the database. Project, user and consumer information all are
stored in separate tables in the DB and represent actual things in the
placement data modeling. Giving them actual objects makes that
consistent with the other objects in the data model, including resource
providers, allocations, inventories, resource classes and traits.

The patch modifies the allocation handler to always ensure that a
consumer record exists for the supplied consumer UUID and an associated
projects and users table record exists for that consumer. If an
allocation is created using API microversion <1.8, which doesn't supply
the project or user for the consumer, we use the value of two new CONF
options that indicate the project and user ID for incomplete consumer
records.

Includes an online data migration for the nova-manage
online_data_migrations command that creates consumer records for
incomplete consumers.

Change-Id: Id609789ef6b4a4c745550cde80dd49cabe03869a
---
 nova/api/openstack/placement/exception.py     |  28 ++++
 .../placement/handlers/allocation.py          |   1 +
 .../openstack/placement/objects/consumer.py   | 153 ++++++++++++++++++
 .../openstack/placement/objects/project.py    |  92 +++++++++++
 .../placement/objects/resource_provider.py    |  18 +--
 nova/api/openstack/placement/objects/user.py  |  92 +++++++++++
 nova/api/openstack/placement/util.py          |  57 +++++++
 nova/cmd/manage.py                            |   3 +
 nova/conf/placement.py                        |  21 +++
 .../openstack/placement/db/test_consumer.py   | 133 +++++++++++++++
 .../openstack/placement/db/test_project.py    |  30 ++++
 .../api/openstack/placement/db/test_user.py   |  30 ++++
 .../placement/gabbits/ensure-consumer.yaml    |  41 +++++
 .../unit/api/openstack/placement/test_util.py |  89 ++++++++++
 ...nsumer-configuration-b775dac1bcd34f9d.yaml |  11 ++
 15 files changed, 790 insertions(+), 9 deletions(-)
 create mode 100644 nova/api/openstack/placement/objects/consumer.py
 create mode 100644 nova/api/openstack/placement/objects/project.py
 create mode 100644 nova/api/openstack/placement/objects/user.py
 create mode 100644 nova/tests/functional/api/openstack/placement/db/test_consumer.py
 create mode 100644 nova/tests/functional/api/openstack/placement/db/test_project.py
 create mode 100644 nova/tests/functional/api/openstack/placement/db/test_user.py
 create mode 100644 nova/tests/functional/api/openstack/placement/gabbits/ensure-consumer.yaml
 create mode 100644 releasenotes/notes/placement-incomplete-consumer-configuration-b775dac1bcd34f9d.yaml

diff --git a/nova/api/openstack/placement/exception.py b/nova/api/openstack/placement/exception.py
index 231c6e7a37ae..faf5673e55af 100644
--- a/nova/api/openstack/placement/exception.py
+++ b/nova/api/openstack/placement/exception.py
@@ -66,6 +66,10 @@ class NotFound(_BaseException):
     msg_fmt = _("Resource could not be found.")
 
 
+class Exists(_BaseException):
+        msg_fmt = _("Resource already exists.")
+
+
 class InvalidInventory(_BaseException):
     msg_fmt = _("Inventory for '%(resource_class)s' on "
                 "resource provider '%(resource_provider)s' invalid.")
@@ -172,3 +176,27 @@ class TraitInUse(_BaseException):
 
 class TraitNotFound(NotFound):
     msg_fmt = _("No such trait(s): %(names)s.")
+
+
+class ProjectNotFound(NotFound):
+    msg_fmt = _("No such project(s): %(external_id)s.")
+
+
+class ProjectExists(Exists):
+        msg_fmt = _("The project %(external_id)s already exists.")
+
+
+class UserNotFound(NotFound):
+    msg_fmt = _("No such user(s): %(external_id)s.")
+
+
+class UserExists(Exists):
+        msg_fmt = _("The user %(external_id)s already exists.")
+
+
+class ConsumerNotFound(NotFound):
+    msg_fmt = _("No such consumer(s): %(uuid)s.")
+
+
+class ConsumerExists(Exists):
+    msg_fmt = _("The consumer %(uuid)s already exists.")
diff --git a/nova/api/openstack/placement/handlers/allocation.py b/nova/api/openstack/placement/handlers/allocation.py
index 3ec1273d619f..13c46558bbaf 100644
--- a/nova/api/openstack/placement/handlers/allocation.py
+++ b/nova/api/openstack/placement/handlers/allocation.py
@@ -217,6 +217,7 @@ def _new_allocations(context, resource_provider_uuid, consumer_uuid,
             _("Allocation for resource provider '%(rp_uuid)s' "
               "that does not exist.") %
             {'rp_uuid': resource_provider_uuid})
+    util.ensure_consumer(context, consumer_uuid, project_id, user_id)
     for resource_class in resources:
         allocation = rp_obj.Allocation(
             resource_provider=resource_provider,
diff --git a/nova/api/openstack/placement/objects/consumer.py b/nova/api/openstack/placement/objects/consumer.py
new file mode 100644
index 000000000000..26eac2624fbc
--- /dev/null
+++ b/nova/api/openstack/placement/objects/consumer.py
@@ -0,0 +1,153 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_db import exception as db_exc
+from oslo_versionedobjects import base
+from oslo_versionedobjects import fields
+import sqlalchemy as sa
+
+from nova.api.openstack.placement import exception
+from nova.api.openstack.placement.objects import project as project_obj
+from nova.api.openstack.placement.objects import resource_provider as rp_obj
+from nova.api.openstack.placement.objects import user as user_obj
+from nova.db.sqlalchemy import api as db_api
+from nova.db.sqlalchemy import api_models as models
+
+CONSUMER_TBL = models.Consumer.__table__
+
+
+@db_api.api_context_manager.writer
+def create_incomplete_consumers(ctx, batch_size):
+    """Finds all the consumer records that are missing for allocations and
+    creates consumer records for them, using the "incomplete consumer" project
+    and user CONF options.
+
+    Returns a tuple containing two identical elements with the number of
+    consumer records created, since this is the expected return format for data
+    migration routines.
+    """
+    # Create a record in the projects table for our incomplete project
+    incomplete_proj_id = project_obj.ensure_incomplete_project(ctx)
+
+    # Create a record in the users table for our incomplete user
+    incomplete_user_id = user_obj.ensure_incomplete_user(ctx)
+
+    # Create a consumer table record for all consumers where
+    # allocations.consumer_id doesn't exist in the consumers table. Use the
+    # incomplete consumer project and user ID.
+    alloc_to_consumer = sa.outerjoin(
+        rp_obj._ALLOC_TBL, CONSUMER_TBL,
+        rp_obj._ALLOC_TBL.c.consumer_id == CONSUMER_TBL.c.uuid)
+    cols = [
+        rp_obj._ALLOC_TBL.c.consumer_id,
+        incomplete_proj_id,
+        incomplete_user_id,
+    ]
+    sel = sa.select(cols)
+    sel = sel.select_from(alloc_to_consumer)
+    sel = sel.where(CONSUMER_TBL.c.id.is_(None))
+    sel = sel.limit(batch_size)
+    target_cols = ['uuid', 'project_id', 'user_id']
+    ins_stmt = CONSUMER_TBL.insert().from_select(target_cols, sel)
+    res = ctx.session.execute(ins_stmt)
+    return res.rowcount, res.rowcount
+
+
+@db_api.api_context_manager.reader
+def _get_consumer_by_uuid(ctx, uuid):
+    # The SQL for this looks like the following:
+    # SELECT
+    #   c.id, c.uuid,
+    #   p.id AS project_id, p.external_id AS project_external_id,
+    #   u.id AS user_id, u.external_id AS user_external_id,
+    #   c.updated_at, c.created_at
+    # FROM consumers c
+    # INNER JOIN projects p
+    #  ON c.project_id = p.id
+    # INNER JOIN users u
+    #  ON c.user_id = u.id
+    # WHERE c.uuid = $uuid
+    consumers = sa.alias(CONSUMER_TBL, name="c")
+    projects = sa.alias(project_obj.PROJECT_TBL, name="p")
+    users = sa.alias(user_obj.USER_TBL, name="u")
+    cols = [
+        consumers.c.id,
+        consumers.c.uuid,
+        projects.c.id.label("project_id"),
+        projects.c.external_id.label("project_external_id"),
+        users.c.id.label("user_id"),
+        users.c.external_id.label("user_external_id"),
+        consumers.c.updated_at,
+        consumers.c.created_at
+    ]
+    c_to_p_join = sa.join(
+        consumers, projects, consumers.c.project_id == projects.c.id)
+    c_to_u_join = sa.join(
+        c_to_p_join, users, consumers.c.user_id == users.c.id)
+    sel = sa.select(cols).select_from(c_to_u_join)
+    sel = sel.where(consumers.c.uuid == uuid)
+    res = ctx.session.execute(sel).fetchone()
+    if not res:
+        raise exception.ConsumerNotFound(uuid=uuid)
+
+    return dict(res)
+
+
+@base.VersionedObjectRegistry.register_if(False)
+class Consumer(base.VersionedObject, base.TimestampedObject):
+
+    fields = {
+        'id': fields.IntegerField(read_only=True),
+        'uuid': fields.UUIDField(nullable=False),
+        'project': fields.ObjectField('Project', nullable=False),
+        'user': fields.ObjectField('User', nullable=False),
+    }
+
+    @staticmethod
+    def _from_db_object(ctx, target, source):
+        target.id = source['id']
+        target.uuid = source['uuid']
+        target.created_at = source['created_at']
+        target.updated_at = source['updated_at']
+
+        target.project = project_obj.Project(
+            ctx, id=source['project_id'],
+            external_id=source['project_external_id'])
+        target.user = user_obj.User(
+            ctx, id=source['user_id'],
+            external_id=source['user_external_id'])
+
+        target._context = ctx
+        target.obj_reset_changes()
+        return target
+
+    @classmethod
+    def get_by_uuid(cls, ctx, uuid):
+        res = _get_consumer_by_uuid(ctx, uuid)
+        return cls._from_db_object(ctx, cls(ctx), res)
+
+    def create(self):
+        @db_api.api_context_manager.writer
+        def _create_in_db(ctx):
+            db_obj = models.Consumer(
+                uuid=self.uuid, project_id=self.project.id,
+                user_id=self.user.id)
+            try:
+                db_obj.save(ctx.session)
+                # NOTE(jaypipes): We don't do the normal _from_db_object()
+                # thing here because models.Consumer doesn't have a
+                # project_external_id or user_external_id attribute.
+                self.id = db_obj.id
+            except db_exc.DBDuplicateEntry:
+                raise exception.ConsumerExists(uuid=self.uuid)
+        _create_in_db(self._context)
+        self.obj_reset_changes()
diff --git a/nova/api/openstack/placement/objects/project.py b/nova/api/openstack/placement/objects/project.py
new file mode 100644
index 000000000000..eac3c7ff357d
--- /dev/null
+++ b/nova/api/openstack/placement/objects/project.py
@@ -0,0 +1,92 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+from oslo_db import exception as db_exc
+from oslo_versionedobjects import base
+from oslo_versionedobjects import fields
+import sqlalchemy as sa
+
+from nova.api.openstack.placement import exception
+from nova.db.sqlalchemy import api as db_api
+from nova.db.sqlalchemy import api_models as models
+
+CONF = cfg.CONF
+PROJECT_TBL = models.Project.__table__
+
+
+@db_api.api_context_manager.writer
+def ensure_incomplete_project(ctx):
+    """Ensures that a project record is created for the "incomplete consumer
+    project". Returns the internal ID of that record.
+    """
+    incomplete_id = CONF.placement.incomplete_consumer_project_id
+    sel = sa.select([PROJECT_TBL.c.id]).where(
+        PROJECT_TBL.c.external_id == incomplete_id)
+    res = ctx.session.execute(sel).fetchone()
+    if res:
+        return res[0]
+    ins = PROJECT_TBL.insert().values(external_id=incomplete_id)
+    res = ctx.session.execute(ins)
+    return res.inserted_primary_key[0]
+
+
+@db_api.api_context_manager.reader
+def _get_project_by_external_id(ctx, external_id):
+    projects = sa.alias(PROJECT_TBL, name="p")
+    cols = [
+        projects.c.id,
+        projects.c.external_id,
+        projects.c.updated_at,
+        projects.c.created_at
+    ]
+    sel = sa.select(cols)
+    sel = sel.where(projects.c.external_id == external_id)
+    res = ctx.session.execute(sel).fetchone()
+    if not res:
+        raise exception.ProjectNotFound(external_id=external_id)
+
+    return dict(res)
+
+
+@base.VersionedObjectRegistry.register_if(False)
+class Project(base.VersionedObject):
+
+    fields = {
+        'id': fields.IntegerField(read_only=True),
+        'external_id': fields.StringField(nullable=False),
+    }
+
+    @staticmethod
+    def _from_db_object(ctx, target, source):
+        for field in target.fields:
+            setattr(target, field, source[field])
+
+        target._context = ctx
+        target.obj_reset_changes()
+        return target
+
+    @classmethod
+    def get_by_external_id(cls, ctx, external_id):
+        res = _get_project_by_external_id(ctx, external_id)
+        return cls._from_db_object(ctx, cls(ctx), res)
+
+    def create(self):
+        @db_api.api_context_manager.writer
+        def _create_in_db(ctx):
+            db_obj = models.Project(external_id=self.external_id)
+            try:
+                db_obj.save(ctx.session)
+            except db_exc.DBDuplicateEntry:
+                raise exception.ProjectExists(external_id=self.external_id)
+            self._from_db_object(ctx, self, db_obj)
+        _create_in_db(self._context)
diff --git a/nova/api/openstack/placement/objects/resource_provider.py b/nova/api/openstack/placement/objects/resource_provider.py
index dffcdcedee1b..0c5af550c38a 100644
--- a/nova/api/openstack/placement/objects/resource_provider.py
+++ b/nova/api/openstack/placement/objects/resource_provider.py
@@ -1555,15 +1555,15 @@ class Allocation(base.VersionedObject, base.TimestampedObject):
         :param ctx: `nova.context.RequestContext` object that has the oslo.db
                     Session object in it
         """
-        # If project_id and user_id are not set then silently
-        # move on. This allows microversion <1.8 to continue to work. Since
-        # then the fields are required and the enforcement is at the HTTP
-        # API layer.
-        if not ('project_id' in self and
-                self.project_id is not None and
-                'user_id' in self and
-                self.user_id is not None):
-            return
+        # If project_id and user_id are not set then create a consumer record
+        # pointing to the incomplete consumer project and user ID.
+        # This allows microversion <1.8 to continue to work. Since then the
+        # fields are required and the enforcement is at the HTTP API layer.
+        if 'project_id' not in self or self.project_id is None:
+            self.project_id = CONF.placement.incomplete_consumer_project_id
+        if 'user_id' not in self or self.user_id is None:
+            self.user_id = CONF.placement.incomplete_consumer_user_id
+
         # Grab the project internal ID if it exists in the projects table
         pid = _ensure_project(ctx, self.project_id)
         # Grab the user internal ID if it exists in the users table
diff --git a/nova/api/openstack/placement/objects/user.py b/nova/api/openstack/placement/objects/user.py
new file mode 100644
index 000000000000..4a12ee2a35d8
--- /dev/null
+++ b/nova/api/openstack/placement/objects/user.py
@@ -0,0 +1,92 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+from oslo_db import exception as db_exc
+from oslo_versionedobjects import base
+from oslo_versionedobjects import fields
+import sqlalchemy as sa
+
+from nova.api.openstack.placement import exception
+from nova.db.sqlalchemy import api as db_api
+from nova.db.sqlalchemy import api_models as models
+
+CONF = cfg.CONF
+USER_TBL = models.User.__table__
+
+
+@db_api.api_context_manager.writer
+def ensure_incomplete_user(ctx):
+    """Ensures that a user record is created for the "incomplete consumer
+    user". Returns the internal ID of that record.
+    """
+    incomplete_id = CONF.placement.incomplete_consumer_user_id
+    sel = sa.select([USER_TBL.c.id]).where(
+        USER_TBL.c.external_id == incomplete_id)
+    res = ctx.session.execute(sel).fetchone()
+    if res:
+        return res[0]
+    ins = USER_TBL.insert().values(external_id=incomplete_id)
+    res = ctx.session.execute(ins)
+    return res.inserted_primary_key[0]
+
+
+@db_api.api_context_manager.reader
+def _get_user_by_external_id(ctx, external_id):
+    users = sa.alias(USER_TBL, name="u")
+    cols = [
+        users.c.id,
+        users.c.external_id,
+        users.c.updated_at,
+        users.c.created_at
+    ]
+    sel = sa.select(cols)
+    sel = sel.where(users.c.external_id == external_id)
+    res = ctx.session.execute(sel).fetchone()
+    if not res:
+        raise exception.UserNotFound(external_id=external_id)
+
+    return dict(res)
+
+
+@base.VersionedObjectRegistry.register_if(False)
+class User(base.VersionedObject):
+
+    fields = {
+        'id': fields.IntegerField(read_only=True),
+        'external_id': fields.StringField(nullable=False),
+    }
+
+    @staticmethod
+    def _from_db_object(ctx, target, source):
+        for field in target.fields:
+            setattr(target, field, source[field])
+
+        target._context = ctx
+        target.obj_reset_changes()
+        return target
+
+    @classmethod
+    def get_by_external_id(cls, ctx, external_id):
+        res = _get_user_by_external_id(ctx, external_id)
+        return cls._from_db_object(ctx, cls(ctx), res)
+
+    def create(self):
+        @db_api.api_context_manager.writer
+        def _create_in_db(ctx):
+            db_obj = models.User(external_id=self.external_id)
+            try:
+                db_obj.save(ctx.session)
+            except db_exc.DBDuplicateEntry:
+                raise exception.UserExists(external_id=self.external_id)
+            self._from_db_object(ctx, self, db_obj)
+        _create_in_db(self._context)
diff --git a/nova/api/openstack/placement/util.py b/nova/api/openstack/placement/util.py
index e2f35a9c3161..6acf03fe2143 100644
--- a/nova/api/openstack/placement/util.py
+++ b/nova/api/openstack/placement/util.py
@@ -15,6 +15,7 @@ import functools
 import re
 
 import jsonschema
+from oslo_config import cfg
 from oslo_middleware import request_id
 from oslo_serialization import jsonutils
 from oslo_utils import timeutils
@@ -22,12 +23,18 @@ from oslo_utils import uuidutils
 import webob
 
 from nova.api.openstack.placement import errors
+from nova.api.openstack.placement import exception
 from nova.api.openstack.placement import lib as placement_lib
 # NOTE(cdent): avoid cyclical import conflict between util and
 # microversion
 import nova.api.openstack.placement.microversion
+from nova.api.openstack.placement.objects import consumer as consumer_obj
+from nova.api.openstack.placement.objects import project as project_obj
+from nova.api.openstack.placement.objects import user as user_obj
 from nova.i18n import _
 
+CONF = cfg.CONF
+
 # Error code handling constants
 ENV_ERROR_CODE = 'placement.error_code'
 ERROR_CODE_MICROVERSION = (1, 23)
@@ -568,3 +575,53 @@ def parse_qs_request_groups(req):
             raise webob.exc.HTTPBadRequest(msg % ', '.join(conflicting_traits))
 
     return by_suffix
+
+
+def ensure_consumer(ctx, consumer_uuid, project_id, user_id):
+    """Ensures there are records in the consumers, projects and users table for
+    the supplied external identifiers.
+
+    Returns a populated Consumer object containing Project and User sub-objects
+
+    :param ctx: The request context.
+    :param consumer_uuid: The uuid of the consumer of the resources.
+    :param project_id: The external ID of the project consuming the resources.
+    :param user_id: The external ID of the user consuming the resources.
+    """
+    if project_id is None:
+        project_id = CONF.placement.incomplete_consumer_project_id
+        user_id = CONF.placement.incomplete_consumer_user_id
+    try:
+        proj = project_obj.Project.get_by_external_id(ctx, project_id)
+    except exception.NotFound:
+        # Auto-create the project if we found no record of it...
+        try:
+            proj = project_obj.Project(ctx, external_id=project_id)
+            proj.create()
+        except exception.ProjectExists:
+            # No worries, another thread created this project already
+            proj = project_obj.Project.get_by_external_id(ctx, project_id)
+    try:
+        user = user_obj.User.get_by_external_id(ctx, user_id)
+    except exception.NotFound:
+        # Auto-create the user if we found no record of it...
+        try:
+            user = user_obj.User(ctx, external_id=user_id)
+            user.create()
+        except exception.UserExists:
+            # No worries, another thread created this user already
+            user = user_obj.User.get_by_external_id(ctx, user_id)
+
+    try:
+        consumer = consumer_obj.Consumer.get_by_uuid(ctx, consumer_uuid)
+    except exception.NotFound:
+        # No such consumer. This is common for new allocations. Create the
+        # consumer record
+        try:
+            consumer = consumer_obj.Consumer(
+                ctx, uuid=consumer_uuid, project=proj, user=user)
+            consumer.create()
+        except exception.ConsumerExists:
+            # No worries, another thread created this user already
+            consumer = consumer_obj.Consumer.get_by_uuid(ctx, consumer_uuid)
+    return consumer
diff --git a/nova/cmd/manage.py b/nova/cmd/manage.py
index cba54fbd6869..fca8a8245738 100644
--- a/nova/cmd/manage.py
+++ b/nova/cmd/manage.py
@@ -44,6 +44,7 @@ import six
 import six.moves.urllib.parse as urlparse
 from sqlalchemy.engine import url as sqla_url
 
+from nova.api.openstack.placement.objects import consumer as consumer_obj
 from nova.cmd import common as cmd_common
 import nova.conf
 from nova import config
@@ -400,6 +401,8 @@ class DbCommands(object):
         # Queens and Pike since instance.avz of instances before Pike
         # need to be populated if it was not specified during boot time.
         instance_obj.populate_missing_availability_zones,
+        # Added in Rocky
+        consumer_obj.create_incomplete_consumers,
     )
 
     def __init__(self):
diff --git a/nova/conf/placement.py b/nova/conf/placement.py
index 7d03a72785e1..417a17ec78ac 100644
--- a/nova/conf/placement.py
+++ b/nova/conf/placement.py
@@ -17,6 +17,7 @@ from nova.conf import utils as confutils
 
 
 DEFAULT_SERVICE_TYPE = 'placement'
+DEFAULT_CONSUMER_MISSING_ID = '00000000-0000-0000-0000-0000000000000'
 
 placement_group = cfg.OptGroup(
     'placement',
@@ -44,6 +45,26 @@ is determined.
         default='placement-policy.yaml',
         help='The file that defines placement policies. This can be an '
              'absolute path or relative to the configuration file.'),
+    cfg.StrOpt(
+        'incomplete_consumer_project_id',
+        default=DEFAULT_CONSUMER_MISSING_ID,
+        help="""
+Early API microversions (<1.8) allowed creating allocations and not specifying
+a project or user identifier for the consumer. In cleaning up the data
+modeling, we no longer allow missing project and user information. if an older
+client makes an allocation, we'll use this in place of the information it
+doesn't provide.
+"""),
+cfg.StrOpt(
+        'incomplete_consumer_user_id',
+        default=DEFAULT_CONSUMER_MISSING_ID,
+        help="""
+Early API microversions (<1.8) allowed creating allocations and not specifying
+a project or user identifier for the consumer. In cleaning up the data
+modeling, we no longer allow missing project and user information. if an older
+client makes an allocation, we'll use this in place of the information it
+doesn't provide.
+"""),
 ]
 
 
diff --git a/nova/tests/functional/api/openstack/placement/db/test_consumer.py b/nova/tests/functional/api/openstack/placement/db/test_consumer.py
new file mode 100644
index 000000000000..db8b6cba4705
--- /dev/null
+++ b/nova/tests/functional/api/openstack/placement/db/test_consumer.py
@@ -0,0 +1,133 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+import sqlalchemy as sa
+
+from nova.api.openstack.placement import exception
+from nova.api.openstack.placement.objects import consumer as consumer_obj
+from nova.api.openstack.placement.objects import project as project_obj
+from nova.api.openstack.placement.objects import resource_provider as rp_obj
+from nova.api.openstack.placement.objects import user as user_obj
+from nova.db.sqlalchemy import api as db_api
+from nova.tests.functional.api.openstack.placement.db import test_base as tb
+from nova.tests import uuidsentinel as uuids
+
+CONF = cfg.CONF
+CONSUMER_TBL = consumer_obj.CONSUMER_TBL
+PROJECT_TBL = project_obj.PROJECT_TBL
+USER_TBL = user_obj.USER_TBL
+ALLOC_TBL = rp_obj._ALLOC_TBL
+
+
+class ConsumerTestCase(tb.PlacementDbBaseTestCase):
+    def test_non_existing_consumer(self):
+        self.assertRaises(exception.ConsumerNotFound,
+            consumer_obj.Consumer.get_by_uuid, self.ctx,
+            uuids.non_existing_consumer)
+
+    def test_create_and_get(self):
+        u = user_obj.User(self.ctx, external_id='fake-user')
+        u.create()
+        p = project_obj.Project(self.ctx, external_id='fake-project')
+        p.create()
+        c = consumer_obj.Consumer(
+            self.ctx, uuid=uuids.consumer, user=u, project=p)
+        c.create()
+        c = consumer_obj.Consumer.get_by_uuid(self.ctx, uuids.consumer)
+        self.assertEqual(1, c.id)
+        self.assertEqual(1, c.project.id)
+        self.assertEqual(1, c.user.id)
+        self.assertRaises(exception.ConsumerExists, c.create)
+
+
+class CreateIncompleteConsumersTestCase(tb.PlacementDbBaseTestCase):
+    @db_api.api_context_manager.writer
+    def _create_incomplete_allocations(self, ctx):
+        # Create some allocations with consumers that don't exist in the
+        # consumers table to represent old allocations that we expect to be
+        # "cleaned up" with consumers table records that point to the sentinel
+        # project/user records.
+        c1_missing_uuid = uuids.c1_missing
+        c2_missing_uuid = uuids.c2_missing
+        ins_stmt = ALLOC_TBL.insert().values(
+            resource_provider_id=1, resource_class_id=0,
+            consumer_id=c1_missing_uuid, used=1)
+        ctx.session.execute(ins_stmt)
+        ins_stmt = ALLOC_TBL.insert().values(
+            resource_provider_id=1, resource_class_id=0,
+            consumer_id=c2_missing_uuid, used=1)
+        ctx.session.execute(ins_stmt)
+        # Verify there are no records in the projects/users table
+        project_count = ctx.session.scalar(
+            sa.select([sa.func.count('*')]).select_from(PROJECT_TBL))
+        self.assertEqual(0, project_count)
+        user_count = ctx.session.scalar(
+            sa.select([sa.func.count('*')]).select_from(USER_TBL))
+        self.assertEqual(0, user_count)
+        # Verify there are no consumer records for the missing consumers
+        sel = CONSUMER_TBL.select(
+            CONSUMER_TBL.c.uuid.in_([c1_missing_uuid, c2_missing_uuid]))
+        res = ctx.session.execute(sel).fetchall()
+        self.assertEqual(0, len(res))
+
+    @db_api.api_context_manager.reader
+    def _check_incomplete_consumers(self, ctx):
+        incomplete_external_id = CONF.placement.incomplete_consumer_project_id
+
+        # Verify we have a record in projects for the missing sentinel
+        sel = PROJECT_TBL.select(
+            PROJECT_TBL.c.external_id == incomplete_external_id)
+        rec = ctx.session.execute(sel).first()
+        self.assertEqual(incomplete_external_id, rec['external_id'])
+        incomplete_proj_id = rec['id']
+
+        # Verify we have a record in users for the missing sentinel
+        sel = user_obj.USER_TBL.select(
+            USER_TBL.c.external_id == incomplete_external_id)
+        rec = ctx.session.execute(sel).first()
+        self.assertEqual(incomplete_external_id, rec['external_id'])
+        incomplete_user_id = rec['id']
+
+        # Verify there are records in the consumers table for our old
+        # allocation records created in the pre-migration setup and that the
+        # projects and users referenced in those consumer records point to the
+        # incomplete project/user
+        sel = CONSUMER_TBL.select(CONSUMER_TBL.c.uuid == uuids.c1_missing)
+        missing_c1 = ctx.session.execute(sel).first()
+        self.assertEqual(incomplete_proj_id, missing_c1['project_id'])
+        self.assertEqual(incomplete_user_id, missing_c1['user_id'])
+        sel = CONSUMER_TBL.select(CONSUMER_TBL.c.uuid == uuids.c2_missing)
+        missing_c2 = ctx.session.execute(sel).first()
+        self.assertEqual(incomplete_proj_id, missing_c2['project_id'])
+        self.assertEqual(incomplete_user_id, missing_c2['user_id'])
+
+        # Ensure there are no more allocations with incomplete consumers
+        alloc_to_consumer = sa.outerjoin(
+            ALLOC_TBL, CONSUMER_TBL,
+            ALLOC_TBL.c.consumer_id == CONSUMER_TBL.c.uuid)
+        sel = sa.select([ALLOC_TBL])
+        sel = sel.select_from(alloc_to_consumer)
+        sel = sel.where(CONSUMER_TBL.c.id.is_(None))
+        res = ctx.session.execute(sel).fetchall()
+        self.assertEqual(0, len(res))
+
+    def test_create_incomplete_consumers(self):
+        """Test the online data migration that creates incomplete consumer
+        records along with the incomplete consumer project/user records.
+        """
+        self._create_incomplete_allocations(self.ctx)
+        res = consumer_obj.create_incomplete_consumers(self.ctx, 10)
+        self.assertEqual((2, 2), res)
+        self._check_incomplete_consumers(self.ctx)
+        res = consumer_obj.create_incomplete_consumers(self.ctx, 10)
+        self.assertEqual((0, 0), res)
diff --git a/nova/tests/functional/api/openstack/placement/db/test_project.py b/nova/tests/functional/api/openstack/placement/db/test_project.py
new file mode 100644
index 000000000000..8e640b6f961a
--- /dev/null
+++ b/nova/tests/functional/api/openstack/placement/db/test_project.py
@@ -0,0 +1,30 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from nova.api.openstack.placement import exception
+from nova.api.openstack.placement.objects import project as project_obj
+from nova.tests.functional.api.openstack.placement.db import test_base as tb
+from nova.tests import uuidsentinel as uuids
+
+
+class ProjectTestCase(tb.PlacementDbBaseTestCase):
+    def test_non_existing_project(self):
+        self.assertRaises(
+            exception.ProjectNotFound, project_obj.Project.get_by_external_id,
+            self.ctx, uuids.non_existing_project)
+
+    def test_create_and_get(self):
+        p = project_obj.Project(self.ctx, external_id='fake-project')
+        p.create()
+        p = project_obj.Project.get_by_external_id(self.ctx, 'fake-project')
+        self.assertEqual(1, p.id)
+        self.assertRaises(exception.ProjectExists, p.create)
diff --git a/nova/tests/functional/api/openstack/placement/db/test_user.py b/nova/tests/functional/api/openstack/placement/db/test_user.py
new file mode 100644
index 000000000000..1e35bfa13bbf
--- /dev/null
+++ b/nova/tests/functional/api/openstack/placement/db/test_user.py
@@ -0,0 +1,30 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from nova.api.openstack.placement import exception
+from nova.api.openstack.placement.objects import user as user_obj
+from nova.tests.functional.api.openstack.placement.db import test_base as tb
+from nova.tests import uuidsentinel as uuids
+
+
+class UserTestCase(tb.PlacementDbBaseTestCase):
+    def test_non_existing_user(self):
+        self.assertRaises(
+            exception.UserNotFound, user_obj.User.get_by_external_id,
+            self.ctx, uuids.non_existing_user)
+
+    def test_create_and_get(self):
+        u = user_obj.User(self.ctx, external_id='fake-user')
+        u.create()
+        u = user_obj.User.get_by_external_id(self.ctx, 'fake-user')
+        self.assertEqual(1, u.id)
+        self.assertRaises(exception.UserExists, u.create)
diff --git a/nova/tests/functional/api/openstack/placement/gabbits/ensure-consumer.yaml b/nova/tests/functional/api/openstack/placement/gabbits/ensure-consumer.yaml
new file mode 100644
index 000000000000..2c574df99b65
--- /dev/null
+++ b/nova/tests/functional/api/openstack/placement/gabbits/ensure-consumer.yaml
@@ -0,0 +1,41 @@
+# Tests of the ensure consumer behaviour for versions of the API from <1.7 to
+# 1.8-1.26 and finally 1.27+
+fixtures:
+    - AllocationFixture
+
+defaults:
+    request_headers:
+        x-auth-token: admin
+        accept: application/json
+        openstack-api-version: placement 1.7
+
+vars:
+- &default_incomplete_id 00000000-0000-0000-0000-0000000000000
+- &consumer_id fbad1a87-c341-4ac0-be49-777b21ce1b7b
+tests:
+
+- name: put an allocation without project/user (1.7)
+  PUT: /allocations/*consumer_id
+  request_headers:
+      content-type: application/json
+      openstack-api-version: placement 1.7
+  data:
+      allocations:
+          - resource_provider:
+                uuid: $ENVIRON['RP_UUID']
+            resources:
+                DISK_GB: 10
+  status: 204
+
+# We now ALWAYS create a consumer record, and if project or user isn't
+# specified (as was the case in 1.7) we should get the project/user
+# corresponding to the CONF option for incomplete consumers when asking for the
+# allocation information at a microversion that shows project/user information
+# (1.12+)
+- name: get with 1.12 microversion and check project and user are filled
+  GET: /allocations/*consumer_id
+  request_headers:
+      openstack-api-version: placement 1.12
+  response_json_paths:
+      $.project_id: *default_incomplete_id
+      $.user_id: *default_incomplete_id
diff --git a/nova/tests/unit/api/openstack/placement/test_util.py b/nova/tests/unit/api/openstack/placement/test_util.py
index cf932b6b1aef..5a823482d453 100644
--- a/nova/tests/unit/api/openstack/placement/test_util.py
+++ b/nova/tests/unit/api/openstack/placement/test_util.py
@@ -18,19 +18,25 @@ import datetime
 import fixtures
 import microversion_parse
 import mock
+from oslo_config import cfg
 from oslo_middleware import request_id
 from oslo_utils import timeutils
 import webob
 
 import six
 
+from nova.api.openstack.placement import exception
 from nova.api.openstack.placement import lib as pl
 from nova.api.openstack.placement import microversion
+from nova.api.openstack.placement.objects import project as project_obj
 from nova.api.openstack.placement.objects import resource_provider as rp_obj
+from nova.api.openstack.placement.objects import user as user_obj
 from nova.api.openstack.placement import util
 from nova import test
 from nova.tests import uuidsentinel
 
+CONF = cfg.CONF
+
 
 class TestCheckAccept(test.NoDBTestCase):
     """Confirm behavior of util.check_accept."""
@@ -894,3 +900,86 @@ class TestPickLastModified(test.NoDBTestCase):
                 None, self.resource_provider)
             self.assertEqual(now, chosen_time)
             mock_utc.assert_called_once_with(with_timezone=True)
+
+
+class TestEnsureConsumer(test.NoDBTestCase):
+    def setUp(self):
+        super(TestEnsureConsumer, self).setUp()
+        self.mock_project_get = self.useFixture(fixtures.MockPatch(
+            'nova.api.openstack.placement.objects.project.'
+            'Project.get_by_external_id')).mock
+        self.mock_user_get = self.useFixture(fixtures.MockPatch(
+            'nova.api.openstack.placement.objects.user.'
+            'User.get_by_external_id')).mock
+        self.mock_consumer_get = self.useFixture(fixtures.MockPatch(
+            'nova.api.openstack.placement.objects.consumer.'
+            'Consumer.get_by_uuid')).mock
+        self.mock_project_create = self.useFixture(fixtures.MockPatch(
+            'nova.api.openstack.placement.objects.project.'
+            'Project.create')).mock
+        self.mock_user_create = self.useFixture(fixtures.MockPatch(
+            'nova.api.openstack.placement.objects.user.'
+            'User.create')).mock
+        self.mock_consumer_create = self.useFixture(fixtures.MockPatch(
+            'nova.api.openstack.placement.objects.consumer.'
+            'Consumer.create')).mock
+        self.ctx = mock.sentinel.ctx
+        self.consumer_id = uuidsentinel.consumer
+        self.project_id = uuidsentinel.project
+        self.user_id = uuidsentinel.user
+
+    def test_no_existing_project_user_consumer(self):
+        self.mock_project_get.side_effect = exception.NotFound
+        self.mock_user_get.side_effect = exception.NotFound
+        self.mock_consumer_get.side_effect = exception.NotFound
+
+        util.ensure_consumer(
+            self.ctx, self.consumer_id, self.project_id, self.user_id)
+
+        self.mock_project_get.assert_called_once_with(
+            self.ctx, self.project_id)
+        self.mock_user_get.assert_called_once_with(
+            self.ctx, self.user_id)
+        self.mock_consumer_get.assert_called_once_with(
+            self.ctx, self.consumer_id)
+        self.mock_project_create.assert_called_once()
+        self.mock_user_create.assert_called_once()
+        self.mock_consumer_create.assert_called_once()
+
+    def test_no_existing_project_user_consumer_use_incomplete(self):
+        """Verify that if the project_id arg is None, that we fall back to the
+        CONF options for incomplete project and user ID.
+        """
+        self.mock_project_get.side_effect = exception.NotFound
+        self.mock_user_get.side_effect = exception.NotFound
+        self.mock_consumer_get.side_effect = exception.NotFound
+
+        util.ensure_consumer(
+            self.ctx, self.consumer_id, None, None)
+
+        self.mock_project_get.assert_called_once_with(
+            self.ctx, CONF.placement.incomplete_consumer_project_id)
+        self.mock_user_get.assert_called_once_with(
+            self.ctx, CONF.placement.incomplete_consumer_user_id)
+        self.mock_consumer_get.assert_called_once_with(
+            self.ctx, self.consumer_id)
+        self.mock_project_create.assert_called_once()
+        self.mock_user_create.assert_called_once()
+        self.mock_consumer_create.assert_called_once()
+
+    def test_existing_project_user_no_existing_consumer(self):
+        """Check that if we find an existing project and user, that we use
+        those found objects in creating the consumer.
+        """
+        proj = project_obj.Project(self.ctx, id=1, external_id=self.project_id)
+        self.mock_project_get.return_value = proj
+        user = user_obj.User(self.ctx, id=1, external_id=self.user_id)
+        self.mock_user_get.return_value = user
+        self.mock_consumer_get.side_effect = exception.NotFound
+
+        util.ensure_consumer(
+            self.ctx, self.consumer_id, self.project_id, self.user_id)
+
+        self.mock_project_create.assert_not_called()
+        self.mock_user_create.assert_not_called()
+        self.mock_consumer_create.assert_called_once()
diff --git a/releasenotes/notes/placement-incomplete-consumer-configuration-b775dac1bcd34f9d.yaml b/releasenotes/notes/placement-incomplete-consumer-configuration-b775dac1bcd34f9d.yaml
new file mode 100644
index 000000000000..ec45ad2008df
--- /dev/null
+++ b/releasenotes/notes/placement-incomplete-consumer-configuration-b775dac1bcd34f9d.yaml
@@ -0,0 +1,11 @@
+---
+features:
+    - |
+      Prior to microversion 1.8 of the placement API, one could create
+      allocations and not supply a project or user ID for the consumer of the
+      allocated resources. While this is no longer allowed after placement API
+      1.8, older allocations exist and we now ensure that a consumer record is
+      created for these older allocations. Use the two new CONF options
+      ``CONF.placement.incomplete_consumer_project_id`` and
+      ``CONF.placement.incomplete_consumer_user_id`` to control the project and
+      user identifiers that are written for these incomplete consumer records.