Stack list does direct stack object query

Currently calls to get a collection of stacks via the stack object
have the following data access behaviour:
- One query to fetch the stack records
- One query per stack to fetch the raw template
- One query per stack to fetch the tags

This causes excessive database round trips when there are many stacks.
In addition, the list_stacks call results in a collection of full
stack objects to be created which builds a fully parsed stack - most
of this information is then discarded by the RPC and middleware
formatters so this overhead is for no benefit.

This change is the first step in changing the data access patterns by
querying the Stack versioned object directly instead of via the full
Stack object. A future change will apply an eager fetch and caching
approach to avoid unnecessary queries.

This change does the following:
- Service list_stacks calls stack_object.Stack.get_all directly
- Service list_stacks formats with new function
  api.format_stack_db_object with the exact fields required by the CFN
  and REST API list stacks formatters
- Since the description field is the only one that requires full stack
  parsing, it is now set to an empty string for stack listings via API
  or calls to aws cloudformation list-stacks [1]

This last point may be controversial, my attempts to find uses of the
stack description in stack listings found none that do, and the following
API uses which *don't* show the description:
- heat stack-list
- openstack stack list
- horizon stack list
- rackspace control panel stack list

If we want to add the description back later we could always add a
description field back to the Stack table to store the denormalised
description.

[1] http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-describing-stacks.html

Partial-Bug: #1578851
Change-Id: I88b6f705e87ee7ff4acb7830dcddfc188ae6b3b2

heat/engine/api.py

@@ -217,7 +217,7 @@ def format_stack(stack, preview=False, resolve_outputs=True):
         rpc_api.STACK_ID: dict(stack.identifier()),
         rpc_api.STACK_CREATION_TIME: created_time.isoformat(),
         rpc_api.STACK_UPDATED_TIME: updated_time,
-        rpc_api.STACK_NOTIFICATION_TOPICS: [],  # TODO(?) Not implemented yet
+        rpc_api.STACK_NOTIFICATION_TOPICS: [],  # TODO(therve) Not implemented
         rpc_api.STACK_PARAMETERS: stack.parameters.map(six.text_type),
         rpc_api.STACK_DESCRIPTION: stack.t[stack.t.DESCRIPTION],
         rpc_api.STACK_TMPL_DESCRIPTION: stack.t[stack.t.DESCRIPTION],
@@ -247,6 +247,35 @@ def format_stack(stack, preview=False, resolve_outputs=True):
     return info
 
 
+def format_stack_db_object(stack):
+    """Return a summary representation of the given stack.
+
+    Given a stack versioned db object, return a representation of the given
+    stack for a stack listing.
+    """
+    updated_time = stack.updated_at and stack.updated_at.isoformat()
+    created_time = stack.created_at
+    tags = None
+    if stack.tags:
+        tags = [t.tag for t in stack.tags]
+    info = {
+        rpc_api.STACK_ID: dict(stack.identifier()),
+        rpc_api.STACK_NAME: stack.name,
+        rpc_api.STACK_DESCRIPTION: '',
+        rpc_api.STACK_ACTION: stack.action,
+        rpc_api.STACK_STATUS: stack.status,
+        rpc_api.STACK_STATUS_DATA: stack.status_reason,
+        rpc_api.STACK_CREATION_TIME: created_time.isoformat(),
+        rpc_api.STACK_UPDATED_TIME: updated_time,
+        rpc_api.STACK_OWNER: stack.username,
+        rpc_api.STACK_PARENT: stack.owner_id,
+        rpc_api.STACK_USER_PROJECT_ID: stack.stack_user_project_id,
+        rpc_api.STACK_TAGS: tags,
+    }
+
+    return info
+
+
 def format_resource_attributes(resource, with_attr=None):
     resolver = resource.attributes
     if not with_attr:

heat/engine/service.py

@@ -559,15 +559,22 @@ class EngineService(service.Service):
         if filters is not None:
             filters = api.translate_filters(filters)
 
-        stacks = parser.Stack.load_all(cnxt, limit, marker, sort_keys,
-                                       sort_dir, filters, tenant_safe,
-                                       show_deleted, resolve_data=False,
-                                       show_nested=show_nested,
-                                       show_hidden=show_hidden,
-                                       tags=tags, tags_any=tags_any,
-                                       not_tags=not_tags,
-                                       not_tags_any=not_tags_any)
-        return [api.format_stack(stack) for stack in stacks]
+        stacks = stack_object.Stack.get_all(
+            cnxt,
+            limit,
+            sort_keys,
+            marker,
+            sort_dir,
+            filters,
+            tenant_safe,
+            show_deleted,
+            show_nested,
+            show_hidden,
+            tags,
+            tags_any,
+            not_tags,
+            not_tags_any) or []
+        return [api.format_stack_db_object(stack) for stack in stacks]
 
     @context.request_context
     def count_stacks(self, cnxt, filters=None, tenant_safe=True,

heat/objects/stack.py

@@ -22,6 +22,7 @@ from oslo_versionedobjects import fields
 
 from heat.common import exception
 from heat.common.i18n import _
+from heat.common import identifier
 from heat.db import api as db_api
 from heat.objects import base as heat_base
 from heat.objects import fields as heat_fields
@@ -210,3 +211,7 @@ class Stack(
     def get_status(cls, context, stack_id):
         """Return action and status for the given stack."""
         return db_api.stack_get_status(context, stack_id)
+
+    def identifier(self):
+        """Return an identifier for this stack."""
+        return identifier.HeatIdentifier(self.tenant, self.name, self.id)

heat/tests/test_engine_service.py

@@ -483,13 +483,6 @@ class StackServiceTest(common.HeatTestCase):
 
     @tools.stack_context('service_list_all_test_stack')
     def test_stack_list_all(self):
-        self.m.StubOutWithMock(parser.Stack, '_from_db')
-        parser.Stack._from_db(
-            self.ctx, mox.IgnoreArg(),
-            resolve_data=False
-        ).AndReturn(self.stack)
-
-        self.m.ReplayAll()
         sl = self.eng.list_stacks(self.ctx)
 
         self.assertEqual(1, len(sl))
@@ -503,9 +496,7 @@ class StackServiceTest(common.HeatTestCase):
             self.assertIn('stack_status', s)
             self.assertIn('stack_status_reason', s)
             self.assertIn('description', s)
-            self.assertIn('WordPress', s['description'])
-
-        self.m.VerifyAll()
+            self.assertEqual('', s['description'])
 
     @mock.patch.object(stack_object.Stack, 'get_all')
     def test_stack_list_passes_marker_info(self, mock_stack_get_all):