Merge deployments data for collectors heat, request

This is required so that a swift-enabled TripleO undercloud can switch
to polling for metadata from a TempURL rather than heat.

Change-Id: I73ac9e01f85e0c72ce7411e2c61c545322f3dccc
Closes-Bug: #1424913

os_collect_config/cfn.py

@@ -47,10 +47,8 @@ opts = [
                help='Access Key ID'),
     cfg.MultiStrOpt('deployment-key',
                     default=['deployments'],
-                    help='Key(s) to explode into multiple collected outputs. '
-                    'Parsed according to the expected Metadata created by '
-                    'OS::Heat::StructuredDeployment. Only Exploded if seen at '
-                    'the root of the Metadata.')
+                    help='DEPRECATED, use global configuration option '
+                         '"deployment-key"')
 ]
 name = 'cfn'
 

os_collect_config/collect.py

@@ -76,6 +76,12 @@ opts = [
                 help='Query normally, print the resulting configs as a json'
                 ' map, and exit immediately without running command if it is'
                 ' configured.'),
+    cfg.MultiStrOpt('deployment-key',
+                    default=['deployments'],
+                    help='Key(s) to explode into multiple collected outputs. '
+                    'Parsed according to the expected Metadata created by '
+                    'OS::Heat::StructuredDeployment. Only Exploded if seen at '
+                    'the root of the Metadata.')
 ]
 
 CONF = cfg.CONF

os_collect_config/heat.py

@@ -18,6 +18,7 @@ from oslo_config import cfg
 
 from os_collect_config import exc
 from os_collect_config import keystone
+from os_collect_config import merger
 from os_collect_config.openstack.common import log
 
 CONF = cfg.CONF
@@ -82,7 +83,10 @@ class Collector(object):
             r = heat.resources.metadata(CONF.heat.stack_id,
                                         CONF.heat.resource_name)
 
-            return [('heat', r)]
+            final_list = merger.merged_list_from_content(
+                r, cfg.CONF.deployment_key, name)
+            return final_list
+
         except Exception as e:
             logger.warn(str(e))
             raise exc.HeatMetadataNotAvailable

os_collect_config/request.py

@@ -21,6 +21,7 @@ from oslo_config import cfg
 
 from os_collect_config import common
 from os_collect_config import exc
+from os_collect_config import merger
 from os_collect_config.openstack.common import log
 
 CONF = cfg.CONF
@@ -90,4 +91,6 @@ class Collector(object):
             raise exc.RequestMetadataNotAvailable
         final_content.update(value)
 
-        return [('request', final_content)]
+        final_list = merger.merged_list_from_content(
+            final_content, cfg.CONF.deployment_key, name)
+        return final_list

os_collect_config/tests/test_heat.py

@@ -113,6 +113,12 @@ class FakeHeatClient(object):
         return META_DATA
 
 
+class FakeHeatClientSoftwareConfig(FakeHeatClient):
+
+    def metadata(self, stack_id, resource_name):
+        return SOFTWARE_CONFIG_DATA
+
+
 class TestHeatBase(testtools.TestCase):
     def setUp(self):
         super(TestHeatBase, self).setUp()
@@ -195,3 +201,21 @@ class TestHeat(TestHeatBase):
         heat_collect = heat.Collector()
         self.assertRaises(exc.HeatMetadataNotConfigured, heat_collect.collect)
         self.assertIn('No resource_name configured', self.log.output)
+
+
+class TestHeatSoftwareConfig(TestHeatBase):
+    @mock.patch.object(ks_discover.Discover, '__init__')
+    @mock.patch.object(ks_discover.Discover, 'url_for')
+    def test_collect_heat(self, mock_url_for, mock___init__):
+        mock___init__.return_value = None
+        mock_url_for.return_value = cfg.CONF.heat.auth_url
+        heat_md = heat.Collector(
+            keystoneclient=FakeKeystoneClient(self),
+            heatclient=FakeHeatClientSoftwareConfig(self)).collect()
+        self.assertThat(heat_md, matchers.IsInstance(list))
+        self.assertEqual(2, len(heat_md))
+        self.assertEqual('heat', heat_md[0][0])
+        self.assertEqual(
+            SOFTWARE_CONFIG_DATA['deployments'], heat_md[0][1]['deployments'])
+        self.assertEqual(
+            ('dep-name1', {'config1': 'value1'}), heat_md[1])

os_collect_config/tests/test_request.py

@@ -36,6 +36,68 @@ META_DATA = {u'int1': 1,
              }}
 
 
+SOFTWARE_CONFIG_DATA = {
+    u'old-style': u'value',
+    u'deployments': [
+        {
+            u'inputs': [
+                {
+                    u'type': u'String',
+                    u'name': u'input1',
+                    u'value': u'value1'
+                }
+            ],
+            u'group': 'Heat::Ungrouped',
+            u'name': 'dep-name1',
+            u'outputs': None,
+            u'options': None,
+            u'config': {
+                u'config1': 'value1'
+            }
+        },
+        {
+            u'inputs': [
+                {
+                    u'type': u'String',
+                    u'name': u'input1',
+                    u'value': u'value1'
+                }
+            ],
+            u'group': 'os-apply-config',
+            u'name': 'dep-name2',
+            u'outputs': None,
+            u'options': None,
+            u'config': {
+                u'config2': 'value2'
+            }
+        },
+        {
+            u'inputs': [
+                {
+                    u'type': u'String',
+                    u'name': u'input1',
+                    u'value': u'value1'
+                }
+            ],
+            u'name': 'dep-name3',
+            u'outputs': None,
+            u'options': None,
+            u'config': {
+                u'config3': 'value3'
+            }
+        },
+        {
+            u'inputs': [],
+            u'group': 'ignore_me',
+            u'name': 'ignore_me_name',
+            u'outputs': None,
+            u'options': None,
+            u'config': 'ignore_me_config'
+        }
+    ]
+}
+
+
 class FakeResponse(dict):
     def __init__(self, text, headers=None):
         self.text = text
@@ -69,6 +131,18 @@ class FakeFailRequests(object):
             raise requests.exceptions.HTTPError(403, 'Forbidden')
 
 
+class FakeRequestsSoftwareConfig(object):
+
+    class Session(object):
+        def get(self, url):
+            return FakeResponse(json.dumps(SOFTWARE_CONFIG_DATA))
+
+        def head(self, url):
+            return FakeResponse('', headers={
+                'last-modified': time.strftime(
+                    "%a, %d %b %Y %H:%M:%S %Z", time.gmtime())})
+
+
 class TestRequestBase(testtools.TestCase):
     def setUp(self):
         super(TestRequestBase, self).setUp()
@@ -146,3 +220,20 @@ class TestRequest(TestRequestBase):
 
         # run no last-modified header, collects
         self.assertIsNone(req_collect.check_fetch_content({}))
+
+
+class TestRequestSoftwareConfig(TestRequestBase):
+
+    def test_collect_request(self):
+        req_collect = request.Collector(
+            requests_impl=FakeRequestsSoftwareConfig)
+        req_md = req_collect.collect()
+        self.assertEqual(4, len(req_md))
+        self.assertEqual(
+            SOFTWARE_CONFIG_DATA['deployments'], req_md[0][1]['deployments'])
+        self.assertEqual(
+            ('dep-name1', {'config1': 'value1'}), req_md[1])
+        self.assertEqual(
+            ('dep-name2', {'config2': 'value2'}), req_md[2])
+        self.assertEqual(
+            ('dep-name3', {'config3': 'value3'}), req_md[3])