Treat ec2 collector data as immutable

Currently the ec2 collector polls the nova metadata service every
$polling_period even though the data is not expected to change and no
known config actions have been written to respond to changes in these
values.

With larger overclouds, this metadata polling will cause noticeable
load on the undercloud, especially nova-api and neutron (for port
lookups).

This change will check for data in /var/lib/os-collect-config/ec2.json
and simply return that if it exists.

Change-Id: I84156f8b005b319fccf8ac17af8943aefb0a8f6d
Closes-Bug: #1619072

os_collect_config/ec2.py

@@ -13,9 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
+import os
+
 from oslo_config import cfg
 from oslo_log import log
 
+from os_collect_config import cache
 from os_collect_config import common
 from os_collect_config import exc
 
@@ -60,5 +64,11 @@ class Collector(object):
         return content
 
     def collect(self):
+        cache_path = cache.get_path('ec2')
+        if os.path.exists(cache_path):
+            with open(cache_path) as f:
+                metadata = json.load(f)
+                if metadata:
+                    return [('ec2', metadata)]
         root_url = '%s/' % (CONF.ec2.metadata_url)
         return [('ec2', self._fetch_metadata(root_url, CONF.ec2.timeout))]

os_collect_config/tests/test_collect.py

@@ -476,9 +476,16 @@ class TestCollectAll(testtools.TestCase):
         # Commit
         for changed in changed_keys:
             cache.commit(changed)
+
+        # Replace the ec2 requests with a failing one to simulate a transient
+        # network failure
+        soft_config_map['ec2'] = {'requests_impl': test_ec2.FakeFailRequests}
         (changed_keys, paths2) = self._call_collect_all(
             store=True, collector_kwargs_map=soft_config_map)
         self.assertEqual(set(), changed_keys)
+
+        # check the second collect includes cached ec2 data despite network
+        # failure
         self.assertEqual(paths, paths2)
 
     def test_collect_all_nostore(self):

os_collect_config/tests/test_ec2.py

@@ -13,9 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
+import os
 import uuid
 
 import fixtures
+from oslo_config import cfg
 import requests
 import six.moves.urllib.parse as urlparse
 import testtools
@@ -114,3 +117,15 @@ class TestEc2(testtools.TestCase):
         collect_ec2 = ec2.Collector(requests_impl=FakeFailRequests)
         self.assertRaises(exc.Ec2MetadataNotAvailable, collect_ec2.collect)
         self.assertIn('Forbidden', self.log.output)
+
+    def test_collect_ec2_collected(self):
+        collect.setup_conf()
+        cache_dir = self.useFixture(fixtures.TempDir())
+        self.addCleanup(cfg.CONF.reset)
+        cfg.CONF.set_override('cachedir', cache_dir.path)
+        ec2_path = os.path.join(cache_dir.path, 'ec2.json')
+        with open(ec2_path, 'w') as f:
+            json.dump(META_DATA, f)
+
+        collect_ec2 = ec2.Collector(requests_impl=FakeFailRequests)
+        self.assertEqual([('ec2', META_DATA)], collect_ec2.collect())