From a950edb320d8ad571bd4132465010a4f85435c2e Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Tue, 11 Oct 2016 11:01:14 +1300 Subject: [PATCH] Treat ec2 collector data as immutable Currently the ec2 collector polls the nova metadata service every $polling_period even though the data is not expected to change and no known config actions have been written to respond to changes in these values. With larger overclouds, this metadata polling will cause noticeable load on the undercloud, especially nova-api and neutron (for port lookups). This change will check for data in /var/lib/os-collect-config/ec2.json and simply return that if it exists. Change-Id: I84156f8b005b319fccf8ac17af8943aefb0a8f6d Closes-Bug: #1619072 --- os_collect_config/ec2.py | 10 ++++++++++ os_collect_config/tests/test_collect.py | 7 +++++++ os_collect_config/tests/test_ec2.py | 15 +++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/os_collect_config/ec2.py b/os_collect_config/ec2.py index d652922..07fc604 100644 --- a/os_collect_config/ec2.py +++ b/os_collect_config/ec2.py @@ -13,9 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json +import os + from oslo_config import cfg from oslo_log import log +from os_collect_config import cache from os_collect_config import common from os_collect_config import exc @@ -60,5 +64,11 @@ class Collector(object): return content def collect(self): + cache_path = cache.get_path('ec2') + if os.path.exists(cache_path): + with open(cache_path) as f: + metadata = json.load(f) + if metadata: + return [('ec2', metadata)] root_url = '%s/' % (CONF.ec2.metadata_url) return [('ec2', self._fetch_metadata(root_url, CONF.ec2.timeout))] diff --git a/os_collect_config/tests/test_collect.py b/os_collect_config/tests/test_collect.py index 84ad306..1c9cefe 100644 --- a/os_collect_config/tests/test_collect.py +++ b/os_collect_config/tests/test_collect.py @@ -476,9 +476,16 @@ class TestCollectAll(testtools.TestCase): # Commit for changed in changed_keys: cache.commit(changed) + + # Replace the ec2 requests with a failing one to simulate a transient + # network failure + soft_config_map['ec2'] = {'requests_impl': test_ec2.FakeFailRequests} (changed_keys, paths2) = self._call_collect_all( store=True, collector_kwargs_map=soft_config_map) self.assertEqual(set(), changed_keys) + + # check the second collect includes cached ec2 data despite network + # failure self.assertEqual(paths, paths2) def test_collect_all_nostore(self): diff --git a/os_collect_config/tests/test_ec2.py b/os_collect_config/tests/test_ec2.py index 7b27d96..6e12d03 100644 --- a/os_collect_config/tests/test_ec2.py +++ b/os_collect_config/tests/test_ec2.py @@ -13,9 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json +import os import uuid import fixtures +from oslo_config import cfg import requests import six.moves.urllib.parse as urlparse import testtools @@ -114,3 +117,15 @@ class TestEc2(testtools.TestCase): collect_ec2 = ec2.Collector(requests_impl=FakeFailRequests) self.assertRaises(exc.Ec2MetadataNotAvailable, collect_ec2.collect) self.assertIn('Forbidden', self.log.output) + + def test_collect_ec2_collected(self): + collect.setup_conf() + cache_dir = self.useFixture(fixtures.TempDir()) + self.addCleanup(cfg.CONF.reset) + cfg.CONF.set_override('cachedir', cache_dir.path) + ec2_path = os.path.join(cache_dir.path, 'ec2.json') + with open(ec2_path, 'w') as f: + json.dump(META_DATA, f) + + collect_ec2 = ec2.Collector(requests_impl=FakeFailRequests) + self.assertEqual([('ec2', META_DATA)], collect_ec2.collect())