diff --git a/os_collect_config/collect.py b/os_collect_config/collect.py index fcf5e0a..11e4364 100644 --- a/os_collect_config/collect.py +++ b/os_collect_config/collect.py @@ -25,14 +25,14 @@ import time from openstack.common import log from os_collect_config import cache from os_collect_config import cfn -from os_collect_config import common from os_collect_config import ec2 from os_collect_config import exc +from os_collect_config import heat from os_collect_config import heat_local from os_collect_config import version from oslo.config import cfg -DEFAULT_COLLECTORS = ['heat_local', 'ec2', 'cfn'] +DEFAULT_COLLECTORS = ['heat_local', 'ec2', 'cfn', 'heat'] opts = [ cfg.StrOpt('command', short='c', help='Command to run on metadata changes. If specified,' @@ -79,6 +79,7 @@ logger = log.getLogger('os-collect-config') COLLECTORS = {ec2.name: ec2, cfn.name: cfn, + heat.name: heat, heat_local.name: heat_local} @@ -92,17 +93,22 @@ def setup_conf(): heat_local_group = cfg.OptGroup(name='heat_local', title='Heat Local Metadata options') + heat_group = cfg.OptGroup(name='heat', + title='Heat Metadata options') + CONF.register_group(ec2_group) CONF.register_group(cfn_group) CONF.register_group(heat_local_group) + CONF.register_group(heat_group) CONF.register_cli_opts(ec2.opts, group='ec2') CONF.register_cli_opts(cfn.opts, group='cfn') CONF.register_cli_opts(heat_local.opts, group='heat_local') + CONF.register_cli_opts(heat.opts, group='heat') CONF.register_cli_opts(opts) -def collect_all(collectors, store=False, requests_impl_map=None): +def collect_all(collectors, store=False, collector_kwargs_map=None): changed_keys = set() all_keys = list() if store: @@ -112,14 +118,13 @@ def collect_all(collectors, store=False, requests_impl_map=None): for collector in collectors: module = COLLECTORS[collector] - if requests_impl_map and collector in requests_impl_map: - requests_impl = requests_impl_map[collector] + if collector_kwargs_map and collector in collector_kwargs_map: + collector_kwargs = collector_kwargs_map[collector] else: - requests_impl = common.requests + collector_kwargs = {} try: - content = module.Collector( - requests_impl=requests_impl).collect() + content = module.Collector(**collector_kwargs).collect() except exc.SourceNotAvailable: logger.warn('Source [%s] Unavailable.' % collector) continue @@ -178,7 +183,7 @@ def getfilehash(files): return m.hexdigest() -def __main__(args=sys.argv, requests_impl_map=None): +def __main__(args=sys.argv, collector_kwargs_map=None): signal.signal(signal.SIGHUP, reexec_self) setup_conf() CONF(args=args[1:], prog="os-collect-config", @@ -210,7 +215,7 @@ def __main__(args=sys.argv, requests_impl_map=None): (changed_keys, content) = collect_all( cfg.CONF.collectors, store=store_and_run, - requests_impl_map=requests_impl_map) + collector_kwargs_map=collector_kwargs_map) if store_and_run: if changed_keys or CONF.force: # ignore HUP now since we will reexec after commit anyway diff --git a/os_collect_config/exc.py b/os_collect_config/exc.py index 481351c..88a6485 100644 --- a/os_collect_config/exc.py +++ b/os_collect_config/exc.py @@ -26,10 +26,18 @@ class CfnMetadataNotAvailable(SourceNotAvailable): """The cfn metadata service is not available.""" +class HeatMetadataNotAvailable(SourceNotAvailable): + """The heat metadata service is not available.""" + + class CfnMetadataNotConfigured(SourceNotAvailable): """The cfn metadata service is not fully configured.""" +class HeatMetadataNotConfigured(SourceNotAvailable): + """The heat metadata service is not fully configured.""" + + class HeatLocalMetadataNotAvailable(SourceNotAvailable): """The local Heat metadata is not available.""" diff --git a/os_collect_config/heat.py b/os_collect_config/heat.py new file mode 100644 index 0000000..8c2f926 --- /dev/null +++ b/os_collect_config/heat.py @@ -0,0 +1,86 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from heatclient import client as heatclient +from keystoneclient.v3 import client as keystoneclient +from oslo.config import cfg + +from openstack.common import log +from os_collect_config import exc + +CONF = cfg.CONF +logger = log.getLogger(__name__) + +opts = [ + cfg.StrOpt('user-id', + help='User ID for API authentication'), + cfg.StrOpt('password', + help='Password for API authentication'), + cfg.StrOpt('project-id', + help='ID of project for API authentication'), + cfg.StrOpt('auth-url', + help='URL for API authentication'), + cfg.StrOpt('stack-id', + help='ID of the stack this deployment belongs to'), + cfg.StrOpt('resource-name', + help='Name of resource in the stack to be polled'), +] +name = 'heat' + + +class Collector(object): + def __init__(self, + keystoneclient=keystoneclient, + heatclient=heatclient): + self.keystoneclient = keystoneclient + self.heatclient = heatclient + + def collect(self): + if CONF.heat.auth_url is None: + logger.warn('No auth_url configured.') + raise exc.HeatMetadataNotConfigured + if CONF.heat.password is None: + logger.warn('No password configured.') + raise exc.HeatMetadataNotConfigured + if CONF.heat.project_id is None: + logger.warn('No project_id configured.') + raise exc.HeatMetadataNotConfigured + if CONF.heat.user_id is None: + logger.warn('No user_id configured.') + raise exc.HeatMetadataNotConfigured + if CONF.heat.stack_id is None: + logger.warn('No stack_id configured.') + raise exc.HeatMetadataNotConfigured + if CONF.heat.resource_name is None: + logger.warn('No resource_name configured.') + raise exc.HeatMetadataNotConfigured + + try: + ks = self.keystoneclient.Client( + auth_url=CONF.heat.auth_url, + user_id=CONF.heat.user_id, + password=CONF.heat.password, + project_id=CONF.heat.project_id) + endpoint = ks.service_catalog.url_for( + service_type='orchestration', endpoint_type='publicURL') + logger.debug('Fetching metadata from %s' % endpoint) + heat = self.heatclient.Client( + '1', endpoint, token=ks.auth_token) + r = heat.resources.metadata(CONF.heat.stack_id, + CONF.heat.resource_name) + + return [('heat', r)] + except Exception as e: + logger.warn(str(e)) + raise exc.HeatMetadataNotAvailable diff --git a/os_collect_config/tests/test_collect.py b/os_collect_config/tests/test_collect.py index 1758557..ff9351a 100644 --- a/os_collect_config/tests/test_collect.py +++ b/os_collect_config/tests/test_collect.py @@ -31,6 +31,7 @@ from os_collect_config import collect from os_collect_config import exc from os_collect_config.tests import test_cfn from os_collect_config.tests import test_ec2 +from os_collect_config.tests import test_heat from os_collect_config.tests import test_heat_local @@ -53,9 +54,16 @@ class TestCollect(testtools.TestCase): # make sure we don't run forever! if '--one-time' not in fake_args: fake_args.append('--one-time') - requests_impl_map = {'ec2': test_ec2.FakeRequests, - 'cfn': test_cfn.FakeRequests(self)} - collect.__main__(args=fake_args, requests_impl_map=requests_impl_map) + collector_kwargs_map = { + 'ec2': {'requests_impl': test_ec2.FakeRequests}, + 'cfn': {'requests_impl': test_cfn.FakeRequests(self)}, + 'heat': { + 'keystoneclient': test_heat.FakeKeystoneClient(self), + 'heatclient': test_heat.FakeHeatClient(self) + } + } + collect.__main__(args=fake_args, + collector_kwargs_map=collector_kwargs_map) def _fake_popen_call_main(self, occ_args): calls = [] @@ -94,6 +102,18 @@ class TestCollect(testtools.TestCase): 'FEDCBA9876543210', '--heat_local-path', fake_metadata, + '--heat-user-id', + 'FEDCBA9876543210', + '--heat-password', + '0123456789ABCDEF', + '--heat-project-id', + '9f6b09df-4d7f-4a33-8ec3-9924d8f46f10', + '--heat-auth-url', + 'http://127.0.0.1:5000/v3', + '--heat-stack-id', + 'a/c482680f-7238-403d-8f76-36acf0c8e0aa', + '--heat-resource-name', + 'server' ] calls = self._fake_popen_call_main(occ_args) proc_args = calls[0] @@ -297,25 +317,37 @@ class TestCollectAll(testtools.TestCase): cfg.CONF.cfn.access_key_id = '0123456789ABCDEF' cfg.CONF.cfn.secret_access_key = 'FEDCBA9876543210' cfg.CONF.heat_local.path = [_setup_local_metadata(self)] + cfg.CONF.heat.auth_url = 'http://127.0.0.1:5000/v3' + cfg.CONF.heat.user_id = '0123456789ABCDEF' + cfg.CONF.heat.password = 'FEDCBA9876543210' + cfg.CONF.heat.project_id = '9f6b09df-4d7f-4a33-8ec3-9924d8f46f10' + cfg.CONF.heat.stack_id = 'a/c482680f-7238-403d-8f76-36acf0c8e0aa' + cfg.CONF.heat.resource_name = 'server' def _call_collect_all( - self, store, requests_impl_map=None, collectors=None): - if requests_impl_map is None: - requests_impl_map = {'ec2': test_ec2.FakeRequests, - 'cfn': test_cfn.FakeRequests(self)} + self, store, collector_kwargs_map=None, collectors=None): + if collector_kwargs_map is None: + collector_kwargs_map = { + 'ec2': {'requests_impl': test_ec2.FakeRequests}, + 'cfn': {'requests_impl': test_cfn.FakeRequests(self)}, + 'heat': { + 'keystoneclient': test_heat.FakeKeystoneClient(self), + 'heatclient': test_heat.FakeHeatClient(self) + } + } if collectors is None: collectors = cfg.CONF.collectors return collect.collect_all( collectors, store=store, - requests_impl_map=requests_impl_map) + collector_kwargs_map=collector_kwargs_map) - def _test_collect_all_store(self, requests_impl_map=None, + def _test_collect_all_store(self, collector_kwargs_map=None, expected_changed=None): (changed_keys, paths) = self._call_collect_all( - store=True, requests_impl_map=requests_impl_map) + store=True, collector_kwargs_map=collector_kwargs_map) if expected_changed is None: - expected_changed = set(['heat_local', 'cfn', 'ec2']) + expected_changed = set(['heat_local', 'cfn', 'ec2', 'heat']) self.assertEqual(expected_changed, changed_keys) self.assertThat(paths, matchers.IsInstance(list)) for path in paths: @@ -326,10 +358,18 @@ class TestCollectAll(testtools.TestCase): self._test_collect_all_store() def test_collect_all_store_softwareconfig(self): - soft_config_map = {'ec2': test_ec2.FakeRequests, - 'cfn': test_cfn.FakeRequestsSoftwareConfig(self)} - expected_changed = set(('heat_local', 'ec2', 'cfn', 'dep-name1')) - self._test_collect_all_store(requests_impl_map=soft_config_map, + soft_config_map = { + 'ec2': {'requests_impl': test_ec2.FakeRequests}, + 'cfn': { + 'requests_impl': test_cfn.FakeRequestsSoftwareConfig(self)}, + 'heat': { + 'keystoneclient': test_heat.FakeKeystoneClient(self), + 'heatclient': test_heat.FakeHeatClient(self) + } + } + expected_changed = set(( + 'heat_local', 'ec2', 'cfn', 'heat', 'dep-name1')) + self._test_collect_all_store(collector_kwargs_map=soft_config_map, expected_changed=expected_changed) def test_collect_all_store_alt_order(self): @@ -355,10 +395,17 @@ class TestCollectAll(testtools.TestCase): self.assertEqual(paths, paths2) def test_collect_all_no_change_softwareconfig(self): - soft_config_map = {'ec2': test_ec2.FakeRequests, - 'cfn': test_cfn.FakeRequestsSoftwareConfig(self)} + soft_config_map = { + 'ec2': {'requests_impl': test_ec2.FakeRequests}, + 'cfn': { + 'requests_impl': test_cfn.FakeRequestsSoftwareConfig(self)}, + 'heat': { + 'keystoneclient': test_heat.FakeKeystoneClient(self), + 'heatclient': test_heat.FakeHeatClient(self) + } + } (changed_keys, paths) = self._call_collect_all( - store=True, requests_impl_map=soft_config_map) + store=True, collector_kwargs_map=soft_config_map) expected_changed = set(cfg.CONF.collectors) expected_changed.add('dep-name1') self.assertEqual(expected_changed, changed_keys) @@ -366,7 +413,7 @@ class TestCollectAll(testtools.TestCase): for changed in changed_keys: cache.commit(changed) (changed_keys, paths2) = self._call_collect_all( - store=True, requests_impl_map=soft_config_map) + store=True, collector_kwargs_map=soft_config_map) self.assertEqual(set(), changed_keys) self.assertEqual(paths, paths2) @@ -379,10 +426,12 @@ class TestCollectAll(testtools.TestCase): self.assertThat(content[collector], matchers.IsInstance(dict)) def test_collect_all_ec2_unavailable(self): - requests_impl_map = {'ec2': test_ec2.FakeFailRequests, - 'cfn': test_cfn.FakeRequests(self)} + collector_kwargs_map = { + 'ec2': {'requests_impl': test_ec2.FakeFailRequests}, + 'cfn': {'requests_impl': test_cfn.FakeRequests(self)} + } (changed_keys, content) = self._call_collect_all( - store=False, requests_impl_map=requests_impl_map) + store=False, collector_kwargs_map=collector_kwargs_map) self.assertEqual(set(), changed_keys) self.assertThat(content, matchers.IsInstance(dict)) self.assertNotIn('ec2', content) diff --git a/os_collect_config/tests/test_heat.py b/os_collect_config/tests/test_heat.py new file mode 100644 index 0000000..5d627a3 --- /dev/null +++ b/os_collect_config/tests/test_heat.py @@ -0,0 +1,177 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fixtures +from keystoneclient import exceptions as ks_exc +from oslo.config import cfg +import testtools +from testtools import matchers + +from os_collect_config import collect +from os_collect_config import exc +from os_collect_config import heat + + +META_DATA = {u'int1': 1, + u'strfoo': u'foo', + u'map_ab': { + u'a': 'apple', + u'b': 'banana', + }} + + +SOFTWARE_CONFIG_DATA = { + u'old-style': u'value', + u'deployments': [ + { + u'inputs': [ + { + u'type': u'String', + u'name': u'input1', + u'value': u'value1' + } + ], + u'group': 'Heat::Ungrouped', + u'name': 'dep-name1', + u'outputs': None, + u'options': None, + u'config': { + u'config1': 'value1' + } + } + ] +} + + +SOFTWARE_CONFIG_IMPOSTER_DATA = { + u'old-style': u'value', + u'deployments': { + u"not": u"a list" + } +} + + +class FakeKeystoneClient(object): + + def __init__(self, testcase): + self._test = testcase + self.service_catalog = self + self.auth_token = 'atoken' + + def Client(self, auth_url, user_id, password, project_id): + self._test.assertEqual(cfg.CONF.heat.auth_url, auth_url) + self._test.assertEqual(cfg.CONF.heat.user_id, user_id) + self._test.assertEqual(cfg.CONF.heat.password, password) + self._test.assertEqual(cfg.CONF.heat.project_id, project_id) + return self + + def url_for(self, service_type, endpoint_type): + self._test.assertEqual('orchestration', service_type) + self._test.assertEqual('publicURL', endpoint_type) + return 'http://127.0.0.1:8004/v1' + + +class FakeFailKeystoneClient(FakeKeystoneClient): + + def Client(self, auth_url, user_id, password, project_id): + raise ks_exc.AuthorizationFailure('Forbidden') + + +class FakeHeatClient(object): + def __init__(self, testcase): + self._test = testcase + self.resources = self + + def Client(self, version, endpoint, token): + self._test.assertEqual('1', version) + self._test.assertEqual('http://127.0.0.1:8004/v1', endpoint) + self._test.assertEqual('atoken', token) + return self + + def metadata(self, stack_id, resource_name): + self._test.assertEqual(cfg.CONF.heat.stack_id, stack_id) + self._test.assertEqual(cfg.CONF.heat.resource_name, resource_name) + return META_DATA + + +class TestHeatBase(testtools.TestCase): + def setUp(self): + super(TestHeatBase, self).setUp() + self.log = self.useFixture(fixtures.FakeLogger()) + self.useFixture(fixtures.NestedTempfile()) + collect.setup_conf() + cfg.CONF.heat.auth_url = 'http://127.0.0.1:5000/v3' + cfg.CONF.heat.user_id = '0123456789ABCDEF' + cfg.CONF.heat.password = 'FEDCBA9876543210' + cfg.CONF.heat.project_id = '9f6b09df-4d7f-4a33-8ec3-9924d8f46f10' + cfg.CONF.heat.stack_id = 'a/c482680f-7238-403d-8f76-36acf0c8e0aa' + cfg.CONF.heat.resource_name = 'server' + + +class TestHeat(TestHeatBase): + def test_collect_heat(self): + heat_md = heat.Collector(keystoneclient=FakeKeystoneClient(self), + heatclient=FakeHeatClient(self)).collect() + self.assertThat(heat_md, matchers.IsInstance(list)) + self.assertEqual('heat', heat_md[0][0]) + heat_md = heat_md[0][1] + + for k in ('int1', 'strfoo', 'map_ab'): + self.assertIn(k, heat_md) + self.assertEqual(heat_md[k], META_DATA[k]) + + self.assertEqual('', self.log.output) + + def test_collect_heat_fail(self): + heat_collect = heat.Collector( + keystoneclient=FakeFailKeystoneClient(self), + heatclient=FakeHeatClient(self)) + self.assertRaises(exc.HeatMetadataNotAvailable, heat_collect.collect) + self.assertIn('Forbidden', self.log.output) + + def test_collect_heat_no_auth_url(self): + cfg.CONF.heat.auth_url = None + heat_collect = heat.Collector() + self.assertRaises(exc.HeatMetadataNotConfigured, heat_collect.collect) + self.assertIn('No auth_url configured', self.log.output) + + def test_collect_heat_no_password(self): + cfg.CONF.heat.password = None + heat_collect = heat.Collector() + self.assertRaises(exc.HeatMetadataNotConfigured, heat_collect.collect) + self.assertIn('No password configured', self.log.output) + + def test_collect_heat_no_project_id(self): + cfg.CONF.heat.project_id = None + heat_collect = heat.Collector() + self.assertRaises(exc.HeatMetadataNotConfigured, heat_collect.collect) + self.assertIn('No project_id configured', self.log.output) + + def test_collect_heat_no_user_id(self): + cfg.CONF.heat.user_id = None + heat_collect = heat.Collector() + self.assertRaises(exc.HeatMetadataNotConfigured, heat_collect.collect) + self.assertIn('No user_id configured', self.log.output) + + def test_collect_heat_no_stack_id(self): + cfg.CONF.heat.stack_id = None + heat_collect = heat.Collector() + self.assertRaises(exc.HeatMetadataNotConfigured, heat_collect.collect) + self.assertIn('No stack_id configured', self.log.output) + + def test_collect_heat_no_resource_name(self): + cfg.CONF.heat.resource_name = None + heat_collect = heat.Collector() + self.assertRaises(exc.HeatMetadataNotConfigured, heat_collect.collect) + self.assertIn('No resource_name configured', self.log.output) diff --git a/requirements.txt b/requirements.txt index 1f767b8..81003e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ anyjson>=0.3.3 argparse eventlet>=0.13.0 python-keystoneclient>=0.7.0 +python-heatclient>=0.2.3 requests>=1.1 iso8601>=0.1.9 lxml>=2.3