openstack
/
os-collect-config
Code Issues Proposed changes

Merge "Add collector for heat API resource metadata"

This commit is contained in:
Jenkins 2014-04-29 20:43:34 +00:00 committed by Gerrit Code Review
parent 6a549c2bcd 80c98a33ee
commit 08a3237484
6 changed files with 358 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
os_collect_config/collect.py
View File

@ -25,14 +25,14 @@ import time
from openstack.common import log
from os_collect_config import cache
from os_collect_config import cfn
from os_collect_config import common
from os_collect_config import ec2
from os_collect_config import exc
from os_collect_config import heat
from os_collect_config import heat_local
from os_collect_config import version
from oslo.config import cfg
DEFAULT_COLLECTORS = ['heat_local', 'ec2', 'cfn']
DEFAULT_COLLECTORS = ['heat_local', 'ec2', 'cfn', 'heat']
opts = [
cfg.StrOpt('command', short='c',
help='Command to run on metadata changes. If specified,'
@ -79,6 +79,7 @@ logger = log.getLogger('os-collect-config')
COLLECTORS = {ec2.name: ec2,
cfn.name: cfn,
heat.name: heat,
heat_local.name: heat_local}
@ -92,17 +93,22 @@ def setup_conf():
heat_local_group = cfg.OptGroup(name='heat_local',
title='Heat Local Metadata options')
heat_group = cfg.OptGroup(name='heat',
title='Heat Metadata options')
CONF.register_group(ec2_group)
CONF.register_group(cfn_group)
CONF.register_group(heat_local_group)
CONF.register_group(heat_group)
CONF.register_cli_opts(ec2.opts, group='ec2')
CONF.register_cli_opts(cfn.opts, group='cfn')
CONF.register_cli_opts(heat_local.opts, group='heat_local')
CONF.register_cli_opts(heat.opts, group='heat')
CONF.register_cli_opts(opts)
def collect_all(collectors, store=False, requests_impl_map=None):
def collect_all(collectors, store=False, collector_kwargs_map=None):
changed_keys = set()
all_keys = list()
if store:
@ -112,14 +118,13 @@ def collect_all(collectors, store=False, requests_impl_map=None):
for collector in collectors:
module = COLLECTORS[collector]
if requests_impl_map and collector in requests_impl_map:
requests_impl = requests_impl_map[collector]
if collector_kwargs_map and collector in collector_kwargs_map:
collector_kwargs = collector_kwargs_map[collector]
else:
requests_impl = common.requests
collector_kwargs = {}
try:
content = module.Collector(
requests_impl=requests_impl).collect()
content = module.Collector(**collector_kwargs).collect()
except exc.SourceNotAvailable:
logger.warn('Source [%s] Unavailable.' % collector)
continue
@ -178,7 +183,7 @@ def getfilehash(files):
return m.hexdigest()
def __main__(args=sys.argv, requests_impl_map=None):
def __main__(args=sys.argv, collector_kwargs_map=None):
signal.signal(signal.SIGHUP, reexec_self)
setup_conf()
CONF(args=args[1:], prog="os-collect-config",
@ -210,7 +215,7 @@ def __main__(args=sys.argv, requests_impl_map=None):
(changed_keys, content) = collect_all(
cfg.CONF.collectors,
store=store_and_run,
requests_impl_map=requests_impl_map)
collector_kwargs_map=collector_kwargs_map)
if store_and_run:
if changed_keys or CONF.force:
# ignore HUP now since we will reexec after commit anyway

8
os_collect_config/exc.py
View File

@ -26,10 +26,18 @@ class CfnMetadataNotAvailable(SourceNotAvailable):
"""The cfn metadata service is not available."""
class HeatMetadataNotAvailable(SourceNotAvailable):
"""The heat metadata service is not available."""
class CfnMetadataNotConfigured(SourceNotAvailable):
"""The cfn metadata service is not fully configured."""
class HeatMetadataNotConfigured(SourceNotAvailable):
"""The heat metadata service is not fully configured."""
class HeatLocalMetadataNotAvailable(SourceNotAvailable):
"""The local Heat metadata is not available."""

86
os_collect_config/heat.py Normal file
View File

@ -0,0 +1,86 @@
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from heatclient import client as heatclient
from keystoneclient.v3 import client as keystoneclient
from oslo.config import cfg
from openstack.common import log
from os_collect_config import exc
CONF = cfg.CONF
logger = log.getLogger(__name__)
opts = [
cfg.StrOpt('user-id',
help='User ID for API authentication'),
cfg.StrOpt('password',
help='Password for API authentication'),
cfg.StrOpt('project-id',
help='ID of project for API authentication'),
cfg.StrOpt('auth-url',
help='URL for API authentication'),
cfg.StrOpt('stack-id',
help='ID of the stack this deployment belongs to'),
cfg.StrOpt('resource-name',
help='Name of resource in the stack to be polled'),
]
name = 'heat'
class Collector(object):
def __init__(self,
keystoneclient=keystoneclient,
heatclient=heatclient):
self.keystoneclient = keystoneclient
self.heatclient = heatclient
def collect(self):
if CONF.heat.auth_url is None:
logger.warn('No auth_url configured.')
raise exc.HeatMetadataNotConfigured
if CONF.heat.password is None:
logger.warn('No password configured.')
raise exc.HeatMetadataNotConfigured
if CONF.heat.project_id is None:
logger.warn('No project_id configured.')
raise exc.HeatMetadataNotConfigured
if CONF.heat.user_id is None:
logger.warn('No user_id configured.')
raise exc.HeatMetadataNotConfigured
if CONF.heat.stack_id is None:
logger.warn('No stack_id configured.')
raise exc.HeatMetadataNotConfigured
if CONF.heat.resource_name is None:
logger.warn('No resource_name configured.')
raise exc.HeatMetadataNotConfigured
try:
ks = self.keystoneclient.Client(
auth_url=CONF.heat.auth_url,
user_id=CONF.heat.user_id,
password=CONF.heat.password,
project_id=CONF.heat.project_id)
endpoint = ks.service_catalog.url_for(
service_type='orchestration', endpoint_type='publicURL')
logger.debug('Fetching metadata from %s' % endpoint)
heat = self.heatclient.Client(
'1', endpoint, token=ks.auth_token)
r = heat.resources.metadata(CONF.heat.stack_id,
CONF.heat.resource_name)
return [('heat', r)]
except Exception as e:
logger.warn(str(e))
raise exc.HeatMetadataNotAvailable

93
os_collect_config/tests/test_collect.py
View File

@ -31,6 +31,7 @@ from os_collect_config import collect
from os_collect_config import exc
from os_collect_config.tests import test_cfn
from os_collect_config.tests import test_ec2
from os_collect_config.tests import test_heat
from os_collect_config.tests import test_heat_local
@ -53,9 +54,16 @@ class TestCollect(testtools.TestCase):
# make sure we don't run forever!
if '--one-time' not in fake_args:
fake_args.append('--one-time')
requests_impl_map = {'ec2': test_ec2.FakeRequests,
'cfn': test_cfn.FakeRequests(self)}
collect.__main__(args=fake_args, requests_impl_map=requests_impl_map)
collector_kwargs_map = {
'ec2': {'requests_impl': test_ec2.FakeRequests},
'cfn': {'requests_impl': test_cfn.FakeRequests(self)},
'heat': {
'keystoneclient': test_heat.FakeKeystoneClient(self),
'heatclient': test_heat.FakeHeatClient(self)
}
}
collect.__main__(args=fake_args,
collector_kwargs_map=collector_kwargs_map)
def _fake_popen_call_main(self, occ_args):
calls = []
@ -94,6 +102,18 @@ class TestCollect(testtools.TestCase):
'FEDCBA9876543210',
'--heat_local-path',
fake_metadata,
'--heat-user-id',
'FEDCBA9876543210',
'--heat-password',
'0123456789ABCDEF',
'--heat-project-id',
'9f6b09df-4d7f-4a33-8ec3-9924d8f46f10',
'--heat-auth-url',
'http://127.0.0.1:5000/v3',
'--heat-stack-id',
'a/c482680f-7238-403d-8f76-36acf0c8e0aa',
'--heat-resource-name',
'server'
]
calls = self._fake_popen_call_main(occ_args)
proc_args = calls[0]
@ -297,25 +317,37 @@ class TestCollectAll(testtools.TestCase):
cfg.CONF.cfn.access_key_id = '0123456789ABCDEF'
cfg.CONF.cfn.secret_access_key = 'FEDCBA9876543210'
cfg.CONF.heat_local.path = [_setup_local_metadata(self)]
cfg.CONF.heat.auth_url = 'http://127.0.0.1:5000/v3'
cfg.CONF.heat.user_id = '0123456789ABCDEF'
cfg.CONF.heat.password = 'FEDCBA9876543210'
cfg.CONF.heat.project_id = '9f6b09df-4d7f-4a33-8ec3-9924d8f46f10'
cfg.CONF.heat.stack_id = 'a/c482680f-7238-403d-8f76-36acf0c8e0aa'
cfg.CONF.heat.resource_name = 'server'
def _call_collect_all(
self, store, requests_impl_map=None, collectors=None):
if requests_impl_map is None:
requests_impl_map = {'ec2': test_ec2.FakeRequests,
'cfn': test_cfn.FakeRequests(self)}
self, store, collector_kwargs_map=None, collectors=None):
if collector_kwargs_map is None:
collector_kwargs_map = {
'ec2': {'requests_impl': test_ec2.FakeRequests},
'cfn': {'requests_impl': test_cfn.FakeRequests(self)},
'heat': {
'keystoneclient': test_heat.FakeKeystoneClient(self),
'heatclient': test_heat.FakeHeatClient(self)
}
}
if collectors is None:
collectors = cfg.CONF.collectors
return collect.collect_all(
collectors,
store=store,
requests_impl_map=requests_impl_map)
collector_kwargs_map=collector_kwargs_map)
def _test_collect_all_store(self, requests_impl_map=None,
def _test_collect_all_store(self, collector_kwargs_map=None,
expected_changed=None):
(changed_keys, paths) = self._call_collect_all(
store=True, requests_impl_map=requests_impl_map)
store=True, collector_kwargs_map=collector_kwargs_map)
if expected_changed is None:
expected_changed = set(['heat_local', 'cfn', 'ec2'])
expected_changed = set(['heat_local', 'cfn', 'ec2', 'heat'])
self.assertEqual(expected_changed, changed_keys)
self.assertThat(paths, matchers.IsInstance(list))
for path in paths:
@ -326,10 +358,18 @@ class TestCollectAll(testtools.TestCase):
self._test_collect_all_store()
def test_collect_all_store_softwareconfig(self):
soft_config_map = {'ec2': test_ec2.FakeRequests,
'cfn': test_cfn.FakeRequestsSoftwareConfig(self)}
expected_changed = set(('heat_local', 'ec2', 'cfn', 'dep-name1'))
self._test_collect_all_store(requests_impl_map=soft_config_map,
soft_config_map = {
'ec2': {'requests_impl': test_ec2.FakeRequests},
'cfn': {
'requests_impl': test_cfn.FakeRequestsSoftwareConfig(self)},
'heat': {
'keystoneclient': test_heat.FakeKeystoneClient(self),
'heatclient': test_heat.FakeHeatClient(self)
}
}
expected_changed = set((
'heat_local', 'ec2', 'cfn', 'heat', 'dep-name1'))
self._test_collect_all_store(collector_kwargs_map=soft_config_map,
expected_changed=expected_changed)
def test_collect_all_store_alt_order(self):
@ -355,10 +395,17 @@ class TestCollectAll(testtools.TestCase):
self.assertEqual(paths, paths2)
def test_collect_all_no_change_softwareconfig(self):
soft_config_map = {'ec2': test_ec2.FakeRequests,
'cfn': test_cfn.FakeRequestsSoftwareConfig(self)}
soft_config_map = {
'ec2': {'requests_impl': test_ec2.FakeRequests},
'cfn': {
'requests_impl': test_cfn.FakeRequestsSoftwareConfig(self)},
'heat': {
'keystoneclient': test_heat.FakeKeystoneClient(self),
'heatclient': test_heat.FakeHeatClient(self)
}
}
(changed_keys, paths) = self._call_collect_all(
store=True, requests_impl_map=soft_config_map)
store=True, collector_kwargs_map=soft_config_map)
expected_changed = set(cfg.CONF.collectors)
expected_changed.add('dep-name1')
self.assertEqual(expected_changed, changed_keys)
@ -366,7 +413,7 @@ class TestCollectAll(testtools.TestCase):
for changed in changed_keys:
cache.commit(changed)
(changed_keys, paths2) = self._call_collect_all(
store=True, requests_impl_map=soft_config_map)
store=True, collector_kwargs_map=soft_config_map)
self.assertEqual(set(), changed_keys)
self.assertEqual(paths, paths2)
@ -379,10 +426,12 @@ class TestCollectAll(testtools.TestCase):
self.assertThat(content[collector], matchers.IsInstance(dict))
def test_collect_all_ec2_unavailable(self):
requests_impl_map = {'ec2': test_ec2.FakeFailRequests,
'cfn': test_cfn.FakeRequests(self)}
collector_kwargs_map = {
'ec2': {'requests_impl': test_ec2.FakeFailRequests},
'cfn': {'requests_impl': test_cfn.FakeRequests(self)}
}
(changed_keys, content) = self._call_collect_all(
store=False, requests_impl_map=requests_impl_map)
store=False, collector_kwargs_map=collector_kwargs_map)
self.assertEqual(set(), changed_keys)
self.assertThat(content, matchers.IsInstance(dict))
self.assertNotIn('ec2', content)

177
os_collect_config/tests/test_heat.py Normal file
View File

@ -0,0 +1,177 @@
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import fixtures
from keystoneclient import exceptions as ks_exc
from oslo.config import cfg
import testtools
from testtools import matchers
from os_collect_config import collect
from os_collect_config import exc
from os_collect_config import heat
META_DATA = {u'int1': 1,
u'strfoo': u'foo',
u'map_ab': {
u'a': 'apple',
u'b': 'banana',
}}
SOFTWARE_CONFIG_DATA = {
u'old-style': u'value',
u'deployments': [
{
u'inputs': [
{
u'type': u'String',
u'name': u'input1',
u'value': u'value1'
}
],
u'group': 'Heat::Ungrouped',
u'name': 'dep-name1',
u'outputs': None,
u'options': None,
u'config': {
u'config1': 'value1'
}
}
]
}
SOFTWARE_CONFIG_IMPOSTER_DATA = {
u'old-style': u'value',
u'deployments': {
u"not": u"a list"
}
}
class FakeKeystoneClient(object):
def __init__(self, testcase):
self._test = testcase
self.service_catalog = self
self.auth_token = 'atoken'
def Client(self, auth_url, user_id, password, project_id):
self._test.assertEqual(cfg.CONF.heat.auth_url, auth_url)
self._test.assertEqual(cfg.CONF.heat.user_id, user_id)
self._test.assertEqual(cfg.CONF.heat.password, password)
self._test.assertEqual(cfg.CONF.heat.project_id, project_id)
return self
def url_for(self, service_type, endpoint_type):
self._test.assertEqual('orchestration', service_type)
self._test.assertEqual('publicURL', endpoint_type)
return 'http://127.0.0.1:8004/v1'
class FakeFailKeystoneClient(FakeKeystoneClient):
def Client(self, auth_url, user_id, password, project_id):
raise ks_exc.AuthorizationFailure('Forbidden')
class FakeHeatClient(object):
def __init__(self, testcase):
self._test = testcase
self.resources = self
def Client(self, version, endpoint, token):
self._test.assertEqual('1', version)
self._test.assertEqual('http://127.0.0.1:8004/v1', endpoint)
self._test.assertEqual('atoken', token)
return self
def metadata(self, stack_id, resource_name):
self._test.assertEqual(cfg.CONF.heat.stack_id, stack_id)
self._test.assertEqual(cfg.CONF.heat.resource_name, resource_name)
return META_DATA
class TestHeatBase(testtools.TestCase):
def setUp(self):
super(TestHeatBase, self).setUp()
self.log = self.useFixture(fixtures.FakeLogger())
self.useFixture(fixtures.NestedTempfile())
collect.setup_conf()
cfg.CONF.heat.auth_url = 'http://127.0.0.1:5000/v3'
cfg.CONF.heat.user_id = '0123456789ABCDEF'
cfg.CONF.heat.password = 'FEDCBA9876543210'
cfg.CONF.heat.project_id = '9f6b09df-4d7f-4a33-8ec3-9924d8f46f10'
cfg.CONF.heat.stack_id = 'a/c482680f-7238-403d-8f76-36acf0c8e0aa'
cfg.CONF.heat.resource_name = 'server'
class TestHeat(TestHeatBase):
def test_collect_heat(self):
heat_md = heat.Collector(keystoneclient=FakeKeystoneClient(self),
heatclient=FakeHeatClient(self)).collect()
self.assertThat(heat_md, matchers.IsInstance(list))
self.assertEqual('heat', heat_md[0][0])
heat_md = heat_md[0][1]
for k in ('int1', 'strfoo', 'map_ab'):
self.assertIn(k, heat_md)
self.assertEqual(heat_md[k], META_DATA[k])
self.assertEqual('', self.log.output)
def test_collect_heat_fail(self):
heat_collect = heat.Collector(
keystoneclient=FakeFailKeystoneClient(self),
heatclient=FakeHeatClient(self))
self.assertRaises(exc.HeatMetadataNotAvailable, heat_collect.collect)
self.assertIn('Forbidden', self.log.output)
def test_collect_heat_no_auth_url(self):
cfg.CONF.heat.auth_url = None
heat_collect = heat.Collector()
self.assertRaises(exc.HeatMetadataNotConfigured, heat_collect.collect)
self.assertIn('No auth_url configured', self.log.output)
def test_collect_heat_no_password(self):
cfg.CONF.heat.password = None
heat_collect = heat.Collector()
self.assertRaises(exc.HeatMetadataNotConfigured, heat_collect.collect)
self.assertIn('No password configured', self.log.output)
def test_collect_heat_no_project_id(self):
cfg.CONF.heat.project_id = None
heat_collect = heat.Collector()
self.assertRaises(exc.HeatMetadataNotConfigured, heat_collect.collect)
self.assertIn('No project_id configured', self.log.output)
def test_collect_heat_no_user_id(self):
cfg.CONF.heat.user_id = None
heat_collect = heat.Collector()
self.assertRaises(exc.HeatMetadataNotConfigured, heat_collect.collect)
self.assertIn('No user_id configured', self.log.output)
def test_collect_heat_no_stack_id(self):
cfg.CONF.heat.stack_id = None
heat_collect = heat.Collector()
self.assertRaises(exc.HeatMetadataNotConfigured, heat_collect.collect)
self.assertIn('No stack_id configured', self.log.output)
def test_collect_heat_no_resource_name(self):
cfg.CONF.heat.resource_name = None
heat_collect = heat.Collector()
self.assertRaises(exc.HeatMetadataNotConfigured, heat_collect.collect)
self.assertIn('No resource_name configured', self.log.output)

1
requirements.txt
View File

@ -4,6 +4,7 @@ anyjson>=0.3.3
argparse
eventlet>=0.13.0
python-keystoneclient>=0.7.0
python-heatclient>=0.2.3
requests>=1.1
iso8601>=0.1.9
lxml>=2.3