From 72522a341ffba5e7ab78a1da258ab67f13674769 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 6 Jul 2017 17:32:19 +0100 Subject: [PATCH] Add support for gnocchi Add new metric-service interface to support use of Gnocchi as a storage backend for resource and metric data. Configure ceilometer to use the gnocchi dispatcher in the event that ceilometer is related to gnocchi. This has the side effect of disabling the ceilometer API - Aodh and Gnocchi API's should be used directly in this deployment topology. Note that Gnocchi is only supported in OpenStack Mitaka or later; 'metrics-service' is added to the required interfaces configuration as an alternative to 'mongodb' for >= Mitaka. Change-Id: Ia31dfefd5efa3fb5ec2ba5d132ee865c567bd8df --- config.yaml | 8 ++- hooks/ceilometer_hooks.py | 43 ++++++++------ hooks/metric-service-relation-broken | 1 + hooks/metric-service-relation-changed | 1 + hooks/metric-service-relation-departed | 1 + hooks/metric-service-relation-joined | 1 + lib/ceilometer_contexts.py | 14 +++++ lib/ceilometer_utils.py | 34 ++++++++++- metadata.yaml | 2 + templates/mitaka/ceilometer.conf | 14 +++++ tests/basic_deployment.py | 81 -------------------------- unit_tests/test_ceilometer_hooks.py | 44 +++++++++++++- unit_tests/test_ceilometer_utils.py | 52 ++++++++++++++++- 13 files changed, 186 insertions(+), 110 deletions(-) create mode 120000 hooks/metric-service-relation-broken create mode 120000 hooks/metric-service-relation-changed create mode 120000 hooks/metric-service-relation-departed create mode 120000 hooks/metric-service-relation-joined diff --git a/config.yaml b/config.yaml index 6c82fa7..3f2c204 100644 --- a/config.yaml +++ b/config.yaml @@ -74,7 +74,7 @@ options: api-workers: type: int default: 1 - description: | + description: | Number of workers for Ceilometer API server. (>= Kilo). # Monitoring config nagios_context: @@ -222,3 +222,9 @@ options: description: | Connect timeout configuration in ms for haproxy, used in HA configurations. If not provided, default value of 5000ms is used. + gnocchi-archive-policy: + type: string + default: low + description: | + Archive retention policy to use when Ceilometer is deployed with + Gnocchi for resource, metric and measures storage. diff --git a/hooks/ceilometer_hooks.py b/hooks/ceilometer_hooks.py index 4619343..2e259bf 100755 --- a/hooks/ceilometer_hooks.py +++ b/hooks/ceilometer_hooks.py @@ -70,6 +70,7 @@ from ceilometer_utils import ( set_shared_secret, assess_status, reload_systemd, + ceilometer_upgrade, ) from ceilometer_contexts import CEILOMETER_PORT from charmhelpers.contrib.openstack.ip import ( @@ -134,32 +135,36 @@ def db_joined(): relation_set(ceilometer_database=CEILOMETER_DB) +@hooks.hook("metric-service-relation-joined") +def metric_service_joined(): + # NOTE(jamespage): gnocchiclient is required to support + # the gnocchi event dispatcher + apt_install(filter_installed_packages(['python-gnocchiclient']), + fatal=True) + + @hooks.hook("amqp-relation-changed", + "amqp-relation-departed", "shared-db-relation-changed", - "shared-db-relation-departed") + "shared-db-relation-departed", + "identity-service-relation-changed", + "identity-service-relation-departed", + "metric-service-relation-changed", + "metric-service-relation-departed") @restart_on_change(restart_map()) def any_changed(): CONFIGS.write_all() configure_https() + for rid in relation_ids('identity-service'): + keystone_joined(relid=rid) ceilometer_joined() - - -@hooks.hook("identity-service-relation-changed") -@restart_on_change(restart_map()) -def identity_service_relation_changed(): - CONFIGS.write_all() - configure_https() - keystone_joined() - ceilometer_joined() - - -@hooks.hook("amqp-relation-departed") -@restart_on_change(restart_map()) -def amqp_departed(): - if 'amqp' not in CONFIGS.complete_contexts(): - log('amqp relation incomplete. Peer not ready?') - return - CONFIGS.write_all() + # NOTE(jamespage): ceilometer@ocata requires both gnocchi + # and mongodb to be configured to successfully + # upgrade the underlying data stores. + if ('metric-service' in CONFIGS.complete_contexts() and + 'identity-service' in CONFIGS.complete_contexts() and + 'mongodb' in CONFIGS.complete_contexts()): + ceilometer_upgrade() def configure_https(): diff --git a/hooks/metric-service-relation-broken b/hooks/metric-service-relation-broken new file mode 120000 index 0000000..c948469 --- /dev/null +++ b/hooks/metric-service-relation-broken @@ -0,0 +1 @@ +ceilometer_hooks.py \ No newline at end of file diff --git a/hooks/metric-service-relation-changed b/hooks/metric-service-relation-changed new file mode 120000 index 0000000..c948469 --- /dev/null +++ b/hooks/metric-service-relation-changed @@ -0,0 +1 @@ +ceilometer_hooks.py \ No newline at end of file diff --git a/hooks/metric-service-relation-departed b/hooks/metric-service-relation-departed new file mode 120000 index 0000000..c948469 --- /dev/null +++ b/hooks/metric-service-relation-departed @@ -0,0 +1 @@ +ceilometer_hooks.py \ No newline at end of file diff --git a/hooks/metric-service-relation-joined b/hooks/metric-service-relation-joined new file mode 120000 index 0000000..c948469 --- /dev/null +++ b/hooks/metric-service-relation-joined @@ -0,0 +1 @@ +ceilometer_hooks.py \ No newline at end of file diff --git a/lib/ceilometer_contexts.py b/lib/ceilometer_contexts.py index d391e54..3068e3d 100644 --- a/lib/ceilometer_contexts.py +++ b/lib/ceilometer_contexts.py @@ -145,3 +145,17 @@ class ApacheSSLContext(SSLContext): external_ports = [CEILOMETER_PORT] service_namespace = "ceilometer" + + +class MetricServiceContext(OSContextGenerator): + interfaces = ['metric-service'] + + def __call__(self): + + for relid in relation_ids('metric-service'): + for unit in related_units(relid): + gnocchi_url = relation_get('gnocchi_url', unit=unit, rid=relid) + if gnocchi_url: + return {'gnocchi_url': gnocchi_url, + 'archive_policy': config('gnocchi-archive-policy')} + return {} diff --git a/lib/ceilometer_utils.py b/lib/ceilometer_utils.py index ae68e1d..b839882 100644 --- a/lib/ceilometer_utils.py +++ b/lib/ceilometer_utils.py @@ -28,6 +28,7 @@ from ceilometer_contexts import ( MongoDBContext, CeilometerContext, HAProxyContext, + MetricServiceContext, CEILOMETER_PORT, ) from charmhelpers.contrib.openstack.utils import ( @@ -43,9 +44,10 @@ from charmhelpers.contrib.openstack.utils import ( enable_memcache, CompareOpenStackReleases, ) -from charmhelpers.core.hookenv import config, log +from charmhelpers.core.hookenv import config, log, is_leader from charmhelpers.fetch import apt_update, apt_install, apt_upgrade from charmhelpers.core.host import init_is_systemd +from charmhelpers.core.decorators import retry_on_exception from copy import deepcopy HAPROXY_CONF = '/etc/haproxy/haproxy.cfg' @@ -119,7 +121,8 @@ CONFIG_FILES = OrderedDict([ CeilometerContext(), context.SyslogContext(), HAProxyContext(), - context.MemcacheContext()], + context.MemcacheContext(), + MetricServiceContext()], 'services': CEILOMETER_BASE_SERVICES }), (CEILOMETER_API_SYSTEMD_CONF, { @@ -363,6 +366,18 @@ def assess_status(configs): os_application_version_set(VERSION_PACKAGE) +def resolve_required_interfaces(): + """Helper function to build a map of required interfaces based on the + OpenStack release being deployed. + + @returns dict - a dictionary keyed by high-level type of interfaces names + """ + required_ints = deepcopy(REQUIRED_INTERFACES) + if CompareOpenStackReleases(os_release('ceilometer-common')) >= 'mitaka': + required_ints['database'].append('metric-service') + return required_ints + + def assess_status_func(configs): """Helper function to create the function that will assess_status() for the unit. @@ -375,7 +390,7 @@ def assess_status_func(configs): @return f() -> None : a function that assesses the unit's workload status """ return make_assess_status_func( - configs, REQUIRED_INTERFACES, + configs, resolve_required_interfaces(), services=services(), ports=determine_ports()) @@ -437,3 +452,16 @@ def disable_package_apache_site(): """ if os.path.exists(PACKAGE_CEILOMETER_API_CONF): subprocess.check_call(['a2dissite', 'ceilometer-api']) + + +@retry_on_exception(5, exc_type=subprocess.CalledProcessError) +def ceilometer_upgrade(): + """Execute ceilometer-upgrade command, with retry on failure if gnocchi + API is not ready for requests""" + if is_leader(): + if (CompareOpenStackReleases(os_release('ceilometer-common')) >= + 'newton'): + cmd = ['ceilometer-upgrade'] + else: + cmd = ['ceilometer-dbsync'] + subprocess.check_call(cmd) diff --git a/metadata.yaml b/metadata.yaml index b428cd1..0e68c0d 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -39,6 +39,8 @@ requires: ha: interface: hacluster scope: container + metric-service: + interface: gnocchi peers: cluster: interface: ceilometer-ha diff --git a/templates/mitaka/ceilometer.conf b/templates/mitaka/ceilometer.conf index 6c7c920..8e6ef1d 100644 --- a/templates/mitaka/ceilometer.conf +++ b/templates/mitaka/ceilometer.conf @@ -10,6 +10,11 @@ verbose = {{ verbose }} use_syslog = {{ use_syslog }} event_pipeline_cfg_file = /etc/ceilometer/event_pipeline_alarm.yaml +{% if gnocchi_url -%} +meter_dispatchers = gnocchi +event_dispatchers = gnocchi +{%- endif %} + [api] port = {{ port }} workers = {{ api_workers }} @@ -30,6 +35,7 @@ user_domain_name = default auth_type = password {% endif -%} +{% if db_host or db_mongo_servers -%} [database] {% if db_replset: -%} connection = mongodb://{{ db_mongo_servers }}/{{ db_name }}?readPreference=primaryPreferred&replicaSet={{ db_replset }} @@ -39,10 +45,18 @@ connection = mongodb://{{ db_host }}:{{ db_port }}/{{ db_name }} {% endif %} metering_time_to_live = {{ metering_time_to_live }} event_time_to_live = {{ event_time_to_live }} +{%- endif %} [publisher] telemetry_secret = {{ metering_secret }} +{% if gnocchi_url -%} +[dispatcher_gnocchi] +filter_service_activity = False +archive_policy = {{ archive_policy }} +url = {{ gnocchi_url }} +{%- endif %} + {% include "section-keystone-authtoken-mitaka" %} {% include "section-rabbitmq-oslo" %} diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 500fa5a..81ab51e 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -438,44 +438,6 @@ class CeilometerBasicDeployment(OpenStackAmuletDeployment): u.log.debug('OK') - def test_209_nova_compute_ceilometer_agent_relation(self): - """Verify the nova-compute to ceilometer relation data""" - u.log.debug('Checking nova-compute:ceilometer relation data...') - unit = self.nova_sentry - relation = ['nova-ceilometer', 'ceilometer-agent:nova-ceilometer'] - expected = {'private-address': u.valid_ip} - - ret = u.validate_relation_data(unit, relation, expected) - if ret: - message = u.relation_error('ceilometer-service', ret) - amulet.raise_status(amulet.FAIL, msg=message) - - u.log.debug('OK') - - def test_210_ceilometer_agent_nova_compute_relation(self): - """Verify the ceilometer to nova-compute relation data""" - u.log.debug('Checking ceilometer:nova-compute relation data...') - unit = self.ceil_agent_sentry - relation = ['nova-ceilometer', 'nova-compute:nova-ceilometer'] - sub = ('{"nova": {"/etc/nova/nova.conf": {"sections": {"DEFAULT": ' - '[["instance_usage_audit", "True"], ' - '["instance_usage_audit_period", "hour"], ' - '["notify_on_state_change", "vm_and_task_state"], ' - '["notification_driver", "ceilometer.compute.nova_notifier"], ' - '["notification_driver", ' - '"nova.openstack.common.notifier.rpc_notifier"]]}}}}') - expected = { - 'subordinate_configuration': sub, - 'private-address': u.valid_ip - } - - ret = u.validate_relation_data(unit, relation, expected) - if ret: - message = u.relation_error('ceilometer-service', ret) - amulet.raise_status(amulet.FAIL, msg=message) - - u.log.debug('OK') - def test_300_ceilometer_config(self): """Verify the data in the ceilometer config file.""" u.log.debug('Checking ceilometer config file data...') @@ -549,49 +511,6 @@ class CeilometerBasicDeployment(OpenStackAmuletDeployment): u.log.debug('OK') - def test_301_nova_config(self): - """Verify data in the nova compute nova config file""" - u.log.debug('Checking nova compute config file...') - unit = self.nova_sentry - conf = '/etc/nova/nova.conf' - expected = { - 'DEFAULT': { - 'verbose': 'False', - 'debug': 'False', - 'use_syslog': 'False', - 'my_ip': u.valid_ip, - } - } - - # NOTE(beisner): notification_driver is not checked like the - # others, as configparser does not support duplicate config - # options, and dicts cant have duplicate keys. - # Ex. from conf file: - # notification_driver = ceilometer.compute.nova_notifier - # notification_driver = nova.openstack.common.notifier.rpc_notifier - for section, pairs in expected.iteritems(): - ret = u.validate_config_data(unit, conf, section, pairs) - if ret: - message = "ceilometer config error: {}".format(ret) - amulet.raise_status(amulet.FAIL, msg=message) - - # Check notification_driver existence via simple grep cmd - lines = [('notification_driver = ' - 'ceilometer.compute.nova_notifier'), - ('notification_driver = ' - 'nova.openstack.common.notifier.rpc_notifier')] - - sentry_units = [unit] - cmds = [] - for line in lines: - cmds.append('grep "{}" {}'.format(line, conf)) - - ret = u.check_commands_on_units(cmds, sentry_units) - if ret: - amulet.raise_status(amulet.FAIL, msg=ret) - - u.log.debug('OK') - def test_400_api_connection(self): """Simple api calls to check service is up and responding""" u.log.debug('Checking api functionality...') diff --git a/unit_tests/test_ceilometer_hooks.py b/unit_tests/test_ceilometer_hooks.py index a2c818d..97891f9 100644 --- a/unit_tests/test_ceilometer_hooks.py +++ b/unit_tests/test_ceilometer_hooks.py @@ -135,12 +135,38 @@ class CeilometerHooksTest(CharmTestCase): self.relation_set.assert_called_with( ceilometer_database='ceilometer') + @patch.object(hooks, 'ceilometer_upgrade') + @patch.object(hooks, 'keystone_joined') @patch('charmhelpers.core.hookenv.config') @patch.object(hooks, 'ceilometer_joined') - def test_any_changed(self, joined, mock_config): + def test_any_changed_with_metrics(self, ceilometer_joined, mock_config, + keystone_joined, ceilometer_upgrade): + self.CONFIGS.complete_contexts.return_value = [ + 'metric-service', + 'identity-service', + 'mongodb' + ] + self.relation_ids.return_value = ['identity-service:1'] + hooks.hooks.execute(['hooks/shared-db-relation-changed']) + self.CONFIGS.write_all.assert_called_once() + ceilometer_joined.assert_called_once() + keystone_joined.assert_called_with(relid='identity-service:1') + ceilometer_upgrade.assert_called_once() + self.configure_https.assert_called_once() + + @patch.object(hooks, 'ceilometer_upgrade') + @patch.object(hooks, 'keystone_joined') + @patch('charmhelpers.core.hookenv.config') + @patch.object(hooks, 'ceilometer_joined') + def test_any_changed(self, ceilometer_joined, mock_config, + keystone_joined, ceilometer_upgrade): + self.relation_ids.return_value = ['identity-service:1'] hooks.hooks.execute(['hooks/shared-db-relation-changed']) self.assertTrue(self.CONFIGS.write_all.called) - self.assertTrue(joined.called) + self.assertTrue(ceilometer_joined.called) + keystone_joined.assert_called_with(relid='identity-service:1') + ceilometer_upgrade.assert_not_called() + self.configure_https.assert_called_once() @patch('charmhelpers.core.hookenv.config') @patch.object(hooks, 'install') @@ -150,14 +176,17 @@ class CeilometerHooksTest(CharmTestCase): self.assertTrue(changed.called) self.assertTrue(install.called) + @patch.object(hooks, 'any_changed') @patch('charmhelpers.core.hookenv.config') @patch.object(hooks, 'cluster_joined') - def test_upgrade_charm_with_cluster(self, cluster_joined, mock_config): + def test_upgrade_charm_with_cluster(self, cluster_joined, mock_config, + any_changed): self.relation_ids.return_value = ['ceilometer/0', 'ceilometer/1', 'ceilometer/2'] hooks.hooks.execute(['hooks/upgrade-charm']) self.assertEquals(cluster_joined.call_count, 3) + any_changed.assert_called_once() @patch.object(hooks, 'install_event_pipeline_setting') @patch('charmhelpers.core.hookenv.config') @@ -494,3 +523,12 @@ class CeilometerHooksTest(CharmTestCase): self.relation_ids.return_value = ['identity-service/0'] hooks.hooks.execute(['hooks/ha-relation-changed']) self.assertEquals(mock_keystone_joined.call_count, 1) + + def test_metric_service_joined(self): + self.filter_installed_packages.return_value = ['python-gnocchiclient'] + hooks.hooks.execute(['hooks/metric-service-relation-joined']) + self.filter_installed_packages.assert_called_with( + ['python-gnocchiclient'] + ) + self.apt_install.assert_called_with(['python-gnocchiclient'], + fatal=True) diff --git a/unit_tests/test_ceilometer_utils.py b/unit_tests/test_ceilometer_utils.py index b78128b..ffba908 100644 --- a/unit_tests/test_ceilometer_utils.py +++ b/unit_tests/test_ceilometer_utils.py @@ -37,6 +37,7 @@ TO_PATCH = [ 'enable_memcache', 'token_cache_pkgs', 'os_release', + 'is_leader', ] @@ -209,7 +210,7 @@ class CeilometerUtilsTest(CharmTestCase): utils.VERSION_PACKAGE ) - @patch.object(utils, 'REQUIRED_INTERFACES') + @patch.object(utils, 'resolve_required_interfaces') @patch.object(utils, 'services') @patch.object(utils, 'determine_ports') @patch.object(utils, 'make_assess_status_func') @@ -217,12 +218,13 @@ class CeilometerUtilsTest(CharmTestCase): make_assess_status_func, determine_ports, services, - REQUIRED_INTERFACES): + resolve_required_interfaces): services.return_value = 's1' determine_ports.return_value = 'p1' + resolve_required_interfaces.return_value = {'a': ['b']} utils.assess_status_func('test-config') make_assess_status_func.assert_called_once_with( - 'test-config', REQUIRED_INTERFACES, services='s1', ports='p1') + 'test-config', {'a': ['b']}, services='s1', ports='p1') def test_pause_unit_helper(self): with patch.object(utils, '_pause_resume_helper') as prh: @@ -243,3 +245,47 @@ class CeilometerUtilsTest(CharmTestCase): utils._pause_resume_helper(f, 'some-config') asf.assert_called_once_with('some-config') f.assert_called_once_with('assessor', services='s1', ports='p1') + + def test_resolve_required_interfaces(self): + self.os_release.side_effect = None + self.os_release.return_value = 'icehouse' + self.assertEqual( + utils.resolve_required_interfaces(), + { + 'database': ['mongodb'], + 'messaging': ['amqp'], + 'identity': ['identity-service'], + } + ) + + def test_resolve_required_interfaces_mitaka(self): + self.os_release.side_effect = None + self.os_release.return_value = 'mitaka' + self.assertEqual( + utils.resolve_required_interfaces(), + { + 'database': ['mongodb', 'metric-service'], + 'messaging': ['amqp'], + 'identity': ['identity-service'], + } + ) + + @patch.object(utils, 'subprocess') + def test_ceilometer_upgrade(self, mock_subprocess): + self.is_leader.return_value = True + self.os_release.return_value = 'ocata' + utils.ceilometer_upgrade() + mock_subprocess.check_call.assert_called_with(['ceilometer-upgrade']) + + @patch.object(utils, 'subprocess') + def test_ceilometer_upgrade_mitaka(self, mock_subprocess): + self.is_leader.return_value = True + self.os_release.return_value = 'mitaka' + utils.ceilometer_upgrade() + mock_subprocess.check_call.assert_called_with(['ceilometer-dbsync']) + + @patch.object(utils, 'subprocess') + def test_ceilometer_upgrade_follower(self, mock_subprocess): + self.is_leader.return_value = False + utils.ceilometer_upgrade() + mock_subprocess.check_call.assert_not_called()