From 668a28986270dfd30546c7c1d52947e15b66049a Mon Sep 17 00:00:00 2001 From: Michael Skalka Date: Fri, 4 May 2018 03:52:18 -0400 Subject: [PATCH] Add support for remote consumers of Ceilometer event data. There is a use-case to forward events received by ceilometer from OpenStack services to an external system via http/udp. It comes from the fact that aodh does not include all information about an event alarm into POSTed data. Raw event data can be published by ceilometer via http/udp to an external system without giving a user direct access to rabbitmq. This patch adds a context generator, associated templates, and configuration entry for configuring multiple event-sinks in /etc/ceilometer/event_pipeline.yaml. This also modifies the ceilometer.conf templates to reflect the correct naming of the same file to align with upstream Ceilometer. Also adds support for a future Panko charm event consumer using the event-service interface in the above context with associated metadata and hook symlinks. This change will only effect Mitaka and later clouds. Closes Bug: 1763321 Change-Id: I931438c720272bd9a3d2b958ebabcd3584790bd0 --- config.yaml | 10 +++++++ hooks/ceilometer_hooks.py | 4 ++- hooks/event-service-relation-broken | 1 + hooks/event-service-relation-changed | 1 + hooks/event-service-relation-departed | 1 + hooks/event-service-relation-joined | 1 + lib/ceilometer_contexts.py | 20 ++++++++++++++ lib/ceilometer_utils.py | 17 ++++++++++++ metadata.yaml | 2 ++ templates/mitaka/ceilometer.conf | 2 +- templates/mitaka/event_pipeline.yaml | 37 ++++++++++++++++++++++++++ templates/ocata/ceilometer.conf | 2 +- tests/basic_deployment.py | 2 +- unit_tests/test_ceilometer_contexts.py | 30 +++++++++++++++++++++ unit_tests/test_ceilometer_utils.py | 18 +++++++++++++ 15 files changed, 144 insertions(+), 4 deletions(-) create mode 120000 hooks/event-service-relation-broken create mode 120000 hooks/event-service-relation-changed create mode 120000 hooks/event-service-relation-departed create mode 120000 hooks/event-service-relation-joined create mode 100644 templates/mitaka/event_pipeline.yaml diff --git a/config.yaml b/config.yaml index 7b069c2..17b09fd 100644 --- a/config.yaml +++ b/config.yaml @@ -239,3 +239,13 @@ options: Openstack mostly defaults to using public endpoints for internal communication between services. If set to True this option will configure services to use internal endpoints where possible. + remote-sink: + type: string + default: + description: | + Space delimited list of remote consumers of Ceilometer event reporting + which reside outside of the deployed model. Only supported for Mitaka + and later clouds. e.g. + . + 'udp://:/' + 'prometheus://pushgateway-host:9091/metrics/job/openstack-telemetry' diff --git a/hooks/ceilometer_hooks.py b/hooks/ceilometer_hooks.py index 54c6b17..87fd8d1 100755 --- a/hooks/ceilometer_hooks.py +++ b/hooks/ceilometer_hooks.py @@ -153,7 +153,9 @@ def metric_service_joined(): "identity-credentials-relation-changed", "identity-credentials-relation-departed", "metric-service-relation-changed", - "metric-service-relation-departed") + "metric-service-relation-departed", + "event-service-relation-changed", + "event-service-relation-departed",) @restart_on_change(restart_map()) def any_changed(): CONFIGS.write_all() diff --git a/hooks/event-service-relation-broken b/hooks/event-service-relation-broken new file mode 120000 index 0000000..c948469 --- /dev/null +++ b/hooks/event-service-relation-broken @@ -0,0 +1 @@ +ceilometer_hooks.py \ No newline at end of file diff --git a/hooks/event-service-relation-changed b/hooks/event-service-relation-changed new file mode 120000 index 0000000..c948469 --- /dev/null +++ b/hooks/event-service-relation-changed @@ -0,0 +1 @@ +ceilometer_hooks.py \ No newline at end of file diff --git a/hooks/event-service-relation-departed b/hooks/event-service-relation-departed new file mode 120000 index 0000000..c948469 --- /dev/null +++ b/hooks/event-service-relation-departed @@ -0,0 +1 @@ +ceilometer_hooks.py \ No newline at end of file diff --git a/hooks/event-service-relation-joined b/hooks/event-service-relation-joined new file mode 120000 index 0000000..c948469 --- /dev/null +++ b/hooks/event-service-relation-joined @@ -0,0 +1 @@ +ceilometer_hooks.py \ No newline at end of file diff --git a/lib/ceilometer_contexts.py b/lib/ceilometer_contexts.py index 7852a58..89e3571 100644 --- a/lib/ceilometer_contexts.py +++ b/lib/ceilometer_contexts.py @@ -140,6 +140,26 @@ class HAProxyContext(OSContextGenerator): return ctxt +class RemoteSinksContext(OSContextGenerator): + interfaces = ['event-service'] + + def __call__(self): + '''Generates context for remote sinks for Panko and other compatible + remote consumers of Ceilometer event data. + ''' + ctxt = {} + if config('remote-sink'): + ctxt['remote_sinks'] = config('remote-sink').split(' ') + for relid in relation_ids('event-service'): + for unit in related_units(relid): + publisher = relation_get('publisher', unit=unit, rid=relid) + if publisher: + if not ctxt.get('internal_sinks'): + ctxt['internal_sinks'] = {} + ctxt['internal_sinks'][unit.split('/')[0]] = publisher + return ctxt + + class ApacheSSLContext(SSLContext): external_ports = [CEILOMETER_PORT] diff --git a/lib/ceilometer_utils.py b/lib/ceilometer_utils.py index 8c659cd..fa052e5 100644 --- a/lib/ceilometer_utils.py +++ b/lib/ceilometer_utils.py @@ -31,6 +31,7 @@ from ceilometer_contexts import ( HAProxyContext, MetricServiceContext, CEILOMETER_PORT, + RemoteSinksContext, ) from charmhelpers.contrib.openstack.utils import ( get_os_codename_package, @@ -51,6 +52,7 @@ from charmhelpers.core.hookenv import ( is_leader, log, DEBUG, + relation_ids, ) from charmhelpers.fetch import apt_update, apt_install, apt_upgrade from charmhelpers.core.host import init_is_systemd @@ -67,6 +69,7 @@ HTTPS_APACHE_24_CONF = "/etc/apache2/sites-available/" \ "openstack_https_frontend.conf" CLUSTER_RES = 'grp_ceilometer_vips' MEMCACHED_CONF = '/etc/memcached.conf' +PIPELINE_CONF = '/etc/ceilometer/event_pipeline.yaml' CEILOMETER_BASE_SERVICES = [ 'ceilometer-agent-central', @@ -238,6 +241,8 @@ def register_configs(): CeilometerContext(), HAProxyContext()] ) + if CompareOpenStackReleases(release) >= 'mitaka': + configs.register(PIPELINE_CONF, [RemoteSinksContext()]) return configs @@ -420,6 +425,17 @@ def set_shared_secret(secret): secret_file.write(secret) +def get_optional_relations(): + """Return a dictionary of optional relations. + + @returns {relation: relation_name} + """ + optional_interfaces = {} + if relation_ids('event-service'): + optional_interfaces['event-service'] = ['event-service'] + return optional_interfaces + + def assess_status(configs): """Assess status of current unit @@ -444,6 +460,7 @@ def resolve_required_interfaces(): @returns dict - a dictionary keyed by high-level type of interfaces names """ required_ints = deepcopy(REQUIRED_INTERFACES) + required_ints.update(get_optional_relations()) if CompareOpenStackReleases(os_release('ceilometer-common')) >= 'mitaka': required_ints['database'].append('metric-service') if CompareOpenStackReleases(os_release('ceilometer-common')) >= 'queens': diff --git a/metadata.yaml b/metadata.yaml index 2e4d6df..47b1c37 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -43,6 +43,8 @@ requires: scope: container metric-service: interface: gnocchi + event-service: + interface: event-service peers: cluster: interface: ceilometer-ha diff --git a/templates/mitaka/ceilometer.conf b/templates/mitaka/ceilometer.conf index 4810c29..3895705 100644 --- a/templates/mitaka/ceilometer.conf +++ b/templates/mitaka/ceilometer.conf @@ -12,7 +12,7 @@ api_workers = {{ workers }} collector_workers = {{ workers }} notification_workers = {{ workers }} -event_pipeline_cfg_file = /etc/ceilometer/event_pipeline_alarm.yaml +event_pipeline_cfg_file = /etc/ceilometer/event_pipeline.yaml {% if gnocchi_url -%} meter_dispatchers = gnocchi diff --git a/templates/mitaka/event_pipeline.yaml b/templates/mitaka/event_pipeline.yaml new file mode 100644 index 0000000..9290e93 --- /dev/null +++ b/templates/mitaka/event_pipeline.yaml @@ -0,0 +1,37 @@ +--- +sources: + - name: event_source + events: + - "*" + sinks: + - event_sink +{%- if remote_sinks %} + - remote_sink +{% endif %} +{%- if internal_sinks %} +{%- for item in internal_sinks.keys() %} + - {{ item }} +{% endfor -%} +{% endif %} +sinks: +{%- if remote_sinks %} + - name: remote_sink + transformers: + publishers: + {% for item in remote_sinks -%} + - {{ item }} + {% endfor %} +{%- endif -%} +{%- if internal_sinks %} +{%- for item, target in internal_sinks.items() -%} + - name: {{ item }} + transformers: + publishers: + - {{ target }} +{%- endfor %} +{% endif %} + - name: event_sink + transformers: + publishers: + - notifier:// + - notifier://?topic=alarm.all diff --git a/templates/ocata/ceilometer.conf b/templates/ocata/ceilometer.conf index 469ed30..77f7334 100644 --- a/templates/ocata/ceilometer.conf +++ b/templates/ocata/ceilometer.conf @@ -8,7 +8,7 @@ debug = {{ debug }} verbose = {{ verbose }} use_syslog = {{ use_syslog }} -event_pipeline_cfg_file = /etc/ceilometer/event_pipeline_alarm.yaml +event_pipeline_cfg_file = /etc/ceilometer/event_pipeline.yaml {% if gnocchi_url -%} meter_dispatchers = gnocchi diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 80ebc8b..43ebc87 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -141,7 +141,7 @@ class CeilometerBasicDeployment(OpenStackAmuletDeployment): } if self._get_openstack_release() >= self.xenial_pike: configs['ceph-osd'] = {'osd-devices': '/dev/vdb', - 'osd-reformat': 'yes', + 'osd-reformat': True, 'ephemeral-unmount': '/mnt'} super(CeilometerBasicDeployment, self)._configure_services(configs) diff --git a/unit_tests/test_ceilometer_contexts.py b/unit_tests/test_ceilometer_contexts.py index e74bc50..61a2234 100644 --- a/unit_tests/test_ceilometer_contexts.py +++ b/unit_tests/test_ceilometer_contexts.py @@ -193,3 +193,33 @@ class CeilometerContextsTest(CharmTestCase): 'port': api_port } self.assertEqual(contexts.HAProxyContext()(), expected) + + def test_remote_sink_context_no_config(self): + self.relation_ids.return_value = [] + self.os_release.return_value = 'mitaka' + self.assertEqual(contexts.RemoteSinksContext()(), {}) + + def test_remote_sink_context_event_service_relation(self): + self.relation_ids.return_value = ['event-service:0'] + self.related_units.return_value = ['panko/0'] + self.os_release.return_value = 'mitaka' + data = { + 'publisher': 'panko://' + } + self.test_relation.set(data) + self.assertEqual(contexts.RemoteSinksContext()(), + {'internal_sinks': {'panko': 'panko://'}}) + + def test_remote_sink_context_with_single_config(self): + self.relation_ids.return_value = [] + self.os_release.return_value = 'mitaka' + self.test_config.set('remote-sink', 'http://foo') + self.assertEqual(contexts.RemoteSinksContext()(), + {'remote_sinks': ['http://foo']}) + + def test_remote_sink_context_with_multiple_config(self): + self.relation_ids.return_value = [] + self.os_release.return_value = 'mitaka' + self.test_config.set('remote-sink', 'http://foo http://bar') + self.assertEqual(contexts.RemoteSinksContext()(), + {'remote_sinks': ['http://foo', 'http://bar']}) diff --git a/unit_tests/test_ceilometer_utils.py b/unit_tests/test_ceilometer_utils.py index b20abc9..119653d 100644 --- a/unit_tests/test_ceilometer_utils.py +++ b/unit_tests/test_ceilometer_utils.py @@ -39,6 +39,7 @@ TO_PATCH = [ 'os_release', 'is_leader', 'reset_os_release', + 'relation_ids', ] @@ -275,6 +276,7 @@ class CeilometerUtilsTest(CharmTestCase): def test_resolve_required_interfaces(self): self.os_release.side_effect = None self.os_release.return_value = 'icehouse' + self.relation_ids.return_value = None self.assertEqual( utils.resolve_required_interfaces(), { @@ -287,6 +289,7 @@ class CeilometerUtilsTest(CharmTestCase): def test_resolve_required_interfaces_mitaka(self): self.os_release.side_effect = None self.os_release.return_value = 'mitaka' + self.relation_ids.return_value = None self.assertEqual( utils.resolve_required_interfaces(), { @@ -299,6 +302,7 @@ class CeilometerUtilsTest(CharmTestCase): def test_resolve_required_interfaces_queens(self): self.os_release.side_effect = None self.os_release.return_value = 'queens' + self.relation_ids.return_value = None self.assertEqual( utils.resolve_required_interfaces(), { @@ -308,6 +312,20 @@ class CeilometerUtilsTest(CharmTestCase): } ) + def test_resolve_optional_interfaces(self): + self.os_release.side_effect = None + self.os_release.return_value = 'icehouse' + self.relation_ids.return_value = [0] + self.assertEqual( + utils.resolve_required_interfaces(), + { + 'database': ['mongodb'], + 'messaging': ['amqp'], + 'identity': ['identity-service'], + 'event-service': ['event-service'], + } + ) + @patch.object(utils, 'subprocess') def test_ceilometer_upgrade(self, mock_subprocess): self.is_leader.return_value = True