From 473dea341fd8373deaeb52d11ad9f94347f5abb1 Mon Sep 17 00:00:00 2001 From: Martin Chacon Piza Date: Wed, 5 Sep 2018 15:07:27 +0200 Subject: [PATCH] Parse envelope fields from kafka to prepare the ES index - Change the default index prefix name for events. Story: 2003955 Task: 26880 Depends-On: https://review.openstack.org/#/c/600030/ Change-Id: Idb6913229d3ac4e391eda0d6129eb2590bc7a678 --- monasca_persister/conf/elasticsearch.py | 4 +- .../elasticsearch/events_repository.py | 11 +- monasca_persister/repositories/utils.py | 11 +- monasca_persister/tests/events.json | 146 +++++++++++------- monasca_persister/tests/test_events.py | 7 +- 5 files changed, 113 insertions(+), 66 deletions(-) diff --git a/monasca_persister/conf/elasticsearch.py b/monasca_persister/conf/elasticsearch.py index 1480cdf4..cb90989f 100644 --- a/monasca_persister/conf/elasticsearch.py +++ b/monasca_persister/conf/elasticsearch.py @@ -20,8 +20,8 @@ from monasca_persister.conf import types elasticsearch_opts = [ cfg.StrOpt( 'index_name', - help='Index name where events are stored', - default='monevents'), + help='Index prefix name where events are stored', + default='events'), cfg.ListOpt( 'hosts', help='List of Elasticsearch nodes in format host[:port]', diff --git a/monasca_persister/repositories/elasticsearch/events_repository.py b/monasca_persister/repositories/elasticsearch/events_repository.py index 6ea2c1a2..ca0362ee 100644 --- a/monasca_persister/repositories/elasticsearch/events_repository.py +++ b/monasca_persister/repositories/elasticsearch/events_repository.py @@ -43,16 +43,17 @@ class ElasticSearchEventsRepository(abstract_repository.AbstractRepository): def write_batch(self, data_points): for data_point in data_points: - (tenant_id, timestamp, event_type, payload) = data_point + (project_id, timestamp, event_type, payload, dimensions) = data_point - index = '%s-%s-%s' % (self.conf.index_name, tenant_id, + index = '%s-%s-%s' % (self.conf.index_name, project_id, ElasticSearchEventsRepository._normalize_timestamp(timestamp)) body = { - 'tenant_id': tenant_id, - 'timestamp': timestamp, + 'project_id': project_id, + '@timestamp': timestamp, 'event_type': event_type, - 'payload': payload + 'payload': payload, + 'dimensions': dimensions } self.es.create( diff --git a/monasca_persister/repositories/utils.py b/monasca_persister/repositories/utils.py index 0f165ffb..6d2673b8 100644 --- a/monasca_persister/repositories/utils.py +++ b/monasca_persister/repositories/utils.py @@ -104,9 +104,10 @@ def parse_alarm_state_hist_message(message): def parse_events_message(message): decoded_message = json.loads(message.message.value) - event_type = decoded_message['event_type'] - timestamp = decoded_message['timestamp'] - payload = decoded_message['payload'] - tenant_id = payload['tenant_id'] + event_type = decoded_message['event']['event_type'] + timestamp = decoded_message['event']['timestamp'] + payload = decoded_message['event']['payload'] + project_id = decoded_message['meta']['project_id'] + dimensions = decoded_message['event']['dimensions'] - return tenant_id, timestamp, event_type, payload + return project_id, timestamp, event_type, payload, dimensions diff --git a/monasca_persister/tests/events.json b/monasca_persister/tests/events.json index edbf57f8..447b0f6f 100644 --- a/monasca_persister/tests/events.json +++ b/monasca_persister/tests/events.json @@ -1,54 +1,96 @@ { - "event_1": { - "event_type": "compute.instance.create.start", - "timestamp": "2017-06-01 09:15:11.494606", - "payload": { - "state_description": "", - "availability_zone": "nova", - "terminated_at": "", - "ephemeral_gb": 0, - "instance_type_id": 1, - "deleted_at": "", - "reservation_id": "r-74ndofdp", - "instance_id": "cb724671-cc36-49cd-9987-d08f2c8356b9", - "display_name": "fred", - "hostname": "fred", - "state": "building", - "progress": "", - "launched_at": "", - "metadata": { + "event_1":{ + "meta":{ + "project_id":"de98fbff448f4f278a56e9929db70b03" }, - "node": null, - "ramdisk_id": "", - "access_ip_v6": null, - "disk_gb": 1, - "access_ip_v4": null, - "kernel_id": "", - "image_name": "cirros", - "host": null, - "user_id": "92e0ceb0f3d648ddabeae1bfde4071b2", - "image_ref_url": "http://d00-19-99-b3-7e-2e.st.est.fujitsu.com:9292/images/e08428a7-aa34-42bd-9e91-6fe15e0ed2ae", - "cell_name": "", - "root_gb": 1, - "tenant_id": "de98fbff448f4f278a56e9929db70b03", - "created_at": "2017-06-01 09:15:10+00:00", - "memory_mb": 512, - "instance_type": "m1.tiny", - "vcpus": 1, - "image_meta": { - "container_format": "bare", - "min_ram": "0", - "disk_format": "qcow2", - "architecture": "x86_64", - "min_disk": "1", - "base_image_ref": "e08428a7-aa34-42bd-9e91-6fe15e0ed2ae" - }, - "architecture": "x86_64", - "os_type": null, - "instance_flavor_id": "1" - }, - "priority": "INFO", - "publisher_id": "compute.d00-26-2d-0c-d5-64", - "message_id": "5e60c9f1-1cf9-4f5d-9826-91fa329a79c1" - } -} \ No newline at end of file + "creation_time":1536050124, + "event":{ + "dimensions":{ + "service":"compute", + "topic":"notification.sample", + "hostname":"nova-compute:compute" + }, + "priority":"INFO", + "timestamp":"2017-06-01 09:15:11.494606", + "publisher_id":"nova-compute:compute", + "event_type":"compute.instance.create.start", + "payload":{ + "nova_object.version":"1.3", + "nova_object.name":"InstanceActionPayload", + "nova_object.data":{ + "availability_zone":"nova", + "terminated_at":null, + "ip_addresses":[ + { + "nova_object.version":"1.0", + "nova_object.name":"IpPayload", + "nova_object.namespace":"nova", + "nova_object.data":{ + "port_uuid":"ce531f90-199f-48c0-816c-13e38010b442", + "label":"private-network", + "mac":"fa:16:3e:4c:2c:30", + "version":4, + "meta":{ + + }, + "address":"192.168.1.3", + "device_name":"tapce531f90-19" + } + } + ], + "ramdisk_id":"", + "image_uuid":"155d900f-4e14-4e4c-a73d-069cbf4541e6", + "flavor":{ + "nova_object.version":"1.3", + "nova_object.name":"FlavorPayload", + "nova_object.data":{ + "memory_mb":512, + "root_gb":1, + "name":"test_flavor", + "ephemeral_gb":0, + "disabled":false, + "vcpus":1, + "extra_specs":{ + "hw:watchdog_action":"disabled" + }, + "swap":0, + "rxtx_factor":1.0, + "is_public":true, + "flavorid":"a22d5517-147c-4147-a0d1-e698df5cd4e3", + "vcpu_weight":0, + "projects":null + }, + "nova_object.namespace":"nova" + }, + "deleted_at":null, + "reservation_id":"r-npxv0e40", + "user_id":"fake", + "uuid":"178b0921-8f85-4257-88b6-2e743b5a975c", + "display_description":"some-server", + "state":"active", + "power_state":"running", + "progress":0, + "launched_at":"2018-07-18T10:02:11Z", + "metadata":{ + + }, + "node":"fake-mini", + "kernel_id":"", + "key_name":"my-key", + "host":"compute", + "display_name":"some-server", + "task_state":null, + "architecture":"x86_64", + "locked":false, + "fault":null, + "created_at":"2018-07-18T10:01:11Z", + "host_name":"some-server", + "tenant_id":"12345678123456781234567812345678", + "auto_disk_config":"MANUAL", + "os_type":null + }, + "nova_object.namespace":"nova" + } + } + } +} diff --git a/monasca_persister/tests/test_events.py b/monasca_persister/tests/test_events.py index 85a6c344..95115dca 100644 --- a/monasca_persister/tests/test_events.py +++ b/monasca_persister/tests/test_events.py @@ -36,10 +36,13 @@ class TestEvents(base.BaseTestCase): def test_parse_event(self): event = self._load_event('event_1') - tenant_id, timestamp, event_type, payload = utils.parse_events_message(event) - self.assertEqual('de98fbff448f4f278a56e9929db70b03', tenant_id) + project_id, timestamp, event_type, payload, dimensions = utils.parse_events_message(event) + self.assertEqual('de98fbff448f4f278a56e9929db70b03', project_id) self.assertEqual('2017-06-01 09:15:11.494606', timestamp) self.assertEqual('compute.instance.create.start', event_type) + self.assertEqual('compute', dimensions['service']) + self.assertEqual('notification.sample', dimensions['topic']) + self.assertEqual('nova-compute:compute', dimensions['hostname']) self.assertIsNotNone(payload) self.assertThat(len(payload), matchers.GreaterThan(0))