diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index ee1da6efe5..12c845f67d 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -449,6 +449,9 @@ monasca [monasca-log-transformer:children] monasca +[monasca-log-persister:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index 1caccd0709..c4c9848654 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -458,6 +458,9 @@ monasca [monasca-log-transformer:children] monasca +[monasca-log-persister:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml index 4db2feb1c6..c8dc213438 100644 --- a/ansible/roles/monasca/defaults/main.yml +++ b/ansible/roles/monasca/defaults/main.yml @@ -27,6 +27,15 @@ monasca_services: - "{{ node_config_directory }}/monasca-log-transformer/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla" + monasca-log-persister: + container_name: monasca_log_persister + group: monasca-log-persister + enabled: true + image: "{{ monasca_logstash_image_full }}" + volumes: + - "{{ node_config_directory }}/monasca-log-persister/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "kolla_logs:/var/log/kolla" #################### # Databases @@ -45,6 +54,7 @@ monasca_influxdb_http_port: "{{ influxdb_http_port }}" monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ kafka_port }}{% if not loop.last %},{% endif %}{% endfor %}" monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}" monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}" +monasca_elasticsearch_servers: "{% for host in groups['elasticsearch'] %}'{{ internal_protocol }}://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }}'{% if not loop.last %},{% endif %}{% endfor %}" monasca_metrics_topic: "metrics" monasca_raw_logs_topic: "logs" diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml index 6caadb6df0..c9db3faf06 100644 --- a/ansible/roles/monasca/handlers/main.yml +++ b/ansible/roles/monasca/handlers/main.yml @@ -62,3 +62,24 @@ - config_json.changed | bool or monasca_log_transformer_confs.changed | bool or monasca_log_transformer_container.changed | bool + +- name: Restart monasca-log-persister container + vars: + service_name: "monasca-log-persister" + service: "{{ monasca_services[service_name] }}" + config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + monasca_log_persister_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + when: + - action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or monasca_log_persister_confs.changed | bool + or monasca_log_persister_elasticsearch_template.changed | bool + or monasca_log_persister_container.changed | bool diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml index 42c40f38dc..66217b5124 100644 --- a/ansible/roles/monasca/tasks/config.yml +++ b/ansible/roles/monasca/tasks/config.yml @@ -118,6 +118,45 @@ notify: - Restart monasca-log-transformer container +- name: Copying over monasca-log-persister config + vars: + service: "{{ monasca_services['monasca-log-persister'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/monasca-log-persister/log-persister.conf" + mode: "0660" + become: true + register: monasca_log_persister_confs + with_first_found: + - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/log-persister.conf" + - "{{ node_custom_config }}/monasca/log-persister.conf" + - "{{ role_path }}/templates/monasca-log-persister/log-persister.conf.j2" + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart monasca-log-persister container + +- name: Copying over monasca-log-persister elasticsearch template + vars: + service: "{{ monasca_services['monasca-log-persister'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/monasca-log-persister/elasticsearch-template.json" + mode: "0660" + become: true + register: monasca_log_persister_elasticsearch_template + with_first_found: + - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/elasticsearch-template.json" + - "{{ node_custom_config }}/monasca/elasticsearch-template.json" + - "{{ role_path }}/templates/monasca-log-persister/elasticsearch-template.json" + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart monasca-log-persister container + + - name: Check monasca containers become: true kolla_docker: diff --git a/ansible/roles/monasca/tasks/deploy.yml b/ansible/roles/monasca/tasks/deploy.yml index b386d15e3f..d470d08cfa 100644 --- a/ansible/roles/monasca/tasks/deploy.yml +++ b/ansible/roles/monasca/tasks/deploy.yml @@ -6,7 +6,8 @@ - include: config.yml when: inventory_hostname in groups['monasca-api'] or inventory_hostname in groups['monasca-log-api'] or - inventory_hostname in groups['monasca-log-transformer'] + inventory_hostname in groups['monasca-log-transformer'] or + inventory_hostname in groups['monasca-log-persister'] - include: bootstrap.yml when: inventory_hostname in groups['monasca-api'] @@ -17,4 +18,5 @@ - include: check.yml when: inventory_hostname in groups['monasca-api'] or inventory_hostname in groups['monasca-log-api'] or - inventory_hostname in groups['monasca-log-transformer'] + inventory_hostname in groups['monasca-log-transformer'] or + inventory_hostname in groups['monasca-log-persister'] diff --git a/ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json b/ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json new file mode 100644 index 0000000000..c882454cb7 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-persister/elasticsearch-template.json @@ -0,0 +1,56 @@ +{ + "aliases": {}, + "mappings": { + "log": { + "_all": { + "enabled": true, + "omit_norms": true + }, + "dynamic_templates": [ + { + "message_field": { + "mapping": { + "fielddata": { + "format": "disabled" + }, + "index": "analyzed", + "omit_norms": true, + "type": "string" + }, + "match": "message", + "match_mapping_type": "string" + } + }, + { + "other_fields": { + "mapping": { + "index": "not_analyzed", + "type": "string" + }, + "match": "*", + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "@version": { + "index": "not_analyzed", + "type": "string" + }, + "creation_time": { + "type": "date" + } + } + } + }, + "order": 0, + "settings": { + "index": { + "refresh_interval": "5s" + } + }, + "template": "monasca-*" +} diff --git a/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 new file mode 100644 index 0000000000..31b08d3c5f --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-persister/log-persister.conf.j2 @@ -0,0 +1,19 @@ +# Persist transformed logs to Elasticsearch + +input { + kafka { + zk_connect => "{{ monasca_zookeeper_servers }}" + topic_id => "{{ monasca_transformed_logs_topic }}" + group_id => "transformer-logstash-consumer" + } +} + +output { + elasticsearch { + index => "monasca-%{[meta][tenantId]}-%{+YYYY.MM.dd}" + hosts => [{{ monasca_elasticsearch_servers }}] + document_type => "log" + template_name => "monasca" + template => "/etc/logstash/elasticsearch-template.json" + } +} diff --git a/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2 b/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2 new file mode 100644 index 0000000000..3eb19fa42d --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-persister/monasca-log-persister.json.j2 @@ -0,0 +1,24 @@ +{ + "command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/monasca/monasca-log-persister.log -f /etc/logstash/conf.d/log-persister.conf", + "config_files": [ + { + "source": "{{ container_config_directory }}/log-persister.conf", + "dest": "/etc/logstash/conf.d/log-persister.conf", + "owner": "logstash", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/elasticsearch-template.json", + "dest": "/etc/logstash/elasticsearch-template.json", + "owner": "logstash", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/monasca", + "owner": "logstash:kolla", + "recurse": true + } + ] +} diff --git a/releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml b/releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml new file mode 100644 index 0000000000..cc0a4a5c5b --- /dev/null +++ b/releasenotes/notes/add-monasca-log-persister-f4da4370a0c5777e.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Add support for deploying the Monasca Log Persister. The Log + Persister is responsible for reading logs from the Kafka processed + logs topic and writing them to Elasticsearch.