openstack
/
kolla-ansible
Code Issues Proposed changes

Update Monasca Log Transformer config 

* Convert all log levels to lowercase
* Make number of threads configurable
* Improve naming
* Update path and log options for Logstash 2.4
* Change worker count from string to integer for the Kafka output plugin supplied
  with Logstash 2.4.

Partially-Implements: blueprint monasca-roles
Change-Id: I9379969ee639dff0b525d7a1a03df59bcf18b98b
This commit is contained in:
Doug Szumski 2018-09-24 17:06:51 +00:00
parent 380418f351
commit 063518d072
2 changed files with 16 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
ansible/roles/monasca/templates/monasca-log-transformer/log-transformer.conf.j2
View File

@ -5,7 +5,9 @@ input {
kafka {
zk_connect => "{{ monasca_zookeeper_servers }}"
topic_id => "{{ monasca_raw_logs_topic }}"
group_id => "transformer-logstash-consumer"
group_id => "log_transformer"
consumer_id => "log_transformer_{{ ansible_hostname }}"
consumer_threads => "{{ monasca_log_pipeline_threads }}"
}
}
@ -15,11 +17,23 @@ filter {
match => [ "[log][dimensions][timestamp]", "yyyy-MM-dd HH:mm:ss +0000", "ISO8601"]
remove_field => [ "[log][dimensions][timestamp]", "[log][dimensions][Timestamp]" ]
}
# OpenStack log levels are uppercase, and syslog are lowercase.
# Furthermore, syslog has more log levels that OpenStack. To avoid
# mapping syslog log levels to OpenStack log levels, we standardise
# on the syslog style here.
if [log][dimensions][log_level] {
mutate {
lowercase => [ "[log][dimensions][log_level]" ]
}
}
}
output {
kafka {
bootstrap_servers => "{{ monasca_kafka_servers }}"
topic_id => "{{ monasca_transformed_logs_topic }}"
client_id => "log_transformer_{{ ansible_hostname }}"
workers => {{ monasca_log_pipeline_threads|int }}
}
}

2
ansible/roles/monasca/templates/monasca-log-transformer/monasca-log-transformer.json.j2
View File

@ -1,5 +1,5 @@
{
"command": "/usr/share/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf",
"command": "/opt/logstash/bin/logstash --log-in-json --log /var/log/kolla/logstash/monasca-log-transformer.log -f /etc/logstash/conf.d/log-transformer.conf",
"config_files": [
{
"source": "{{ container_config_directory }}/log-transformer.conf",