Upgrade Logstash to 1.2.1.
* modules/logstash/manifests/init.pp: Download and install Logstash 1.2.1. * modules/openstack_project/files/logstash/log-gearman-client.py: Logstash 1.2.1 comes with a new schema. Update the job data sent to log push workers to better accomodate the new schema. * modules/openstack_project/files/logstash/log-gearman-worker.py: Push Logstash 1.2.1 schema compliant JSON to the Logstash TCP input. * modules/openstack_project/templates/logstash/indexer.conf.erb: Logstash 1.2.1 comes with a new schema and many input and filter changes. Use the newly supported features like conditionals to keep the config up to date. * modules/kibana/templates/config.rb.erb: Change the default field for kibana to 'message'. It was @message which is deprecated in the new logstash schema. Change-Id: Id19fc05bcce8d42c5c0cf33df3da7e95f5794107
This commit is contained in:
Clark Boylan
parent
d2d138c09c
commit
4698b088a2
204
indexer.conf.erb
204
indexer.conf.erb
|
|
@ -2,132 +2,108 @@ input {
|
||||||
tcp {
|
tcp {
|
||||||
host => "localhost"
|
host => "localhost"
|
||||||
port => 9999
|
port => 9999
|
||||||
format => "json"
|
codec => line {}
|
||||||
message_format => "%{event_message}"
|
|
||||||
type => "jenkins"
|
type => "jenkins"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
# You can check grok patterns at http://grokdebug.herokuapp.com/
|
# You can check grok patterns at http://grokdebug.herokuapp.com/
|
||||||
filter {
|
filter {
|
||||||
grep {
|
# This is a work around for a bug. We should be able to set the tcp
|
||||||
# Remove unneeded html tags.
|
# input codec to json, but that codec doesn't support streaming.
|
||||||
type => "jenkins"
|
# Convert to json here instead.
|
||||||
tags => ["console.html"]
|
json {
|
||||||
# Drop matches.
|
source => "message"
|
||||||
negate => true
|
|
||||||
match => ["@message", "^</?pre>$"]
|
|
||||||
}
|
}
|
||||||
grep {
|
if "screen" in [tags] and [message] =~ "^\+ " {
|
||||||
# Remove screen log headers.
|
drop {}
|
||||||
type => "jenkins"
|
|
||||||
tags => ["screen"]
|
|
||||||
# Drop matches.
|
|
||||||
negate => true
|
|
||||||
match => ["@message", "^\+ "]
|
|
||||||
}
|
}
|
||||||
grep {
|
if "console.html" in [tags] {
|
||||||
# Remove blank lines.
|
if [message] == "<pre>" or [message] == "</pre>" {
|
||||||
type => "jenkins"
|
drop {}
|
||||||
tags => ["keystonefmt"]
|
}
|
||||||
# Drop matches.
|
multiline {
|
||||||
negate => true
|
negate => true
|
||||||
match => ["@message", "^$"]
|
pattern => "^%{TIMESTAMP_ISO8601} \|"
|
||||||
}
|
what => "previous"
|
||||||
multiline {
|
stream_identity => "%{host}.%{filename}"
|
||||||
type => "jenkins"
|
}
|
||||||
tags => ["console.html"]
|
grok {
|
||||||
negate => true
|
# Do multiline matching as the above mutliline filter may add newlines
|
||||||
pattern => "^%{DATESTAMP} \|"
|
# to the log messages.
|
||||||
what => "previous"
|
match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate} \| %{GREEDYDATA:logmessage}" }
|
||||||
stream_identity => "%{@source_host}.%{filename}"
|
add_field => { "received_at" => "%{@timestamp}" }
|
||||||
}
|
}
|
||||||
multiline {
|
} else if "oslofmt" in [tags] {
|
||||||
type => "jenkins"
|
multiline {
|
||||||
tags => ["oslofmt"]
|
negate => true
|
||||||
negate => true
|
pattern => "^%{TIMESTAMP_ISO8601} "
|
||||||
pattern => "^%{DATESTAMP} "
|
what => "previous"
|
||||||
what => "previous"
|
stream_identity => "%{host}.%{filename}"
|
||||||
stream_identity => "%{@source_host}.%{filename}"
|
}
|
||||||
}
|
multiline {
|
||||||
multiline {
|
negate => false
|
||||||
type => "jenkins"
|
pattern => "^%{TIMESTAMP_ISO8601}%{SPACE}%{NUMBER}?%{SPACE}?TRACE"
|
||||||
tags => ["oslofmt"]
|
what => "previous"
|
||||||
negate => false
|
stream_identity => "%{host}.%{filename}"
|
||||||
pattern => "^%{DATESTAMP}%{SPACE}%{NUMBER}?%{SPACE}?TRACE"
|
}
|
||||||
what => "previous"
|
grok {
|
||||||
stream_identity => "%{@source_host}.%{filename}"
|
# Do multiline matching as the above mutliline filter may add newlines
|
||||||
}
|
# to the log messages.
|
||||||
multiline {
|
# TODO move the LOGLEVELs into a proper grok pattern.
|
||||||
type => "jenkins"
|
match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" }
|
||||||
tags => ["keystonefmt"]
|
add_field => { "received_at" => "%{@timestamp}" }
|
||||||
negate => true
|
}
|
||||||
pattern => "^\(\b%{NOTSPACE}\b\):"
|
} else if "keystonefmt" in [tags] {
|
||||||
what => "previous"
|
if [message] == "" {
|
||||||
stream_identity => "%{@source_host}.%{filename}"
|
drop {}
|
||||||
}
|
}
|
||||||
grok {
|
multiline {
|
||||||
type => "jenkins"
|
negate => true
|
||||||
tags => ["console.html"]
|
pattern => "^\(\b%{NOTSPACE}\b\):"
|
||||||
# Do multiline matching as the above mutliline filter may add newlines
|
what => "previous"
|
||||||
# to the log messages.
|
stream_identity => "%{host}.%{filename}"
|
||||||
pattern => [ "(?m)^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ]
|
}
|
||||||
add_field => [ "received_at", "%{@timestamp}" ]
|
grok {
|
||||||
}
|
# Do multiline matching as the above mutliline filter may add newlines
|
||||||
grok {
|
# to the log messages.
|
||||||
type => "jenkins"
|
# TODO move the LOGLEVELs into a proper grok pattern.
|
||||||
tags => ["oslofmt"]
|
match => { "message" => "(?m)^\(\b%{NOTSPACE:module}\b\):%{SPACE}%{TIMESTAMP_ISO8601:logdate}%{SPACE}(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR)%{SPACE}%{GREEDYDATA:logmessage}" }
|
||||||
# Do multiline matching as the above mutliline filter may add newlines
|
add_field => { "received_at" => "%{@timestamp}" }
|
||||||
# to the log messages.
|
}
|
||||||
# TODO move the LOGLEVELs into a proper grok pattern.
|
} else if "apachecombined" in [tags] {
|
||||||
pattern => [ "(?m)^%{DATESTAMP:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" ]
|
grok {
|
||||||
add_field => [ "received_at", "%{@timestamp}" ]
|
match => { "message" => "%{COMBINEDAPACHELOG}" }
|
||||||
}
|
add_field => { "received_at" => "%{@timestamp}" }
|
||||||
grok {
|
add_field => { "logdate" => "%{timestamp}" }
|
||||||
type => "jenkins"
|
add_field => { "logmessage" => "%{verb} %{request} %{response}" }
|
||||||
tags => ["keystonefmt"]
|
}
|
||||||
# Do multiline matching as the above mutliline filter may add newlines
|
} else if "syslog" in [tags] {
|
||||||
# to the log messages.
|
grok {
|
||||||
# TODO move the LOGLEVELs into a proper grok pattern.
|
# Syslog grok filter adapted from
|
||||||
pattern => [ "(?m)^\(\b%{NOTSPACE:module}\b\):%{SPACE}%{DATESTAMP:logdate}%{SPACE}(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR)%{SPACE}%{GREEDYDATA:logmessage}" ]
|
# http://cookbook.logstash.net/recipes/syslog-pri/syslog.conf
|
||||||
add_field => [ "received_at", "%{@timestamp}" ]
|
match => { "message" => "%{SYSLOGTIMESTAMP:logdate}%{SPACE}%{SYSLOGHOST:syslog_host}?%{SPACE}%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:? %{GREEDYDATA:logmessage}" }
|
||||||
}
|
add_field => { "received_at" => "%{@timestamp}" }
|
||||||
grok {
|
}
|
||||||
type => "jenkins"
|
|
||||||
tags => ["apachecombined"]
|
|
||||||
pattern => [ "%{COMBINEDAPACHELOG}" ]
|
|
||||||
add_field => [ "received_at", "%{@timestamp}", "logdate", "%{timestamp}", "logmessage", "%{verb} %{request} %{response}" ]
|
|
||||||
}
|
|
||||||
grok {
|
|
||||||
type => "jenkins"
|
|
||||||
tags => ["syslog"]
|
|
||||||
# Syslog grok filter adapted from
|
|
||||||
# http://cookbook.logstash.net/recipes/syslog-pri/syslog.conf
|
|
||||||
pattern => [ "%{SYSLOGTIMESTAMP:logdate}%{SPACE}%{SYSLOGHOST:syslog_host}?%{SPACE}%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:? %{GREEDYDATA:logmessage}" ]
|
|
||||||
add_field => [ "received_at", "%{@timestamp}" ]
|
|
||||||
}
|
|
||||||
# Remove DEBUG logs to reduce the amount of data that needs to be processed.
|
|
||||||
grep {
|
|
||||||
type => "jenkins"
|
|
||||||
negate => true
|
|
||||||
match => [ "loglevel", "DEBUG" ]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Filters below here should be consistent for all Jenkins log formats.
|
# Filters below here should be consistent for all Jenkins log formats.
|
||||||
date {
|
# Remove DEBUG logs to reduce the amount of data that needs to be processed.
|
||||||
type => "jenkins"
|
if [loglevel] == "DEBUG" {
|
||||||
exclude_tags => "_grokparsefailure"
|
drop {}
|
||||||
match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "dd/MMM/yyyy:HH:mm:ss Z" ]
|
|
||||||
}
|
}
|
||||||
mutate {
|
|
||||||
type => "jenkins"
|
if ! ("_grokparsefailure" in [tags]) {
|
||||||
exclude_tags => "_grokparsefailure"
|
date {
|
||||||
replace => [ "@message", "%{logmessage}" ]
|
match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "dd/MMM/yyyy:HH:mm:ss Z" ]
|
||||||
}
|
timezone => "UTC"
|
||||||
mutate {
|
}
|
||||||
type => "jenkins"
|
mutate {
|
||||||
exclude_tags => "_grokparsefailure"
|
replace => { "message" => "%{logmessage}" }
|
||||||
remove => [ "logdate", "logmessage", "event_message" ]
|
}
|
||||||
|
mutate {
|
||||||
|
remove_field => [ "logdate", "logmessage" ]
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue