openstack
/
logstash-filters
Code Issues Proposed changes

Add cinder and quantum screen logs to logstash. 

* modules/openstack_project/files/logstash/jenkins-log-pusher.yaml:
Add the cinder and quantum screen log files to the log pusher. Tag them
with screen and oslofmt (screen because they are screen logs and oslofmt
because they use the oslo log message format).

* modules/openstack_project/templates/logstash/indexer.conf.erb:
Add a grep filter to remove the screen log header lines. Add a multiline
filter to handle oslo log format multi line events. Add a grok filter to
parse the oslo format logs. Handle timestamps without millisecond
precision. Remove event_message field if that message was properly
parsed.

Change-Id: Icd18e252a512416e0cce5ee0e27942b072a25e09
Reviewed-on: https://review.openstack.org/29985
Reviewed-by: James E. Blair <corvus@inaugust.com>
Approved: Clark Boylan <clark.boylan@gmail.com>
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Tested-by: Jenkins
This commit is contained in:
Clark Boylan 2013-05-21 12:59:57 -07:00 committed by Jenkins
parent 017aba78ea
commit 8fb3f9d288
1 changed files with 27 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
indexer.conf.erb
View File

@ -11,12 +11,21 @@ input {
# You can check grok patterns at http://grokdebug.herokuapp.com/
filter {
grep {
# Remove unneeded html tags.
type => "jenkins"
tags => ["console.html"]
# Drop matches.
negate => true
match => ["@message", "^</?pre>$"]
}
grep {
# Remove screen log headers.
type => "jenkins"
tags => ["screen"]
# Drop matches.
negate => true
match => ["@message", "^\+ "]
}
multiline {
type => "jenkins"
tags => ["console.html"]
@ -24,6 +33,13 @@ filter {
pattern => "^%{DATESTAMP} \|"
what => "previous"
}
multiline {
type => "jenkins"
tags => ["oslofmt"]
negate => true
pattern => "^%{DATESTAMP} "
what => "previous"
}
grok {
type => "jenkins"
tags => ["console.html"]
@ -32,10 +48,19 @@ filter {
pattern => [ "(?m)^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ]
add_field => [ "received_at", "%{@timestamp}" ]
}
grok {
type => "jenkins"
tags => ["oslofmt"]
# Do multiline matching as the above mutliline filter may add newlines
# to the log messages.
# TODO move the LOGLEVELs into a proper grok pattern.
pattern => [ "(?m)^%{DATESTAMP:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" ]
add_field => [ "received_at", "%{@timestamp}" ]
}
date {
type => "jenkins"
exclude_tags => "_grokparsefailure"
match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS" ]
match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss" ]
}
mutate {
type => "jenkins"
@ -45,7 +70,7 @@ filter {
mutate {
type => "jenkins"
exclude_tags => "_grokparsefailure"
remove => [ "logdate", "logmessage" ]
remove => [ "logdate", "logmessage", "event_message" ]
}
}