From 8fb3f9d28831c68f4fb61a06e94b4e21b43a465a Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Tue, 21 May 2013 12:59:57 -0700
Subject: [PATCH] Add cinder and quantum screen logs to logstash.

* modules/openstack_project/files/logstash/jenkins-log-pusher.yaml:
Add the cinder and quantum screen log files to the log pusher. Tag them
with screen and oslofmt (screen because they are screen logs and oslofmt
because they use the oslo log message format).

* modules/openstack_project/templates/logstash/indexer.conf.erb:
Add a grep filter to remove the screen log header lines. Add a multiline
filter to handle oslo log format multi line events. Add a grok filter to
parse the oslo format logs. Handle timestamps without millisecond
precision. Remove event_message field if that message was properly
parsed.

Change-Id: Icd18e252a512416e0cce5ee0e27942b072a25e09
Reviewed-on: https://review.openstack.org/29985
Reviewed-by: James E. Blair <corvus@inaugust.com>
Approved: Clark Boylan <clark.boylan@gmail.com>
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Tested-by: Jenkins
---
 indexer.conf.erb | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/indexer.conf.erb b/indexer.conf.erb
index c03ffcc..940f77a 100644
--- a/indexer.conf.erb
+++ b/indexer.conf.erb
@@ -11,12 +11,21 @@ input {
 # You can check grok patterns at http://grokdebug.herokuapp.com/
 filter {
   grep {
+    # Remove unneeded html tags.
     type => "jenkins"
     tags => ["console.html"]
     # Drop matches.
     negate => true
     match => ["@message", "^</?pre>$"]
   }
+  grep {
+    # Remove screen log headers.
+    type => "jenkins"
+    tags => ["screen"]
+    # Drop matches.
+    negate => true
+    match => ["@message", "^\+ "]
+  }
   multiline {
     type => "jenkins"
     tags => ["console.html"]
@@ -24,6 +33,13 @@ filter {
     pattern => "^%{DATESTAMP} \|"
     what => "previous"
   }
+  multiline {
+    type => "jenkins"
+    tags => ["oslofmt"]
+    negate => true
+    pattern => "^%{DATESTAMP} "
+    what => "previous"
+  }
   grok {
     type => "jenkins"
     tags => ["console.html"]
@@ -32,10 +48,19 @@ filter {
     pattern => [ "(?m)^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ]
     add_field => [ "received_at", "%{@timestamp}" ]
   }
+  grok {
+    type => "jenkins"
+    tags => ["oslofmt"]
+    # Do multiline matching as the above mutliline filter may add newlines
+    # to the log messages.
+    # TODO move the LOGLEVELs into a proper grok pattern.
+    pattern => [ "(?m)^%{DATESTAMP:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" ]
+    add_field => [ "received_at", "%{@timestamp}" ]
+  }
   date {
     type => "jenkins"
     exclude_tags => "_grokparsefailure"
-    match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS" ]
+    match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss" ]
   }
   mutate {
     type => "jenkins"
@@ -45,7 +70,7 @@ filter {
   mutate {
     type => "jenkins"
     exclude_tags => "_grokparsefailure"
-    remove => [ "logdate", "logmessage" ]
+    remove => [ "logdate", "logmessage", "event_message" ]
   }
 }