From 929f152d494ed7faecba213aebb1b8f688ad0fd5 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Thu, 2 May 2013 16:02:19 -0700
Subject: [PATCH] Use log-pusher injected tags during logstash grok

* modules/openstack_project/templates/logstash/indexer.conf.erb: The
log-pusher.py script tags events with the filename of the log generating
the event. Use these values instead of different types to differentiate
Jenkins console logs from nova logs and so on.

Note that filters must match all of the values in the tags array (they
are ANDed together). This may mean that as the logstash filter rulesets
grow we will need a set of initial mutates to convert
'logs/screen-n-api.txt' and 'logs/screen-n-cpu.txt' tags to 'nova' to
allow for common grok and parsing filters.

Change-Id: I2769bc05a2e9cc7e8dbc46849e052146b9fee75e
Reviewed-on: https://review.openstack.org/28119
Reviewed-by: James E. Blair <corvus@inaugust.com>
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: Jeremy Stanley <fungi@yuggoth.org>
Tested-by: Jenkins
---
 indexer.conf.erb | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/indexer.conf.erb b/indexer.conf.erb
index 66b195c..54c200e 100644
--- a/indexer.conf.erb
+++ b/indexer.conf.erb
@@ -4,42 +4,44 @@ input {
     port => 9999
     format => "json"
     message_format => "%{event_message}"
-    tags => ["jenkins", "console"]
-    type => "jenkins_console"
+    type => "jenkins"
   }
 }
 
 # You can check grok patterns at http://grokdebug.herokuapp.com/
 filter {
   grep {
-    type => "jenkins_console"
+    type => "jenkins"
+    tags => ["console.html"]
     # Drop matches.
     negate => true
     match => ["@message", "^</?pre>$"]
   }
   multiline {
-    type => "jenkins_console"
+    type => "jenkins"
+    tags => ["console.html"]
     negate => true
     pattern => "^%{DATESTAMP} \|"
     what => "previous"
   }
   grok {
-    type => "jenkins_console"
+    type => "jenkins"
+    tags => ["console.html"]
     pattern => [ "^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ]
     add_field => [ "received_at", "%{@timestamp}" ]
   }
   date {
-    type => "jenkins_console"
+    type => "jenkins"
     exclude_tags => "_grokparsefailure"
     match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS" ]
   }
   mutate {
-    type => "jenkins_console"
+    type => "jenkins"
     exclude_tags => "_grokparsefailure"
     replace => [ "@message", "%{logmessage}" ]
   }
   mutate {
-    type => "jenkins_console"
+    type => "jenkins"
     exclude_tags => "_grokparsefailure"
     remove => [ "logdate", "logmessage" ]
   }