From d2d138c09caa62b549b1de102e29a52b2537aa76 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Tue, 6 Aug 2013 11:52:26 -0700
Subject: [PATCH] Don't index logs with DEBUG log level.

ElasticSearch has a hard time performing queries on large amounts of
data. It must load the fields it is searching on into memory and that
can cause the node to use all of the memory allocated to it which in
turn causes a lot of garbage collection essentially taking the node
offline. Filter out DEBUG log messages in the devstack logs to reduce
the amount of data that must be loaded into memory when performing
searches.

Change-Id: Icfe9c8c17ccef4f9379c774eef791f43463dcf6d
---
 indexer.conf.erb | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/indexer.conf.erb b/indexer.conf.erb
index d555059..fec7b5d 100644
--- a/indexer.conf.erb
+++ b/indexer.conf.erb
@@ -106,6 +106,12 @@ filter {
     pattern => [ "%{SYSLOGTIMESTAMP:logdate}%{SPACE}%{SYSLOGHOST:syslog_host}?%{SPACE}%{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:? %{GREEDYDATA:logmessage}" ]
     add_field => [ "received_at", "%{@timestamp}" ]
   }
+  # Remove DEBUG logs to reduce the amount of data that needs to be processed.
+  grep {
+    type => "jenkins"
+    negate => true
+    match => [ "loglevel", "DEBUG" ]
+  }
 
   # Filters below here should be consistent for all Jenkins log formats.
   date {