From 3a3e0a1f22b914e232422cb6fd2db3023b02c0a2 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Wed, 24 Apr 2013 15:20:14 -0700
Subject: [PATCH] Run single node logstash without redis.

logstash.o.o is running into memory contention. The three logstash
processes (agent, indexer, web), redis, and elasticsearch are all memory
hogs. Remove redis which allows us to consolidate the work of the agent
and indexer into a single process.

If we decide to add remote logstash agents we will want to add redis to
the mix again. This is probably best done on a second host dedicated to
redis.

Change-Id: I5ba96b4a27d91cb73c9380ed47d0495b9005effa
Reviewed-on: https://review.openstack.org/27452
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Reviewed-by: James E. Blair <corvus@inaugust.com>
Approved: Clark Boylan <clark.boylan@gmail.com>
Tested-by: Jenkins
---
 indexer.conf.erb | 49 ++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 41 insertions(+), 8 deletions(-)

diff --git a/indexer.conf.erb b/indexer.conf.erb
index 386dbd3..5ab6825 100644
--- a/indexer.conf.erb
+++ b/indexer.conf.erb
@@ -1,13 +1,46 @@
 input {
-  redis {
-    host => "127.0.0.1"
-    type => "redis-input"
-    # these settings should match the output of the agent
-    data_type => "list"
-    key => "logstash"
+  pipe {
+    command => "python3 /usr/local/bin/log-pusher.py -r -z tcp://jenkins.openstack.org:8888 -l http://logs.openstack.org -f console.html"
+    format => "json"
+    message_format => "%{event_message}"
+    tags => ["jenkins", "console"]
+    type => "jenkins_console"
+  }
+}
 
-    # We use json_event here since the sender is a logstash agent
-    format => "json_event"
+# You can check grok patterns at http://grokdebug.herokuapp.com/
+filter {
+  grep {
+    type => "jenkins_console"
+    # Drop matches.
+    negate => true
+    match => ["@message", "^</?pre>$"]
+  }
+  multiline {
+    type => "jenkins_console"
+    negate => true
+    pattern => "^%{DATESTAMP} \|"
+    what => "previous"
+  }
+  grok {
+    type => "jenkins_console"
+    pattern => [ "^%{DATESTAMP:logdate} \| %{GREEDYDATA:logmessage}" ]
+    add_field => [ "received_at", "%{@timestamp}" ]
+  }
+  date {
+    type => "jenkins_console"
+    exclude_tags => "_grokparsefailure"
+    match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS" ]
+  }
+  mutate {
+    type => "jenkins_console"
+    exclude_tags => "_grokparsefailure"
+    replace => [ "@message", "%{logmessage}" ]
+  }
+  mutate {
+    type => "jenkins_console"
+    exclude_tags => "_grokparsefailure"
+    remove => [ "logdate", "logmessage" ]
   }
 }