From f8e624d1c293697ede6f3c268f147c56f736f38e Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Thu, 30 May 2013 11:28:14 -0700
Subject: [PATCH] Add keystone logs to Logstash.

* modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml:
Add the keystone log file to the list of files to be processed by
logstash-worker1.

* modules/openstack_project/templates/logstash/indexer.conf.erb: Add
Logstash filters for keystone format files.

Change-Id: I5a72fc17ed1f37b816581faabe44f26f8cc36db2
Reviewed-on: https://review.openstack.org/31096
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Approved: James E. Blair <corvus@inaugust.com>
Reviewed-by: James E. Blair <corvus@inaugust.com>
Tested-by: Jenkins
---
 .../logstash-worker1/jenkins-log-pusher.yaml  |  6 +++-
 .../templates/logstash/indexer.conf.erb       | 29 ++++++++++++++++++-
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml b/modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml
index 5be9257ddb..47ed63b519 100644
--- a/modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml
+++ b/modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml
@@ -26,4 +26,8 @@ source-files:
     tags:
       - screen
       - oslofmt
-# TODO(clarkb) Add keystone, swift, and syslog logs here.
+  - name: logs/screen-key.txt
+    tags:
+      - screen
+      - keystonefmt
+# TODO(clarkb) Add swift and syslog logs here.
diff --git a/modules/openstack_project/templates/logstash/indexer.conf.erb b/modules/openstack_project/templates/logstash/indexer.conf.erb
index 0adf897f4a..60f8739472 100644
--- a/modules/openstack_project/templates/logstash/indexer.conf.erb
+++ b/modules/openstack_project/templates/logstash/indexer.conf.erb
@@ -26,6 +26,14 @@ filter {
     negate => true
     match => ["@message", "^\+ "]
   }
+  grep {
+    # Remove blank lines.
+    type => "jenkins"
+    tags => ["keystonefmt"]
+    # Drop matches.
+    negate => true
+    match => ["@message", "^$"]
+  }
   multiline {
     type => "jenkins"
     tags => ["console.html"]
@@ -42,6 +50,14 @@ filter {
     what => "previous"
     stream_identity => "%{@source_host}.%{filename}"
   }
+  multiline {
+    type => "jenkins"
+    tags => ["keystonefmt"]
+    negate => true
+    pattern => "^\(\b%{NOTSPACE}\b\):"
+    what => "previous"
+    stream_identity => "%{@source_host}.%{filename}"
+  }
   grok {
     type => "jenkins"
     tags => ["console.html"]
@@ -59,10 +75,21 @@ filter {
     pattern => [ "(?m)^%{DATESTAMP:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" ]
     add_field => [ "received_at", "%{@timestamp}" ]
   }
+  grok {
+    type => "jenkins"
+    tags => ["keystonefmt"]
+    # Do multiline matching as the above mutliline filter may add newlines
+    # to the log messages.
+    # TODO move the LOGLEVELs into a proper grok pattern.
+    pattern => [ "(?m)^\(\b%{NOTSPACE:module}\b\):%{SPACE}%{DATESTAMP:logdate}%{SPACE}(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR)%{SPACE}%{GREEDYDATA:logmessage}" ]
+    add_field => [ "received_at", "%{@timestamp}" ]
+  }
+
+  # Filters below here should be consistent for all Jenkins log formats.
   date {
     type => "jenkins"
     exclude_tags => "_grokparsefailure"
-    match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss" ]
+    match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss" ]
   }
   mutate {
     type => "jenkins"