From 98c90a0a531dc62ba79d81672adf0d0acbf9be94 Mon Sep 17 00:00:00 2001
From: David Wahlstrom <david.wahlstrom@dreamhost.com>
Date: Fri, 26 Feb 2016 11:14:07 -0800
Subject: [PATCH] Add logstash files for log parsing

This commit provides a couple of template files to be used with logstash
for parsing log files generated by the rug.  As provided, these files
will attempt to send metrics to graphite, but should prove useful as a
basis for other rug related logstash activities.

Change-Id: Ie3ab18c2b8bfd9765f066bcf34c5f572956b3f72
---
 tools/logstash/README.md             |  3 +++
 tools/logstash/logstash-rug-patterns |  4 +++
 tools/logstash/rug-filter.conf       | 39 ++++++++++++++++++++++++++++
 3 files changed, 46 insertions(+)
 create mode 100644 tools/logstash/README.md
 create mode 100644 tools/logstash/logstash-rug-patterns
 create mode 100644 tools/logstash/rug-filter.conf

diff --git a/tools/logstash/README.md b/tools/logstash/README.md
new file mode 100644
index 00000000..efe81c7c
--- /dev/null
+++ b/tools/logstash/README.md
@@ -0,0 +1,3 @@
+These files are used for logstash to read the astara-orchestrator log and send
+metrics to a graphite server.  Logstash has many output options, so these files
+serve as a template for parsing the astara-orchestrator log file.
diff --git a/tools/logstash/logstash-rug-patterns b/tools/logstash/logstash-rug-patterns
new file mode 100644
index 00000000..f7c981e1
--- /dev/null
+++ b/tools/logstash/logstash-rug-patterns
@@ -0,0 +1,4 @@
+WORKER p[0-9][0-9]
+THREAD tmain|t[0-9][0-9]
+ASTARA_STATE_METHOD (?:[a-zA-Z]*\.)*
+METHOD (?:astara.openstack.common.rpc.amqp|astara.worker|astara.api.configuration|astara.tenant|astara.state.|astara.scheduler)
diff --git a/tools/logstash/rug-filter.conf b/tools/logstash/rug-filter.conf
new file mode 100644
index 00000000..5a50a098
--- /dev/null
+++ b/tools/logstash/rug-filter.conf
@@ -0,0 +1,39 @@
+input {
+  file {
+     path => '/var/log/akrug/akrug.log'
+  }
+}
+
+filter {
+  grok {
+    patterns_dir => "/etc/logstash/patterns/logstash-rug-patterns"
+    match => {"message" => "%{TIMESTAMP_ISO8601:akanda_timestamp}:%{LOGLEVEL:loglevel}:%{ASTARA_STATE_METHOD:method}%{UUID:router_id}:%{POSINT:pid}:%{WORKER:worker}:%{THREAD:thread}:incoming message brings queue length to %{INT:queue_length}"}
+    add_tag => [ "queue_length" ]
+  }
+
+  grok {
+    patterns_dir => "/etc/logstash/patterns/logstash-rug-patterns"
+    match => {"message" => "%{TIMESTAMP_ISO8601:akanda_timestamp}:%{LOGLEVEL:loglevel}:%{METHOD:method}:%{POSINT:pid}:%{WORKER:worker}:%{THREAD:thread}"}
+    add_tag => [ "threads" ]
+  }
+
+  date {
+    match => [ "akanda_timestamp", "YYYY-MM-dd HH:mm:ss" ]
+  }
+}
+
+output {
+  if "queue_length" in [tags] {
+    graphite {
+      host => "127.0.0.1"
+      metrics => { "%{router_id}/queue_length" => "%{queue_length}" }
+      metrics_format => "openstack.astara.rug.routers.*"
+    }
+  } else if "threads" in [tags] {
+    graphite {
+      host => "127.0.0.1"
+      metrics => { "%{worker}/%{thread}" => "1" }
+      metrics_format => "openstack.astara.rug.workers.*"
+    }
+  }
+}