Basic logstash and kibana dashboard configs

May not [probably won't] be applicable in all environments.
Encourage others to submit PRs against this to make it a more
generic config that should work for most setups.
This commit is contained in:
Mike Dorman 2014-10-07 09:43:57 -06:00
parent d67a31751f
commit 173fda225d
2 changed files with 657 additions and 0 deletions

452
logstash/kibana.json Normal file
View File

@ -0,0 +1,452 @@
{
"title": "Openstack Logs",
"services": {
"query": {
"list": {
"0": {
"id": 0,
"color": "#7EB26D",
"alias": "INFO",
"pin": false,
"type": "lucene",
"enable": true,
"query": "loglevel:INFO"
},
"1": {
"id": 1,
"color": "#EAB839",
"alias": "TRACE",
"pin": false,
"type": "lucene",
"enable": true,
"query": "loglevel:TRACE"
},
"2": {
"id": 2,
"color": "#6ED0E0",
"alias": "DEBUG",
"pin": false,
"type": "lucene",
"enable": true,
"query": "loglevel:DEBUG"
},
"3": {
"id": 3,
"color": "#5195CE",
"alias": "AUDIT",
"pin": false,
"type": "lucene",
"enable": true,
"query": "loglevel:AUDIT"
},
"4": {
"id": 4,
"color": "#C15C17",
"alias": "WARNING",
"pin": false,
"type": "lucene",
"enable": true,
"query": "loglevel:WARNING"
},
"9": {
"id": 9,
"type": "lucene",
"query": "loglevel:ERROR",
"alias": "ERROR",
"color": "#BF1B00",
"pin": false,
"enable": true
}
},
"ids": [
0,
1,
2,
3,
4,
9
]
},
"filter": {
"list": {
"0": {
"type": "time",
"field": "@timestamp",
"from": "now-1h",
"to": "now",
"mandate": "must",
"active": true,
"alias": "",
"id": 0
}
},
"ids": [
0
]
}
},
"rows": [
{
"title": "Histogram",
"height": "150px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 12,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "count",
"time_field": "@timestamp",
"value_field": null,
"x-axis": true,
"y-axis": true,
"scale": 1,
"y_format": "none",
"grid": {
"max": null,
"min": 0
},
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
4,
9
]
},
"annotate": {
"enable": false,
"query": "*",
"size": 20,
"field": "_type",
"sort": [
"_score",
"desc"
]
},
"auto_int": true,
"resolution": 100,
"interval": "30s",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1y"
],
"lines": false,
"fill": 0,
"linewidth": 3,
"points": false,
"pointradius": 5,
"bars": true,
"stack": true,
"spyable": true,
"zoomlinks": true,
"options": true,
"legend": true,
"show_query": true,
"interactive": true,
"legend_counts": true,
"timezone": "browser",
"percentage": false,
"zerofill": true,
"derivative": false,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": true
},
"title": "Events by Time",
"scaleSeconds": false
}
],
"notice": false
},
{
"title": "Graph",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 4,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "loglevel",
"exclude": [
""
],
"missing": false,
"other": false,
"size": 18,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "table",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
4,
9
]
},
"title": "Events",
"tmode": "terms",
"tstat": "total",
"valuefield": ""
},
{
"error": false,
"span": 4,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "host",
"exclude": [],
"missing": false,
"other": false,
"size": 10,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "table",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
4,
9
]
},
"title": "Events by Host",
"tmode": "terms",
"tstat": "total",
"valuefield": ""
},
{
"error": false,
"span": 4,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "_type",
"exclude": [],
"missing": false,
"other": false,
"size": 10,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "pie",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
4,
9
]
},
"title": "Events Type",
"tmode": "terms",
"tstat": "total",
"valuefield": ""
}
],
"notice": false
},
{
"title": "Events",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"title": "All events",
"error": false,
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "table",
"size": 100,
"pages": 5,
"offset": 0,
"sort": [
"@timestamp",
"desc"
],
"style": {
"font-size": "8pt"
},
"overflow": "min-height",
"fields": [
"@timestamp",
"host",
"loglevel",
"_type",
"module",
"logmessage"
],
"localTime": true,
"timeField": "@timestamp",
"highlight": [],
"sortable": true,
"header": true,
"paging": true,
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
4,
9
]
},
"field_list": false,
"status": "Stable",
"trimFactor": 700,
"normTimes": true,
"all_fields": false
}
],
"notice": false
}
],
"editable": true,
"failover": false,
"index": {
"interval": "day",
"pattern": "[logstash-]YYYY.MM.DD",
"default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED",
"warm_fields": true
},
"style": "dark",
"panel_hints": true,
"pulldowns": [
{
"type": "query",
"collapse": true,
"notice": false,
"query": "*",
"pinned": true,
"history": [
"loglevel:ERROR",
"loglevel:WARNING",
"loglevel:AUDIT",
"loglevel:DEBUG",
"loglevel:TRACE",
"loglevel:INFO",
],
"remember": 10,
"enable": true
},
{
"type": "filtering",
"collapse": false,
"notice": true,
"enable": true
}
],
"nav": [
{
"type": "timepicker",
"collapse": false,
"notice": false,
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"timefield": "@timestamp",
"now": true,
"filter_id": 0,
"enable": true
}
],
"loader": {
"save_gist": true,
"save_elasticsearch": true,
"save_local": true,
"save_default": true,
"save_temp": true,
"save_temp_ttl_enable": true,
"save_temp_ttl": "30d",
"load_gist": true,
"load_elasticsearch": true,
"load_elasticsearch_size": 20,
"load_local": true,
"hide": false
},
"refresh": "1m"
}

205
logstash/logstash.conf Normal file
View File

@ -0,0 +1,205 @@
# logstash.conf
#
# Basic logstash config and filters for injesting most logs from Nova,
# Keystone, Glance, Ceilometer, Heat, and Neutron services, as well as
# Apache (Horizon) and syslog.
#
# Author: Kris Lindgren <klindgren@godaddy.com>
#
# Copyright (c) 2014 Go Daddy Operating Company, LLC
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
#
input {
file {
path => ['/var/log/nova/nova-api.log']
tags => ['nova', 'oslofmt']
type => "nova"
}
file {
path => ['/var/log/nova/nova-conductor.log']
tags => ['nova', 'oslofmt']
type => "nova"
}
file {
path => ['/var/log/nova/nova-manage.log']
tags => ['nova', 'oslofmt']
type => "nova"
}
file {
path => ['/var/log/nova/nova-scheduler.log']
tags => ['nova', 'oslofmt']
type => "nova"
}
file {
path => ['/var/log/nova/nova-spicehtml5proxy.log']
tags => ['nova', 'oslofmt']
type => "nova"
}
file {
path => ['/var/log/keystone/keystone-all.log']
tags => ['keystone', 'keystonefmt']
type => "keystone"
}
file {
path => ['/var/log/keystone/keystone-manage.log']
tags => ['keystone', 'keystonefmt']
type => "keystone"
}
file {
path => ['/var/log/glance/api.log']
tags => ['glance', 'oslofmt']
type => "glance"
}
file {
path => ['/var/log/glance/registry.log']
tags => ['glance', 'oslofmt']
type => "glance"
}
file {
path => ['/var/log/glance/scrubber.log']
tags => ['glance', 'oslofmt']
type => "glance"
}
file {
path => ['/var/log/ceilometer/ceilometer-agent-central.log']
tags => ['ceilometer', 'oslofmt']
type => "ceilometer"
}
file {
path => ['/var/log/ceilometer/ceilometer-alarm-notifier.log']
tags => ['ceilometer', 'oslofmt']
type => "ceilometer"
}
file {
path => ['/var/log/ceilometer/ceilometer-api.log']
tags => ['ceilometer', 'oslofmt']
type => "ceilometer"
}
file {
path => ['/var/log/ceilometer/ceilometer-alarm-evaluator.log']
tags => ['ceilometer', 'oslofmt']
type => "ceilometer"
}
file {
path => ['/var/log/ceilometer/ceilometer-collector.log']
tags => ['ceilometer', 'oslofmt']
type => "ceilometer"
}
file {
path => ['/var/log/heat/heat.log']
tags => ['heat', 'oslofmt']
type => "heat"
}
file {
path => ['/var/log/neutron/neutron-server.log']
tags => ['neutron', 'oslofmt']
type => "neutron"
}
# Not collecting RabbitMQ logs for the moment
# file {
# path => ['/var/log/rabbitmq/rabbit@<%= @hostname %>.log']
# tags => ['rabbitmq', 'oslofmt']
# type => "rabbitmq"
# }
file {
path => ['/var/log/httpd/access_log']
tags => ['horizon']
type => "horizon"
}
file {
path => ['/var/log/httpd/error_log']
tags => ['horizon']
type => "horizon"
}
file {
path => ['/var/log/httpd/horizon_access_log']
tags => ['horizon']
type => "horizon"
}
file {
path => ['/var/log/httpd/horizon_error_log']
tags => ['horizon']
type => "horizon"
}
}
filter {
if "oslofmt" in [tags] {
multiline {
negate => true
pattern => "^%{TIMESTAMP_ISO8601} "
what => "previous"
}
multiline {
negate => false
pattern => "^%{TIMESTAMP_ISO8601}%{SPACE}%{NUMBER}?%{SPACE}?TRACE"
what => "previous"
}
grok {
# Do multiline matching as the above mutliline filter may add newlines
# to the log messages.
# TODO move the LOGLEVELs into a proper grok pattern.
match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" }
add_field => { "received_at" => "%{@timestamp}" }
}
} else if "keystonefmt" in [tags] {
grok {
# Do multiline matching as the above mutliline filter may add newlines
# to the log messages.
# TODO move the LOGLEVELs into a proper grok pattern.
match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}%{SPACE}%{NUMBER:pid}?%{SPACE}?(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) \[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" }
add_field => { "received_at" => "%{@timestamp}" }
}
if [module] == "iso8601.iso8601" {
#log message for each part of the date? Really?
drop {}
}
} else if "libvirt" in [tags] {
grok {
match => { "message" => "(?m)^%{TIMESTAMP_ISO8601:logdate}:%{SPACE}%{NUMBER:code}:?%{SPACE}\[?\b%{NOTSPACE:loglevel}\b\]?%{SPACE}?:?%{SPACE}\[?\b%{NOTSPACE:module}\b\]?%{SPACE}?%{GREEDYDATA:logmessage}?" }
add_field => { "received_at" => "%{@timestamp}"}
}
mutate {
uppercase => [ "loglevel" ]
}
} else if [type] == "syslog" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:logmessage}" }
add_field => [ "received_at", "%{@timestamp}" ]
}
syslog_pri {
severity_labels => ["ERROR", "ERROR", "ERROR", "ERROR", "WARNING", "INFO", "INFO", "DEBUG" ]
}
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
if !("_grokparsefailure" in [tags]) {
mutate {
replace => [ "@source_host", "%{syslog_hostname}" ]
}
}
mutate {
remove_field => [ "syslog_hostname", "syslog_timestamp" ]
add_field => [ "loglevel", "%{syslog_severity}" ]
add_field => [ "module", "%{syslog_program}" ]
}
}
}