Merge pull request #2 from Chealion/logstash-dashboards

Show examples from Cybera
This commit is contained in:
Mike Dorman 2015-01-06 14:05:29 -07:00
commit 0352cc1903
12 changed files with 1977 additions and 0 deletions

View File

@ -2,3 +2,7 @@ tools-logging
=============
OpenStack Logging Tools
logstash
Example dashboards for Kibana and logging configurations for logstash.

4
logstash/basic/README.md Normal file
View File

@ -0,0 +1,4 @@
# Basic
Basic logstash config and filters for ingesting most logs from OpenStack services. Courtesy Kris Lindgren from GoDaddy.

View File

@ -0,0 +1,496 @@
{
"title": "Default View",
"services": {
"query": {
"list": {
"0": {
"query": "\"region1\"",
"alias": "region1 Logs",
"color": "#7EB26D",
"id": 0,
"pin": false,
"type": "lucene",
"enable": true
},
"1": {
"id": 1,
"color": "#EAB839",
"alias": "Region2 Logs",
"pin": false,
"type": "lucene",
"enable": true,
"query": "\"region2\""
},
"2": {
"id": 2,
"color": "#7EB26D",
"alias": "Instances Spawned",
"pin": false,
"type": "lucene",
"enable": true,
"query": "\"Instance spawned successfully\""
},
"3": {
"id": 3,
"color": "#EAB839",
"alias": "Instances Destroyed",
"pin": false,
"type": "lucene",
"enable": true,
"query": "\"Instance destroyed successfully\""
},
"4": {
"id": 4,
"color": "#6ED0E0",
"alias": "Snapshots created",
"pin": false,
"type": "lucene",
"enable": true,
"query": "\"Snapshot image upload complete\""
},
"5": {
"id": 5,
"color": "#1F78C1",
"alias": "Volumes Created",
"pin": false,
"type": "lucene",
"enable": true,
"query": "\"created -using\" AND loglevel:\"INFO\" AND module:\"cinder.volume.flows.create_volume\""
},
"6": {
"id": 6,
"color": "#BA43A9",
"alias": "Volumes Deleted",
"pin": false,
"type": "lucene",
"enable": true,
"query": "\"deleted\" AND loglevel:\"INFO\" AND module:\"cinder.volume.manager\""
}
},
"ids": [
0,
1,
2,
3,
4,
5,
6
]
},
"filter": {
"list": {
"0": {
"type": "time",
"field": "@timestamp",
"from": "now-30d",
"to": "now",
"mandate": "must",
"active": true,
"alias": "",
"id": 0
}
},
"ids": [
0
]
}
},
"rows": [
{
"title": "Graph",
"height": "250px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "histogram",
"mode": "count",
"time_field": "@timestamp",
"value_field": null,
"auto_int": false,
"resolution": 100,
"interval": "1d",
"fill": 3,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": true,
"lines": false,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"queries": {
"mode": "selected",
"ids": [
2,
3,
4,
5,
6
]
},
"title": "Events over time",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": true
},
"scale": 1,
"y_format": "none",
"grid": {
"max": null,
"min": 0
},
"annotate": {
"enable": false,
"query": "*",
"size": 20,
"field": "_type",
"sort": [
"_score",
"desc"
]
},
"pointradius": 5,
"show_query": true,
"legend_counts": true,
"zerofill": false,
"derivative": false
},
{
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "histogram",
"mode": "count",
"time_field": "@timestamp",
"value_field": null,
"auto_int": true,
"resolution": 100,
"interval": "12h",
"fill": 3,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": false,
"lines": true,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"queries": {
"mode": "selected",
"ids": [
0,
1
]
},
"title": "Events over time",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": true
},
"scale": 1,
"y_format": "none",
"grid": {
"max": null,
"min": 0
},
"annotate": {
"enable": false,
"query": "*",
"size": 20,
"field": "_type",
"sort": [
"_score",
"desc"
]
},
"pointradius": 5,
"show_query": true,
"legend_counts": true,
"zerofill": true,
"derivative": false
}
],
"notice": false
},
{
"title": "Instances/Volumes",
"height": "250",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 12,
"editable": true,
"type": "trends",
"loadingEditor": false,
"ago": "1d",
"arrangement": "horizontal",
"reverse": false,
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
4,
5,
6
]
},
"style": {
"font-size": "16pt"
},
"title": "Compared to Yesterday..."
},
{
"span": 12,
"editable": true,
"type": "trends",
"loadingEditor": false,
"ago": "1w",
"arrangement": "horizontal",
"reverse": false,
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
4,
5,
6
]
},
"style": {
"font-size": "16pt"
},
"title": "Compared to Last Week..."
},
{
"span": 12,
"editable": true,
"type": "trends",
"loadingEditor": false,
"ago": "4w",
"arrangement": "horizontal",
"reverse": false,
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
4,
5,
6
]
},
"style": {
"font-size": "16pt"
},
"title": "Compared to Last Month..."
}
],
"notice": false
},
{
"title": "Events",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"title": "All events",
"error": false,
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "table",
"size": 100,
"pages": 5,
"offset": 0,
"sort": [
"@timestamp",
"desc"
],
"style": {
"font-size": "9pt"
},
"overflow": "min-height",
"fields": [
"@timestamp",
"logmessage",
"syslog_hostname"
],
"localTime": true,
"timeField": "@timestamp",
"highlight": [],
"sortable": true,
"header": true,
"paging": true,
"spyable": true,
"queries": {
"mode": "pinned",
"ids": []
},
"field_list": true,
"status": "Stable",
"trimFactor": 300,
"normTimes": true,
"all_fields": false
}
],
"notice": false
}
],
"editable": true,
"failover": false,
"index": {
"interval": "day",
"pattern": "[logstash-]YYYY.MM.DD",
"default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED",
"warm_fields": true
},
"style": "dark",
"panel_hints": true,
"pulldowns": [
{
"type": "query",
"collapse": true,
"notice": false,
"query": "*",
"pinned": true,
"history": [
"\"deleted\" AND loglevel:\"INFO\" AND module:\"cinder.volume.manager\"",
"\"created -using\" AND loglevel:\"INFO\" AND module:\"cinder.volume.flows.create_volume\"",
"\"Snapshot image upload complete\"",
"\"Instance destroyed successfully\"",
"\"Instance spawned successfully\"",
"\"rac-yeg\"",
"\"rac-yyc\"",
"\"deleted\" AND loglevel:\"INFO\" AND module:\"cinder.volume\"",
"\"created -using\" AND loglevel:\"INFO\" AND module:\"cinder\"",
"\"deleted\" AND loglevel:\"INFO\" AND module:\"cinder\""
],
"remember": 10,
"enable": true
},
{
"type": "filtering",
"collapse": true,
"notice": true,
"enable": true
}
],
"nav": [
{
"type": "timepicker",
"collapse": false,
"notice": false,
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"timefield": "@timestamp",
"now": true,
"filter_id": 0,
"enable": true
}
],
"loader": {
"save_gist": false,
"save_elasticsearch": true,
"save_local": true,
"save_default": true,
"save_temp": true,
"save_temp_ttl_enable": true,
"save_temp_ttl": "30d",
"load_gist": true,
"load_elasticsearch": true,
"load_elasticsearch_size": 20,
"load_local": true,
"hide": false
},
"refresh": "15m"
}

View File

@ -0,0 +1,319 @@
{
"title": "Instance Spawns and Destroys",
"services": {
"query": {
"list": {
"0": {
"query": "\"Instance spawned successfully\"",
"alias": "Instances Spawned",
"color": "#7EB26D",
"id": 0,
"pin": false,
"type": "lucene",
"enable": true
},
"1": {
"id": 1,
"color": "#EAB839",
"alias": "Instances Destroyed",
"pin": false,
"type": "lucene",
"enable": true,
"query": "\"Instance destroyed successfully\""
},
"2": {
"id": 2,
"color": "#6ED0E0",
"alias": "Snapshots created",
"pin": false,
"type": "lucene",
"enable": true,
"query": "\"Snapshot image upload complete\""
}
},
"ids": [
0,
1,
2
]
},
"filter": {
"list": {
"0": {
"type": "time",
"field": "@timestamp",
"from": "now-30d",
"to": "now",
"mandate": "must",
"active": true,
"alias": "",
"id": 0
}
},
"ids": [
0
]
}
},
"rows": [
{
"title": "Graph",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "histogram",
"mode": "count",
"time_field": "@timestamp",
"value_field": null,
"auto_int": false,
"resolution": 100,
"interval": "24h",
"fill": 0,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": true,
"lines": false,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2
]
},
"title": "Events over time",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": true
},
"scale": 1,
"y_format": "none",
"grid": {
"max": null,
"min": 0
},
"annotate": {
"enable": false,
"query": "*",
"size": 20,
"field": "_type",
"sort": [
"_score",
"desc"
]
},
"pointradius": 3,
"show_query": true,
"legend_counts": true,
"zerofill": false,
"derivative": false,
"scaleSeconds": false
}
],
"notice": false
},
{
"title": "Stats",
"height": "50px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 4,
"editable": true,
"type": "trends",
"loadingEditor": false,
"ago": "1w",
"arrangement": "horizontal",
"reverse": false,
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2
]
},
"style": {
"font-size": "14pt"
},
"title": "Compared to last week..."
}
],
"notice": false
},
{
"title": "Events",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"title": "All events",
"error": false,
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "table",
"size": 100,
"pages": 5,
"offset": 0,
"sort": [
"@timestamp",
"desc"
],
"style": {
"font-size": "9pt"
},
"overflow": "min-height",
"fields": [],
"localTime": true,
"timeField": "@timestamp",
"highlight": [],
"sortable": true,
"header": true,
"paging": true,
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2
]
},
"field_list": true,
"status": "Stable",
"trimFactor": 300,
"normTimes": true,
"all_fields": false
}
],
"notice": false
}
],
"editable": true,
"failover": false,
"index": {
"interval": "day",
"pattern": "[logstash-]YYYY.MM.DD",
"default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED",
"warm_fields": true
},
"style": "dark",
"panel_hints": true,
"pulldowns": [
{
"type": "query",
"collapse": true,
"notice": false,
"query": "*",
"pinned": true,
"history": [
"\"Snapshot image upload complete\"",
"\"Instance destroyed successfully\"",
"\"Instance spawned successfully\"",
"\"Snapshot upload complete\"",
"\"Instance Spawned Successfully\"",
"Instance",
"Created",
],
"remember": 10,
"enable": true
},
{
"type": "filtering",
"collapse": true,
"notice": false,
"enable": true
}
],
"nav": [
{
"type": "timepicker",
"collapse": false,
"notice": false,
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"timefield": "@timestamp",
"now": true,
"filter_id": 0,
"enable": true
}
],
"loader": {
"save_gist": false,
"save_elasticsearch": true,
"save_local": true,
"save_default": true,
"save_temp": true,
"save_temp_ttl_enable": true,
"save_temp_ttl": "30d",
"load_gist": true,
"load_elasticsearch": true,
"load_elasticsearch_size": 20,
"load_local": true,
"hide": false
},
"refresh": false
}

View File

@ -0,0 +1,298 @@
{
"title": "Migrations",
"services": {
"query": {
"list": {
"0": {
"query": "\"Going to try to live migrate instance to\"",
"alias": "Migration Attempts",
"color": "#B7DBAB",
"id": 0,
"pin": false,
"type": "lucene",
"enable": true
},
"1": {
"id": 1,
"color": "#7EB26D",
"alias": "Migration Success",
"pin": false,
"type": "lucene",
"enable": true,
"query": "\"base_of_fqdn_goes_here finished successfully\""
},
"2": {
"id": 2,
"color": "#890F02",
"alias": "Migration Failures",
"pin": false,
"type": "lucene",
"enable": true,
"query": "\"Live Migration failure\""
}
},
"ids": [
0,
1,
2
]
},
"filter": {
"list": {
"0": {
"from": "2014-09-03T19:02:17.256Z",
"to": "now",
"type": "time",
"field": "@timestamp",
"mandate": "must",
"active": true,
"alias": "",
"id": 0
},
"1": {
"type": "time",
"from": "2014-09-03T19:25:36.941Z",
"to": "2014-09-03T20:54:09.058Z",
"field": "@timestamp",
"mandate": "must",
"active": true,
"alias": "",
"id": 1
}
},
"ids": [
0,
1
]
}
},
"rows": [
{
"title": "Graph",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "histogram",
"mode": "count",
"time_field": "@timestamp",
"value_field": null,
"auto_int": true,
"resolution": 100,
"interval": "1m",
"fill": 3,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": true,
"stack": true,
"points": false,
"lines": false,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2
]
},
"title": "Events over time",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"options": true,
"tooltip": {
"value_type": "individual",
"query_as_alias": true
},
"scale": 1,
"y_format": "short",
"grid": {
"max": null,
"min": 0
},
"annotate": {
"enable": false,
"query": "*",
"size": 20,
"field": "_type",
"sort": [
"_score",
"desc"
]
},
"pointradius": 5,
"show_query": true,
"legend_counts": true,
"zerofill": false,
"derivative": false
}
],
"notice": false
},
{
"title": "Events",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"title": "All events",
"error": false,
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "table",
"size": 100,
"pages": 5,
"offset": 0,
"sort": [
"@timestamp",
"desc"
],
"style": {
"font-size": "9pt"
},
"overflow": "min-height",
"fields": [
"@timestamp",
"logmessage",
"@source_host"
],
"localTime": true,
"timeField": "@timestamp",
"highlight": [],
"sortable": true,
"header": true,
"paging": true,
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2
]
},
"field_list": true,
"status": "Stable",
"trimFactor": 300,
"normTimes": true,
"all_fields": false
}
],
"notice": false
}
],
"editable": true,
"failover": false,
"index": {
"interval": "day",
"pattern": "[logstash-]YYYY.MM.DD",
"default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED",
"warm_fields": true
},
"style": "dark",
"panel_hints": true,
"pulldowns": [
{
"type": "query",
"collapse": false,
"notice": false,
"query": "*",
"pinned": true,
"history": [
"\"Live Migration failure\"",
"\"Going to try to live migrate instance to\"",
"\"Migrate instance to\"",
"\"Migrate instance to\" AND \"finished successfully\""
],
"remember": 10,
"enable": true
},
{
"type": "filtering",
"collapse": true,
"notice": true,
"enable": true
}
],
"nav": [
{
"type": "timepicker",
"collapse": false,
"notice": false,
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"timefield": "@timestamp",
"now": false,
"filter_id": 0,
"enable": true
}
],
"loader": {
"save_gist": false,
"save_elasticsearch": true,
"save_local": true,
"save_default": true,
"save_temp": true,
"save_temp_ttl_enable": true,
"save_temp_ttl": "30d",
"load_gist": true,
"load_elasticsearch": true,
"load_elasticsearch_size": 20,
"load_local": true,
"hide": false
},
"refresh": false
}

50
logstash/cybera/README.md Normal file
View File

@ -0,0 +1,50 @@
# Cybera
Example config and dashboards developed at Cybera for our public clouds. Most of the dashboards are largely to see what kind of information can be pulled from the logs or as alternatives to watching for specific log entries in a very busy `tail -f` stream.
Setup:
All services are set to DEBUG and to log to syslog, and the nodes then forward to a central rsyslog server that runs [beaver](https://github.com/josegonzalez/python-beaver) to push and tag the logs to our Rabbit cluster. The logstash agent then pulls the logs from Rabbit.
Caveats:
Beaver can only manage pushing up to 350 events/sec due to the way the Pika (Rabbit) library is used. If better performance is needed - look at the Redis options.
## Dashboards
### DefaultView.json
The Default View gives an overview of the number of logs between regions, along with some log based counts of instance creation/deletion, volume creation/deletion, and snapshot creation.
<Screenshot>
### SnapshotCheckpoints.json
Shows the "checkpoints" of instance snapshotting.
<Screenshot>
### InstanceCRUD.json
Shows instance creation and deletion along with snapshot creation points.
<Screenshot>
### Migrations.json
Shows the "checkpoints" of instance migration. One of the queries needs to be changed to the base of your compute node's fqdn. (node1.example.com would be just example.com)
<Screenshot>
### VolumeCRUD.json
Shows volume creation and deletion.
<Screenshot>
## Logstash
Logstash.conf - example downloading from rabbit.
## Beaver
beaver.conf - The example beaver config showing what we tag logs with.

View File

@ -0,0 +1,356 @@
{
"title": "Instance Checkpoint Checks",
"services": {
"query": {
"list": {
"0": {
"query": "\"instance snapshotting\"",
"alias": "",
"color": "#7EB26D",
"id": 0,
"pin": false,
"type": "lucene",
"enable": true
},
"1": {
"id": 1,
"color": "#EAB839",
"alias": "",
"pin": false,
"type": "lucene",
"enable": true,
"query": "\"Beginning live snapshot process\""
},
"2": {
"id": 2,
"color": "#6ED0E0",
"alias": "",
"pin": false,
"type": "lucene",
"enable": true,
"query": "\"Snapshot extracted\""
},
"3": {
"id": 3,
"color": "#EF843C",
"alias": "",
"pin": false,
"type": "lucene",
"enable": true,
"query": "\"Uploading image data for image\""
},
"4": {
"id": 4,
"color": "#E24D42",
"alias": "",
"pin": false,
"type": "lucene",
"enable": true,
"query": "\"bytes to /var/lib/glance/images\""
},
"5": {
"id": 5,
"color": "#1F78C1",
"alias": "",
"pin": false,
"type": "lucene",
"enable": true,
"query": "\"Snapshot image upload complete\""
}
},
"ids": [
0,
1,
2,
3,
4,
5
]
},
"filter": {
"list": {
"0": {
"type": "time",
"field": "@timestamp",
"from": "now-24h",
"to": "now",
"mandate": "must",
"active": true,
"alias": "",
"id": 0
},
"1": {
"type": "field",
"field": "logmessage",
"query": "\"UUID GOES HERE\"",
"mandate": "must",
"active": true,
"alias": "",
"id": 1
}
},
"ids": [
0,
1
]
}
},
"rows": [
{
"title": "Instructions",
"height": "50px",
"editable": true,
"collapse": false,
"collapsable": false,
"panels": [
{
"error": false,
"span": 12,
"editable": true,
"type": "text",
"loadingEditor": false,
"mode": "markdown",
"content": "Add the **instance** UUID to the logmessage filter to monitor the times the snapshot hits it's Glance checkpoints. Alternatively remove the filter to see all instances.\n\nPlease note the last two points (image saved and bytes saved) will show results when an image is uploaded as well.",
"style": {},
"title": "Instructions"
}
],
"notice": false
},
{
"title": "Graph",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "histogram",
"mode": "count",
"time_field": "@timestamp",
"value_field": null,
"auto_int": true,
"resolution": 100,
"interval": "10m",
"fill": 3,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": true,
"stack": true,
"points": false,
"lines": false,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
4,
5
]
},
"title": "Events over time",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"options": true,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": true
},
"scale": 1,
"y_format": "none",
"grid": {
"max": null,
"min": 0
},
"annotate": {
"enable": false,
"query": "*",
"size": 20,
"field": "_type",
"sort": [
"_score",
"desc"
]
},
"pointradius": 5,
"show_query": true,
"legend_counts": true,
"zerofill": false,
"derivative": false
}
],
"notice": false
},
{
"title": "Events",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"title": "All events",
"error": false,
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "table",
"size": 100,
"pages": 5,
"offset": 0,
"sort": [
"@timestamp",
"desc"
],
"style": {
"font-size": "9pt"
},
"overflow": "min-height",
"fields": [],
"localTime": true,
"timeField": "@timestamp",
"highlight": [],
"sortable": true,
"header": true,
"paging": true,
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1,
2,
3,
4,
5
]
},
"field_list": true,
"status": "Stable",
"trimFactor": 300,
"normTimes": true,
"all_fields": false
}
],
"notice": false
}
],
"editable": true,
"failover": false,
"index": {
"interval": "day",
"pattern": "[logstash-]YYYY.MM.DD",
"default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED",
"warm_fields": true
},
"style": "dark",
"panel_hints": true,
"pulldowns": [
{
"type": "query",
"collapse": true,
"notice": false,
"query": "*",
"pinned": true,
"history": [
"\"Snapshot image upload complete\"",
"\"bytes to /var/lib/glance/images\"",
"\"Uploading image data for image\"",
"\"Snapshot extracted\"",
"\"Beginning live snapshot process\"",
"\"instance snapshotting\"",
"Snapshot extracted",
"Beginning live snapshot process",
"instance snapshotting",
"[instance: *] Beginning live snapshot process"
],
"remember": 10,
"enable": true
},
{
"type": "filtering",
"collapse": false,
"notice": true,
"enable": true
}
],
"nav": [
{
"type": "timepicker",
"collapse": false,
"notice": false,
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"timefield": "@timestamp",
"now": true,
"filter_id": 0,
"enable": true
}
],
"loader": {
"save_gist": false,
"save_elasticsearch": true,
"save_local": true,
"save_default": true,
"save_temp": true,
"save_temp_ttl_enable": true,
"save_temp_ttl": "30d",
"load_gist": true,
"load_elasticsearch": true,
"load_elasticsearch_size": 20,
"load_local": true,
"hide": false
},
"refresh": false
}

View File

@ -0,0 +1,297 @@
{
"title": "Volume Creation and Deletion",
"services": {
"query": {
"list": {
"0": {
"query": "created -using",
"alias": "Created",
"color": "#7EB26D",
"id": 0,
"pin": false,
"type": "lucene",
"enable": true
},
"1": {
"id": 1,
"color": "#EAB839",
"alias": "Deleted",
"pin": false,
"type": "lucene",
"enable": true,
"query": "deleted"
}
},
"ids": [
0,
1
]
},
"filter": {
"list": {
"0": {
"type": "time",
"field": "@timestamp",
"from": "now-30d",
"to": "now",
"mandate": "must",
"active": true,
"alias": "",
"id": 0
},
"1": {
"type": "field",
"field": "module",
"query": "\"cinder.volume.flows.create_volume\" or \"cinder.volume.manager\"",
"mandate": "either",
"active": true,
"alias": "",
"id": 1
},
"2": {
"type": "field",
"field": "loglevel",
"query": "\"INFO\"",
"mandate": "must",
"active": true,
"alias": "",
"id": 2
}
},
"ids": [
0,
1,
2
]
}
},
"rows": [
{
"title": "Graph",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "histogram",
"mode": "count",
"time_field": "@timestamp",
"value_field": null,
"auto_int": false,
"resolution": 100,
"interval": "24h",
"fill": 3,
"linewidth": 3,
"timezone": "browser",
"spyable": true,
"zoomlinks": true,
"bars": false,
"stack": false,
"points": true,
"lines": false,
"legend": true,
"x-axis": true,
"y-axis": true,
"percentage": false,
"interactive": true,
"queries": {
"mode": "all",
"ids": [
0,
1
]
},
"title": "Volume Events",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1M",
"1y"
],
"options": true,
"tooltip": {
"value_type": "individual",
"query_as_alias": true
},
"scale": 1,
"y_format": "none",
"grid": {
"max": null,
"min": 0
},
"annotate": {
"enable": false,
"query": "*",
"size": 20,
"field": "_type",
"sort": [
"_score",
"desc"
]
},
"pointradius": 5,
"show_query": true,
"legend_counts": true,
"zerofill": false,
"derivative": false
}
],
"notice": false
},
{
"title": "Events",
"height": "350px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"title": "All events",
"error": false,
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "table",
"size": 100,
"pages": 5,
"offset": 0,
"sort": [
"@timestamp",
"desc"
],
"style": {
"font-size": "9pt"
},
"overflow": "min-height",
"fields": [],
"localTime": true,
"timeField": "@timestamp",
"highlight": [],
"sortable": true,
"header": true,
"paging": true,
"spyable": true,
"queries": {
"mode": "all",
"ids": [
0,
1
]
},
"field_list": true,
"status": "Stable",
"trimFactor": 300,
"normTimes": true,
"all_fields": false
}
],
"notice": false
}
],
"editable": true,
"failover": false,
"index": {
"interval": "day",
"pattern": "[logstash-]YYYY.MM.DD",
"default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED",
"warm_fields": true
},
"style": "dark",
"panel_hints": true,
"pulldowns": [
{
"type": "query",
"collapse": false,
"notice": false,
"query": "*",
"pinned": true,
"history": [
"deleted",
"created -using",
"created",
"successfully",
"f2b10018-f9eb-424b-ad7b-669cc691687b",
"created successfully",
"\"cinder.volume.flows.create_volume\" + message:\"created successfully\"",
"\"cinder.volume.flows.create_volume\" message:\"created successfully\"",
"\"cinder.volume.flows.create_volume\" message:succesfully",
"\"cinder.volume.flows.create_volume\" + succesfully"
],
"remember": 10,
"enable": true
},
{
"type": "filtering",
"collapse": true,
"notice": true,
"enable": true
}
],
"nav": [
{
"type": "timepicker",
"collapse": false,
"notice": false,
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"timefield": "@timestamp",
"now": true,
"filter_id": 0,
"enable": true
}
],
"loader": {
"save_gist": false,
"save_elasticsearch": true,
"save_local": true,
"save_default": true,
"save_temp": true,
"save_temp_ttl_enable": true,
"save_temp_ttl": "30d",
"load_gist": true,
"load_elasticsearch": true,
"load_elasticsearch_size": 20,
"load_local": true,
"hide": false
},
"refresh": false
}

View File

@ -0,0 +1,37 @@
[beaver]
rabbitmq_host = rabbitmqcluster_fqdn
rabbitmq_password = password
format = msgpack
rabbitmq_vhost = rsyslog
rabbitmq_exchange_type = direct
rabbitmq_queue_durable = 1
rabbitmq_username = logstash
rabbitmq_ssl = 1
logstash_version = 1
rabbitmq_queue = logstash
rabbitmq_exchange_durable = 0
rabbitmq_exchange = region1-logs
[/var/log/rsyslog/swift.log]
tags = cloud,region1,openstack,swift,swiftfmt
type = swift
[/var/log/rsyslog/nova.log]
tags = cloud,region1,openstack,nova,oslofmt
type = nova
[/var/log/rsyslog/syslog.log]
tags = cloud,region1,syslogfmt
type = syslog
[/var/log/rsyslog/cinder.log]
type = cinder
tags = cloud,region1,openstack,cinder,oslofmt
[/var/log/rsyslog/keystone.log]
tags = cloud,region1,openstack,keystone,oslofmt
type = keystone
[/var/log/rsyslog/glance.log]
tags = cloud,region1,openstack,glance,oslofmt
type = glance

View File

@ -0,0 +1,116 @@
input {
# Region 1
rabbitmq {
codec => "msgpack"
debug => true
host => "region1.cybera.ca"
exchange => "region1-logs"
user => "logstash"
password => "password"
ssl => true
port => "5672"
vhost => "rsyslog"
auto_delete => false
durable => true
key => 'logstash'
exclusive => false
passive => true
queue => 'logstash'
}
# Region 2
rabbitmq {
codec => "msgpack"
debug => true
host => "region2.cybera.ca"
exchange => "region1-logs"
user => "logstash"
password => "password"
ssl => true
port => "5672"
vhost => "rsyslog"
auto_delete => false
durable => true
key => 'logstash'
exclusive => false
passive => true
queue => 'logstash'
}
}
filter {
if "oslofmt" in [tags] {
grok {
match => { "message" => "^%{TIMESTAMP_ISO8601:logdate} %{SYSLOGHOST:syslog_hostname} %{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{NUMBER:syslog_pid} (?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) %{NOTSPACE:module} (?<ref_id_group>\[%{NOTSPACE:ref_id}?%{DATA:ref_id2}\]) %{GREEDYDATA:logmessage}" }
add_field => { "received_at" => "%{@timestamp}" }
}
if !("_grokparsefailure" in [tags]) {
mutate {
replace => [ "@source_host", "%{syslog_hostname}" ]
gsub => [ "message", "#012", "\
"]
}
}
# Make sure we set @timestamp to the log date
date {
match => [ "logdate", "ISO8601" ]
locale => "en"
target => "@timestamp"
}
} else if "syslogfmt" in [tags] {
grok {
match => { "message" => "^%{TIMESTAMP_ISO8601:logdate} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:logmessage}" }
add_field => [ "received_at", "%{@timestamp}" ]
}
date {
match => [ "logdate", "ISO8601" ]
locale => "en"
target => "@timestamp"
}
syslog_pri {
severity_labels => ["ERROR", "ERROR", "ERROR", "ERROR", "WARNING", "INFO", "INFO", "DEBUG" ]
}
if !("_grokparsefailure" in [tags]) {
mutate {
replace => [ "@source_host", "%{syslog_hostname}" ]
}
}
mutate {
add_field => [ "loglevel", "%{syslog_severity}" ]
add_field => [ "module", "%{syslog_program}" ]
}
} else if "swiftfmt" in [tags] {
grok {
match => { "message" => "^%{TIMESTAMP_ISO8601:logdate} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program} %{GREEDYDATA:logmessage}" }
add_field => [ "received_at", "%{@timestamp}" ]
}
syslog_pri {
severity_labels => ["ERROR", "ERROR", "ERROR", "ERROR", "WARNING", "INFO", "INFO", "DEBUG" ]
}
if !("_grokparsefailure" in [tags]) {
mutate {
replace => [ "@source_host", "%{syslog_hostname}" ]
}
}
mutate {
add_field => [ "loglevel", "%{syslog_severity}" ]
add_field => [ "module", "%{syslog_program}" ]
}
date {
match => [ "logdate", "ISO8601" ]
locale => "en"
target => "@timestamp"
}
}
}
output {
elasticsearch_http {
host => "127.0.0.1"
}
}