From a0d274373c30f677e463aca484e81835d7963730 Mon Sep 17 00:00:00 2001 From: Micheal Jones Date: Tue, 6 Jan 2015 12:42:54 -0700 Subject: [PATCH] Show examples from Cybera. --- README.md | 4 + logstash/basic/README.md | 4 + logstash/{ => basic}/kibana.json | 0 logstash/{ => basic}/logstash.conf | 0 logstash/cybera/DefaultView.json | 496 +++++++++++++++++++++++ logstash/cybera/InstanceCRUD.json | 319 +++++++++++++++ logstash/cybera/Migrations.json | 298 ++++++++++++++ logstash/cybera/README.md | 50 +++ logstash/cybera/SnapshotCheckpoints.json | 356 ++++++++++++++++ logstash/cybera/VolumeCRUD.json | 297 ++++++++++++++ logstash/cybera/beaver.conf | 37 ++ logstash/cybera/logstash.conf | 116 ++++++ 12 files changed, 1977 insertions(+) create mode 100644 logstash/basic/README.md rename logstash/{ => basic}/kibana.json (100%) rename logstash/{ => basic}/logstash.conf (100%) create mode 100644 logstash/cybera/DefaultView.json create mode 100644 logstash/cybera/InstanceCRUD.json create mode 100644 logstash/cybera/Migrations.json create mode 100644 logstash/cybera/README.md create mode 100644 logstash/cybera/SnapshotCheckpoints.json create mode 100644 logstash/cybera/VolumeCRUD.json create mode 100644 logstash/cybera/beaver.conf create mode 100644 logstash/cybera/logstash.conf diff --git a/README.md b/README.md index bbd4e3d..de6d814 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,7 @@ tools-logging ============= OpenStack Logging Tools + +logstash + +Example dashboards for Kibana and logging configurations for logstash. diff --git a/logstash/basic/README.md b/logstash/basic/README.md new file mode 100644 index 0000000..420ced5 --- /dev/null +++ b/logstash/basic/README.md @@ -0,0 +1,4 @@ +# Basic + +Basic logstash config and filters for ingesting most logs from OpenStack services. Courtesy Kris Lindgren from GoDaddy. + diff --git a/logstash/kibana.json b/logstash/basic/kibana.json similarity index 100% rename from logstash/kibana.json rename to logstash/basic/kibana.json diff --git a/logstash/logstash.conf b/logstash/basic/logstash.conf similarity index 100% rename from logstash/logstash.conf rename to logstash/basic/logstash.conf diff --git a/logstash/cybera/DefaultView.json b/logstash/cybera/DefaultView.json new file mode 100644 index 0000000..48f0654 --- /dev/null +++ b/logstash/cybera/DefaultView.json @@ -0,0 +1,496 @@ +{ + "title": "Default View", + "services": { + "query": { + "list": { + "0": { + "query": "\"region1\"", + "alias": "region1 Logs", + "color": "#7EB26D", + "id": 0, + "pin": false, + "type": "lucene", + "enable": true + }, + "1": { + "id": 1, + "color": "#EAB839", + "alias": "Region2 Logs", + "pin": false, + "type": "lucene", + "enable": true, + "query": "\"region2\"" + }, + "2": { + "id": 2, + "color": "#7EB26D", + "alias": "Instances Spawned", + "pin": false, + "type": "lucene", + "enable": true, + "query": "\"Instance spawned successfully\"" + }, + "3": { + "id": 3, + "color": "#EAB839", + "alias": "Instances Destroyed", + "pin": false, + "type": "lucene", + "enable": true, + "query": "\"Instance destroyed successfully\"" + }, + "4": { + "id": 4, + "color": "#6ED0E0", + "alias": "Snapshots created", + "pin": false, + "type": "lucene", + "enable": true, + "query": "\"Snapshot image upload complete\"" + }, + "5": { + "id": 5, + "color": "#1F78C1", + "alias": "Volumes Created", + "pin": false, + "type": "lucene", + "enable": true, + "query": "\"created -using\" AND loglevel:\"INFO\" AND module:\"cinder.volume.flows.create_volume\"" + }, + "6": { + "id": 6, + "color": "#BA43A9", + "alias": "Volumes Deleted", + "pin": false, + "type": "lucene", + "enable": true, + "query": "\"deleted\" AND loglevel:\"INFO\" AND module:\"cinder.volume.manager\"" + } + }, + "ids": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6 + ] + }, + "filter": { + "list": { + "0": { + "type": "time", + "field": "@timestamp", + "from": "now-30d", + "to": "now", + "mandate": "must", + "active": true, + "alias": "", + "id": 0 + } + }, + "ids": [ + 0 + ] + } + }, + "rows": [ + { + "title": "Graph", + "height": "250px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 12, + "editable": true, + "group": [ + "default" + ], + "type": "histogram", + "mode": "count", + "time_field": "@timestamp", + "value_field": null, + "auto_int": false, + "resolution": 100, + "interval": "1d", + "fill": 3, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": true, + "lines": false, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "queries": { + "mode": "selected", + "ids": [ + 2, + 3, + 4, + 5, + 6 + ] + }, + "title": "Events over time", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": true + }, + "scale": 1, + "y_format": "none", + "grid": { + "max": null, + "min": 0 + }, + "annotate": { + "enable": false, + "query": "*", + "size": 20, + "field": "_type", + "sort": [ + "_score", + "desc" + ] + }, + "pointradius": 5, + "show_query": true, + "legend_counts": true, + "zerofill": false, + "derivative": false + }, + { + "span": 12, + "editable": true, + "group": [ + "default" + ], + "type": "histogram", + "mode": "count", + "time_field": "@timestamp", + "value_field": null, + "auto_int": true, + "resolution": 100, + "interval": "12h", + "fill": 3, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": false, + "lines": true, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "queries": { + "mode": "selected", + "ids": [ + 0, + 1 + ] + }, + "title": "Events over time", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": true + }, + "scale": 1, + "y_format": "none", + "grid": { + "max": null, + "min": 0 + }, + "annotate": { + "enable": false, + "query": "*", + "size": 20, + "field": "_type", + "sort": [ + "_score", + "desc" + ] + }, + "pointradius": 5, + "show_query": true, + "legend_counts": true, + "zerofill": true, + "derivative": false + } + ], + "notice": false + }, + { + "title": "Instances/Volumes", + "height": "250", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 12, + "editable": true, + "type": "trends", + "loadingEditor": false, + "ago": "1d", + "arrangement": "horizontal", + "reverse": false, + "spyable": true, + "queries": { + "mode": "all", + "ids": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6 + ] + }, + "style": { + "font-size": "16pt" + }, + "title": "Compared to Yesterday..." + }, + { + "span": 12, + "editable": true, + "type": "trends", + "loadingEditor": false, + "ago": "1w", + "arrangement": "horizontal", + "reverse": false, + "spyable": true, + "queries": { + "mode": "all", + "ids": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6 + ] + }, + "style": { + "font-size": "16pt" + }, + "title": "Compared to Last Week..." + }, + { + "span": 12, + "editable": true, + "type": "trends", + "loadingEditor": false, + "ago": "4w", + "arrangement": "horizontal", + "reverse": false, + "spyable": true, + "queries": { + "mode": "all", + "ids": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6 + ] + }, + "style": { + "font-size": "16pt" + }, + "title": "Compared to Last Month..." + } + ], + "notice": false + }, + { + "title": "Events", + "height": "350px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "title": "All events", + "error": false, + "span": 12, + "editable": true, + "group": [ + "default" + ], + "type": "table", + "size": 100, + "pages": 5, + "offset": 0, + "sort": [ + "@timestamp", + "desc" + ], + "style": { + "font-size": "9pt" + }, + "overflow": "min-height", + "fields": [ + "@timestamp", + "logmessage", + "syslog_hostname" + ], + "localTime": true, + "timeField": "@timestamp", + "highlight": [], + "sortable": true, + "header": true, + "paging": true, + "spyable": true, + "queries": { + "mode": "pinned", + "ids": [] + }, + "field_list": true, + "status": "Stable", + "trimFactor": 300, + "normTimes": true, + "all_fields": false + } + ], + "notice": false + } + ], + "editable": true, + "failover": false, + "index": { + "interval": "day", + "pattern": "[logstash-]YYYY.MM.DD", + "default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED", + "warm_fields": true + }, + "style": "dark", + "panel_hints": true, + "pulldowns": [ + { + "type": "query", + "collapse": true, + "notice": false, + "query": "*", + "pinned": true, + "history": [ + "\"deleted\" AND loglevel:\"INFO\" AND module:\"cinder.volume.manager\"", + "\"created -using\" AND loglevel:\"INFO\" AND module:\"cinder.volume.flows.create_volume\"", + "\"Snapshot image upload complete\"", + "\"Instance destroyed successfully\"", + "\"Instance spawned successfully\"", + "\"rac-yeg\"", + "\"rac-yyc\"", + "\"deleted\" AND loglevel:\"INFO\" AND module:\"cinder.volume\"", + "\"created -using\" AND loglevel:\"INFO\" AND module:\"cinder\"", + "\"deleted\" AND loglevel:\"INFO\" AND module:\"cinder\"" + ], + "remember": 10, + "enable": true + }, + { + "type": "filtering", + "collapse": true, + "notice": true, + "enable": true + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "notice": false, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "timefield": "@timestamp", + "now": true, + "filter_id": 0, + "enable": true + } + ], + "loader": { + "save_gist": false, + "save_elasticsearch": true, + "save_local": true, + "save_default": true, + "save_temp": true, + "save_temp_ttl_enable": true, + "save_temp_ttl": "30d", + "load_gist": true, + "load_elasticsearch": true, + "load_elasticsearch_size": 20, + "load_local": true, + "hide": false + }, + "refresh": "15m" +} diff --git a/logstash/cybera/InstanceCRUD.json b/logstash/cybera/InstanceCRUD.json new file mode 100644 index 0000000..7e96cf6 --- /dev/null +++ b/logstash/cybera/InstanceCRUD.json @@ -0,0 +1,319 @@ +{ + "title": "Instance Spawns and Destroys", + "services": { + "query": { + "list": { + "0": { + "query": "\"Instance spawned successfully\"", + "alias": "Instances Spawned", + "color": "#7EB26D", + "id": 0, + "pin": false, + "type": "lucene", + "enable": true + }, + "1": { + "id": 1, + "color": "#EAB839", + "alias": "Instances Destroyed", + "pin": false, + "type": "lucene", + "enable": true, + "query": "\"Instance destroyed successfully\"" + }, + "2": { + "id": 2, + "color": "#6ED0E0", + "alias": "Snapshots created", + "pin": false, + "type": "lucene", + "enable": true, + "query": "\"Snapshot image upload complete\"" + } + }, + "ids": [ + 0, + 1, + 2 + ] + }, + "filter": { + "list": { + "0": { + "type": "time", + "field": "@timestamp", + "from": "now-30d", + "to": "now", + "mandate": "must", + "active": true, + "alias": "", + "id": 0 + } + }, + "ids": [ + 0 + ] + } + }, + "rows": [ + { + "title": "Graph", + "height": "350px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 12, + "editable": true, + "group": [ + "default" + ], + "type": "histogram", + "mode": "count", + "time_field": "@timestamp", + "value_field": null, + "auto_int": false, + "resolution": 100, + "interval": "24h", + "fill": 0, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": true, + "lines": false, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "queries": { + "mode": "all", + "ids": [ + 0, + 1, + 2 + ] + }, + "title": "Events over time", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": true + }, + "scale": 1, + "y_format": "none", + "grid": { + "max": null, + "min": 0 + }, + "annotate": { + "enable": false, + "query": "*", + "size": 20, + "field": "_type", + "sort": [ + "_score", + "desc" + ] + }, + "pointradius": 3, + "show_query": true, + "legend_counts": true, + "zerofill": false, + "derivative": false, + "scaleSeconds": false + } + ], + "notice": false + }, + { + "title": "Stats", + "height": "50px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 4, + "editable": true, + "type": "trends", + "loadingEditor": false, + "ago": "1w", + "arrangement": "horizontal", + "reverse": false, + "spyable": true, + "queries": { + "mode": "all", + "ids": [ + 0, + 1, + 2 + ] + }, + "style": { + "font-size": "14pt" + }, + "title": "Compared to last week..." + } + ], + "notice": false + }, + { + "title": "Events", + "height": "350px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "title": "All events", + "error": false, + "span": 12, + "editable": true, + "group": [ + "default" + ], + "type": "table", + "size": 100, + "pages": 5, + "offset": 0, + "sort": [ + "@timestamp", + "desc" + ], + "style": { + "font-size": "9pt" + }, + "overflow": "min-height", + "fields": [], + "localTime": true, + "timeField": "@timestamp", + "highlight": [], + "sortable": true, + "header": true, + "paging": true, + "spyable": true, + "queries": { + "mode": "all", + "ids": [ + 0, + 1, + 2 + ] + }, + "field_list": true, + "status": "Stable", + "trimFactor": 300, + "normTimes": true, + "all_fields": false + } + ], + "notice": false + } + ], + "editable": true, + "failover": false, + "index": { + "interval": "day", + "pattern": "[logstash-]YYYY.MM.DD", + "default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED", + "warm_fields": true + }, + "style": "dark", + "panel_hints": true, + "pulldowns": [ + { + "type": "query", + "collapse": true, + "notice": false, + "query": "*", + "pinned": true, + "history": [ + "\"Snapshot image upload complete\"", + "\"Instance destroyed successfully\"", + "\"Instance spawned successfully\"", + "\"Snapshot upload complete\"", + "\"Instance Spawned Successfully\"", + "Instance", + "Created", + ], + "remember": 10, + "enable": true + }, + { + "type": "filtering", + "collapse": true, + "notice": false, + "enable": true + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "notice": false, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "timefield": "@timestamp", + "now": true, + "filter_id": 0, + "enable": true + } + ], + "loader": { + "save_gist": false, + "save_elasticsearch": true, + "save_local": true, + "save_default": true, + "save_temp": true, + "save_temp_ttl_enable": true, + "save_temp_ttl": "30d", + "load_gist": true, + "load_elasticsearch": true, + "load_elasticsearch_size": 20, + "load_local": true, + "hide": false + }, + "refresh": false +} diff --git a/logstash/cybera/Migrations.json b/logstash/cybera/Migrations.json new file mode 100644 index 0000000..eb4b5a8 --- /dev/null +++ b/logstash/cybera/Migrations.json @@ -0,0 +1,298 @@ +{ + "title": "Migrations", + "services": { + "query": { + "list": { + "0": { + "query": "\"Going to try to live migrate instance to\"", + "alias": "Migration Attempts", + "color": "#B7DBAB", + "id": 0, + "pin": false, + "type": "lucene", + "enable": true + }, + "1": { + "id": 1, + "color": "#7EB26D", + "alias": "Migration Success", + "pin": false, + "type": "lucene", + "enable": true, + "query": "\"base_of_fqdn_goes_here finished successfully\"" + }, + "2": { + "id": 2, + "color": "#890F02", + "alias": "Migration Failures", + "pin": false, + "type": "lucene", + "enable": true, + "query": "\"Live Migration failure\"" + } + }, + "ids": [ + 0, + 1, + 2 + ] + }, + "filter": { + "list": { + "0": { + "from": "2014-09-03T19:02:17.256Z", + "to": "now", + "type": "time", + "field": "@timestamp", + "mandate": "must", + "active": true, + "alias": "", + "id": 0 + }, + "1": { + "type": "time", + "from": "2014-09-03T19:25:36.941Z", + "to": "2014-09-03T20:54:09.058Z", + "field": "@timestamp", + "mandate": "must", + "active": true, + "alias": "", + "id": 1 + } + }, + "ids": [ + 0, + 1 + ] + } + }, + "rows": [ + { + "title": "Graph", + "height": "350px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 12, + "editable": true, + "group": [ + "default" + ], + "type": "histogram", + "mode": "count", + "time_field": "@timestamp", + "value_field": null, + "auto_int": true, + "resolution": 100, + "interval": "1m", + "fill": 3, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": true, + "stack": true, + "points": false, + "lines": false, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "queries": { + "mode": "all", + "ids": [ + 0, + 1, + 2 + ] + }, + "title": "Events over time", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "options": true, + "tooltip": { + "value_type": "individual", + "query_as_alias": true + }, + "scale": 1, + "y_format": "short", + "grid": { + "max": null, + "min": 0 + }, + "annotate": { + "enable": false, + "query": "*", + "size": 20, + "field": "_type", + "sort": [ + "_score", + "desc" + ] + }, + "pointradius": 5, + "show_query": true, + "legend_counts": true, + "zerofill": false, + "derivative": false + } + ], + "notice": false + }, + { + "title": "Events", + "height": "350px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "title": "All events", + "error": false, + "span": 12, + "editable": true, + "group": [ + "default" + ], + "type": "table", + "size": 100, + "pages": 5, + "offset": 0, + "sort": [ + "@timestamp", + "desc" + ], + "style": { + "font-size": "9pt" + }, + "overflow": "min-height", + "fields": [ + "@timestamp", + "logmessage", + "@source_host" + ], + "localTime": true, + "timeField": "@timestamp", + "highlight": [], + "sortable": true, + "header": true, + "paging": true, + "spyable": true, + "queries": { + "mode": "all", + "ids": [ + 0, + 1, + 2 + ] + }, + "field_list": true, + "status": "Stable", + "trimFactor": 300, + "normTimes": true, + "all_fields": false + } + ], + "notice": false + } + ], + "editable": true, + "failover": false, + "index": { + "interval": "day", + "pattern": "[logstash-]YYYY.MM.DD", + "default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED", + "warm_fields": true + }, + "style": "dark", + "panel_hints": true, + "pulldowns": [ + { + "type": "query", + "collapse": false, + "notice": false, + "query": "*", + "pinned": true, + "history": [ + "\"Live Migration failure\"", + "\"Going to try to live migrate instance to\"", + "\"Migrate instance to\"", + "\"Migrate instance to\" AND \"finished successfully\"" + ], + "remember": 10, + "enable": true + }, + { + "type": "filtering", + "collapse": true, + "notice": true, + "enable": true + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "notice": false, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "timefield": "@timestamp", + "now": false, + "filter_id": 0, + "enable": true + } + ], + "loader": { + "save_gist": false, + "save_elasticsearch": true, + "save_local": true, + "save_default": true, + "save_temp": true, + "save_temp_ttl_enable": true, + "save_temp_ttl": "30d", + "load_gist": true, + "load_elasticsearch": true, + "load_elasticsearch_size": 20, + "load_local": true, + "hide": false + }, + "refresh": false +} diff --git a/logstash/cybera/README.md b/logstash/cybera/README.md new file mode 100644 index 0000000..f22119e --- /dev/null +++ b/logstash/cybera/README.md @@ -0,0 +1,50 @@ +# Cybera + +Example config and dashboards developed at Cybera for our public clouds. Most of the dashboards are largely to see what kind of information can be pulled from the logs or as alternatives to watching for specific log entries in a very busy `tail -f` stream. + +Setup: +All services are set to DEBUG and to log to syslog, and the nodes then forward to a central rsyslog server that runs [beaver](https://github.com/josegonzalez/python-beaver) to push and tag the logs to our Rabbit cluster. The logstash agent then pulls the logs from Rabbit. + +Caveats: +Beaver can only manage pushing up to 350 events/sec due to the way the Pika (Rabbit) library is used. If better performance is needed - look at the Redis options. + +## Dashboards + +### DefaultView.json + +The Default View gives an overview of the number of logs between regions, along with some log based counts of instance creation/deletion, volume creation/deletion, and snapshot creation. + + + +### SnapshotCheckpoints.json + +Shows the "checkpoints" of instance snapshotting. + + + +### InstanceCRUD.json + +Shows instance creation and deletion along with snapshot creation points. + + + +### Migrations.json + +Shows the "checkpoints" of instance migration. One of the queries needs to be changed to the base of your compute node's fqdn. (node1.example.com would be just example.com) + + + +### VolumeCRUD.json + +Shows volume creation and deletion. + + + +## Logstash + +Logstash.conf - example downloading from rabbit. + +## Beaver + +beaver.conf - The example beaver config showing what we tag logs with. + diff --git a/logstash/cybera/SnapshotCheckpoints.json b/logstash/cybera/SnapshotCheckpoints.json new file mode 100644 index 0000000..ecabccb --- /dev/null +++ b/logstash/cybera/SnapshotCheckpoints.json @@ -0,0 +1,356 @@ +{ + "title": "Instance Checkpoint Checks", + "services": { + "query": { + "list": { + "0": { + "query": "\"instance snapshotting\"", + "alias": "", + "color": "#7EB26D", + "id": 0, + "pin": false, + "type": "lucene", + "enable": true + }, + "1": { + "id": 1, + "color": "#EAB839", + "alias": "", + "pin": false, + "type": "lucene", + "enable": true, + "query": "\"Beginning live snapshot process\"" + }, + "2": { + "id": 2, + "color": "#6ED0E0", + "alias": "", + "pin": false, + "type": "lucene", + "enable": true, + "query": "\"Snapshot extracted\"" + }, + "3": { + "id": 3, + "color": "#EF843C", + "alias": "", + "pin": false, + "type": "lucene", + "enable": true, + "query": "\"Uploading image data for image\"" + }, + "4": { + "id": 4, + "color": "#E24D42", + "alias": "", + "pin": false, + "type": "lucene", + "enable": true, + "query": "\"bytes to /var/lib/glance/images\"" + }, + "5": { + "id": 5, + "color": "#1F78C1", + "alias": "", + "pin": false, + "type": "lucene", + "enable": true, + "query": "\"Snapshot image upload complete\"" + } + }, + "ids": [ + 0, + 1, + 2, + 3, + 4, + 5 + ] + }, + "filter": { + "list": { + "0": { + "type": "time", + "field": "@timestamp", + "from": "now-24h", + "to": "now", + "mandate": "must", + "active": true, + "alias": "", + "id": 0 + }, + "1": { + "type": "field", + "field": "logmessage", + "query": "\"UUID GOES HERE\"", + "mandate": "must", + "active": true, + "alias": "", + "id": 1 + } + }, + "ids": [ + 0, + 1 + ] + } + }, + "rows": [ + { + "title": "Instructions", + "height": "50px", + "editable": true, + "collapse": false, + "collapsable": false, + "panels": [ + { + "error": false, + "span": 12, + "editable": true, + "type": "text", + "loadingEditor": false, + "mode": "markdown", + "content": "Add the **instance** UUID to the logmessage filter to monitor the times the snapshot hits it's Glance checkpoints. Alternatively remove the filter to see all instances.\n\nPlease note the last two points (image saved and bytes saved) will show results when an image is uploaded as well.", + "style": {}, + "title": "Instructions" + } + ], + "notice": false + }, + { + "title": "Graph", + "height": "350px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 12, + "editable": true, + "group": [ + "default" + ], + "type": "histogram", + "mode": "count", + "time_field": "@timestamp", + "value_field": null, + "auto_int": true, + "resolution": 100, + "interval": "10m", + "fill": 3, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": true, + "stack": true, + "points": false, + "lines": false, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "queries": { + "mode": "all", + "ids": [ + 0, + 1, + 2, + 3, + 4, + 5 + ] + }, + "title": "Events over time", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "options": true, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": true + }, + "scale": 1, + "y_format": "none", + "grid": { + "max": null, + "min": 0 + }, + "annotate": { + "enable": false, + "query": "*", + "size": 20, + "field": "_type", + "sort": [ + "_score", + "desc" + ] + }, + "pointradius": 5, + "show_query": true, + "legend_counts": true, + "zerofill": false, + "derivative": false + } + ], + "notice": false + }, + { + "title": "Events", + "height": "350px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "title": "All events", + "error": false, + "span": 12, + "editable": true, + "group": [ + "default" + ], + "type": "table", + "size": 100, + "pages": 5, + "offset": 0, + "sort": [ + "@timestamp", + "desc" + ], + "style": { + "font-size": "9pt" + }, + "overflow": "min-height", + "fields": [], + "localTime": true, + "timeField": "@timestamp", + "highlight": [], + "sortable": true, + "header": true, + "paging": true, + "spyable": true, + "queries": { + "mode": "all", + "ids": [ + 0, + 1, + 2, + 3, + 4, + 5 + ] + }, + "field_list": true, + "status": "Stable", + "trimFactor": 300, + "normTimes": true, + "all_fields": false + } + ], + "notice": false + } + ], + "editable": true, + "failover": false, + "index": { + "interval": "day", + "pattern": "[logstash-]YYYY.MM.DD", + "default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED", + "warm_fields": true + }, + "style": "dark", + "panel_hints": true, + "pulldowns": [ + { + "type": "query", + "collapse": true, + "notice": false, + "query": "*", + "pinned": true, + "history": [ + "\"Snapshot image upload complete\"", + "\"bytes to /var/lib/glance/images\"", + "\"Uploading image data for image\"", + "\"Snapshot extracted\"", + "\"Beginning live snapshot process\"", + "\"instance snapshotting\"", + "Snapshot extracted", + "Beginning live snapshot process", + "instance snapshotting", + "[instance: *] Beginning live snapshot process" + ], + "remember": 10, + "enable": true + }, + { + "type": "filtering", + "collapse": false, + "notice": true, + "enable": true + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "notice": false, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "timefield": "@timestamp", + "now": true, + "filter_id": 0, + "enable": true + } + ], + "loader": { + "save_gist": false, + "save_elasticsearch": true, + "save_local": true, + "save_default": true, + "save_temp": true, + "save_temp_ttl_enable": true, + "save_temp_ttl": "30d", + "load_gist": true, + "load_elasticsearch": true, + "load_elasticsearch_size": 20, + "load_local": true, + "hide": false + }, + "refresh": false +} \ No newline at end of file diff --git a/logstash/cybera/VolumeCRUD.json b/logstash/cybera/VolumeCRUD.json new file mode 100644 index 0000000..f9422d4 --- /dev/null +++ b/logstash/cybera/VolumeCRUD.json @@ -0,0 +1,297 @@ +{ + "title": "Volume Creation and Deletion", + "services": { + "query": { + "list": { + "0": { + "query": "created -using", + "alias": "Created", + "color": "#7EB26D", + "id": 0, + "pin": false, + "type": "lucene", + "enable": true + }, + "1": { + "id": 1, + "color": "#EAB839", + "alias": "Deleted", + "pin": false, + "type": "lucene", + "enable": true, + "query": "deleted" + } + }, + "ids": [ + 0, + 1 + ] + }, + "filter": { + "list": { + "0": { + "type": "time", + "field": "@timestamp", + "from": "now-30d", + "to": "now", + "mandate": "must", + "active": true, + "alias": "", + "id": 0 + }, + "1": { + "type": "field", + "field": "module", + "query": "\"cinder.volume.flows.create_volume\" or \"cinder.volume.manager\"", + "mandate": "either", + "active": true, + "alias": "", + "id": 1 + }, + "2": { + "type": "field", + "field": "loglevel", + "query": "\"INFO\"", + "mandate": "must", + "active": true, + "alias": "", + "id": 2 + } + }, + "ids": [ + 0, + 1, + 2 + ] + } + }, + "rows": [ + { + "title": "Graph", + "height": "350px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 12, + "editable": true, + "group": [ + "default" + ], + "type": "histogram", + "mode": "count", + "time_field": "@timestamp", + "value_field": null, + "auto_int": false, + "resolution": 100, + "interval": "24h", + "fill": 3, + "linewidth": 3, + "timezone": "browser", + "spyable": true, + "zoomlinks": true, + "bars": false, + "stack": false, + "points": true, + "lines": false, + "legend": true, + "x-axis": true, + "y-axis": true, + "percentage": false, + "interactive": true, + "queries": { + "mode": "all", + "ids": [ + 0, + 1 + ] + }, + "title": "Volume Events", + "intervals": [ + "auto", + "1s", + "1m", + "5m", + "10m", + "30m", + "1h", + "3h", + "12h", + "1d", + "1w", + "1M", + "1y" + ], + "options": true, + "tooltip": { + "value_type": "individual", + "query_as_alias": true + }, + "scale": 1, + "y_format": "none", + "grid": { + "max": null, + "min": 0 + }, + "annotate": { + "enable": false, + "query": "*", + "size": 20, + "field": "_type", + "sort": [ + "_score", + "desc" + ] + }, + "pointradius": 5, + "show_query": true, + "legend_counts": true, + "zerofill": false, + "derivative": false + } + ], + "notice": false + }, + { + "title": "Events", + "height": "350px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "title": "All events", + "error": false, + "span": 12, + "editable": true, + "group": [ + "default" + ], + "type": "table", + "size": 100, + "pages": 5, + "offset": 0, + "sort": [ + "@timestamp", + "desc" + ], + "style": { + "font-size": "9pt" + }, + "overflow": "min-height", + "fields": [], + "localTime": true, + "timeField": "@timestamp", + "highlight": [], + "sortable": true, + "header": true, + "paging": true, + "spyable": true, + "queries": { + "mode": "all", + "ids": [ + 0, + 1 + ] + }, + "field_list": true, + "status": "Stable", + "trimFactor": 300, + "normTimes": true, + "all_fields": false + } + ], + "notice": false + } + ], + "editable": true, + "failover": false, + "index": { + "interval": "day", + "pattern": "[logstash-]YYYY.MM.DD", + "default": "NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED", + "warm_fields": true + }, + "style": "dark", + "panel_hints": true, + "pulldowns": [ + { + "type": "query", + "collapse": false, + "notice": false, + "query": "*", + "pinned": true, + "history": [ + "deleted", + "created -using", + "created", + "successfully", + "f2b10018-f9eb-424b-ad7b-669cc691687b", + "created successfully", + "\"cinder.volume.flows.create_volume\" + message:\"created successfully\"", + "\"cinder.volume.flows.create_volume\" message:\"created successfully\"", + "\"cinder.volume.flows.create_volume\" message:succesfully", + "\"cinder.volume.flows.create_volume\" + succesfully" + ], + "remember": 10, + "enable": true + }, + { + "type": "filtering", + "collapse": true, + "notice": true, + "enable": true + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "notice": false, + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "timefield": "@timestamp", + "now": true, + "filter_id": 0, + "enable": true + } + ], + "loader": { + "save_gist": false, + "save_elasticsearch": true, + "save_local": true, + "save_default": true, + "save_temp": true, + "save_temp_ttl_enable": true, + "save_temp_ttl": "30d", + "load_gist": true, + "load_elasticsearch": true, + "load_elasticsearch_size": 20, + "load_local": true, + "hide": false + }, + "refresh": false +} \ No newline at end of file diff --git a/logstash/cybera/beaver.conf b/logstash/cybera/beaver.conf new file mode 100644 index 0000000..a1cf9ce --- /dev/null +++ b/logstash/cybera/beaver.conf @@ -0,0 +1,37 @@ +[beaver] +rabbitmq_host = rabbitmqcluster_fqdn +rabbitmq_password = password +format = msgpack +rabbitmq_vhost = rsyslog +rabbitmq_exchange_type = direct +rabbitmq_queue_durable = 1 +rabbitmq_username = logstash +rabbitmq_ssl = 1 +logstash_version = 1 +rabbitmq_queue = logstash +rabbitmq_exchange_durable = 0 +rabbitmq_exchange = region1-logs + +[/var/log/rsyslog/swift.log] +tags = cloud,region1,openstack,swift,swiftfmt +type = swift + +[/var/log/rsyslog/nova.log] +tags = cloud,region1,openstack,nova,oslofmt +type = nova + +[/var/log/rsyslog/syslog.log] +tags = cloud,region1,syslogfmt +type = syslog + +[/var/log/rsyslog/cinder.log] +type = cinder +tags = cloud,region1,openstack,cinder,oslofmt + +[/var/log/rsyslog/keystone.log] +tags = cloud,region1,openstack,keystone,oslofmt +type = keystone + +[/var/log/rsyslog/glance.log] +tags = cloud,region1,openstack,glance,oslofmt +type = glance diff --git a/logstash/cybera/logstash.conf b/logstash/cybera/logstash.conf new file mode 100644 index 0000000..f241605 --- /dev/null +++ b/logstash/cybera/logstash.conf @@ -0,0 +1,116 @@ +input { + + # Region 1 + rabbitmq { + codec => "msgpack" + debug => true + host => "region1.cybera.ca" + exchange => "region1-logs" + user => "logstash" + password => "password" + ssl => true + port => "5672" + vhost => "rsyslog" + auto_delete => false + durable => true + key => 'logstash' + exclusive => false + passive => true + queue => 'logstash' + } + + # Region 2 + rabbitmq { + codec => "msgpack" + debug => true + host => "region2.cybera.ca" + exchange => "region1-logs" + user => "logstash" + password => "password" + ssl => true + port => "5672" + vhost => "rsyslog" + auto_delete => false + durable => true + key => 'logstash' + exclusive => false + passive => true + queue => 'logstash' + } + +} + + +filter { + if "oslofmt" in [tags] { + grok { + match => { "message" => "^%{TIMESTAMP_ISO8601:logdate} %{SYSLOGHOST:syslog_hostname} %{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{NUMBER:syslog_pid} (?AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR) %{NOTSPACE:module} (?\[%{NOTSPACE:ref_id}?%{DATA:ref_id2}\]) %{GREEDYDATA:logmessage}" } + add_field => { "received_at" => "%{@timestamp}" } + } + if !("_grokparsefailure" in [tags]) { + mutate { + replace => [ "@source_host", "%{syslog_hostname}" ] + gsub => [ "message", "#012", "\ +"] + } + } + # Make sure we set @timestamp to the log date + date { + match => [ "logdate", "ISO8601" ] + locale => "en" + target => "@timestamp" + } + } else if "syslogfmt" in [tags] { + grok { + match => { "message" => "^%{TIMESTAMP_ISO8601:logdate} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:logmessage}" } + add_field => [ "received_at", "%{@timestamp}" ] + } + date { + match => [ "logdate", "ISO8601" ] + locale => "en" + target => "@timestamp" + } + syslog_pri { + severity_labels => ["ERROR", "ERROR", "ERROR", "ERROR", "WARNING", "INFO", "INFO", "DEBUG" ] + } + if !("_grokparsefailure" in [tags]) { + mutate { + replace => [ "@source_host", "%{syslog_hostname}" ] + } + } + mutate { + add_field => [ "loglevel", "%{syslog_severity}" ] + add_field => [ "module", "%{syslog_program}" ] + } + } else if "swiftfmt" in [tags] { + grok { + match => { "message" => "^%{TIMESTAMP_ISO8601:logdate} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program} %{GREEDYDATA:logmessage}" } + add_field => [ "received_at", "%{@timestamp}" ] + } + syslog_pri { + severity_labels => ["ERROR", "ERROR", "ERROR", "ERROR", "WARNING", "INFO", "INFO", "DEBUG" ] + } + if !("_grokparsefailure" in [tags]) { + mutate { + replace => [ "@source_host", "%{syslog_hostname}" ] + } + } + mutate { + add_field => [ "loglevel", "%{syslog_severity}" ] + add_field => [ "module", "%{syslog_program}" ] + } + date { + match => [ "logdate", "ISO8601" ] + locale => "en" + target => "@timestamp" + } + } +} + + +output { + elasticsearch_http { + host => "127.0.0.1" + } +} +