From 0804dc9dc01e320f5d0df9a5eea12166b8f4d9b6 Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Thu, 9 Nov 2017 18:03:06 -0500 Subject: [PATCH] Add Ping Plugin This plugin helps each controller ping the other 2 controllers over the InternalAPI interface. This helps us to indetify the latency and jutter between the controllers in situations where REST calls take too long or clustered services go down. Change-Id: I24a2ad9cafeb6900932e982b4b1bb456f8ad691f --- ansible/install/group_vars/all.yml | 7 ++ .../collectd-openstack/defaults/main.yml | 7 ++ .../roles/collectd-openstack/tasks/main.yml | 1 + .../templates/controller.collectd.conf.j2 | 23 +++++- ...enstack_general_system_performance.json.j2 | 75 +++++++++++++++++++ 5 files changed, 112 insertions(+), 1 deletion(-) diff --git a/ansible/install/group_vars/all.yml b/ansible/install/group_vars/all.yml index eac388d78..980b8861c 100644 --- a/ansible/install/group_vars/all.yml +++ b/ansible/install/group_vars/all.yml @@ -260,6 +260,13 @@ regex_warn: false regex_info: false +######################################################## +# Ping Plugin for Latency and Jitter between controllers +######################################################## +# Might result in more network traffic +ping_plugin: false +ping_interval: 1 + ############################ # OpenDaylight JAVA Plugin ########################### diff --git a/ansible/install/roles/collectd-openstack/defaults/main.yml b/ansible/install/roles/collectd-openstack/defaults/main.yml index 419b0460d..702af0cfe 100644 --- a/ansible/install/roles/collectd-openstack/defaults/main.yml +++ b/ansible/install/roles/collectd-openstack/defaults/main.yml @@ -145,3 +145,10 @@ karaf_password: karaf # Determines if WARN/INFO messages are also counted regex_warn: false regex_info: false + +######################################################## +# Ping Plugin for Latency and Jitter between controllers +######################################################## +# Might result in more network traffic +ping_plugin: false +ping_interval: 1 diff --git a/ansible/install/roles/collectd-openstack/tasks/main.yml b/ansible/install/roles/collectd-openstack/tasks/main.yml index 11684a29c..adb08b01a 100644 --- a/ansible/install/roles/collectd-openstack/tasks/main.yml +++ b/ansible/install/roles/collectd-openstack/tasks/main.yml @@ -37,6 +37,7 @@ - collectd-ceph - collectd-mysql - collectd-turbostat + - collectd-ping # (sai) Separating out collectd java rpms as they have a lot of dependencies and # are only required for ODL monitoring on controllers only diff --git a/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 b/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 index 05bbbb6f3..6bc418e35 100644 --- a/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 +++ b/ansible/install/roles/collectd-openstack/templates/controller.collectd.conf.j2 @@ -51,11 +51,13 @@ LoadPlugin tail LoadPlugin turbostat LoadPlugin unixsock LoadPlugin uptime +{% if ping_plugin %} +LoadPlugin ping +{% endif %} {% if opendaylight_java_plugin %} LoadPlugin java {% endif %} - # Open unix domain socket for collectdctl SocketFile "/var/run/collectd-unixsock" @@ -423,6 +425,25 @@ PreCacheChain "PreCache" ValuesPercentage true +# ping plugin +{% if ping_plugin %} +{% if groups['controller'] | length > 1 %} + +{% if inventory_hostname == groups['controller'][0] %} + Host "{{groups['controller'][1]}}" + Host "{{groups['controller'][2]}}" +{% elif inventory_hostname == groups['controller'][1] %} + Host "{{groups['controller'][0]}}" + Host "{{groups['controller'][2]}}" +{% elif inventory_hostname == groups['controller'][2] %} + Host "{{groups['controller'][0]}}" + Host "{{groups['controller'][1]}}" +{% endif %} + Interval {{ping_interval}} + +{% endif %} +{% endif %} + # Tail plugin configuration diff --git a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 index aa26c639a..d47ea3e1f 100644 --- a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 +++ b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.json.j2 @@ -2670,6 +2670,81 @@ "short", "percent" ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + {% if vars.update({'panel_idx': (vars.panel_idx + 1)}) %} {% endif %} + "id": {{vars.panel_idx}}, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode($Cloud.$Node.ping.*, 3)" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ping- Latency and Jitter", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] } ], "showTitle": true,