From da6bdafd65b028d3db6abab7f1a63ff203596688 Mon Sep 17 00:00:00 2001
From: Andrew Bonney <andrew.bonney@bbc.co.uk>
Date: Thu, 15 Apr 2021 10:17:57 +0100
Subject: [PATCH] Listen on an additional port for monitoring/diagnostic
 purposes

If the Galera cluster hits its configured max_connections value
then only the super-users can still connect for debug purposes.
As the monitoring user cannot connect, this can cause a cascading
failure as HAProxy marks the instance as unreachable.

This configuration adds an extra listening port with a limited
number of connections to allow the monitoring user to connect at
all times.

Change-Id: I57187bab2ee35521c275f0f0b99c1ca8fd1830ad
---
 defaults/main.yml                                   |  2 ++
 releasenotes/notes/extra-port-47d2ba4074ab6a62.yaml |  9 +++++++++
 templates/clustercheck.j2                           | 12 ++++++++++++
 templates/my.cnf.j2                                 |  4 ++++
 4 files changed, 27 insertions(+)
 create mode 100644 releasenotes/notes/extra-port-47d2ba4074ab6a62.yaml

diff --git a/defaults/main.yml b/defaults/main.yml
index f32dfc50..cb199dab 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -65,6 +65,8 @@ galera_gpg_keys: "{{ _galera_gpg_keys | default([]) }}"
 
 galera_monitoring_user: monitoring
 galera_monitoring_user_password: ""
+galera_monitoring_port: 3307
+galera_monitoring_max_connections: 10
 
 # WARNING: Set this to open xinetd rules for galera monitoring.
 # This is REQUIRED to run a working openstack-ansible deployment.
diff --git a/releasenotes/notes/extra-port-47d2ba4074ab6a62.yaml b/releasenotes/notes/extra-port-47d2ba4074ab6a62.yaml
new file mode 100644
index 00000000..e6e24577
--- /dev/null
+++ b/releasenotes/notes/extra-port-47d2ba4074ab6a62.yaml
@@ -0,0 +1,9 @@
+---
+upgrade:
+  - |
+    Galera will now additionally listen on port 3307 by default, with this port
+    being used by the monitoring user to check cluster status. Ensure that any
+    firewall rules permit access to this port before upgrading. If an
+    'extra_port' was already configured, ensure that any conflicting
+    configuration is removed and set your preferred values via
+    'galera_monitoring_port' and 'galera_monitoring_max_connections'.
diff --git a/templates/clustercheck.j2 b/templates/clustercheck.j2
index 2c77e779..8d7ca629 100644
--- a/templates/clustercheck.j2
+++ b/templates/clustercheck.j2
@@ -33,6 +33,8 @@ fi
 
 MYSQL_USERNAME="${1-{{ galera_monitoring_user }}}"
 MYSQL_PASSWORD="${2-{{ galera_monitoring_user_password }}}"
+MYSQL_HOST="{{ galera_server_bind_address }}"
+MYSQL_PORT="{{ galera_monitoring_port }}"
 AVAILABLE_WHEN_DONOR=${3:-0}
 ERR_FILE="${4:-/dev/null}"
 AVAILABLE_WHEN_READONLY=${5:-1}
@@ -52,6 +54,16 @@ else
     EXTRA_ARGS="$EXTRA_ARGS --password="
 fi
 
+if [[ -n "$MYSQL_PORT" ]]; then
+    # Port does not apply unless host is set
+    if [[ "$MYSQL_HOST" != "0.0.0.0" ]]; then
+        EXTRA_ARGS="$EXTRA_ARGS --host=${MYSQL_HOST}"
+    else
+        EXTRA_ARGS="$EXTRA_ARGS --host=127.0.0.1"
+    fi
+    EXTRA_ARGS="$EXTRA_ARGS --port=${MYSQL_PORT}"
+fi
+
 if [[ -r $DEFAULTS_EXTRA_FILE ]]; then
     MYSQL_CMDLINE="mysql --defaults-extra-file=$DEFAULTS_EXTRA_FILE -nNE --connect-timeout=$TIMEOUT \
                     ${EXTRA_ARGS}"
diff --git a/templates/my.cnf.j2 b/templates/my.cnf.j2
index 2d0ed1be..9ac4166b 100644
--- a/templates/my.cnf.j2
+++ b/templates/my.cnf.j2
@@ -49,6 +49,10 @@ expire-logs-days = 7
 log_slave_updates = 1
 log_bin_trust_function_creators = 1
 
+# MONITORING #
+extra_port = {{ galera_monitoring_port }}
+extra_max_connections = {{ galera_monitoring_max_connections }}
+
 # SAFETY #
 max-allowed-packet = 16M
 max-connect-errors = 1000000