Allow setting gmcast.peer_timeout value

In some resource constrained environments particularly during deploy
time percona-cluster nodes can experience time outs during inter-node
communication.

This changes makes the gmcast.peer_timeout configurable based on the
galara cluster documentation:
http://galeracluster.com/documentation-webpages/galeraparameters.html

Warning: Changing this value from the default may have unintended
consequences. This should only be used when constraint's call for it.

Closes-Bug: #1742683
Change-Id: If93d6ba9d0e99b6af59358a7f3ea40e3aa2a6dbc

config.yaml

@@ -308,3 +308,13 @@ options:
       collisions. Known wait is the amount of time between one node executing
       an operation and another. On slower hardware this value may need to be
       larger than the default of 30 seconds.
+  peer-timeout:
+    type: string
+    default:
+    description: |
+        This setting sets the gmcast.peer_timeout value. Possible values are documented
+        on the galera cluster site http://galeracluster.com/documentation-webpages/galeraparameters.html
+        For very busy clouds or in resource restricted environments this value can be changed.
+        WARNING Please read all documentation before changing the default value which may have
+        unintended consequences. It may be necessary to set this value higher during deploy time
+        (PTS15) and subsequently change it back to the default (PT3S) after deployment.

hooks/percona_hooks.py

@@ -108,6 +108,7 @@ from percona_utils import (
     LeaderNoBootstrapUUIDError,
     update_root_password,
     cluster_wait,
+    get_wsrep_provider_options,
 )
 
 from charmhelpers.core.unitdata import kv
@@ -197,11 +198,14 @@ def render_config(clustered=False, hosts=None):
         # See lp 1380747 for more info. This is intended as a stop gap until
         # percona package is fixed to support ipv6.
         context['bind_address'] = '::'
-        context['wsrep_provider_options'] = 'gmcast.listen_addr=tcp://:::4567;'
         context['ipv6'] = True
     else:
         context['ipv6'] = False
 
+    wsrep_provider_options = get_wsrep_provider_options()
+    if wsrep_provider_options:
+        context['wsrep_provider_options'] = wsrep_provider_options
+
     context.update(PerconaClusterHelper().parse_config())
     render(os.path.basename(config_file),
            config_file, context, perms=0o444)

hooks/percona_utils.py

@@ -925,3 +925,22 @@ def cluster_wait():
         for rid in relation_ids('cluster'):
             num_nodes += len(related_units(rid))
     distributed_wait(modulo=num_nodes, wait=wait)
+
+
+def get_wsrep_provider_options():
+
+    wsrep_provider_options = []
+
+    if config('prefer-ipv6'):
+        wsrep_provider_options.append('gmcast.listen_addr=tcp://:::4567')
+
+    peer_timeout = config('peer-timeout')
+    if peer_timeout and(not peer_timeout.startswith('PT') or
+                        not peer_timeout.endswith('S')):
+        raise ValueError("Invalid gcast.peer_timeout value: {}"
+                         .format(peer_timeout))
+    elif peer_timeout:
+        wsrep_provider_options.append('gmcast.peer_timeout={}'
+                                      .format(config('peer-timeout')))
+
+    return ';'.join(wsrep_provider_options)

unit_tests/test_percona_utils.py

@@ -241,6 +241,40 @@ class UtilsTests(unittest.TestCase):
         _wsrep_value.side_effect = [True, 2]
         self.assertTrue(percona_utils.cluster_in_sync())
 
+    @mock.patch("percona_utils.config")
+    def test_get_wsrep_provider_options(self, mock_config):
+        # Empty
+        _config = {"min-cluster-size": 3}
+        mock_config.side_effect = lambda key: _config.get(key)
+        expected = ""
+        self.assertEqual(percona_utils.get_wsrep_provider_options(),
+                         expected)
+
+        # IPv6 only
+        _config = {"prefer-ipv6": True}
+        mock_config.side_effect = lambda key: _config.get(key)
+        expected = "gmcast.listen_addr=tcp://:::4567"
+        self.assertEqual(percona_utils.get_wsrep_provider_options(),
+                         expected)
+        # ipv6 and peer_timeout
+        _config = {"peer-timeout": "PT15S",
+                   "prefer-ipv6": True}
+        mock_config.side_effect = lambda key: _config.get(key)
+        expected = ("gmcast.listen_addr=tcp://:::4567;"
+                    "gmcast.peer_timeout=PT15S")
+        self.assertEqual(percona_utils.get_wsrep_provider_options(),
+                         expected)
+
+        # peer_timeout bad setting
+        _config = {"peer-timeout": "10"}
+        mock_config.side_effect = lambda key: _config.get(key)
+        with self.assertRaises(ValueError):
+            percona_utils.get_wsrep_provider_options()
+        _config = {"peer-timeout": "PT10M"}
+        mock_config.side_effect = lambda key: _config.get(key)
+        with self.assertRaises(ValueError):
+            percona_utils.get_wsrep_provider_options()
+
 
 TO_PATCH = [
     'is_sufficient_peers',