[hopem,r=james-page]

Fix bootstrap clustering issues

Current code assumes the cluster relation will
fire prior to others if peers exist but this is
not a safe assumption. This patch ensures that
however we mark the cluster as bootstrapped we
ensure at least the leader is marked seeded so
as to avoid unnecessary restarts.

hooks/percona_hooks.py

@@ -60,6 +60,7 @@ from percona_utils import (
     is_sufficient_peers,
     notify_bootstrapped,
     is_bootstrapped,
+    get_wsrep_value,
 )
 from charmhelpers.contrib.database.mysql import (
     PerconaClusterHelper,
@@ -180,8 +181,11 @@ def render_config_restart_on_changed(clustered, hosts, bootstrap=False):
                 time.sleep(delay)
                 delay += 2
                 attempts += 1
-            else:
-                mark_seeded()
+
+        # If we get here we assume prior actions have succeeded to always
+        # this unit is marked as seeded so that subsequent calls don't result
+        # in a restart.
+        mark_seeded()
 
 
 def update_shared_db_rels():
@@ -191,6 +195,28 @@ def update_shared_db_rels():
 
 
 @hooks.hook('upgrade-charm')
+def upgrade():
+    check_bootstrap = False
+    try:
+        if is_leader():
+            check_bootstrap = True
+    except:
+        if oldest_peer(peer_units()):
+            check_bootstrap = True
+
+    if check_bootstrap and not is_bootstrapped() and is_sufficient_peers():
+        # If this is the leader but we have not yet broadcast the cluster uuid
+        # then do so now.
+        wsrep_ready = get_wsrep_value('wsrep_ready') or ""
+        if wsrep_ready.lower() in ['on', 'ready']:
+            cluster_state_uuid = get_wsrep_value('wsrep_cluster_state_uuid')
+            if cluster_state_uuid:
+                mark_seeded()
+                notify_bootstrapped(cluster_uuid=cluster_state_uuid)
+
+    config_changed()
+
+
 @hooks.hook('config-changed')
 def config_changed():
     if config('prefer-ipv6'):

hooks/percona_utils.py

@@ -25,11 +25,6 @@ from charmhelpers.core.hookenv import (
     INFO,
     WARNING,
     ERROR,
-    is_leader,
-)
-from charmhelpers.contrib.hahelpers.cluster import (
-    oldest_peer,
-    peer_units,
 )
 from charmhelpers.fetch import (
     apt_install,
@@ -342,23 +337,6 @@ def is_bootstrapped():
 
         return True
 
-    try:
-        if not is_leader():
-            return False
-    except:
-        oldest = oldest_peer(peer_units())
-        if not oldest:
-            return False
-
-    # If this is the leader but we have not yet broadcast the cluster uuid then
-    # do so now.
-    wsrep_ready = get_wsrep_value('wsrep_ready') or ""
-    if wsrep_ready.lower() in ['on', 'ready']:
-        cluster_state_uuid = get_wsrep_value('wsrep_cluster_state_uuid')
-        if cluster_state_uuid:
-            notify_bootstrapped(cluster_uuid=cluster_state_uuid)
-            return True
-
     return False
 
 

unit_tests/test_percona_utils.py

@@ -129,6 +129,7 @@ class UtilsTests(unittest.TestCase):
         mock_rel_get.assert_called_with(rid=2, unit=4)
         self.assertEqual(hosts, ['hostA', 'hostB'])
 
+    @mock.patch.object(percona_utils, 'log', lambda *args, **kwargs: None)
     @mock.patch.object(percona_utils, 'related_units')
     @mock.patch.object(percona_utils, 'relation_ids')
     @mock.patch.object(percona_utils, 'config')