Guarantee timing of installation and render

Previous attempts to solve Bug #1738896 missed the root cause. The
root cause problem is when the configuration file is rendered before
percona is installed. The rendering includes clustering configuration
which causes percona-cluster to automatically do a single bootstrap
when percona-cluster packages are installed leading to the UUID
mismatch.

The timing and ordering of installation, rendering of the
configuration and restart of mysql is critical across all hook
executions.

This change is a partial reversion of Change ID
I95e56bd28152c934f413025a22dd6821b2ad8e94. The change primarily
guarantees percona-cluster is not installed on non-leader nodes
before the leader is bootstrapped and makes sure the configuration
does not get rendered prior to installation.

is_leader_bootstrapped is introduced and guarantees all data expected
from the leader is available to guard on various tasks.

Closes-Bug: #1744961
Closes-Bug: #1738896
Change-Id: Ifeb1520dba3b14fc1b51a586141905a385f2b2c1

hooks/percona_hooks.py

@@ -89,7 +89,7 @@ from percona_utils import (
     mark_seeded, seeded,
     install_mysql_ocf,
     notify_bootstrapped,
-    is_bootstrapped,
+    is_leader_bootstrapped,
     get_wsrep_value,
     assess_status,
     register_configs,
@@ -129,6 +129,10 @@ def install_percona_xtradb_cluster():
         log('MySQL already installed, skipping')
         return
 
+    if not is_leader() and not is_leader_bootstrapped():
+        log('Non-leader waiting on leader bootstrap, skipping percona install')
+        return
+
     _root_password = root_password()
     _sst_password = sst_password()
     if not _root_password or not _sst_password:
@@ -211,6 +215,9 @@ def render_config_restart_on_changed(clustered, hosts, bootstrap=False):
     it is started so long as the new node to be added is guaranteed to have
     been restarted so as to apply the new config.
     """
+    if not is_leader() and not is_leader_bootstrapped():
+        log('Non-leader waiting on leader bootstrap, skipping render')
+        return
     config_file = resolve_cnf_file()
     pre_hash = file_hash(config_file)
     render_config(clustered, hosts)
@@ -304,6 +311,8 @@ def upgrade():
 @hooks.hook('config-changed')
 @harden()
 def config_changed():
+    install_percona_xtradb_cluster()
+
     # if we are paused, delay doing any config changed hooks.  It is forced on
     # the resume.
     if is_unit_paused_set():
@@ -314,7 +323,7 @@ def config_changed():
 
     hosts = get_cluster_hosts()
     clustered = len(hosts) > 1
-    bootstrapped = is_bootstrapped()
+    bootstrapped = is_leader_bootstrapped()
 
     # NOTE: only configure the cluster if we have sufficient peers. This only
     # applies if min-cluster-size is provided and is used to avoid extraneous
@@ -729,10 +738,11 @@ def ha_relation_changed():
 @hooks.hook('leader-settings-changed')
 def leader_settings_changed():
     '''Re-trigger install once leader has seeded passwords into install'''
+    if not is_leader_bootstrapped():
+        log('Leader is not fully bootstrapped, '
+            'skipping leader-setting-changed', level='DEBUG')
+        return
     install_percona_xtradb_cluster()
-    # Need to render the template files
-    config_changed()
-    log('leader-settings-changed', level='DEBUG')
     try:
         update_bootstrap_uuid()
     except LeaderNoBootstrapUUIDError:
@@ -741,6 +751,8 @@ def leader_settings_changed():
         # to the user.
         status_set('waiting', "Waiting for bootstrap-uuid set by leader")
 
+    config_changed()
+
 
 @hooks.hook('nrpe-external-master-relation-joined',
             'nrpe-external-master-relation-changed')

hooks/percona_utils.py

@@ -404,20 +404,51 @@ def get_wsrep_value(key):
     return None
 
 
+def is_leader_bootstrapped():
+    """ Check that the leader is bootstrapped and has set required settings
+
+    :side_effect: calls leader_get
+    :returns: boolean
+    """
+
+    check_settings = ['bootstrap-uuid', 'mysql.passwd',
+                      'root-password', 'sst-password']
+    leader_settings = leader_get()
+
+    # Is the leader bootstrapped?
+    for setting in check_settings:
+        if leader_settings.get(setting) is None:
+            log("Leader is NOT bootstrapped {}: {}".format(setting,
+                leader_settings.get('bootstrap-uuid')), DEBUG)
+            return False
+
+    log("Leader is bootstrapped uuid: {}".format(
+        leader_settings.get('bootstrap-uuid')), DEBUG)
+    return True
+
+
 def is_bootstrapped():
     """ Check that this unit is bootstrapped
 
     @returns boolean
     """
+    uuids = []
+    rids = relation_ids('cluster') or []
+    for rid in rids:
+        units = related_units(rid)
+        units.append(local_unit())
+        for unit in units:
+            id = relation_get('bootstrap-uuid', unit=unit, rid=rid)
+            if id:
+                uuids.append(id)
+
+    if uuids:
+        if len(set(uuids)) > 1:
+            log("Found inconsistent bootstrap uuids - %s" % (uuids), WARNING)
 
-    # Is the leader bootstrapped?
-    # Leader settings happen long before relation settings.
-    if leader_get('bootstrap-uuid'):
-        log("Leader is bootstrapped uuid: {}".format(
-            leader_get('bootstrap-uuid')), WARNING)
         return True
-    else:
-        return False
+
+    return False
 
 
 def bootstrap_pxc():
@@ -668,6 +699,9 @@ def _pause_resume_helper(f, configs):
 
 
 def create_binlogs_directory():
+    if not pxc_installed():
+        log("PXC not yet installed. Not setting up binlogs", DEBUG)
+        return
     binlogs_directory = os.path.dirname(config('binlogs-path'))
     data_dir = resolve_data_dir() + '/'
     if binlogs_directory.startswith(data_dir):

unit_tests/test_percona_hooks.py

@@ -24,7 +24,7 @@ TO_PATCH = ['log', 'config',
             'update_nrpe_config',
             'get_iface_for_address',
             'get_netmask_for_address',
-            'is_bootstrapped',
+            'is_leader_bootstrapped',
             'network_get_primary_address',
             'resolve_network_cidr',
             'unit_get',
@@ -291,7 +291,7 @@ class TestConfigChanged(CharmTestCase):
         'config',
         'is_unit_paused_set',
         'get_cluster_hosts',
-        'is_bootstrapped',
+        'is_leader_bootstrapped',
         'is_leader',
         'render_config_restart_on_changed',
         'update_shared_db_rels',
@@ -310,7 +310,7 @@ class TestConfigChanged(CharmTestCase):
     def test_config_changed_open_port(self):
         '''Ensure open_port is called with MySQL default port'''
         self.is_unit_paused_set.return_value = False
-        self.is_bootstrapped.return_value = False
+        self.is_leader_bootstrapped.return_value = False
         self.is_leader.return_value = False
         self.relation_ids.return_value = []
         self.is_relation_made.return_value = False
@@ -331,6 +331,8 @@ class TestInstallPerconaXtraDB(CharmTestCase):
         'configure_sstuser',
         'config',
         'run_mysql_checks',
+        'is_leader_bootstrapped',
+        'is_leader',
     ]
 
     def setUp(self):
@@ -351,6 +353,7 @@ class TestInstallPerconaXtraDB(CharmTestCase):
         self.configure_mysql_root_password.assert_not_called()
         self.configure_sstuser.assert_not_called()
         self.apt_install.assert_not_called()
+        self.is_leader_bootstrapped.return_value = True
 
         self.root_password.return_value = None
         self.sst_password.return_value = 'testpassword'
@@ -363,6 +366,7 @@ class TestInstallPerconaXtraDB(CharmTestCase):
         self.root_password.return_value = 'rootpassword'
         self.sst_password.return_value = 'testpassword'
         self.determine_packages.return_value = ['pxc-5.6']
+        self.is_leader_bootstrapped.return_value = True
         hooks.install_percona_xtradb_cluster()
         self.configure_mysql_root_password.assert_called_with('rootpassword')
         self.configure_sstuser.assert_called_with('testpassword')

unit_tests/test_percona_utils.py

@@ -654,3 +654,24 @@ class TestUpdateBootstrapUUID(CharmTestCase):
 
         percona_utils.update_root_password()
         my_mock.set_mysql_root_password.assert_called_once_with('leaderpass')
+
+    def test_is_leader_bootstrapped_once(self):
+        leader_config = {'bootstrap-uuid': None, 'mysql.passwd': None,
+                         'root-password': None, 'sst-password': None}
+        self.leader_get.return_value = leader_config
+        self.assertFalse(percona_utils.is_leader_bootstrapped())
+
+        leader_config = {'bootstrap-uuid': 'UUID', 'mysql.passwd': None,
+                         'root-password': None, 'sst-password': None}
+        self.leader_get.return_value = leader_config
+        self.assertFalse(percona_utils.is_leader_bootstrapped())
+
+        leader_config = {'bootstrap-uuid': None, 'mysql.passwd': None,
+                         'root-password': 'pass', 'sst-password': None}
+        self.leader_get.return_value = leader_config
+        self.assertFalse(percona_utils.is_leader_bootstrapped())
+
+        leader_config = {'bootstrap-uuid': 'UUID', 'mysql.passwd': 'pass',
+                         'root-password': 'pass', 'sst-password': 'pass'}
+        self.leader_get.return_value = leader_config
+        self.assertTrue(percona_utils.is_leader_bootstrapped())