diff --git a/hooks/percona_hooks.py b/hooks/percona_hooks.py index bf66de5..afc3e25 100755 --- a/hooks/percona_hooks.py +++ b/hooks/percona_hooks.py @@ -27,6 +27,7 @@ from charmhelpers.core.hookenv import ( charm_name, leader_get, open_port, + status_set, ) from charmhelpers.core.host import ( service_restart, @@ -101,6 +102,8 @@ from percona_utils import ( sst_password, root_password, pxc_installed, + update_bootstrap_uuid, + LeaderNoBootstrapUUIDError, ) @@ -261,15 +264,28 @@ def update_shared_db_rels(): @harden() def upgrade(): - if leader_node_is_ready(): - # If this is the leader but we have not yet broadcast the cluster uuid - # then do so now. + if is_leader(): + if is_unit_paused_set(): + log('Unit is paused, skiping upgrade', level=INFO) + return + + # broadcast the bootstrap-uuid wsrep_ready = get_wsrep_value('wsrep_ready') or "" if wsrep_ready.lower() in ['on', 'ready']: cluster_state_uuid = get_wsrep_value('wsrep_cluster_state_uuid') if cluster_state_uuid: mark_seeded() notify_bootstrapped(cluster_uuid=cluster_state_uuid) + else: + # Ensure all the peers have the bootstrap-uuid attribute set + # as this is all happening during the upgrade-charm hook is reasonable + # to expect the cluster is running. + + # Wait until the leader has set the + try: + update_bootstrap_uuid() + except LeaderNoBootstrapUUIDError: + status_set('waiting', "Waiting for bootstrap-uuid set by leader") config_changed() @@ -697,6 +713,14 @@ def leader_settings_changed(): install_percona_xtradb_cluster() # Notify any changes to data in leader storage update_shared_db_rels() + log('leader-settings-changed', level='DEBUG') + try: + update_bootstrap_uuid() + except LeaderNoBootstrapUUIDError: + # until the bootstrap-uuid attribute is not replicated cluster_ready() + # will evaluate to False, so it is necessary to feed back this info + # to the user. + status_set('waiting', "Waiting for bootstrap-uuid set by leader") @hooks.hook('nrpe-external-master-relation-joined', diff --git a/hooks/percona_utils.py b/hooks/percona_utils.py index 57df07a..1c70903 100644 --- a/hooks/percona_utils.py +++ b/hooks/percona_utils.py @@ -77,6 +77,38 @@ DEFAULT_MYSQL_PORT = 3306 REQUIRED_INTERFACES = {} +class LeaderNoBootstrapUUIDError(Exception): + """Raised when the leader doesn't have set the bootstrap-uuid attribute""" + def __init__(self): + super(LeaderNoBootstrapUUIDError, self).__init__( + "the leader doesn't have set the bootstrap-uuid attribute") + + +class InconsistentUUIDError(Exception): + """Raised when the leader and the unit have different UUIDs set""" + def __init__(self, leader_uuid, unit_uuid): + super(InconsistentUUIDError, self).__init__( + "Leader UUID ('%s') != Unit UUID ('%s')" % (leader_uuid, + unit_uuid)) + + +class DesyncedException(Exception): + '''Raised if PXC unit is not in sync with its peers''' + pass + + +class FakeOSConfigRenderer(object): + """This class is to provide to register_configs() as a 'fake' + OSConfigRenderer object that has a complete_contexts method that returns + an empty list. This is so that the pause/resume framework can be used + from charmhelpers that requires configs to be able to run. + This is a bit of a hack, but via Python's duck-typing enables the function + to work. + """ + def complete_contexts(self): + return [] + + def determine_packages(): if lsb_release()['DISTRIB_CODENAME'] >= 'wily': # NOTE(beisner): Use recommended mysql-client package @@ -435,6 +467,48 @@ def notify_bootstrapped(cluster_rid=None, cluster_uuid=None): leader_set(**{'bootstrap-uuid': cluster_uuid}) +def update_bootstrap_uuid(): + """This function verifies if the leader has set the bootstrap-uuid + attribute to then check it against the running cluster uuid, if the check + succeeds the bootstrap-uuid field is set in the cluster relation. + + :returns: True if the cluster UUID was updated, False if the local UUID is + empty. + """ + + lead_cluster_state_uuid = leader_get('bootstrap-uuid') + if not lead_cluster_state_uuid: + log('Leader has not set bootstrap-uuid', level=DEBUG) + raise LeaderNoBootstrapUUIDError() + + wsrep_ready = get_wsrep_value('wsrep_ready') or "" + log("wsrep_ready: '%s'" % wsrep_ready, DEBUG) + if wsrep_ready.lower() in ['on', 'ready']: + cluster_state_uuid = get_wsrep_value('wsrep_cluster_state_uuid') + else: + cluster_state_uuid = None + + if not cluster_state_uuid: + log("UUID is empty: '%s'" % cluster_state_uuid, level=DEBUG) + return False + elif lead_cluster_state_uuid != cluster_state_uuid: + # this may mean 2 things: + # 1) the units have diverged, which it's bad and we do stop. + # 2) cluster_state_uuid could not be retrieved because it + # hasn't been bootstrapped, mysqld is stopped, etc. + log('bootstrap uuid differs: %s != %s' % (lead_cluster_state_uuid, + cluster_state_uuid), + level=ERROR) + raise InconsistentUUIDError(lead_cluster_state_uuid, + cluster_state_uuid) + + for rid in relation_ids('cluster'): + notify_bootstrapped(cluster_rid=rid, + cluster_uuid=cluster_state_uuid) + + return True + + def cluster_in_sync(): ''' Determines whether the current unit is in sync @@ -447,11 +521,6 @@ def cluster_in_sync(): return False -class DesyncedException(Exception): - '''Raised if PXC unit is not in sync with its peers''' - pass - - def charm_check_func(): """Custom function to assess the status of the current unit @@ -506,18 +575,6 @@ def resolve_cnf_file(): return '/etc/mysql/percona-xtradb-cluster.conf.d/mysqld.cnf' -class FakeOSConfigRenderer(object): - """This class is to provide to register_configs() as a 'fake' - OSConfigRenderer object that has a complete_contexts method that returns - an empty list. This is so that the pause/resume framework can be used - from charmhelpers that requires configs to be able to run. - This is a bit of a hack, but via Python's duck-typing enables the function - to work. - """ - def complete_contexts(self): - return [] - - def register_configs(): """Return a OSConfigRenderer object. However, ceph-mon wasn't written using OSConfigRenderer objects to do the diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 75f62a1..8057053 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -99,6 +99,7 @@ class BasicDeployment(OpenStackAmuletDeployment): self.test_pacemaker() self.test_pxc_running() self.test_bootstrapped_and_clustered() + self.test_bootstrap_uuid_set_in_the_relation() self.test_pause_resume() self.test_kill_master() @@ -151,6 +152,25 @@ class BasicDeployment(OpenStackAmuletDeployment): " (wanted=%s, got=%s)" % (self.units, got)) assert got == self.units, msg + def test_bootstrap_uuid_set_in_the_relation(self): + """Verify that the bootstrap-uuid attribute was set by the leader and + all the peers where notified. + """ + (leader_uuid, code) = self.master_unit.run("leader-get bootstrap-uuid") + assert leader_uuid + + cmd_rel_get = ("relation-get -r `relation-ids cluster` " + "bootstrap-uuid %s") + units = self.d.sentry['percona-cluster'] + for unit in units: + for peer in units: + cmd = cmd_rel_get % peer.info['unit_name'] + self.log.debug(cmd) + (output, code) = unit.run(cmd) + assert code == 0 + assert output == leader_uuid, "%s != %s" % (output, + leader_uuid) + def test_pause_resume(self): ''' Ensure pasue/resume actions stop/start mysqld on units diff --git a/unit_tests/test_percona_hooks.py b/unit_tests/test_percona_hooks.py index 8c9fdfa..b904888 100644 --- a/unit_tests/test_percona_hooks.py +++ b/unit_tests/test_percona_hooks.py @@ -1,6 +1,8 @@ -import sys - import mock +import os +import shutil +import sys +import tempfile from test_utils import CharmTestCase @@ -14,6 +16,7 @@ with mock.patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec: lambda *args, **kwargs: f(*args, **kwargs)) import percona_hooks as hooks + TO_PATCH = ['log', 'config', 'get_db_helper', 'relation_ids', @@ -294,3 +297,65 @@ class TestInstallPerconaXtraDB(CharmTestCase): self.configure_sstuser.assert_called_with('testpassword') self.apt_install.assert_called_with(['pxc-5.6'], fatal=True) self.run_mysql_checks.assert_not_called() + + +class TestUpgradeCharm(CharmTestCase): + TO_PATCH = [ + 'config', + 'log', + 'is_leader', + 'is_unit_paused_set', + 'get_wsrep_value', + 'config_changed', + ] + + def print_log(self, msg, level=None): + print('juju-log: %s: %s' % (level, msg)) + + def setUp(self): + CharmTestCase.setUp(self, hooks, self.TO_PATCH) + self.config.side_effect = self.test_config.get + self.log.side_effect = self.print_log + self.tmpdir = tempfile.mkdtemp() + + def tearDown(self): + CharmTestCase.tearDown(self) + try: + shutil.rmtree(self.tmpdir) + except: + pass + + @mock.patch('percona_utils.is_leader') + @mock.patch('percona_utils.leader_set') + @mock.patch('percona_utils.relation_set') + @mock.patch('percona_utils.get_wsrep_value') + @mock.patch('percona_utils.relation_ids') + @mock.patch('percona_utils.resolve_data_dir') + def test_upgrade_charm(self, mock_data_dir, mock_rids, mock_wsrep, + mock_rset, mock_lset, mock_is_leader): + mock_rids.return_value = ['cluster:22'] + mock_is_leader.return_value = True + self.is_leader.return_value = True + self.is_unit_paused_set.return_value = False + + def c(k): + values = {'wsrep_ready': 'on', + 'wsrep_cluster_state_uuid': '1234-abcd'} + return values[k] + + self.get_wsrep_value.side_effect = c + mock_wsrep.side_effect = c + mock_data_dir.return_value = self.tmpdir + + hooks.upgrade() + + seeded_file = os.path.join(self.tmpdir, 'seeded') + self.assertTrue(os.path.isfile(seeded_file), + "%s is not file" % seeded_file) + with open(seeded_file) as f: + self.assertEqual(f.read(), 'done') + + mock_rset.assert_called_with(relation_id='cluster:22', + **{'bootstrap-uuid': '1234-abcd'}) + mock_lset.assert_called_with(**{'bootstrap-uuid': '1234-abcd'}) + self.config_changed.assert_called_with() diff --git a/unit_tests/test_percona_utils.py b/unit_tests/test_percona_utils.py index 75d58fb..0d720e5 100644 --- a/unit_tests/test_percona_utils.py +++ b/unit_tests/test_percona_utils.py @@ -512,3 +512,74 @@ class TestResolveHostnameToIP(CharmTestCase): dns_query.assert_has_calls([ mock.call('myhostname', 'A'), ]) + + +class TestUpdateBootstrapUUID(CharmTestCase): + TO_PATCH = [ + 'log', + 'leader_get', + 'get_wsrep_value', + 'relation_ids', + 'relation_set', + 'is_leader', + 'leader_set', + ] + + def setUp(self): + CharmTestCase.setUp(self, percona_utils, self.TO_PATCH) + self.log.side_effect = self.juju_log + + def juju_log(self, msg, level=None): + print('juju-log %s: %s' % (level, msg)) + + def test_no_bootstrap_uuid(self): + self.leader_get.return_value = None + self.assertRaises(percona_utils.LeaderNoBootstrapUUIDError, + percona_utils.update_bootstrap_uuid) + + def test_bootstrap_uuid_already_set(self): + self.leader_get.return_value = '1234-abcd' + + def fake_wsrep(k): + d = {'wsrep_ready': 'ON', + 'wsrep_cluster_state_uuid': '1234-abcd'} + return d[k] + + self.get_wsrep_value.side_effect = fake_wsrep + self.relation_ids.return_value = ['cluster:2'] + self.is_leader.return_value = False + percona_utils.update_bootstrap_uuid() + self.relation_set.assert_called_with(relation_id='cluster:2', + **{'bootstrap-uuid': '1234-abcd'}) + self.leader_set.assert_not_called() + + self.is_leader.return_value = True + percona_utils.update_bootstrap_uuid() + self.relation_set.assert_called_with(relation_id='cluster:2', + **{'bootstrap-uuid': '1234-abcd'}) + self.leader_set.assert_called_with(**{'bootstrap-uuid': '1234-abcd'}) + + @mock.patch.object(percona_utils, 'notify_bootstrapped') + def test_bootstrap_uuid_could_not_be_retrieved(self, mock_notify): + self.leader_get.return_value = '1234-abcd' + + def fake_wsrep(k): + d = {'wsrep_ready': 'ON', + 'wsrep_cluster_state_uuid': ''} + return d[k] + + self.get_wsrep_value.side_effect = fake_wsrep + self.assertFalse(percona_utils.update_bootstrap_uuid()) + mock_notify.assert_not_called() + + def test_bootstrap_uuid_diffent_uuids(self): + self.leader_get.return_value = '1234-abcd' + + def fake_wsrep(k): + d = {'wsrep_ready': 'ON', + 'wsrep_cluster_state_uuid': '5678-dead-beef'} + return d[k] + + self.get_wsrep_value.side_effect = fake_wsrep + self.assertRaises(percona_utils.InconsistentUUIDError, + percona_utils.update_bootstrap_uuid)