Notify peers bootstrap-uuid during upgrade-charm hook
Since 17.02 the charm relies on the existence of bootstrap-uuid to determine if pxc is bootstraped. The upgrade-charm hook handler uses leader_node_is_ready() which calls cluster_ready(), this function will evaluate to False when bootstrap-uuid is not defined This patch drops leader_node_is_ready() from the upgrade-charm hook to simply rely on is_leader(), for the non-leader unit leader_get('bootstrap-uuid') and then notify_bootstrapped() if it's set. For cases where the non-leader units are upgraded before notify_bootstrapped() is called as part of the leader-settings-changed hooks. Change-Id: I621e23d7666920b91614491927711a85370310ad Closes-Bug: 1674467
This commit is contained in:
parent
bb30f468aa
commit
af74254f20
|
@ -27,6 +27,7 @@ from charmhelpers.core.hookenv import (
|
|||
charm_name,
|
||||
leader_get,
|
||||
open_port,
|
||||
status_set,
|
||||
)
|
||||
from charmhelpers.core.host import (
|
||||
service_restart,
|
||||
|
@ -101,6 +102,8 @@ from percona_utils import (
|
|||
sst_password,
|
||||
root_password,
|
||||
pxc_installed,
|
||||
update_bootstrap_uuid,
|
||||
LeaderNoBootstrapUUIDError,
|
||||
)
|
||||
|
||||
|
||||
|
@ -261,15 +264,28 @@ def update_shared_db_rels():
|
|||
@harden()
|
||||
def upgrade():
|
||||
|
||||
if leader_node_is_ready():
|
||||
# If this is the leader but we have not yet broadcast the cluster uuid
|
||||
# then do so now.
|
||||
if is_leader():
|
||||
if is_unit_paused_set():
|
||||
log('Unit is paused, skiping upgrade', level=INFO)
|
||||
return
|
||||
|
||||
# broadcast the bootstrap-uuid
|
||||
wsrep_ready = get_wsrep_value('wsrep_ready') or ""
|
||||
if wsrep_ready.lower() in ['on', 'ready']:
|
||||
cluster_state_uuid = get_wsrep_value('wsrep_cluster_state_uuid')
|
||||
if cluster_state_uuid:
|
||||
mark_seeded()
|
||||
notify_bootstrapped(cluster_uuid=cluster_state_uuid)
|
||||
else:
|
||||
# Ensure all the peers have the bootstrap-uuid attribute set
|
||||
# as this is all happening during the upgrade-charm hook is reasonable
|
||||
# to expect the cluster is running.
|
||||
|
||||
# Wait until the leader has set the
|
||||
try:
|
||||
update_bootstrap_uuid()
|
||||
except LeaderNoBootstrapUUIDError:
|
||||
status_set('waiting', "Waiting for bootstrap-uuid set by leader")
|
||||
|
||||
config_changed()
|
||||
|
||||
|
@ -697,6 +713,14 @@ def leader_settings_changed():
|
|||
install_percona_xtradb_cluster()
|
||||
# Notify any changes to data in leader storage
|
||||
update_shared_db_rels()
|
||||
log('leader-settings-changed', level='DEBUG')
|
||||
try:
|
||||
update_bootstrap_uuid()
|
||||
except LeaderNoBootstrapUUIDError:
|
||||
# until the bootstrap-uuid attribute is not replicated cluster_ready()
|
||||
# will evaluate to False, so it is necessary to feed back this info
|
||||
# to the user.
|
||||
status_set('waiting', "Waiting for bootstrap-uuid set by leader")
|
||||
|
||||
|
||||
@hooks.hook('nrpe-external-master-relation-joined',
|
||||
|
|
|
@ -77,6 +77,38 @@ DEFAULT_MYSQL_PORT = 3306
|
|||
REQUIRED_INTERFACES = {}
|
||||
|
||||
|
||||
class LeaderNoBootstrapUUIDError(Exception):
|
||||
"""Raised when the leader doesn't have set the bootstrap-uuid attribute"""
|
||||
def __init__(self):
|
||||
super(LeaderNoBootstrapUUIDError, self).__init__(
|
||||
"the leader doesn't have set the bootstrap-uuid attribute")
|
||||
|
||||
|
||||
class InconsistentUUIDError(Exception):
|
||||
"""Raised when the leader and the unit have different UUIDs set"""
|
||||
def __init__(self, leader_uuid, unit_uuid):
|
||||
super(InconsistentUUIDError, self).__init__(
|
||||
"Leader UUID ('%s') != Unit UUID ('%s')" % (leader_uuid,
|
||||
unit_uuid))
|
||||
|
||||
|
||||
class DesyncedException(Exception):
|
||||
'''Raised if PXC unit is not in sync with its peers'''
|
||||
pass
|
||||
|
||||
|
||||
class FakeOSConfigRenderer(object):
|
||||
"""This class is to provide to register_configs() as a 'fake'
|
||||
OSConfigRenderer object that has a complete_contexts method that returns
|
||||
an empty list. This is so that the pause/resume framework can be used
|
||||
from charmhelpers that requires configs to be able to run.
|
||||
This is a bit of a hack, but via Python's duck-typing enables the function
|
||||
to work.
|
||||
"""
|
||||
def complete_contexts(self):
|
||||
return []
|
||||
|
||||
|
||||
def determine_packages():
|
||||
if lsb_release()['DISTRIB_CODENAME'] >= 'wily':
|
||||
# NOTE(beisner): Use recommended mysql-client package
|
||||
|
@ -435,6 +467,48 @@ def notify_bootstrapped(cluster_rid=None, cluster_uuid=None):
|
|||
leader_set(**{'bootstrap-uuid': cluster_uuid})
|
||||
|
||||
|
||||
def update_bootstrap_uuid():
|
||||
"""This function verifies if the leader has set the bootstrap-uuid
|
||||
attribute to then check it against the running cluster uuid, if the check
|
||||
succeeds the bootstrap-uuid field is set in the cluster relation.
|
||||
|
||||
:returns: True if the cluster UUID was updated, False if the local UUID is
|
||||
empty.
|
||||
"""
|
||||
|
||||
lead_cluster_state_uuid = leader_get('bootstrap-uuid')
|
||||
if not lead_cluster_state_uuid:
|
||||
log('Leader has not set bootstrap-uuid', level=DEBUG)
|
||||
raise LeaderNoBootstrapUUIDError()
|
||||
|
||||
wsrep_ready = get_wsrep_value('wsrep_ready') or ""
|
||||
log("wsrep_ready: '%s'" % wsrep_ready, DEBUG)
|
||||
if wsrep_ready.lower() in ['on', 'ready']:
|
||||
cluster_state_uuid = get_wsrep_value('wsrep_cluster_state_uuid')
|
||||
else:
|
||||
cluster_state_uuid = None
|
||||
|
||||
if not cluster_state_uuid:
|
||||
log("UUID is empty: '%s'" % cluster_state_uuid, level=DEBUG)
|
||||
return False
|
||||
elif lead_cluster_state_uuid != cluster_state_uuid:
|
||||
# this may mean 2 things:
|
||||
# 1) the units have diverged, which it's bad and we do stop.
|
||||
# 2) cluster_state_uuid could not be retrieved because it
|
||||
# hasn't been bootstrapped, mysqld is stopped, etc.
|
||||
log('bootstrap uuid differs: %s != %s' % (lead_cluster_state_uuid,
|
||||
cluster_state_uuid),
|
||||
level=ERROR)
|
||||
raise InconsistentUUIDError(lead_cluster_state_uuid,
|
||||
cluster_state_uuid)
|
||||
|
||||
for rid in relation_ids('cluster'):
|
||||
notify_bootstrapped(cluster_rid=rid,
|
||||
cluster_uuid=cluster_state_uuid)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def cluster_in_sync():
|
||||
'''
|
||||
Determines whether the current unit is in sync
|
||||
|
@ -447,11 +521,6 @@ def cluster_in_sync():
|
|||
return False
|
||||
|
||||
|
||||
class DesyncedException(Exception):
|
||||
'''Raised if PXC unit is not in sync with its peers'''
|
||||
pass
|
||||
|
||||
|
||||
def charm_check_func():
|
||||
"""Custom function to assess the status of the current unit
|
||||
|
||||
|
@ -506,18 +575,6 @@ def resolve_cnf_file():
|
|||
return '/etc/mysql/percona-xtradb-cluster.conf.d/mysqld.cnf'
|
||||
|
||||
|
||||
class FakeOSConfigRenderer(object):
|
||||
"""This class is to provide to register_configs() as a 'fake'
|
||||
OSConfigRenderer object that has a complete_contexts method that returns
|
||||
an empty list. This is so that the pause/resume framework can be used
|
||||
from charmhelpers that requires configs to be able to run.
|
||||
This is a bit of a hack, but via Python's duck-typing enables the function
|
||||
to work.
|
||||
"""
|
||||
def complete_contexts(self):
|
||||
return []
|
||||
|
||||
|
||||
def register_configs():
|
||||
"""Return a OSConfigRenderer object.
|
||||
However, ceph-mon wasn't written using OSConfigRenderer objects to do the
|
||||
|
|
|
@ -99,6 +99,7 @@ class BasicDeployment(OpenStackAmuletDeployment):
|
|||
self.test_pacemaker()
|
||||
self.test_pxc_running()
|
||||
self.test_bootstrapped_and_clustered()
|
||||
self.test_bootstrap_uuid_set_in_the_relation()
|
||||
self.test_pause_resume()
|
||||
self.test_kill_master()
|
||||
|
||||
|
@ -151,6 +152,25 @@ class BasicDeployment(OpenStackAmuletDeployment):
|
|||
" (wanted=%s, got=%s)" % (self.units, got))
|
||||
assert got == self.units, msg
|
||||
|
||||
def test_bootstrap_uuid_set_in_the_relation(self):
|
||||
"""Verify that the bootstrap-uuid attribute was set by the leader and
|
||||
all the peers where notified.
|
||||
"""
|
||||
(leader_uuid, code) = self.master_unit.run("leader-get bootstrap-uuid")
|
||||
assert leader_uuid
|
||||
|
||||
cmd_rel_get = ("relation-get -r `relation-ids cluster` "
|
||||
"bootstrap-uuid %s")
|
||||
units = self.d.sentry['percona-cluster']
|
||||
for unit in units:
|
||||
for peer in units:
|
||||
cmd = cmd_rel_get % peer.info['unit_name']
|
||||
self.log.debug(cmd)
|
||||
(output, code) = unit.run(cmd)
|
||||
assert code == 0
|
||||
assert output == leader_uuid, "%s != %s" % (output,
|
||||
leader_uuid)
|
||||
|
||||
def test_pause_resume(self):
|
||||
'''
|
||||
Ensure pasue/resume actions stop/start mysqld on units
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
import sys
|
||||
|
||||
import mock
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from test_utils import CharmTestCase
|
||||
|
||||
|
@ -14,6 +16,7 @@ with mock.patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec:
|
|||
lambda *args, **kwargs: f(*args, **kwargs))
|
||||
import percona_hooks as hooks
|
||||
|
||||
|
||||
TO_PATCH = ['log', 'config',
|
||||
'get_db_helper',
|
||||
'relation_ids',
|
||||
|
@ -294,3 +297,65 @@ class TestInstallPerconaXtraDB(CharmTestCase):
|
|||
self.configure_sstuser.assert_called_with('testpassword')
|
||||
self.apt_install.assert_called_with(['pxc-5.6'], fatal=True)
|
||||
self.run_mysql_checks.assert_not_called()
|
||||
|
||||
|
||||
class TestUpgradeCharm(CharmTestCase):
|
||||
TO_PATCH = [
|
||||
'config',
|
||||
'log',
|
||||
'is_leader',
|
||||
'is_unit_paused_set',
|
||||
'get_wsrep_value',
|
||||
'config_changed',
|
||||
]
|
||||
|
||||
def print_log(self, msg, level=None):
|
||||
print('juju-log: %s: %s' % (level, msg))
|
||||
|
||||
def setUp(self):
|
||||
CharmTestCase.setUp(self, hooks, self.TO_PATCH)
|
||||
self.config.side_effect = self.test_config.get
|
||||
self.log.side_effect = self.print_log
|
||||
self.tmpdir = tempfile.mkdtemp()
|
||||
|
||||
def tearDown(self):
|
||||
CharmTestCase.tearDown(self)
|
||||
try:
|
||||
shutil.rmtree(self.tmpdir)
|
||||
except:
|
||||
pass
|
||||
|
||||
@mock.patch('percona_utils.is_leader')
|
||||
@mock.patch('percona_utils.leader_set')
|
||||
@mock.patch('percona_utils.relation_set')
|
||||
@mock.patch('percona_utils.get_wsrep_value')
|
||||
@mock.patch('percona_utils.relation_ids')
|
||||
@mock.patch('percona_utils.resolve_data_dir')
|
||||
def test_upgrade_charm(self, mock_data_dir, mock_rids, mock_wsrep,
|
||||
mock_rset, mock_lset, mock_is_leader):
|
||||
mock_rids.return_value = ['cluster:22']
|
||||
mock_is_leader.return_value = True
|
||||
self.is_leader.return_value = True
|
||||
self.is_unit_paused_set.return_value = False
|
||||
|
||||
def c(k):
|
||||
values = {'wsrep_ready': 'on',
|
||||
'wsrep_cluster_state_uuid': '1234-abcd'}
|
||||
return values[k]
|
||||
|
||||
self.get_wsrep_value.side_effect = c
|
||||
mock_wsrep.side_effect = c
|
||||
mock_data_dir.return_value = self.tmpdir
|
||||
|
||||
hooks.upgrade()
|
||||
|
||||
seeded_file = os.path.join(self.tmpdir, 'seeded')
|
||||
self.assertTrue(os.path.isfile(seeded_file),
|
||||
"%s is not file" % seeded_file)
|
||||
with open(seeded_file) as f:
|
||||
self.assertEqual(f.read(), 'done')
|
||||
|
||||
mock_rset.assert_called_with(relation_id='cluster:22',
|
||||
**{'bootstrap-uuid': '1234-abcd'})
|
||||
mock_lset.assert_called_with(**{'bootstrap-uuid': '1234-abcd'})
|
||||
self.config_changed.assert_called_with()
|
||||
|
|
|
@ -512,3 +512,74 @@ class TestResolveHostnameToIP(CharmTestCase):
|
|||
dns_query.assert_has_calls([
|
||||
mock.call('myhostname', 'A'),
|
||||
])
|
||||
|
||||
|
||||
class TestUpdateBootstrapUUID(CharmTestCase):
|
||||
TO_PATCH = [
|
||||
'log',
|
||||
'leader_get',
|
||||
'get_wsrep_value',
|
||||
'relation_ids',
|
||||
'relation_set',
|
||||
'is_leader',
|
||||
'leader_set',
|
||||
]
|
||||
|
||||
def setUp(self):
|
||||
CharmTestCase.setUp(self, percona_utils, self.TO_PATCH)
|
||||
self.log.side_effect = self.juju_log
|
||||
|
||||
def juju_log(self, msg, level=None):
|
||||
print('juju-log %s: %s' % (level, msg))
|
||||
|
||||
def test_no_bootstrap_uuid(self):
|
||||
self.leader_get.return_value = None
|
||||
self.assertRaises(percona_utils.LeaderNoBootstrapUUIDError,
|
||||
percona_utils.update_bootstrap_uuid)
|
||||
|
||||
def test_bootstrap_uuid_already_set(self):
|
||||
self.leader_get.return_value = '1234-abcd'
|
||||
|
||||
def fake_wsrep(k):
|
||||
d = {'wsrep_ready': 'ON',
|
||||
'wsrep_cluster_state_uuid': '1234-abcd'}
|
||||
return d[k]
|
||||
|
||||
self.get_wsrep_value.side_effect = fake_wsrep
|
||||
self.relation_ids.return_value = ['cluster:2']
|
||||
self.is_leader.return_value = False
|
||||
percona_utils.update_bootstrap_uuid()
|
||||
self.relation_set.assert_called_with(relation_id='cluster:2',
|
||||
**{'bootstrap-uuid': '1234-abcd'})
|
||||
self.leader_set.assert_not_called()
|
||||
|
||||
self.is_leader.return_value = True
|
||||
percona_utils.update_bootstrap_uuid()
|
||||
self.relation_set.assert_called_with(relation_id='cluster:2',
|
||||
**{'bootstrap-uuid': '1234-abcd'})
|
||||
self.leader_set.assert_called_with(**{'bootstrap-uuid': '1234-abcd'})
|
||||
|
||||
@mock.patch.object(percona_utils, 'notify_bootstrapped')
|
||||
def test_bootstrap_uuid_could_not_be_retrieved(self, mock_notify):
|
||||
self.leader_get.return_value = '1234-abcd'
|
||||
|
||||
def fake_wsrep(k):
|
||||
d = {'wsrep_ready': 'ON',
|
||||
'wsrep_cluster_state_uuid': ''}
|
||||
return d[k]
|
||||
|
||||
self.get_wsrep_value.side_effect = fake_wsrep
|
||||
self.assertFalse(percona_utils.update_bootstrap_uuid())
|
||||
mock_notify.assert_not_called()
|
||||
|
||||
def test_bootstrap_uuid_diffent_uuids(self):
|
||||
self.leader_get.return_value = '1234-abcd'
|
||||
|
||||
def fake_wsrep(k):
|
||||
d = {'wsrep_ready': 'ON',
|
||||
'wsrep_cluster_state_uuid': '5678-dead-beef'}
|
||||
return d[k]
|
||||
|
||||
self.get_wsrep_value.side_effect = fake_wsrep
|
||||
self.assertRaises(percona_utils.InconsistentUUIDError,
|
||||
percona_utils.update_bootstrap_uuid)
|
||||
|
|
Loading…
Reference in New Issue