openstack
/
charm-percona-cluster
Code Issues Proposed changes

Merge "Notify peers bootstrap-uuid during upgrade-charm hook"

This commit is contained in:
Jenkins 2017-04-05 16:10:09 +00:00 committed by Gerrit Code Review
parent 00b5990e74 af74254f20
commit 7f1b953982
5 changed files with 259 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
hooks/percona_hooks.py
View File

@ -27,6 +27,7 @@ from charmhelpers.core.hookenv import (
charm_name,
leader_get,
open_port,
status_set,
)
from charmhelpers.core.host import (
service_restart,
@ -102,6 +103,8 @@ from percona_utils import (
sst_password,
root_password,
pxc_installed,
update_bootstrap_uuid,
LeaderNoBootstrapUUIDError,
)
@ -263,15 +266,28 @@ def update_shared_db_rels():
@harden()
def upgrade():
if leader_node_is_ready():
# If this is the leader but we have not yet broadcast the cluster uuid
# then do so now.
if is_leader():
if is_unit_paused_set():
log('Unit is paused, skiping upgrade', level=INFO)
return
# broadcast the bootstrap-uuid
wsrep_ready = get_wsrep_value('wsrep_ready') or ""
if wsrep_ready.lower() in ['on', 'ready']:
cluster_state_uuid = get_wsrep_value('wsrep_cluster_state_uuid')
if cluster_state_uuid:
mark_seeded()
notify_bootstrapped(cluster_uuid=cluster_state_uuid)
else:
# Ensure all the peers have the bootstrap-uuid attribute set
# as this is all happening during the upgrade-charm hook is reasonable
# to expect the cluster is running.
# Wait until the leader has set the
try:
update_bootstrap_uuid()
except LeaderNoBootstrapUUIDError:
status_set('waiting', "Waiting for bootstrap-uuid set by leader")
config_changed()
@ -699,6 +715,14 @@ def leader_settings_changed():
install_percona_xtradb_cluster()
# Notify any changes to data in leader storage
update_shared_db_rels()
log('leader-settings-changed', level='DEBUG')
try:
update_bootstrap_uuid()
except LeaderNoBootstrapUUIDError:
# until the bootstrap-uuid attribute is not replicated cluster_ready()
# will evaluate to False, so it is necessary to feed back this info
# to the user.
status_set('waiting', "Waiting for bootstrap-uuid set by leader")
@hooks.hook('nrpe-external-master-relation-joined',

91
hooks/percona_utils.py
View File

@ -78,6 +78,38 @@ DEFAULT_MYSQL_PORT = 3306
REQUIRED_INTERFACES = {}
class LeaderNoBootstrapUUIDError(Exception):
"""Raised when the leader doesn't have set the bootstrap-uuid attribute"""
def __init__(self):
super(LeaderNoBootstrapUUIDError, self).__init__(
"the leader doesn't have set the bootstrap-uuid attribute")
class InconsistentUUIDError(Exception):
"""Raised when the leader and the unit have different UUIDs set"""
def __init__(self, leader_uuid, unit_uuid):
super(InconsistentUUIDError, self).__init__(
"Leader UUID ('%s') != Unit UUID ('%s')" % (leader_uuid,
unit_uuid))
class DesyncedException(Exception):
'''Raised if PXC unit is not in sync with its peers'''
pass
class FakeOSConfigRenderer(object):
"""This class is to provide to register_configs() as a 'fake'
OSConfigRenderer object that has a complete_contexts method that returns
an empty list. This is so that the pause/resume framework can be used
from charmhelpers that requires configs to be able to run.
This is a bit of a hack, but via Python's duck-typing enables the function
to work.
"""
def complete_contexts(self):
return []
def determine_packages():
if CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'wily':
# NOTE(beisner): Use recommended mysql-client package
@ -437,6 +469,48 @@ def notify_bootstrapped(cluster_rid=None, cluster_uuid=None):
leader_set(**{'bootstrap-uuid': cluster_uuid})
def update_bootstrap_uuid():
"""This function verifies if the leader has set the bootstrap-uuid
attribute to then check it against the running cluster uuid, if the check
succeeds the bootstrap-uuid field is set in the cluster relation.
:returns: True if the cluster UUID was updated, False if the local UUID is
empty.
"""
lead_cluster_state_uuid = leader_get('bootstrap-uuid')
if not lead_cluster_state_uuid:
log('Leader has not set bootstrap-uuid', level=DEBUG)
raise LeaderNoBootstrapUUIDError()
wsrep_ready = get_wsrep_value('wsrep_ready') or ""
log("wsrep_ready: '%s'" % wsrep_ready, DEBUG)
if wsrep_ready.lower() in ['on', 'ready']:
cluster_state_uuid = get_wsrep_value('wsrep_cluster_state_uuid')
else:
cluster_state_uuid = None
if not cluster_state_uuid:
log("UUID is empty: '%s'" % cluster_state_uuid, level=DEBUG)
return False
elif lead_cluster_state_uuid != cluster_state_uuid:
# this may mean 2 things:
# 1) the units have diverged, which it's bad and we do stop.
# 2) cluster_state_uuid could not be retrieved because it
# hasn't been bootstrapped, mysqld is stopped, etc.
log('bootstrap uuid differs: %s != %s' % (lead_cluster_state_uuid,
cluster_state_uuid),
level=ERROR)
raise InconsistentUUIDError(lead_cluster_state_uuid,
cluster_state_uuid)
for rid in relation_ids('cluster'):
notify_bootstrapped(cluster_rid=rid,
cluster_uuid=cluster_state_uuid)
return True
def cluster_in_sync():
'''
Determines whether the current unit is in sync
@ -449,11 +523,6 @@ def cluster_in_sync():
return False
class DesyncedException(Exception):
'''Raised if PXC unit is not in sync with its peers'''
pass
def charm_check_func():
"""Custom function to assess the status of the current unit
@ -510,18 +579,6 @@ def resolve_cnf_file():
return '/etc/mysql/percona-xtradb-cluster.conf.d/mysqld.cnf'
class FakeOSConfigRenderer(object):
"""This class is to provide to register_configs() as a 'fake'
OSConfigRenderer object that has a complete_contexts method that returns
an empty list. This is so that the pause/resume framework can be used
from charmhelpers that requires configs to be able to run.
This is a bit of a hack, but via Python's duck-typing enables the function
to work.
"""
def complete_contexts(self):
return []
def register_configs():
"""Return a OSConfigRenderer object.
However, ceph-mon wasn't written using OSConfigRenderer objects to do the

20
tests/basic_deployment.py
View File

@ -99,6 +99,7 @@ class BasicDeployment(OpenStackAmuletDeployment):
self.test_pacemaker()
self.test_pxc_running()
self.test_bootstrapped_and_clustered()
self.test_bootstrap_uuid_set_in_the_relation()
self.test_pause_resume()
self.test_kill_master()
@ -151,6 +152,25 @@ class BasicDeployment(OpenStackAmuletDeployment):
" (wanted=%s, got=%s)" % (self.units, got))
assert got == self.units, msg
def test_bootstrap_uuid_set_in_the_relation(self):
"""Verify that the bootstrap-uuid attribute was set by the leader and
all the peers where notified.
"""
(leader_uuid, code) = self.master_unit.run("leader-get bootstrap-uuid")
assert leader_uuid
cmd_rel_get = ("relation-get -r `relation-ids cluster` "
"bootstrap-uuid %s")
units = self.d.sentry['percona-cluster']
for unit in units:
for peer in units:
cmd = cmd_rel_get % peer.info['unit_name']
self.log.debug(cmd)
(output, code) = unit.run(cmd)
assert code == 0
assert output == leader_uuid, "%s != %s" % (output,
leader_uuid)
def test_pause_resume(self):
'''
Ensure pasue/resume actions stop/start mysqld on units

69
unit_tests/test_percona_hooks.py
View File

@ -1,6 +1,8 @@
import sys
import mock
import os
import shutil
import sys
import tempfile
from test_utils import CharmTestCase
@ -14,6 +16,7 @@ with mock.patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec:
lambda *args, **kwargs: f(*args, **kwargs))
import percona_hooks as hooks
TO_PATCH = ['log', 'config',
'get_db_helper',
'relation_ids',
@ -294,3 +297,65 @@ class TestInstallPerconaXtraDB(CharmTestCase):
self.configure_sstuser.assert_called_with('testpassword')
self.apt_install.assert_called_with(['pxc-5.6'], fatal=True)
self.run_mysql_checks.assert_not_called()
class TestUpgradeCharm(CharmTestCase):
TO_PATCH = [
'config',
'log',
'is_leader',
'is_unit_paused_set',
'get_wsrep_value',
'config_changed',
]
def print_log(self, msg, level=None):
print('juju-log: %s: %s' % (level, msg))
def setUp(self):
CharmTestCase.setUp(self, hooks, self.TO_PATCH)
self.config.side_effect = self.test_config.get
self.log.side_effect = self.print_log
self.tmpdir = tempfile.mkdtemp()
def tearDown(self):
CharmTestCase.tearDown(self)
try:
shutil.rmtree(self.tmpdir)
except:
pass
@mock.patch('percona_utils.is_leader')
@mock.patch('percona_utils.leader_set')
@mock.patch('percona_utils.relation_set')
@mock.patch('percona_utils.get_wsrep_value')
@mock.patch('percona_utils.relation_ids')
@mock.patch('percona_utils.resolve_data_dir')
def test_upgrade_charm(self, mock_data_dir, mock_rids, mock_wsrep,
mock_rset, mock_lset, mock_is_leader):
mock_rids.return_value = ['cluster:22']
mock_is_leader.return_value = True
self.is_leader.return_value = True
self.is_unit_paused_set.return_value = False
def c(k):
values = {'wsrep_ready': 'on',
'wsrep_cluster_state_uuid': '1234-abcd'}
return values[k]
self.get_wsrep_value.side_effect = c
mock_wsrep.side_effect = c
mock_data_dir.return_value = self.tmpdir
hooks.upgrade()
seeded_file = os.path.join(self.tmpdir, 'seeded')
self.assertTrue(os.path.isfile(seeded_file),
"%s is not file" % seeded_file)
with open(seeded_file) as f:
self.assertEqual(f.read(), 'done')
mock_rset.assert_called_with(relation_id='cluster:22',
**{'bootstrap-uuid': '1234-abcd'})
mock_lset.assert_called_with(**{'bootstrap-uuid': '1234-abcd'})
self.config_changed.assert_called_with()

71
unit_tests/test_percona_utils.py
View File

@ -512,3 +512,74 @@ class TestResolveHostnameToIP(CharmTestCase):
dns_query.assert_has_calls([
mock.call('myhostname', 'A'),
])
class TestUpdateBootstrapUUID(CharmTestCase):
TO_PATCH = [
'log',
'leader_get',
'get_wsrep_value',
'relation_ids',
'relation_set',
'is_leader',
'leader_set',
]
def setUp(self):
CharmTestCase.setUp(self, percona_utils, self.TO_PATCH)
self.log.side_effect = self.juju_log
def juju_log(self, msg, level=None):
print('juju-log %s: %s' % (level, msg))
def test_no_bootstrap_uuid(self):
self.leader_get.return_value = None
self.assertRaises(percona_utils.LeaderNoBootstrapUUIDError,
percona_utils.update_bootstrap_uuid)
def test_bootstrap_uuid_already_set(self):
self.leader_get.return_value = '1234-abcd'
def fake_wsrep(k):
d = {'wsrep_ready': 'ON',
'wsrep_cluster_state_uuid': '1234-abcd'}
return d[k]
self.get_wsrep_value.side_effect = fake_wsrep
self.relation_ids.return_value = ['cluster:2']
self.is_leader.return_value = False
percona_utils.update_bootstrap_uuid()
self.relation_set.assert_called_with(relation_id='cluster:2',
**{'bootstrap-uuid': '1234-abcd'})
self.leader_set.assert_not_called()
self.is_leader.return_value = True
percona_utils.update_bootstrap_uuid()
self.relation_set.assert_called_with(relation_id='cluster:2',
**{'bootstrap-uuid': '1234-abcd'})
self.leader_set.assert_called_with(**{'bootstrap-uuid': '1234-abcd'})
@mock.patch.object(percona_utils, 'notify_bootstrapped')
def test_bootstrap_uuid_could_not_be_retrieved(self, mock_notify):
self.leader_get.return_value = '1234-abcd'
def fake_wsrep(k):
d = {'wsrep_ready': 'ON',
'wsrep_cluster_state_uuid': ''}
return d[k]
self.get_wsrep_value.side_effect = fake_wsrep
self.assertFalse(percona_utils.update_bootstrap_uuid())
mock_notify.assert_not_called()
def test_bootstrap_uuid_diffent_uuids(self):
self.leader_get.return_value = '1234-abcd'
def fake_wsrep(k):
d = {'wsrep_ready': 'ON',
'wsrep_cluster_state_uuid': '5678-dead-beef'}
return d[k]
self.get_wsrep_value.side_effect = fake_wsrep
self.assertRaises(percona_utils.InconsistentUUIDError,
percona_utils.update_bootstrap_uuid)