From 069153f4e69b0719a6a01b03adcc6f4c51594b5e Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 7 Oct 2015 12:29:00 -0700 Subject: [PATCH 1/8] Add basic status checks --- hooks/percona_hooks.py | 18 ++++++++++++++++++ hooks/percona_utils.py | 10 ++++++++++ hooks/update-status | 1 + 3 files changed, 29 insertions(+) create mode 120000 hooks/update-status diff --git a/hooks/percona_hooks.py b/hooks/percona_hooks.py index 4d4288c..93f6820 100755 --- a/hooks/percona_hooks.py +++ b/hooks/percona_hooks.py @@ -24,6 +24,7 @@ from charmhelpers.core.hookenv import ( INFO, WARNING, is_leader, + status_set, ) from charmhelpers.core.host import ( service, @@ -61,6 +62,7 @@ from percona_utils import ( notify_bootstrapped, is_bootstrapped, get_wsrep_value, + cluster_in_sync, ) from charmhelpers.contrib.database.mysql import ( PerconaClusterHelper, @@ -620,11 +622,27 @@ def update_nrpe_config(): nrpe_setup.write() +def assess_status(): + '''Assess the status of the current unit''' + # Ensure that number of peers > cluster size configuration + if not is_bootstrapped(): + status_set('blocked', 'Insufficient peers to bootstrap cluster') + return + # Once running, ensure that cluster is in sync and has the required peers + # will need to access mysql to determine status. + # Also check to see if hacluster has presented 'clustered' back yet + if is_bootstrapped() and cluster_in_sync(): + status_set('active', 'Unit is ready and in sync') + else: + status_set('blocked', 'Unit is not in sync') + + def main(): try: hooks.execute(sys.argv) except UnregisteredHookError as e: log('Unknown hook {} - skipping.'.format(e)) + assess_status() if __name__ == '__main__': diff --git a/hooks/percona_utils.py b/hooks/percona_utils.py index 80185f5..51b21d8 100644 --- a/hooks/percona_utils.py +++ b/hooks/percona_utils.py @@ -360,3 +360,13 @@ def notify_bootstrapped(cluster_rid=None, cluster_uuid=None): (cluster_uuid), DEBUG) for rid in rids: relation_set(relation_id=rid, **{'bootstrap-uuid': cluster_uuid}) + + +def cluster_in_sync(): + '''Determines whether the current unit is in sync with the rest of the cluster''' + ready = get_wsrep_value('wsrep_ready') or False + sync_status = get_wsrep_value('wsrep_local_state') or 0 + if ready and int(sync_status) == 4: + return True + return False + diff --git a/hooks/update-status b/hooks/update-status new file mode 120000 index 0000000..2af5208 --- /dev/null +++ b/hooks/update-status @@ -0,0 +1 @@ +percona_hooks.py \ No newline at end of file From f2ce96516c74ca09120d0645b7cf1368b5ceb62d Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 8 Oct 2015 04:48:45 -0700 Subject: [PATCH 2/8] Tidy lint, add utils tests for cluster_in_sync --- hooks/percona_utils.py | 8 +++++--- unit_tests/test_percona_utils.py | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/hooks/percona_utils.py b/hooks/percona_utils.py index 51b21d8..7611fd6 100644 --- a/hooks/percona_utils.py +++ b/hooks/percona_utils.py @@ -363,10 +363,12 @@ def notify_bootstrapped(cluster_rid=None, cluster_uuid=None): def cluster_in_sync(): - '''Determines whether the current unit is in sync with the rest of the cluster''' + ''' + Determines whether the current unit is in sync + with the rest of the cluster + ''' ready = get_wsrep_value('wsrep_ready') or False sync_status = get_wsrep_value('wsrep_local_state') or 0 - if ready and int(sync_status) == 4: + if ready and int(sync_status) in [2, 4]: return True return False - diff --git a/unit_tests/test_percona_utils.py b/unit_tests/test_percona_utils.py index cf237a8..6b75845 100644 --- a/unit_tests/test_percona_utils.py +++ b/unit_tests/test_percona_utils.py @@ -160,3 +160,23 @@ class UtilsTests(unittest.TestCase): self.assertEqual(percona_utils.determine_packages(), ['percona-xtradb-cluster-server-5.5', 'percona-xtradb-cluster-client-5.5']) + + @mock.patch.object(percona_utils, 'get_wsrep_value') + def test_cluster_in_sync_not_ready(self, _wsrep_value): + _wsrep_value.side_effect = [None, None] + self.assertFalse(percona_utils.cluster_in_sync()) + + @mock.patch.object(percona_utils, 'get_wsrep_value') + def test_cluster_in_sync_ready_syncing(self, _wsrep_value): + _wsrep_value.side_effect = [True, None] + self.assertFalse(percona_utils.cluster_in_sync()) + + @mock.patch.object(percona_utils, 'get_wsrep_value') + def test_cluster_in_sync_ready_sync(self, _wsrep_value): + _wsrep_value.side_effect = [True, 4] + self.assertTrue(percona_utils.cluster_in_sync()) + + @mock.patch.object(percona_utils, 'get_wsrep_value') + def test_cluster_in_sync_ready_sync_donor(self, _wsrep_value): + _wsrep_value.side_effect = [True, 2] + self.assertTrue(percona_utils.cluster_in_sync()) From dcb800d1b4e72b83112eb9b4e4bab8311d7bfcfb Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 8 Oct 2015 04:52:56 -0700 Subject: [PATCH 3/8] Add unit tests for assess status --- unit_tests/test_percona_hooks.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/unit_tests/test_percona_hooks.py b/unit_tests/test_percona_hooks.py index 92cc2d9..5f24f06 100644 --- a/unit_tests/test_percona_hooks.py +++ b/unit_tests/test_percona_hooks.py @@ -11,7 +11,10 @@ TO_PATCH = ['log', 'config', 'relation_set', 'update_nrpe_config', 'get_iface_for_address', - 'get_netmask_for_address'] + 'get_netmask_for_address', + 'cluster_in_sync', + 'is_bootstrapped', + 'status_set'] class TestHaRelation(CharmTestCase): @@ -127,3 +130,25 @@ class TestHaRelation(CharmTestCase): call_args, call_kwargs = self.relation_set.call_args self.assertEqual(resource_params, call_kwargs['resource_params']) + + +class TestAssessStatus(CharmTestCase): + def setUp(self): + CharmTestCase.setUp(self, hooks, TO_PATCH) + + def test_not_bootstrapped(self): + self.is_bootstrapped.return_value = False + hooks.assess_status() + self.status_set.assert_called_with('blocked', mock.ANY) + + def test_bootstrapped_in_sync(self): + self.is_bootstrapped.return_value = True + self.cluster_in_sync.return_value = True + hooks.assess_status() + self.status_set.assert_called_with('active', mock.ANY) + + def test_bootstrapped_not_in_sync(self): + self.is_bootstrapped.return_value = True + self.cluster_in_sync.return_value = False + hooks.assess_status() + self.status_set.assert_called_with('blocked', mock.ANY) From e87d240357a7dfce08315e7943bc0d3aaa2cd735 Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 8 Oct 2015 05:20:12 -0700 Subject: [PATCH 4/8] Drop a comment --- hooks/percona_hooks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hooks/percona_hooks.py b/hooks/percona_hooks.py index 93f6820..0d57f96 100755 --- a/hooks/percona_hooks.py +++ b/hooks/percona_hooks.py @@ -629,8 +629,6 @@ def assess_status(): status_set('blocked', 'Insufficient peers to bootstrap cluster') return # Once running, ensure that cluster is in sync and has the required peers - # will need to access mysql to determine status. - # Also check to see if hacluster has presented 'clustered' back yet if is_bootstrapped() and cluster_in_sync(): status_set('active', 'Unit is ready and in sync') else: From 7414a1f78fca7d199e03ca96ea17f56ce2db3c0f Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 9 Oct 2015 08:26:12 -0700 Subject: [PATCH 5/8] Set initial state for mysql unit to active, not unknown --- tests/31-test-pause-and-resume.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/31-test-pause-and-resume.py b/tests/31-test-pause-and-resume.py index a89bd83..8dc23e0 100755 --- a/tests/31-test-pause-and-resume.py +++ b/tests/31-test-pause-and-resume.py @@ -14,7 +14,7 @@ class PauseResume(basic_deployment.BasicDeployment): uid = 'percona-cluster/0' unit = self.d.sentry.unit[uid] assert self.is_mysqld_running(unit), 'mysql not running: %s' % uid - assert utils.status_get(unit)[0] == "unknown" + assert utils.status_get(unit)[0] == "active" action_id = utils.run_action(unit, "pause") assert utils.wait_on_action(action_id), "Pause action failed." From 31d9ffe1eeed8c18b888f893c2c2e73e6ccd0d84 Mon Sep 17 00:00:00 2001 From: James Page Date: Sat, 10 Oct 2015 09:01:14 -0700 Subject: [PATCH 6/8] Fixup tests deal with single units --- hooks/percona_hooks.py | 18 +++++++++++++----- unit_tests/test_percona_hooks.py | 25 ++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/hooks/percona_hooks.py b/hooks/percona_hooks.py index 0d57f96..17d471a 100755 --- a/hooks/percona_hooks.py +++ b/hooks/percona_hooks.py @@ -624,15 +624,23 @@ def update_nrpe_config(): def assess_status(): '''Assess the status of the current unit''' + min_size = config('min-cluster-size') # Ensure that number of peers > cluster size configuration - if not is_bootstrapped(): + if not is_sufficient_peers(): status_set('blocked', 'Insufficient peers to bootstrap cluster') return - # Once running, ensure that cluster is in sync and has the required peers - if is_bootstrapped() and cluster_in_sync(): - status_set('active', 'Unit is ready and in sync') + + if min_size and int(min_size) > 1: + # Once running, ensure that cluster is in sync + # and has the required peers + if not is_bootstrapped(): + status_set('waiting', 'Unit waiting for cluster bootstrap') + elif is_bootstrapped() and cluster_in_sync(): + status_set('active', 'Unit is ready and clustered') + else: + status_set('blocked', 'Unit is not in sync') else: - status_set('blocked', 'Unit is not in sync') + status_set('active', 'Unit is ready') def main(): diff --git a/unit_tests/test_percona_hooks.py b/unit_tests/test_percona_hooks.py index 5f24f06..199acf6 100644 --- a/unit_tests/test_percona_hooks.py +++ b/unit_tests/test_percona_hooks.py @@ -14,7 +14,8 @@ TO_PATCH = ['log', 'config', 'get_netmask_for_address', 'cluster_in_sync', 'is_bootstrapped', - 'status_set'] + 'status_set', + 'is_sufficient_peers'] class TestHaRelation(CharmTestCase): @@ -136,18 +137,36 @@ class TestAssessStatus(CharmTestCase): def setUp(self): CharmTestCase.setUp(self, hooks, TO_PATCH) - def test_not_bootstrapped(self): - self.is_bootstrapped.return_value = False + def test_single_unit(self): + self.config.return_value = None + self.is_sufficient_peers.return_value = True + hooks.assess_status() + self.status_set.assert_called_with('active', mock.ANY) + + def test_insufficient_peers(self): + self.config.return_value = 3 + self.is_sufficient_peers.return_value = False hooks.assess_status() self.status_set.assert_called_with('blocked', mock.ANY) + def test_not_bootstrapped(self): + self.config.return_value = 3 + self.is_sufficient_peers.return_value = True + self.is_bootstrapped.return_value = False + hooks.assess_status() + self.status_set.assert_called_with('waiting', mock.ANY) + def test_bootstrapped_in_sync(self): + self.config.return_value = 3 + self.is_sufficient_peers.return_value = True self.is_bootstrapped.return_value = True self.cluster_in_sync.return_value = True hooks.assess_status() self.status_set.assert_called_with('active', mock.ANY) def test_bootstrapped_not_in_sync(self): + self.config.return_value = 3 + self.is_sufficient_peers.return_value = True self.is_bootstrapped.return_value = True self.cluster_in_sync.return_value = False hooks.assess_status() From f736601e49fc9e3f476527b7b78b7a008f3390fe Mon Sep 17 00:00:00 2001 From: James Page Date: Sat, 10 Oct 2015 09:07:05 -0700 Subject: [PATCH 7/8] Rework assess_status location, call on service resume action --- actions/actions.py | 6 ++-- actions/percona_utils.py | 1 + hooks/percona_hooks.py | 24 +-------------- hooks/percona_utils.py | 22 ++++++++++++++ unit_tests/test_percona_hooks.py | 42 -------------------------- unit_tests/test_percona_utils.py | 51 ++++++++++++++++++++++++++++++++ 6 files changed, 79 insertions(+), 67 deletions(-) create mode 120000 actions/percona_utils.py diff --git a/actions/actions.py b/actions/actions.py index ebe789c..f8df69e 100755 --- a/actions/actions.py +++ b/actions/actions.py @@ -5,6 +5,7 @@ import sys from charmhelpers.core.host import service_pause, service_resume from charmhelpers.core.hookenv import action_fail, status_set +from percona_utils import assess_status MYSQL_SERVICE = "mysql" @@ -18,7 +19,8 @@ def pause(args): if not service_pause(MYSQL_SERVICE): raise Exception("Failed to pause MySQL service.") status_set( - "maintenance", "Paused. Use 'resume' action to resume normal service.") + "maintenance", + "Unit paused. Use 'resume' action to resume normal service.") def resume(args): @@ -27,7 +29,7 @@ def resume(args): @raises Exception should the service fail to start.""" if not service_resume(MYSQL_SERVICE): raise Exception("Failed to resume MySQL service.") - status_set("active", "") + assess_status() # A dictionary of all the defined actions to callables (which take diff --git a/actions/percona_utils.py b/actions/percona_utils.py new file mode 120000 index 0000000..1e5682a --- /dev/null +++ b/actions/percona_utils.py @@ -0,0 +1 @@ +../hooks/percona_utils.py \ No newline at end of file diff --git a/hooks/percona_hooks.py b/hooks/percona_hooks.py index 17d471a..20bda10 100755 --- a/hooks/percona_hooks.py +++ b/hooks/percona_hooks.py @@ -24,7 +24,6 @@ from charmhelpers.core.hookenv import ( INFO, WARNING, is_leader, - status_set, ) from charmhelpers.core.host import ( service, @@ -62,7 +61,7 @@ from percona_utils import ( notify_bootstrapped, is_bootstrapped, get_wsrep_value, - cluster_in_sync, + assess_status, ) from charmhelpers.contrib.database.mysql import ( PerconaClusterHelper, @@ -622,27 +621,6 @@ def update_nrpe_config(): nrpe_setup.write() -def assess_status(): - '''Assess the status of the current unit''' - min_size = config('min-cluster-size') - # Ensure that number of peers > cluster size configuration - if not is_sufficient_peers(): - status_set('blocked', 'Insufficient peers to bootstrap cluster') - return - - if min_size and int(min_size) > 1: - # Once running, ensure that cluster is in sync - # and has the required peers - if not is_bootstrapped(): - status_set('waiting', 'Unit waiting for cluster bootstrap') - elif is_bootstrapped() and cluster_in_sync(): - status_set('active', 'Unit is ready and clustered') - else: - status_set('blocked', 'Unit is not in sync') - else: - status_set('active', 'Unit is ready') - - def main(): try: hooks.execute(sys.argv) diff --git a/hooks/percona_utils.py b/hooks/percona_utils.py index 7611fd6..c1fe36d 100644 --- a/hooks/percona_utils.py +++ b/hooks/percona_utils.py @@ -25,6 +25,7 @@ from charmhelpers.core.hookenv import ( INFO, WARNING, ERROR, + status_set, ) from charmhelpers.fetch import ( apt_install, @@ -372,3 +373,24 @@ def cluster_in_sync(): if ready and int(sync_status) in [2, 4]: return True return False + + +def assess_status(): + '''Assess the status of the current unit''' + min_size = config('min-cluster-size') + # Ensure that number of peers > cluster size configuration + if not is_sufficient_peers(): + status_set('blocked', 'Insufficient peers to bootstrap cluster') + return + + if min_size and int(min_size) > 1: + # Once running, ensure that cluster is in sync + # and has the required peers + if not is_bootstrapped(): + status_set('waiting', 'Unit waiting for cluster bootstrap') + elif is_bootstrapped() and cluster_in_sync(): + status_set('active', 'Unit is ready and clustered') + else: + status_set('blocked', 'Unit is not in sync') + else: + status_set('active', 'Unit is ready') diff --git a/unit_tests/test_percona_hooks.py b/unit_tests/test_percona_hooks.py index 199acf6..bed3e39 100644 --- a/unit_tests/test_percona_hooks.py +++ b/unit_tests/test_percona_hooks.py @@ -12,9 +12,7 @@ TO_PATCH = ['log', 'config', 'update_nrpe_config', 'get_iface_for_address', 'get_netmask_for_address', - 'cluster_in_sync', 'is_bootstrapped', - 'status_set', 'is_sufficient_peers'] @@ -131,43 +129,3 @@ class TestHaRelation(CharmTestCase): call_args, call_kwargs = self.relation_set.call_args self.assertEqual(resource_params, call_kwargs['resource_params']) - - -class TestAssessStatus(CharmTestCase): - def setUp(self): - CharmTestCase.setUp(self, hooks, TO_PATCH) - - def test_single_unit(self): - self.config.return_value = None - self.is_sufficient_peers.return_value = True - hooks.assess_status() - self.status_set.assert_called_with('active', mock.ANY) - - def test_insufficient_peers(self): - self.config.return_value = 3 - self.is_sufficient_peers.return_value = False - hooks.assess_status() - self.status_set.assert_called_with('blocked', mock.ANY) - - def test_not_bootstrapped(self): - self.config.return_value = 3 - self.is_sufficient_peers.return_value = True - self.is_bootstrapped.return_value = False - hooks.assess_status() - self.status_set.assert_called_with('waiting', mock.ANY) - - def test_bootstrapped_in_sync(self): - self.config.return_value = 3 - self.is_sufficient_peers.return_value = True - self.is_bootstrapped.return_value = True - self.cluster_in_sync.return_value = True - hooks.assess_status() - self.status_set.assert_called_with('active', mock.ANY) - - def test_bootstrapped_not_in_sync(self): - self.config.return_value = 3 - self.is_sufficient_peers.return_value = True - self.is_bootstrapped.return_value = True - self.cluster_in_sync.return_value = False - hooks.assess_status() - self.status_set.assert_called_with('blocked', mock.ANY) diff --git a/unit_tests/test_percona_utils.py b/unit_tests/test_percona_utils.py index 6b75845..6c1d127 100644 --- a/unit_tests/test_percona_utils.py +++ b/unit_tests/test_percona_utils.py @@ -7,6 +7,8 @@ import sys sys.modules['MySQLdb'] = mock.Mock() import percona_utils +from test_utils import CharmTestCase + class UtilsTests(unittest.TestCase): def setUp(self): @@ -180,3 +182,52 @@ class UtilsTests(unittest.TestCase): def test_cluster_in_sync_ready_sync_donor(self, _wsrep_value): _wsrep_value.side_effect = [True, 2] self.assertTrue(percona_utils.cluster_in_sync()) + + +TO_PATCH = [ + 'status_set', + 'is_sufficient_peers', + 'is_bootstrapped', + 'config', + 'cluster_in_sync', +] + + +class TestAssessStatus(CharmTestCase): + def setUp(self): + CharmTestCase.setUp(self, percona_utils, TO_PATCH) + + def test_single_unit(self): + self.config.return_value = None + self.is_sufficient_peers.return_value = True + percona_utils.assess_status() + self.status_set.assert_called_with('active', mock.ANY) + + def test_insufficient_peers(self): + self.config.return_value = 3 + self.is_sufficient_peers.return_value = False + percona_utils.assess_status() + self.status_set.assert_called_with('blocked', mock.ANY) + + def test_not_bootstrapped(self): + self.config.return_value = 3 + self.is_sufficient_peers.return_value = True + self.is_bootstrapped.return_value = False + percona_utils.assess_status() + self.status_set.assert_called_with('waiting', mock.ANY) + + def test_bootstrapped_in_sync(self): + self.config.return_value = 3 + self.is_sufficient_peers.return_value = True + self.is_bootstrapped.return_value = True + self.cluster_in_sync.return_value = True + percona_utils.assess_status() + self.status_set.assert_called_with('active', mock.ANY) + + def test_bootstrapped_not_in_sync(self): + self.config.return_value = 3 + self.is_sufficient_peers.return_value = True + self.is_bootstrapped.return_value = True + self.cluster_in_sync.return_value = False + percona_utils.assess_status() + self.status_set.assert_called_with('blocked', mock.ANY) From b28c1277d73d0dd789c816b0930e3c638eba745c Mon Sep 17 00:00:00 2001 From: James Page Date: Sat, 10 Oct 2015 09:19:59 -0700 Subject: [PATCH 8/8] Normalize messages in actions --- actions/actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/actions.py b/actions/actions.py index f8df69e..5c6c942 100755 --- a/actions/actions.py +++ b/actions/actions.py @@ -20,7 +20,7 @@ def pause(args): raise Exception("Failed to pause MySQL service.") status_set( "maintenance", - "Unit paused. Use 'resume' action to resume normal service.") + "Unit paused - use 'resume' action to resume normal service") def resume(args):