From b97a0971c22f129b71674a22a65080a65c96af76 Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 10 Jul 2019 12:01:06 -0700 Subject: [PATCH] Bootstrap action after a cold boot After a cold boot, percona-cluster will require administrative intervention. One node will need to bootstrap per upstream Percona Cluster documentation: https://www.percona.com/blog/2014/09/01/galera-replication-how-to-recover-a-pxc-cluster/ This change adds an action to bootstrap a single node. On the other nodes systemd will be attempting to start percona. Once the bootstrapped node is up the others will join automatically. Change-Id: Id9a860edc343ee5dbd7fc8c5ce3b4420ec6e134e Partial-Bug: #1744393 --- actions.yaml | 8 +++++++ actions/actions.py | 47 +++++++++++++++++++++++++++----------- actions/bootstrap-pxc | 1 + unit_tests/test_actions.py | 32 ++++++++++++++------------ 4 files changed, 61 insertions(+), 27 deletions(-) create mode 120000 actions/bootstrap-pxc diff --git a/actions.yaml b/actions.yaml index 73a6957..1adc042 100644 --- a/actions.yaml +++ b/actions.yaml @@ -24,3 +24,11 @@ complete-cluster-series-upgrade: peers for wsrep replication. This action should be performed on the current leader. Note the leader may have changed during the series upgrade process. +bootstrap-pxc: + description: | + Bootstrap this unit of Percona. + *WARNING* This action will bootstrap this unit of Percona cluster. This + should only occur in a recovery scenario. Make sure this unit has the + highest sequence number in grstate.dat or data loss may occur. + See upstream Percona documentation for context + https://www.percona.com/blog/2014/09/01/galera-replication-how-to-recover-a-pxc-cluster/ diff --git a/actions/actions.py b/actions/actions.py index 671f305..4c8d561 100755 --- a/actions/actions.py +++ b/actions/actions.py @@ -15,6 +15,7 @@ def _add_path(path): if path not in sys.path: sys.path.insert(1, path) + _add_path(_hooks) _add_path(_root) @@ -32,13 +33,8 @@ from charmhelpers.core.host import ( lsb_release, ) -from percona_utils import ( - pause_unit_helper, - resume_unit_helper, - register_configs, - _get_password, -) -from percona_hooks import config_changed +import percona_utils +import percona_hooks def pause(args): @@ -46,7 +42,7 @@ def pause(args): @raises Exception should the service fail to stop. """ - pause_unit_helper(register_configs()) + percona_utils.pause_unit_helper(percona_utils.register_configs()) def resume(args): @@ -54,10 +50,10 @@ def resume(args): @raises Exception should the service fail to start. """ - resume_unit_helper(register_configs()) + percona_utils.resume_unit_helper(percona_utils.register_configs()) # NOTE(ajkavanagh) - we force a config_changed pseudo-hook to see if the # unit needs to bootstrap or restart it's services here. - config_changed() + percona_hooks.config_changed() def complete_cluster_series_upgrade(args): @@ -71,14 +67,14 @@ def complete_cluster_series_upgrade(args): # Unset cluster_series_upgrading leader_set(cluster_series_upgrading="") leader_set(cluster_series_upgrade_leader="") - config_changed() + percona_hooks.config_changed() def backup(args): basedir = (action_get("basedir")).lower() compress = action_get("compress") incremental = action_get("incremental") - sstpw = _get_password("sst-password") + sstpw = percona_utils._get_password("sst-password") optionlist = [] # innobackupex will not create recursive dirs that do not already exist, @@ -115,10 +111,35 @@ def backup(args): "and check the status of the database") +def bootstrap_pxc(args): + try: + # Force safe to bootstrap + percona_utils.set_grstate_safe_to_bootstrap() + # Boostrap this node + percona_utils.bootstrap_pxc() + except (percona_utils.GRStateFileNotFound, OSError) as e: + action_set({ + 'output': e.output, + 'return-code': e.returncode}) + action_fail("The GRState file does not exist or cannot be written to.") + except (subprocess.CalledProcessError, Exception) as e: + action_set({ + 'output': e.output, + 'return-code': e.returncode, + 'traceback': traceback.format_exc()}) + action_fail("The bootstrap-pxc failed. " + "See traceback in show-action-output") + action_set({ + 'output': "Bootstrap succeded. " + "Wait for the other units to run update-status"}) + percona_utils.assess_status(percona_utils.register_configs()) + + # A dictionary of all the defined actions to callables (which take # parsed arguments). ACTIONS = {"pause": pause, "resume": resume, "backup": backup, - "complete-cluster-series-upgrade": complete_cluster_series_upgrade} + "complete-cluster-series-upgrade": complete_cluster_series_upgrade, + "bootstrap-pxc": bootstrap_pxc} def main(args): diff --git a/actions/bootstrap-pxc b/actions/bootstrap-pxc new file mode 120000 index 0000000..405a394 --- /dev/null +++ b/actions/bootstrap-pxc @@ -0,0 +1 @@ +actions.py \ No newline at end of file diff --git a/unit_tests/test_actions.py b/unit_tests/test_actions.py index bd4e9cd..02edd08 100644 --- a/unit_tests/test_actions.py +++ b/unit_tests/test_actions.py @@ -6,11 +6,9 @@ from test_utils import CharmTestCase # we have to patch out harden decorator because hooks/percona_hooks.py gets # imported via actions.py and will freak out if it trys to run in the context # of a test. -with patch('percona_utils.register_configs') as configs, \ - patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec: +with patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec: mock_dec.side_effect = (lambda *dargs, **dkwargs: lambda f: lambda *args, **kwargs: f(*args, **kwargs)) - configs.return_value = 'test-config' from actions import actions @@ -18,9 +16,10 @@ class PauseTestCase(CharmTestCase): def setUp(self): super(PauseTestCase, self).setUp( - actions, ["pause_unit_helper"]) + actions.percona_utils, ["pause_unit_helper", "register_configs"]) def test_pauses_services(self): + self.register_configs.return_value = "test-config" actions.pause([]) self.pause_unit_helper.assert_called_once_with('test-config') @@ -29,10 +28,12 @@ class ResumeTestCase(CharmTestCase): def setUp(self): super(ResumeTestCase, self).setUp( - actions, ["resume_unit_helper"]) + actions.percona_utils, ["resume_unit_helper", "register_configs"]) def test_pauses_services(self): - with patch('actions.actions.config_changed') as config_changed: + self.register_configs.return_value = "test-config" + with patch('actions.actions.percona_hooks.config_changed' + ) as config_changed: actions.resume([]) self.resume_unit_helper.assert_called_once_with('test-config') config_changed.assert_called_once_with() @@ -42,22 +43,25 @@ class CompleteClusterSeriesUpgrade(CharmTestCase): def setUp(self): super(CompleteClusterSeriesUpgrade, self).setUp( - actions, ["config_changed", "is_leader", "leader_set"]) + actions, ["is_leader", "leader_set"]) def test_leader_complete_series_upgrade(self): self.is_leader.return_value = True - calls = [mock.call(cluster_series_upgrading=""), mock.call(cluster_series_upgrade_leader="")] - actions.complete_cluster_series_upgrade([]) - self.leader_set.assert_has_calls(calls) - self.config_changed.assert_called_once_with() + with patch('actions.actions.percona_hooks.config_changed' + ) as config_changed: + actions.complete_cluster_series_upgrade([]) + self.leader_set.assert_has_calls(calls) + config_changed.assert_called_once_with() def test_non_leader_complete_series_upgrade(self): self.is_leader.return_value = False - actions.complete_cluster_series_upgrade([]) - self.leader_set.assert_not_called() - self.config_changed.assert_called_once_with() + with patch('actions.actions.percona_hooks.config_changed' + ) as config_changed: + actions.complete_cluster_series_upgrade([]) + self.leader_set.assert_not_called() + config_changed.assert_called_once_with() class MainTestCase(CharmTestCase):