diff --git a/octane/handlers/backup_restore/__init__.py b/octane/handlers/backup_restore/__init__.py index 22aba306..cb11ad44 100644 --- a/octane/handlers/backup_restore/__init__.py +++ b/octane/handlers/backup_restore/__init__.py @@ -18,6 +18,7 @@ from octane.handlers.backup_restore import cobbler from octane.handlers.backup_restore import fuel_keys from octane.handlers.backup_restore import fuel_uuid from octane.handlers.backup_restore import logs +from octane.handlers.backup_restore import mcollective from octane.handlers.backup_restore import mirrors from octane.handlers.backup_restore import nailgun_plugins from octane.handlers.backup_restore import postgres @@ -48,6 +49,7 @@ ARCHIVATORS = [ version.VersionArchivator, nailgun_plugins.NailgunPluginsArchivator, puppet.PuppetApplyTasks, + mcollective.McollectiveArchivator, ] REPO_ARCHIVATORS = [ diff --git a/octane/handlers/backup_restore/mcollective.py b/octane/handlers/backup_restore/mcollective.py new file mode 100644 index 00000000..8592b23b --- /dev/null +++ b/octane/handlers/backup_restore/mcollective.py @@ -0,0 +1,53 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import io +import json +import logging +import tarfile + +from fuelclient import objects + +from octane.handlers.backup_restore import base +from octane.util import fuel_client +from octane.util import mcollective +from octane.util import node as node_util + +LOG = logging.getLogger(__name__) + + +class McollectiveArchivator(base.Base): + filename = "mco/ping.json" + + def backup(self): + status = mcollective.get_mco_ping_status() + content = json.dumps(status) + info = tarfile.TarInfo(self.filename) + info.size = len(content) + fileobj = io.BytesIO(content) + self.archive.addfile(info, fileobj=fileobj) + + def restore(self): + with fuel_client.set_auth_context(self.context): + nodes = objects.Node.get_all() + for node in nodes: + node_util.restart_mcollective(node) + content = self.archive.extractfile(self.filename) + if content is not None: + orig_status = json.load(content) + new_status = mcollective.get_mco_ping_status() + offline = mcollective.compair_mco_ping_statuses(orig_status, + new_status) + if offline: + LOG.warning("Some nodes went offline after the upgrade of the " + "master node (check them manually): %s", + ", ".join(offline)) diff --git a/octane/tests/test_archivators.py b/octane/tests/test_archivators.py index 8187ce86..b09c9ee8 100644 --- a/octane/tests/test_archivators.py +++ b/octane/tests/test_archivators.py @@ -20,6 +20,7 @@ from octane.handlers.backup_restore import base from octane.handlers.backup_restore import cobbler from octane.handlers.backup_restore import fuel_keys from octane.handlers.backup_restore import fuel_uuid +from octane.handlers.backup_restore import mcollective from octane.handlers.backup_restore import mirrors from octane.handlers.backup_restore import nailgun_plugins from octane.handlers.backup_restore import postgres @@ -271,3 +272,18 @@ def test_repos_backup( def test_archivator_name(mocker, name, expected_name): assert expected_name == type(name, (base.Base, ), {})(None).archivator_name + + +def test_mcollective_backup(mocker): + archive = mock.Mock() + mocker.patch("octane.util.mcollective.get_mco_ping_status") + mock_json = mocker.patch("json.dumps") + mock_json.return_value = "{}" + mock_info = mocker.patch("tarfile.TarInfo") + mock_io = mocker.patch("io.BytesIO") + mcollective.McollectiveArchivator(archive).backup() + archive.addfile.assert_called_once_with( + mock_info.return_value, fileobj=mock_io.return_value) + mock_io.assert_called_once_with(mock_json.return_value) + mock_info.assert_called_once_with("mco/ping.json") + assert mock_info.return_value.size == len(mock_json.return_value) diff --git a/octane/tests/test_archivators_restore.py b/octane/tests/test_archivators_restore.py index 7ac31390..f866284d 100644 --- a/octane/tests/test_archivators_restore.py +++ b/octane/tests/test_archivators_restore.py @@ -24,6 +24,7 @@ from octane.handlers.backup_restore import cobbler from octane.handlers.backup_restore import fuel_keys from octane.handlers.backup_restore import fuel_uuid from octane.handlers.backup_restore import logs +from octane.handlers.backup_restore import mcollective from octane.handlers.backup_restore import mirrors from octane.handlers.backup_restore import postgres from octane.handlers.backup_restore import puppet @@ -677,3 +678,43 @@ def test_admin_network_restore(mocker, members, is_exist): mock_puppet.assert_called_once_with('dhcp-ranges') else: mock_puppet.assert_not_called() + + +@pytest.mark.parametrize(("members", "check_status"), [ + ([TestMember("mco/ping.json", True, False)], True), + ([], False), +]) +def test_mcollective_restore(mocker, members, check_status): + nodes = [mock.Mock(), mock.Mock()] + mocker.patch("octane.util.fuel_client.set_auth_context") + mock_get = mocker.patch("fuelclient.objects.Node.get_all") + mock_get.return_value = nodes + mock_restart = mocker.patch("octane.util.node.restart_mcollective") + mock_json = mocker.patch("json.load") + mock_status = mocker.patch("octane.util.mcollective.get_mco_ping_status") + mock_cmp = mocker.patch( + "octane.util.mcollective.compair_mco_ping_statuses") + mock_cmp.return_value = set(["1"]) + mock_log = mocker.patch( + "octane.handlers.backup_restore.mcollective.LOG.warning") + + archive = TestArchive(members, mcollective.McollectiveArchivator) + mcollective.McollectiveArchivator(archive).restore() + assert mock_restart.call_args_list == [ + mock.call(node) for node in nodes + ] + if check_status: + effective = [ + member + for member in members if member.name == "mco/ping.json" + ][-1] + assert effective + mock_json.assert_called_once_with(effective) + mock_status.assert_called_once_with() + mock_cmp.assert_called_once_with( + mock_json.return_value, mock_status.return_value) + mock_log.assert_called_once_with(mock.ANY, "1") + else: + assert not mock_json.called + assert not mock_status.called + assert not mock_cmp.called diff --git a/octane/tests/test_util_mcollective.py b/octane/tests/test_util_mcollective.py new file mode 100644 index 00000000..1ff6af47 --- /dev/null +++ b/octane/tests/test_util_mcollective.py @@ -0,0 +1,36 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import io +import json + +import pytest + +from octane.util import mcollective + + +@pytest.mark.parametrize("status", [ + {"a": "b", "c": "d"}, +]) +def test_get_mco_ping_status(mocker, status): + stdout = io.BytesIO(json.dumps(status)) + mock_popen = mocker.patch("octane.util.subprocess.popen") + mock_popen.return_value.__enter__.return_value.stdout = stdout + result = mcollective.get_mco_ping_status() + assert result == status + + +@pytest.mark.parametrize(("orig", "new", "offline"), [ + ([{"sender": 1}, {"sender": 2}], [{"sender": 1}], set([2])), +]) +def test_compair_mco_ping_statuses(mocker, orig, new, offline): + assert mcollective.compair_mco_ping_statuses(orig, new) == offline diff --git a/octane/tests/test_util_node.py b/octane/tests/test_util_node.py index c88cf8f2..db541edd 100644 --- a/octane/tests/test_util_node.py +++ b/octane/tests/test_util_node.py @@ -242,3 +242,16 @@ def test_get_parameters(mocker, parameters, parameters_to_get, required, mock_get.assert_called_once_with( mock_sftp.return_value.open.return_value.__enter__.return_value, parameters_to_get) + + +@pytest.mark.parametrize(("online", "result", "error"), [ + (True, True, False), + (False, None, False), + (True, False, True), +]) +def test_restart_mcollective(mocker, online, result, error): + node = mock.Mock(data={"online": online, "id": 123}) + mock_ssh = mocker.patch("octane.util.ssh.call") + if error: + mock_ssh.side_effect = Exception() + assert node_util.restart_mcollective(node) == result diff --git a/octane/util/mcollective.py b/octane/util/mcollective.py new file mode 100644 index 00000000..1547a8bc --- /dev/null +++ b/octane/util/mcollective.py @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +from octane.util import subprocess + + +def get_mco_ping_status(node_id=None): + cmd = ["mco", "rpc", "rpcutil", "ping", "--json"] + if node_id is not None: + cmd.extend(["-I", str(node_id)]) + with subprocess.popen(cmd, stdout=subprocess.PIPE) as proc: + return json.load(proc.stdout) + + +def compair_mco_ping_statuses(orig_status, new_status): + # NOTE(akcram): Statuses are present only for alive nodes. + orig_ids = {resp["sender"] for resp in orig_status} + new_ids = {resp["sender"] for resp in new_status} + offline = orig_ids - new_ids + return offline diff --git a/octane/util/node.py b/octane/util/node.py index c45da3f8..88b9e78e 100644 --- a/octane/util/node.py +++ b/octane/util/node.py @@ -212,3 +212,21 @@ def get_parameters(node, filename, parameters_to_get, ensure=True): raise AbsentParametersError( node.data["id"], filename, flat_parameters) return parameters + + +def restart_mcollective(node): + node_id = node.data["id"] + if not node.data["online"]: + LOG.warning("Not possible to restart mcollective on the offline " + "node %s", node_id) + return None + try: + ssh.call(["service", "mcollective", "restart"], node=node) + except Exception as exc: + LOG.warning("Failed to restart mcollective on the node %s: %s", + node_id, exc) + return False + else: + LOG.info("The mcollective service was successfully restarted on " + "the node %s", node_id) + return True