Restart mcollective on slave nodes after restore

Also, on the backup step a status of `mco ping` is serialized in an
upgrade tarball and on the restore step is compared with the actual
status. All nodes that are not respond are logged.

In additional, the status of `mco ping` is archived on the backup step
and is compared on the restore step with the actual ones.

Change-Id: Ibba81102214998d83614a42cdb21c21bebd8284a
Closes-Bug: #1561092

octane/handlers/backup_restore/__init__.py

@@ -16,6 +16,7 @@ from octane.handlers.backup_restore import astute
 from octane.handlers.backup_restore import cobbler
 from octane.handlers.backup_restore import fuel_keys
 from octane.handlers.backup_restore import fuel_uuid
+from octane.handlers.backup_restore import mcollective
 from octane.handlers.backup_restore import mirrors
 from octane.handlers.backup_restore import nailgun_plugins
 from octane.handlers.backup_restore import postgres
@@ -39,6 +40,7 @@ ARCHIVATORS = [
     version.VersionArchivator,
     nailgun_plugins.NailgunPluginsArchivator,
     puppet.PuppetApplyHost,
+    mcollective.McollectiveArchivator,
 ]
 
 REPO_ARCHIVATORS = [

octane/handlers/backup_restore/mcollective.py

@@ -0,0 +1,52 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import io
+import json
+import logging
+import tarfile
+
+from fuelclient import objects
+
+from octane.handlers.backup_restore import base
+from octane.util import fuel_client
+from octane.util import mcollective
+from octane.util import node as node_util
+
+LOG = logging.getLogger(__name__)
+
+
+class McollectiveArchivator(base.Base):
+    filename = "mco/ping.json"
+
+    def backup(self):
+        status = mcollective.get_mco_ping_status()
+        content = json.dumps(status)
+        info = tarfile.TarInfo(self.filename)
+        info.size = len(content)
+        fileobj = io.BytesIO(content)
+        self.archive.addfile(info, fileobj=fileobj)
+
+    def restore(self):
+        with fuel_client.set_auth_context(self.context):
+            nodes = objects.Node.get_all()
+            node_util.restart_mcollective(nodes)
+        content = self.archive.extractfile(self.filename)
+        if content is not None:
+            orig_status = json.load(content)
+            new_status = mcollective.get_mco_ping_status()
+            offline = mcollective.compair_mco_ping_statuses(orig_status,
+                                                            new_status)
+            if offline:
+                LOG.warning("Some nodes went offline after the upgrade of the "
+                            "master node (check them manually): %s",
+                            ", ".join(offline))

octane/tests/test_util_mcollective.py

@@ -0,0 +1,58 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+import mock
+import pytest
+
+from octane.util import mcollective
+from octane.util import subprocess
+
+
+def test_get_mco_ping_status(mocker):
+    popen_mock = mock.MagicMock()
+    popen_mock.return_value.__enter__.return_value = popen_mock
+    data = {"a": 1, "test": True, "b": 2}
+    popen_mock.stdout.read.return_value = json.dumps(data)
+    popen_patch = mocker.patch(
+        "octane.util.docker.in_container", new=popen_mock)
+    assert data == mcollective.get_mco_ping_status()
+    popen_patch.assert_called_once_with(
+        'astute',
+        ["mco", "rpc", "rpcutil", "ping", "--json"],
+        stdout=subprocess.PIPE)
+
+
+@pytest.mark.parametrize('orig,new,result', [
+    (
+        [{'sender': 1}, {'sender': 2}],
+        [],
+        set([])
+    ),
+    (
+        [{'sender': 1}, {'sender': 2}],
+        [{'sender': 1}, {'sender': 2}],
+        set([])
+    ),
+    (
+        [],
+        [{'sender': 1}, {'sender': 2}],
+        set([1, 2])
+    ),
+    (
+        [{'sender': 1}],
+        [{'sender': 1}, {'sender': 2}],
+        set([2])
+    ),
+])
+def test_compair_mco_ping_statuses(orig, new, result):
+    assert result == mcollective.compair_mco_ping_statuses(orig, new)

octane/tests/test_util_node.py

@@ -183,3 +183,75 @@ def test_get_nova_node_handle(mocker, node_data, fuel_version, expected_name):
     else:
         with pytest.raises(Exception):
             node_util.get_nova_node_handle(node)
+
+
+@pytest.mark.parametrize('stdout,nova_services_to_restart', [
+    (
+        "   [ + ] nova-service-1\n"
+        "   [ + ] nova-service-2\n"
+        "   [ - ] nova-service-3\n"
+        "   [ - ] not-nova-service-1\n"
+        "   [ + ] not-nova-service-2\n"
+        "   [ + ] nova-service-4\n",
+        [
+            'nova-service-1',
+            'nova-service-2',
+            'nova-service-4',
+        ]
+    )
+])
+def test_restart_nova_services(mocker, node, stdout, nova_services_to_restart):
+    call_output_mock = mocker.patch(
+        "octane.util.ssh.call_output", return_value=stdout)
+    call_mock = mocker.patch("octane.util.ssh.call")
+    node_util.restart_nova_services(node)
+    for service in nova_services_to_restart:
+        call_mock.assert_any_call(["service", service, "restart"], node=node)
+    call_output_mock.assert_called_once_with(
+        ["service", "--status-all"], node=node)
+
+
+@pytest.mark.parametrize('online', [True, False])
+@pytest.mark.parametrize('exception', [True, False])
+def test_restart_mcollective_node(mocker, online, exception):
+
+    class TestException(Exception):
+        pass
+
+    side_effect = TestException('test exception')
+    log_patch = mocker.patch.object(node_util, 'LOG')
+    node = mock.Mock(id='node_test_id', data={'online': online})
+    ssh_call_mock = mocker.patch('octane.util.ssh.call')
+    if exception:
+        ssh_call_mock.side_effect = side_effect
+    node_util.restart_mcollective_on_node(node)
+
+    if online:
+        ssh_call_mock.assert_called_once_with(
+            ["service", "mcollective", "restart"], node=node)
+    else:
+        assert not ssh_call_mock.called
+    if not online:
+        log_patch.warning.assert_called_once_with(
+            "Not possible to restart mcollective on the offline node {0}",
+            node.id)
+    elif exception:
+        log_patch.warning.assert_called_once_with(
+            "Failed to restart mcollective on the node %s: %s",
+            node.id, side_effect)
+    else:
+        assert not log_patch.warning.called
+
+
+@pytest.mark.parametrize('nodes_count', [0, 1, 10])
+def test_restart_mcollective(mocker, nodes_count):
+    nodes = []
+    calls = []
+    for _ in range(nodes_count):
+        node = mock.Mock()
+        calls.append(mock.call(node))
+        nodes.append(node)
+    mock_restart_mcol = mocker.patch(
+        'octane.util.node.restart_mcollective_on_node')
+    node_util.restart_mcollective(nodes)
+    assert calls == mock_restart_mcol.call_args_list

octane/util/mcollective.py

@@ -0,0 +1,29 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+
+from octane.util import docker
+from octane.util import subprocess
+
+
+def get_mco_ping_status():
+    cmd = ["mco", "rpc", "rpcutil", "ping", "--json"]
+    with docker.in_container('astute', cmd, stdout=subprocess.PIPE) as proc:
+        return json.load(proc.stdout)
+
+
+def compair_mco_ping_statuses(orig_status, new_status):
+    orig_ids = set([resp["sender"] for resp in orig_status])
+    new_ids = set([resp["sender"] for resp in new_status])
+    offline = new_ids - orig_ids
+    return offline

octane/util/node.py

@@ -158,3 +158,41 @@ def is_live_migration_supported(node):
                     and "VIR_MIGRATE_LIVE" in line:
                 return True
     return False
+
+
+def restart_nova_services(node):
+    nova_services = ssh.call_output(["service", "--status-all"], node=node)
+    for service_line in nova_services.splitlines():
+        service_line = service_line.strip()
+        _, status, _, service = service_line.split()
+        if status == "+" and service.startswith("nova"):
+            ssh.call(["service", service, "restart"], node=node)
+
+
+class AbsentParametersError(Exception):
+    msg = "Could not get parameters from the file " \
+          "node-{node_id}[{filename}]: {parameters}"
+
+    def __init__(self, node_id, filename, parameters):
+        super(AbsentParametersError, self).__init__(self.msg.format(
+            node_id=node_id,
+            filename=filename,
+            parameters=", ".join(parameters),
+        ))
+
+
+def restart_mcollective_on_node(node):
+    if not node.data['online']:
+        LOG.warning("Not possible to restart mcollective on the offline "
+                    "node {0}", node.id)
+        return
+    try:
+        ssh.call(["service", "mcollective", "restart"], node=node)
+    except Exception as exc:
+        LOG.warning("Failed to restart mcollective on the node %s: %s",
+                    node.id, exc)
+
+
+def restart_mcollective(nodes):
+    for node in nodes:
+        restart_mcollective_on_node(node)