Restart mcollective on slave nodes after restore
Also, on the backup step a status of `mco ping` is serialized in an upgrade tarball and on the restore step it is compared with the actual status. All nodes that are not respond are logged. In additional, the status of `mco ping` is archived on the backup step and is compared on the restore step with the actual ones. Change-Id: Ibba81102214998d83614a42cdb21c21bebd8284a Related-Bug: #1561092
This commit is contained in:
parent
66e7a8c493
commit
f91ff40264
|
@ -18,6 +18,7 @@ from octane.handlers.backup_restore import cobbler
|
|||
from octane.handlers.backup_restore import fuel_keys
|
||||
from octane.handlers.backup_restore import fuel_uuid
|
||||
from octane.handlers.backup_restore import logs
|
||||
from octane.handlers.backup_restore import mcollective
|
||||
from octane.handlers.backup_restore import mirrors
|
||||
from octane.handlers.backup_restore import nailgun_plugins
|
||||
from octane.handlers.backup_restore import postgres
|
||||
|
@ -48,6 +49,7 @@ ARCHIVATORS = [
|
|||
version.VersionArchivator,
|
||||
nailgun_plugins.NailgunPluginsArchivator,
|
||||
puppet.PuppetApplyTasks,
|
||||
mcollective.McollectiveArchivator,
|
||||
]
|
||||
|
||||
REPO_ARCHIVATORS = [
|
||||
|
|
|
@ -0,0 +1,53 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import tarfile
|
||||
|
||||
from fuelclient import objects
|
||||
|
||||
from octane.handlers.backup_restore import base
|
||||
from octane.util import fuel_client
|
||||
from octane.util import mcollective
|
||||
from octane.util import node as node_util
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class McollectiveArchivator(base.Base):
|
||||
filename = "mco/ping.json"
|
||||
|
||||
def backup(self):
|
||||
status = mcollective.get_mco_ping_status()
|
||||
content = json.dumps(status)
|
||||
info = tarfile.TarInfo(self.filename)
|
||||
info.size = len(content)
|
||||
fileobj = io.BytesIO(content)
|
||||
self.archive.addfile(info, fileobj=fileobj)
|
||||
|
||||
def restore(self):
|
||||
with fuel_client.set_auth_context(self.context):
|
||||
nodes = objects.Node.get_all()
|
||||
for node in nodes:
|
||||
node_util.restart_mcollective(node)
|
||||
content = self.archive.extractfile(self.filename)
|
||||
if content is not None:
|
||||
orig_status = json.load(content)
|
||||
new_status = mcollective.get_mco_ping_status()
|
||||
offline = mcollective.compair_mco_ping_statuses(orig_status,
|
||||
new_status)
|
||||
if offline:
|
||||
LOG.warning("Some nodes went offline after the upgrade of the "
|
||||
"master node (check them manually): %s",
|
||||
", ".join(offline))
|
|
@ -20,6 +20,7 @@ from octane.handlers.backup_restore import base
|
|||
from octane.handlers.backup_restore import cobbler
|
||||
from octane.handlers.backup_restore import fuel_keys
|
||||
from octane.handlers.backup_restore import fuel_uuid
|
||||
from octane.handlers.backup_restore import mcollective
|
||||
from octane.handlers.backup_restore import mirrors
|
||||
from octane.handlers.backup_restore import nailgun_plugins
|
||||
from octane.handlers.backup_restore import postgres
|
||||
|
@ -271,3 +272,18 @@ def test_repos_backup(
|
|||
def test_archivator_name(mocker, name, expected_name):
|
||||
|
||||
assert expected_name == type(name, (base.Base, ), {})(None).archivator_name
|
||||
|
||||
|
||||
def test_mcollective_backup(mocker):
|
||||
archive = mock.Mock()
|
||||
mocker.patch("octane.util.mcollective.get_mco_ping_status")
|
||||
mock_json = mocker.patch("json.dumps")
|
||||
mock_json.return_value = "{}"
|
||||
mock_info = mocker.patch("tarfile.TarInfo")
|
||||
mock_io = mocker.patch("io.BytesIO")
|
||||
mcollective.McollectiveArchivator(archive).backup()
|
||||
archive.addfile.assert_called_once_with(
|
||||
mock_info.return_value, fileobj=mock_io.return_value)
|
||||
mock_io.assert_called_once_with(mock_json.return_value)
|
||||
mock_info.assert_called_once_with("mco/ping.json")
|
||||
assert mock_info.return_value.size == len(mock_json.return_value)
|
||||
|
|
|
@ -24,6 +24,7 @@ from octane.handlers.backup_restore import cobbler
|
|||
from octane.handlers.backup_restore import fuel_keys
|
||||
from octane.handlers.backup_restore import fuel_uuid
|
||||
from octane.handlers.backup_restore import logs
|
||||
from octane.handlers.backup_restore import mcollective
|
||||
from octane.handlers.backup_restore import mirrors
|
||||
from octane.handlers.backup_restore import postgres
|
||||
from octane.handlers.backup_restore import puppet
|
||||
|
@ -677,3 +678,43 @@ def test_admin_network_restore(mocker, members, is_exist):
|
|||
mock_puppet.assert_called_once_with('dhcp-ranges')
|
||||
else:
|
||||
mock_puppet.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("members", "check_status"), [
|
||||
([TestMember("mco/ping.json", True, False)], True),
|
||||
([], False),
|
||||
])
|
||||
def test_mcollective_restore(mocker, members, check_status):
|
||||
nodes = [mock.Mock(), mock.Mock()]
|
||||
mocker.patch("octane.util.fuel_client.set_auth_context")
|
||||
mock_get = mocker.patch("fuelclient.objects.Node.get_all")
|
||||
mock_get.return_value = nodes
|
||||
mock_restart = mocker.patch("octane.util.node.restart_mcollective")
|
||||
mock_json = mocker.patch("json.load")
|
||||
mock_status = mocker.patch("octane.util.mcollective.get_mco_ping_status")
|
||||
mock_cmp = mocker.patch(
|
||||
"octane.util.mcollective.compair_mco_ping_statuses")
|
||||
mock_cmp.return_value = set(["1"])
|
||||
mock_log = mocker.patch(
|
||||
"octane.handlers.backup_restore.mcollective.LOG.warning")
|
||||
|
||||
archive = TestArchive(members, mcollective.McollectiveArchivator)
|
||||
mcollective.McollectiveArchivator(archive).restore()
|
||||
assert mock_restart.call_args_list == [
|
||||
mock.call(node) for node in nodes
|
||||
]
|
||||
if check_status:
|
||||
effective = [
|
||||
member
|
||||
for member in members if member.name == "mco/ping.json"
|
||||
][-1]
|
||||
assert effective
|
||||
mock_json.assert_called_once_with(effective)
|
||||
mock_status.assert_called_once_with()
|
||||
mock_cmp.assert_called_once_with(
|
||||
mock_json.return_value, mock_status.return_value)
|
||||
mock_log.assert_called_once_with(mock.ANY, "1")
|
||||
else:
|
||||
assert not mock_json.called
|
||||
assert not mock_status.called
|
||||
assert not mock_cmp.called
|
||||
|
|
|
@ -0,0 +1,36 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import io
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from octane.util import mcollective
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", [
|
||||
{"a": "b", "c": "d"},
|
||||
])
|
||||
def test_get_mco_ping_status(mocker, status):
|
||||
stdout = io.BytesIO(json.dumps(status))
|
||||
mock_popen = mocker.patch("octane.util.subprocess.popen")
|
||||
mock_popen.return_value.__enter__.return_value.stdout = stdout
|
||||
result = mcollective.get_mco_ping_status()
|
||||
assert result == status
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("orig", "new", "offline"), [
|
||||
([{"sender": 1}, {"sender": 2}], [{"sender": 1}], set([2])),
|
||||
])
|
||||
def test_compair_mco_ping_statuses(mocker, orig, new, offline):
|
||||
assert mcollective.compair_mco_ping_statuses(orig, new) == offline
|
|
@ -242,3 +242,16 @@ def test_get_parameters(mocker, parameters, parameters_to_get, required,
|
|||
mock_get.assert_called_once_with(
|
||||
mock_sftp.return_value.open.return_value.__enter__.return_value,
|
||||
parameters_to_get)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("online", "result", "error"), [
|
||||
(True, True, False),
|
||||
(False, None, False),
|
||||
(True, False, True),
|
||||
])
|
||||
def test_restart_mcollective(mocker, online, result, error):
|
||||
node = mock.Mock(data={"online": online, "id": 123})
|
||||
mock_ssh = mocker.patch("octane.util.ssh.call")
|
||||
if error:
|
||||
mock_ssh.side_effect = Exception()
|
||||
assert node_util.restart_mcollective(node) == result
|
||||
|
|
|
@ -0,0 +1,31 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import json
|
||||
|
||||
from octane.util import subprocess
|
||||
|
||||
|
||||
def get_mco_ping_status(node_id=None):
|
||||
cmd = ["mco", "rpc", "rpcutil", "ping", "--json"]
|
||||
if node_id is not None:
|
||||
cmd.extend(["-I", str(node_id)])
|
||||
with subprocess.popen(cmd, stdout=subprocess.PIPE) as proc:
|
||||
return json.load(proc.stdout)
|
||||
|
||||
|
||||
def compair_mco_ping_statuses(orig_status, new_status):
|
||||
# NOTE(akcram): Statuses are present only for alive nodes.
|
||||
orig_ids = {resp["sender"] for resp in orig_status}
|
||||
new_ids = {resp["sender"] for resp in new_status}
|
||||
offline = orig_ids - new_ids
|
||||
return offline
|
|
@ -212,3 +212,21 @@ def get_parameters(node, filename, parameters_to_get, ensure=True):
|
|||
raise AbsentParametersError(
|
||||
node.data["id"], filename, flat_parameters)
|
||||
return parameters
|
||||
|
||||
|
||||
def restart_mcollective(node):
|
||||
node_id = node.data["id"]
|
||||
if not node.data["online"]:
|
||||
LOG.warning("Not possible to restart mcollective on the offline "
|
||||
"node %s", node_id)
|
||||
return None
|
||||
try:
|
||||
ssh.call(["service", "mcollective", "restart"], node=node)
|
||||
except Exception as exc:
|
||||
LOG.warning("Failed to restart mcollective on the node %s: %s",
|
||||
node_id, exc)
|
||||
return False
|
||||
else:
|
||||
LOG.info("The mcollective service was successfully restarted on "
|
||||
"the node %s", node_id)
|
||||
return True
|
||||
|
|
Loading…
Reference in New Issue