Merge "Restart mcollective on slave nodes after restore"

This commit is contained in:
Jenkins 2016-10-13 10:35:34 +00:00 committed by Gerrit Code Review
commit 9e69e8d0d6
8 changed files with 210 additions and 0 deletions

View File

@ -18,6 +18,7 @@ from octane.handlers.backup_restore import cobbler
from octane.handlers.backup_restore import fuel_keys
from octane.handlers.backup_restore import fuel_uuid
from octane.handlers.backup_restore import logs
from octane.handlers.backup_restore import mcollective
from octane.handlers.backup_restore import mirrors
from octane.handlers.backup_restore import nailgun_plugins
from octane.handlers.backup_restore import postgres
@ -48,6 +49,7 @@ ARCHIVATORS = [
version.VersionArchivator,
nailgun_plugins.NailgunPluginsArchivator,
puppet.PuppetApplyTasks,
mcollective.McollectiveArchivator,
]
REPO_ARCHIVATORS = [

View File

@ -0,0 +1,53 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import io
import json
import logging
import tarfile
from fuelclient import objects
from octane.handlers.backup_restore import base
from octane.util import fuel_client
from octane.util import mcollective
from octane.util import node as node_util
LOG = logging.getLogger(__name__)
class McollectiveArchivator(base.Base):
filename = "mco/ping.json"
def backup(self):
status = mcollective.get_mco_ping_status()
content = json.dumps(status)
info = tarfile.TarInfo(self.filename)
info.size = len(content)
fileobj = io.BytesIO(content)
self.archive.addfile(info, fileobj=fileobj)
def restore(self):
with fuel_client.set_auth_context(self.context):
nodes = objects.Node.get_all()
for node in nodes:
node_util.restart_mcollective(node)
content = self.archive.extractfile(self.filename)
if content is not None:
orig_status = json.load(content)
new_status = mcollective.get_mco_ping_status()
offline = mcollective.compair_mco_ping_statuses(orig_status,
new_status)
if offline:
LOG.warning("Some nodes went offline after the upgrade of the "
"master node (check them manually): %s",
", ".join(offline))

View File

@ -20,6 +20,7 @@ from octane.handlers.backup_restore import base
from octane.handlers.backup_restore import cobbler
from octane.handlers.backup_restore import fuel_keys
from octane.handlers.backup_restore import fuel_uuid
from octane.handlers.backup_restore import mcollective
from octane.handlers.backup_restore import mirrors
from octane.handlers.backup_restore import nailgun_plugins
from octane.handlers.backup_restore import postgres
@ -271,3 +272,18 @@ def test_repos_backup(
def test_archivator_name(mocker, name, expected_name):
assert expected_name == type(name, (base.Base, ), {})(None).archivator_name
def test_mcollective_backup(mocker):
archive = mock.Mock()
mocker.patch("octane.util.mcollective.get_mco_ping_status")
mock_json = mocker.patch("json.dumps")
mock_json.return_value = "{}"
mock_info = mocker.patch("tarfile.TarInfo")
mock_io = mocker.patch("io.BytesIO")
mcollective.McollectiveArchivator(archive).backup()
archive.addfile.assert_called_once_with(
mock_info.return_value, fileobj=mock_io.return_value)
mock_io.assert_called_once_with(mock_json.return_value)
mock_info.assert_called_once_with("mco/ping.json")
assert mock_info.return_value.size == len(mock_json.return_value)

View File

@ -24,6 +24,7 @@ from octane.handlers.backup_restore import cobbler
from octane.handlers.backup_restore import fuel_keys
from octane.handlers.backup_restore import fuel_uuid
from octane.handlers.backup_restore import logs
from octane.handlers.backup_restore import mcollective
from octane.handlers.backup_restore import mirrors
from octane.handlers.backup_restore import postgres
from octane.handlers.backup_restore import puppet
@ -677,3 +678,43 @@ def test_admin_network_restore(mocker, members, is_exist):
mock_puppet.assert_called_once_with('dhcp-ranges')
else:
mock_puppet.assert_not_called()
@pytest.mark.parametrize(("members", "check_status"), [
([TestMember("mco/ping.json", True, False)], True),
([], False),
])
def test_mcollective_restore(mocker, members, check_status):
nodes = [mock.Mock(), mock.Mock()]
mocker.patch("octane.util.fuel_client.set_auth_context")
mock_get = mocker.patch("fuelclient.objects.Node.get_all")
mock_get.return_value = nodes
mock_restart = mocker.patch("octane.util.node.restart_mcollective")
mock_json = mocker.patch("json.load")
mock_status = mocker.patch("octane.util.mcollective.get_mco_ping_status")
mock_cmp = mocker.patch(
"octane.util.mcollective.compair_mco_ping_statuses")
mock_cmp.return_value = set(["1"])
mock_log = mocker.patch(
"octane.handlers.backup_restore.mcollective.LOG.warning")
archive = TestArchive(members, mcollective.McollectiveArchivator)
mcollective.McollectiveArchivator(archive).restore()
assert mock_restart.call_args_list == [
mock.call(node) for node in nodes
]
if check_status:
effective = [
member
for member in members if member.name == "mco/ping.json"
][-1]
assert effective
mock_json.assert_called_once_with(effective)
mock_status.assert_called_once_with()
mock_cmp.assert_called_once_with(
mock_json.return_value, mock_status.return_value)
mock_log.assert_called_once_with(mock.ANY, "1")
else:
assert not mock_json.called
assert not mock_status.called
assert not mock_cmp.called

View File

@ -0,0 +1,36 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import io
import json
import pytest
from octane.util import mcollective
@pytest.mark.parametrize("status", [
{"a": "b", "c": "d"},
])
def test_get_mco_ping_status(mocker, status):
stdout = io.BytesIO(json.dumps(status))
mock_popen = mocker.patch("octane.util.subprocess.popen")
mock_popen.return_value.__enter__.return_value.stdout = stdout
result = mcollective.get_mco_ping_status()
assert result == status
@pytest.mark.parametrize(("orig", "new", "offline"), [
([{"sender": 1}, {"sender": 2}], [{"sender": 1}], set([2])),
])
def test_compair_mco_ping_statuses(mocker, orig, new, offline):
assert mcollective.compair_mco_ping_statuses(orig, new) == offline

View File

@ -242,3 +242,16 @@ def test_get_parameters(mocker, parameters, parameters_to_get, required,
mock_get.assert_called_once_with(
mock_sftp.return_value.open.return_value.__enter__.return_value,
parameters_to_get)
@pytest.mark.parametrize(("online", "result", "error"), [
(True, True, False),
(False, None, False),
(True, False, True),
])
def test_restart_mcollective(mocker, online, result, error):
node = mock.Mock(data={"online": online, "id": 123})
mock_ssh = mocker.patch("octane.util.ssh.call")
if error:
mock_ssh.side_effect = Exception()
assert node_util.restart_mcollective(node) == result

View File

@ -0,0 +1,31 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
from octane.util import subprocess
def get_mco_ping_status(node_id=None):
cmd = ["mco", "rpc", "rpcutil", "ping", "--json"]
if node_id is not None:
cmd.extend(["-I", str(node_id)])
with subprocess.popen(cmd, stdout=subprocess.PIPE) as proc:
return json.load(proc.stdout)
def compair_mco_ping_statuses(orig_status, new_status):
# NOTE(akcram): Statuses are present only for alive nodes.
orig_ids = {resp["sender"] for resp in orig_status}
new_ids = {resp["sender"] for resp in new_status}
offline = orig_ids - new_ids
return offline

View File

@ -212,3 +212,21 @@ def get_parameters(node, filename, parameters_to_get, ensure=True):
raise AbsentParametersError(
node.data["id"], filename, flat_parameters)
return parameters
def restart_mcollective(node):
node_id = node.data["id"]
if not node.data["online"]:
LOG.warning("Not possible to restart mcollective on the offline "
"node %s", node_id)
return None
try:
ssh.call(["service", "mcollective", "restart"], node=node)
except Exception as exc:
LOG.warning("Failed to restart mcollective on the node %s: %s",
node_id, exc)
return False
else:
LOG.info("The mcollective service was successfully restarted on "
"the node %s", node_id)
return True