openstack
/
fuel-octane
Code Issues Proposed changes

Osd upgrade failed if version not changed 

ceph deploy not raise exception if it doesn't upgrade osd version.

Change-Id: Ifcddd822228d78166d59b2ba49852be2e51c79fc
Closes-bug: 1620277
This commit is contained in:
Sergey Abramov 2016-09-05 18:24:37 +03:00
parent 3315d741f2
commit e7fad96f41
2 changed files with 129 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
octane/commands/osd_upgrade.py
View File

@ -11,8 +11,10 @@
# under the License.
import contextlib
import json
import logging
import os
import time
from cliff import command as cmd
@ -105,6 +107,48 @@ def applied_repos(nodes, preference_priority, seed_repos):
sftp.unlink(file_name_to_remove)
def get_current_versions(controller, kind):
stdout = ssh.call_output(
['ceph', 'tell', '{0}.*'.format(kind), 'version', '-f', 'json'],
node=controller)
results = []
for line in stdout.splitlines():
if not line:
continue
if line.startswith(kind):
line = line.split(":", 1)[1]
results.append(json.loads(line))
return {v['version'] for v in results}
def is_same_versions_on_mon_and_osd(controller):
mons = get_current_versions(controller, "mon")
osds = get_current_versions(controller, "osd")
is_equal = mons == osds
if not is_equal:
LOG.info("Installed MONs versions: {0} and OSDs versions: {1}".format(
' '.join(mons), ' '.join(osds)))
return is_equal
def is_ceph_up(controller):
with ssh.popen(['ceph', 'osd', 'tree', '-f', 'json'],
node=controller, stdout=ssh.PIPE) as proc:
data = json.load(proc.stdout)
return all(n['status'] == 'up' for n in data['nodes']
if n['type'] == 'osd')
def waiting_until_ceph_up(controller, delay=5, times=30):
for _ in xrange(times):
if is_ceph_up(controller):
return
time.sleep(delay)
raise Exception(
"After upgrade not all ceph osd nodes ar UP after {0} seconds".format(
delay * times))
def upgrade_osd(orig_env_id, seed_env_id, user, password):
with fuel_client.set_auth_context(
backup_restore.NailgunCredentialsContext(user, password)):
@ -116,6 +160,10 @@ def upgrade_osd(orig_env_id, seed_env_id, user, password):
if not nodes:
LOG.info("Nothing to upgrade")
return
controller = env.get_one_controller(seed_env)
if is_same_versions_on_mon_and_osd(controller):
LOG.warn("MONs and OSDs have the same version, nothing to upgrade.")
return
hostnames = [n.data['hostname'] for n in nodes]
with applied_repos(nodes, preference_priority + 1, seed_repos):
call_node = nodes[0]
@ -125,6 +173,11 @@ def upgrade_osd(orig_env_id, seed_env_id, user, password):
for node in nodes:
ssh.call(["restart", "ceph-osd-all"], node=node)
ssh.call(["ceph", "osd", "unset", "noout"], node=call_node)
waiting_until_ceph_up(controller)
if not is_same_versions_on_mon_and_osd(controller):
msg = "OSDs not upgraded up to MONs version, please fix the problem"
LOG.error(msg)
raise Exception(msg)
class UpgradeOSDCommand(cmd.Command):

80
octane/tests/test_osd_upgrade.py
View File

@ -10,10 +10,12 @@
# License for the specific language governing permissions and limitations
# under the License.
import io
import mock
import pytest
from octane.commands import osd_upgrade
from octane.util import ssh
@pytest.mark.parametrize("orig_env_id", [None, 1])
@ -118,13 +120,18 @@ def test_get_repo_highest_priority(mocker, repos, result):
@pytest.mark.parametrize("password", ["password"])
@pytest.mark.parametrize("nodes_count", [0, 10])
@pytest.mark.parametrize("priority", [100, 500])
@pytest.mark.parametrize(
"is_same_versions_on_mon_and_osd_return_values",
[(True, True), (False, True), (False, False)])
def test_upgrade_osd(
mocker, nodes_count, priority, user, password, orig_id, seed_id):
mocker, nodes_count, priority, user, password, orig_id, seed_id,
is_same_versions_on_mon_and_osd_return_values):
orig_env = mock.Mock()
seed_env = mock.Mock()
nodes = []
hostnames = []
restart_calls = []
controller = mock.Mock()
for idx in range(nodes_count):
hostname = "host_{0}".format(idx)
hostnames.append(hostname)
@ -134,6 +141,7 @@ def test_upgrade_osd(
env_get = mocker.patch("fuelclient.objects.environment.Environment",
side_effect=[orig_env, seed_env])
mocker.patch("octane.util.env.get_nodes", return_value=iter(nodes))
mocker.patch("octane.util.env.get_one_controller", return_value=controller)
mock_creds = mocker.patch(
"octane.handlers.backup_restore.NailgunCredentialsContext")
mock_auth_cntx = mocker.patch("octane.util.fuel_client.set_auth_context")
@ -144,15 +152,24 @@ def test_upgrade_osd(
mock_get_env_repos = mocker.patch(
"octane.commands.osd_upgrade.get_env_repos")
ssh_call_mock = mocker.patch("octane.util.ssh.call")
osd_upgrade.upgrade_osd(orig_id, seed_id, user, password)
mock_is_same_version = mocker.patch(
"octane.commands.osd_upgrade.is_same_versions_on_mon_and_osd",
side_effect=is_same_versions_on_mon_and_osd_return_values)
mock_up_waiter = mocker.patch(
"octane.commands.osd_upgrade.waiting_until_ceph_up")
already_same, upgraded = is_same_versions_on_mon_and_osd_return_values
if not upgraded and not already_same and nodes:
with pytest.raises(Exception):
osd_upgrade.upgrade_osd(orig_id, seed_id, user, password)
else:
osd_upgrade.upgrade_osd(orig_id, seed_id, user, password)
mock_creds.assert_called_once_with(user, password)
mock_auth_cntx.assert_called_once_with(mock_creds.return_value)
env_get.assert_any_call(orig_id)
env_get.assert_any_call(seed_id)
ssh_calls = []
if nodes:
if nodes and not already_same:
ssh_calls.append(
mock.call(["ceph", "osd", "set", "noout"], node=nodes[0]))
ssh_calls.append(
@ -167,6 +184,14 @@ def test_upgrade_osd(
nodes,
priority + 1,
mock_get_env_repos.return_value)
assert [mock.call(controller), mock.call(controller)] == \
mock_is_same_version.call_args_list
mock_up_waiter.assert_called_once_with(controller)
elif nodes and already_same:
mock_is_same_version.assert_called_once_with(controller)
elif not nodes:
assert not mock_is_same_version.called
assert ssh_calls == ssh_call_mock.mock_calls
@ -306,3 +331,50 @@ def test_get_env_repos(attrs, result):
env = mock.MagicMock()
env.get_attributes.return_value = attrs
assert result == osd_upgrade.get_env_repos(env)
@pytest.mark.parametrize("tree,result", [
('\n{"nodes":[{"type":"root"},{"type":"host"},'
'{"type":"osd","status":"up"},'
'{"type":"osd","status":"up"}],"stray":[]}', True),
('{"nodes":[{"type":"root"},{"type":"host"},'
'{"type":"osd","status":"down"},{"type":"osd","status":"up"}],'
'"stray":[]}', False),
])
def test_is_ceph_up(mocker, tree, result):
controller = mock.Mock()
mock_call = mocker.patch("octane.util.ssh.popen")
mock_call.return_value.__enter__.return_value = mock_call.return_value
stdout = io.BytesIO()
stdout.write(tree)
stdout.seek(0)
mock_call.return_value.stdout = stdout
assert result == osd_upgrade.is_ceph_up(controller)
mock_call.assert_called_once_with(['ceph', 'osd', 'tree', '-f', 'json'],
stdout=ssh.PIPE, node=controller)
@pytest.mark.parametrize("running_times", [1, 2, 31])
@pytest.mark.parametrize("delay", [5])
@pytest.mark.parametrize("times", [30])
def test_waiting_until_ceph_up(mocker, running_times, delay, times):
time_calls = []
is_ceph_up_calls = []
is_ceph_up_side_effects = []
controller = mock.Mock()
for idx in range(min(running_times, times)):
is_ceph_up_calls.append(mock.call(controller))
is_ok = idx == (running_times - 1)
is_ceph_up_side_effects.append(is_ok)
if not is_ok:
time_calls.append(mock.call(delay))
time_mock = mocker.patch("time.sleep")
is_ceph_up_mock = mocker.patch("octane.commands.osd_upgrade.is_ceph_up",
side_effect=is_ceph_up_side_effects)
if any(is_ceph_up_side_effects):
osd_upgrade.waiting_until_ceph_up(controller)
else:
with pytest.raises(Exception):
osd_upgrade.waiting_until_ceph_up(controller)
assert time_calls == time_mock.call_args_list
assert is_ceph_up_calls == is_ceph_up_mock.call_args_list