Set up repos for ceph upgrade in separate source
* Set up repos for upgrade ceph on ceph_osd to separate source.
* add preference file for ceph source on upgrade-osd step with
highest Pin-Priority
* add tests
Change-Id: I1781eb4aa3e66b6e464256ab9b24e39f6a6d0b3d
Closes-bug: 1585204
(cherry picked from commit 13043d0eb0)
This commit is contained in:
Sergey Abramov
parent
c66d26e9f6
commit
c1ea3bdc2e
|
|
@ -10,15 +10,17 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
|
||||
from cliff import command as cmd
|
||||
|
||||
from fuelclient.objects import node as node_obj
|
||||
from fuelclient.objects import environment as env_obj
|
||||
|
||||
from octane.handlers import backup_restore
|
||||
from octane import magic_consts
|
||||
from octane.util import env
|
||||
from octane.util import fuel_client
|
||||
from octane.util import helpers
|
||||
from octane.util import ssh
|
||||
|
|
@ -39,46 +41,66 @@ def _get_backup_path(path, node):
|
|||
node=node)
|
||||
|
||||
|
||||
def write_content_to_tmp_file_on_node(node, content, directory, template):
|
||||
tmp_name = ssh.call_output(
|
||||
["mktemp", "-p", directory, "-t", template], node=node).strip()
|
||||
sftp = ssh.sftp(node)
|
||||
with sftp.open(tmp_name, "w") as new:
|
||||
new.write(content)
|
||||
return tmp_name
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def applied_repos(nodes, preference_priority):
|
||||
admin_ip = helpers.get_astute_dict()["ADMIN_NETWORK"]["ipaddress"]
|
||||
packages = " ".join(magic_consts.OSD_UPGRADE_REQUIRED_PACKAGES)
|
||||
preference_content = magic_consts.OSD_UPGADE_PREFERENCE_TEMPLATE.format(
|
||||
packages=packages, priority=preference_priority)
|
||||
source_content = magic_consts.OSD_UPGRADE_SOURCE_TEMPLATE.format(
|
||||
admin_ip=admin_ip)
|
||||
|
||||
node_file_to_clear_list = []
|
||||
try:
|
||||
for node in nodes:
|
||||
source = write_content_to_tmp_file_on_node(
|
||||
node, source_content,
|
||||
"/etc/apt/sources.list.d/", "mos.osd_XXX.list")
|
||||
node_file_to_clear_list.append((node, source))
|
||||
preference = write_content_to_tmp_file_on_node(
|
||||
node, preference_content,
|
||||
"/etc/apt/preferences.d/", "mos.osd_XXX.pref")
|
||||
node_file_to_clear_list.append((node, preference))
|
||||
yield
|
||||
finally:
|
||||
for node, file_name_to_remove in node_file_to_clear_list:
|
||||
sftp = ssh.sftp(node)
|
||||
sftp.unlink(file_name_to_remove)
|
||||
|
||||
|
||||
def get_repo_highest_priority(orig_env):
|
||||
editable = orig_env.get_attributes()['editable']
|
||||
repos = editable['repo_setup']['repos']['value']
|
||||
return max([i['priority'] for i in repos])
|
||||
|
||||
|
||||
def upgrade_osd(env_id, user, password):
|
||||
with fuel_client.set_auth_context(
|
||||
backup_restore.NailgunCredentialsContext(user, password)):
|
||||
nodes = [
|
||||
n for n in node_obj.Node.get_all()
|
||||
if "ceph-osd" in n.data["roles"] and n.data["cluster"] == env_id]
|
||||
orig_env = env_obj.Environment(env_id)
|
||||
nodes = list(env.get_nodes(orig_env, ["ceph-osd"]))
|
||||
if not nodes:
|
||||
LOG.info("Nothing to upgrade")
|
||||
return
|
||||
backup_val = [
|
||||
# (node, path, backup_path)
|
||||
]
|
||||
admin_ip = helpers.get_astute_dict()["ADMIN_NETWORK"]["ipaddress"]
|
||||
try:
|
||||
hostnames = []
|
||||
for node in nodes:
|
||||
sftp = ssh.sftp(node)
|
||||
for path, content in magic_consts.OSD_REPOS_UPDATE:
|
||||
back_path = _get_backup_path(path, node)
|
||||
ssh.call(["cp", path, back_path], node=node)
|
||||
backup_val.append((node, path, back_path))
|
||||
with ssh.update_file(sftp, path) as (_, new):
|
||||
new.write(content.format(admin_ip=admin_ip))
|
||||
hostnames.append(node.data["hostname"])
|
||||
ssh.call(["dpkg", "--configure", "-a"], node=node)
|
||||
preference_priority = get_repo_highest_priority(orig_env)
|
||||
hostnames = [n.data['hostname'] for n in nodes]
|
||||
with applied_repos(nodes, preference_priority + 1):
|
||||
call_node = nodes[0]
|
||||
ssh.call(["ceph", "osd", "set", "noout"], node=call_node)
|
||||
ssh.call(['ceph-deploy', 'install', '--release', 'hammer'] + hostnames,
|
||||
node=call_node, stdout=ssh.PIPE, stderr=ssh.PIPE)
|
||||
for node in nodes:
|
||||
ssh.call(["restart", "ceph-osd-all"], node=node)
|
||||
ssh.call(["ceph", "osd", "unset", "noout"], node=call_node)
|
||||
ssh.call(["ceph", "osd", "stat"], node=call_node)
|
||||
finally:
|
||||
nodes_to_revert = set()
|
||||
for node, path, back_path in backup_val:
|
||||
ssh.call(["mv", back_path, path], node=node)
|
||||
nodes_to_revert.add(node)
|
||||
for node in nodes_to_revert:
|
||||
ssh.call(["dpkg", "--configure", "-a"], node=node)
|
||||
node=call_node)
|
||||
for node in nodes:
|
||||
ssh.call(["restart", "ceph-osd-all"], node=node)
|
||||
ssh.call(["ceph", "osd", "unset", "noout"], node=call_node)
|
||||
|
||||
|
||||
class UpgradeOSDCommand(cmd.Command):
|
||||
|
|
|
|||
|
|
@ -63,18 +63,20 @@ RUNNING_REQUIRED_CONTAINERS = [
|
|||
|
||||
OPENSTACK_FIXTURES = "/usr/share/fuel-openstack-metadata/openstack.yaml"
|
||||
|
||||
OSD_REPOS_UPDATE = [
|
||||
# ("path", "content")
|
||||
(
|
||||
"/etc/apt/sources.list.d/mos.list",
|
||||
"deb http://{admin_ip}:8080/liberty-8.0/ubuntu/x86_64 "
|
||||
"mos8.0 main restricted"
|
||||
),
|
||||
(
|
||||
"/etc/apt/sources.list.d/mos-updates.list",
|
||||
'deb http://{admin_ip}:8080/ubuntu/x86_64/ mos8.0 main restricted',
|
||||
),
|
||||
|
||||
OSD_UPGRADE_REQUIRED_PACKAGES = [
|
||||
"libcephfs1", "librados2", "librbd1", "python-ceph", "python-cephfs",
|
||||
"python-rados", "python-rbd", "ceph", "ceph-common", "ceph-fs-common",
|
||||
"ceph-mds",
|
||||
]
|
||||
OSD_UPGRADE_SOURCE_TEMPLATE = \
|
||||
"deb http://{admin_ip}:8080/liberty-8.0/ubuntu/x86_64 " \
|
||||
"mos8.0 main restricted\n" \
|
||||
"deb http://{admin_ip}:8080/ubuntu/x86_64/ mos8.0 main restricted"
|
||||
|
||||
OSD_UPGADE_PREFERENCE_TEMPLATE = "Package: {packages}\n" \
|
||||
"Pin: release a=mos8.0,n=mos8.0,l=mos8.0\n" \
|
||||
"Pin-Priority: {priority}"
|
||||
COBBLER_DROP_VERSION = "7.0"
|
||||
CEPH_UPSTART_VERSION = "7.0"
|
||||
|
||||
|
|
|
|||
|
|
@ -10,13 +10,10 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import contextlib
|
||||
import mock
|
||||
import pytest
|
||||
|
||||
from octane.commands import osd_upgrade
|
||||
from octane import magic_consts
|
||||
from octane.util import ssh
|
||||
|
||||
|
||||
@pytest.mark.parametrize("env_id", [None, 1])
|
||||
|
|
@ -37,132 +34,137 @@ def test_osd_cmd_upgrade(mocker, octane_app, env_id, admin_pswd):
|
|||
assert not upgrade_osd_mock.called
|
||||
|
||||
|
||||
@pytest.mark.parametrize("node_roles, exception_node", [
|
||||
([('ceph-osd',)] * 10, None),
|
||||
([('ceph-osd', 'compute')] * 10, None),
|
||||
([('ceph-osd',), ('compute',)] * 10, None),
|
||||
([('ceph-osd',), ('compute',), ('controller',)] * 10, None),
|
||||
([], None),
|
||||
([('compute',)] * 10, None),
|
||||
([('ceph-osd',)] * 10, 0),
|
||||
([('ceph-osd',), ('compute',)] * 10, 9),
|
||||
])
|
||||
@pytest.mark.parametrize("user", ["usr", "admin"])
|
||||
@pytest.mark.parametrize("password", ["admin", "pswd"])
|
||||
@pytest.mark.parametrize("env_id", [1, 2, 3])
|
||||
@pytest.mark.parametrize("master_ip", ["10.21.10.2", "10.20.1.2"])
|
||||
def test_upgrade_osd(
|
||||
mocker, node_roles, user, password, exception_node, master_ip, env_id):
|
||||
auth_mock_client = mocker.patch("octane.util.fuel_client.set_auth_context")
|
||||
creds_mock = mocker.patch(
|
||||
"octane.handlers.backup_restore.NailgunCredentialsContext")
|
||||
@pytest.mark.parametrize("content", ["test_content"])
|
||||
@pytest.mark.parametrize("directory", ["/dir/path"])
|
||||
@pytest.mark.parametrize("template", ["templ"])
|
||||
@pytest.mark.parametrize("generated_name", ["gen_name", "\n\n gen_name\n\n"])
|
||||
def test_write_content_to_tmp_file_on_node(
|
||||
mocker, content, directory, template, generated_name):
|
||||
node = mock.MagicMock()
|
||||
sftp_mock = mocker.patch("octane.util.ssh.sftp", return_value=node)
|
||||
node.open.__enter__.return_value = node
|
||||
short_gen_name = generated_name.strip()
|
||||
ssh_mock = mocker.patch("octane.util.ssh.call_output",
|
||||
return_value=generated_name)
|
||||
assert short_gen_name == osd_upgrade.write_content_to_tmp_file_on_node(
|
||||
node, content, directory, template)
|
||||
node.open.assert_called_once_with(short_gen_name, "w")
|
||||
node.write(content)
|
||||
sftp_mock.assert_called_once_with(node)
|
||||
ssh_mock.assert_called_once_with(
|
||||
["mktemp", "-p", directory, "-t", template], node=node)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("nodes_count", [0, 1, 2])
|
||||
@pytest.mark.parametrize("admin_ip,source_tmpl,source", [(
|
||||
"10.10.0.1", "source {admin_ip}", "source 10.10.0.1"
|
||||
)])
|
||||
@pytest.mark.parametrize("packages,priority,pref_tmpl,pref", [(
|
||||
["pack_1", "pack_2"], 1000, "pref {packages} {priority}",
|
||||
"pref pack_1 pack_2 1000"
|
||||
)])
|
||||
@pytest.mark.parametrize("error", [True, False])
|
||||
def test_applied_repos(mocker, nodes_count, admin_ip, source_tmpl, source,
|
||||
packages, priority, pref_tmpl, pref, error):
|
||||
mocker.patch("octane.magic_consts.OSD_UPGRADE_REQUIRED_PACKAGES", packages)
|
||||
mocker.patch(
|
||||
"octane.commands.osd_upgrade._get_backup_path",
|
||||
return_value="backup_path")
|
||||
mocker.patch("octane.magic_consts.OSD_REPOS_UPDATE",
|
||||
[("path", "{admin_ip}")])
|
||||
ssh_call_mock = mocker.patch("octane.util.ssh.call")
|
||||
preinstall_calls = []
|
||||
rollbabk_calls = []
|
||||
dpkg_rollbabk_calls = []
|
||||
nodes = []
|
||||
osd_nodes = []
|
||||
hostnames = []
|
||||
osd_node_idx = 0
|
||||
call_node = None
|
||||
|
||||
class TestException(Exception):
|
||||
pass
|
||||
|
||||
for roles in node_roles:
|
||||
node = mocker.Mock()
|
||||
hostname = "{0}_node.{1}".format("_".join(roles), osd_node_idx)
|
||||
node.data = {"roles": roles, "hostname": hostname, "cluster": env_id}
|
||||
nodes.append(node)
|
||||
new_env_node = mocker.Mock()
|
||||
new_env_node.data = {
|
||||
"roles": roles,
|
||||
"hostname": "{0}_env.{1}".format("_".join(roles), osd_node_idx),
|
||||
"cluster": env_id + 1
|
||||
}
|
||||
nodes.append(new_env_node)
|
||||
if 'ceph-osd' not in roles:
|
||||
continue
|
||||
osd_nodes.append(node)
|
||||
hostnames.append(hostname)
|
||||
call_node = call_node or node
|
||||
for path, _ in magic_consts.OSD_REPOS_UPDATE:
|
||||
preinstall_calls.append((
|
||||
mock.call(["cp", path, "backup_path"], node=node),
|
||||
exception_node == osd_node_idx,
|
||||
))
|
||||
if exception_node == osd_node_idx:
|
||||
break
|
||||
rollbabk_calls.append(
|
||||
(mock.call(["mv", "backup_path", path], node=node), False))
|
||||
if exception_node == osd_node_idx:
|
||||
break
|
||||
preinstall_calls.append(
|
||||
(mock.call(["dpkg", "--configure", "-a"], node=node), False))
|
||||
dpkg_rollbabk_calls.append(
|
||||
(mock.call(["dpkg", "--configure", "-a"], node=node), False))
|
||||
osd_node_idx += 1
|
||||
mocker.patch("fuelclient.objects.node.Node.get_all", return_value=nodes)
|
||||
|
||||
file_mock = mock.Mock()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def update_file(*args, **kwargs):
|
||||
yield (None, file_mock)
|
||||
|
||||
mocker.patch("octane.util.ssh.update_file", side_effect=update_file)
|
||||
mocker.patch("octane.util.ssh.sftp")
|
||||
"octane.magic_consts.OSD_UPGADE_PREFERENCE_TEMPLATE", pref_tmpl)
|
||||
mocker.patch(
|
||||
"octane.magic_consts.OSD_UPGRADE_SOURCE_TEMPLATE", source_tmpl)
|
||||
mock_get_astute = mocker.patch(
|
||||
"octane.util.helpers.get_astute_dict",
|
||||
return_value={"ADMIN_NETWORK": {"ipaddress": master_ip}})
|
||||
update_calls = []
|
||||
return_value={"ADMIN_NETWORK": {"ipaddress": admin_ip}})
|
||||
|
||||
if exception_node is None and osd_node_idx:
|
||||
update_calls.append((
|
||||
mock.call(["ceph", "osd", "set", "noout"], node=call_node), False))
|
||||
update_calls.append((
|
||||
def mock_write_content_to_tmp_side_effect(
|
||||
node, content, directory, template):
|
||||
if directory == "/etc/apt/sources.list.d/":
|
||||
return node.source
|
||||
if directory == "/etc/apt/preferences.d/":
|
||||
return node.preference
|
||||
|
||||
mock_write_content_to_tmp = mocker.patch(
|
||||
"octane.commands.osd_upgrade.write_content_to_tmp_file_on_node",
|
||||
side_effect=mock_write_content_to_tmp_side_effect)
|
||||
|
||||
sftp_mock = mocker.patch("octane.util.ssh.sftp")
|
||||
|
||||
nodes = [mock.MagicMock() for _ in range(nodes_count)]
|
||||
|
||||
with osd_upgrade.applied_repos(nodes, priority):
|
||||
for node in nodes:
|
||||
mock_write_content_to_tmp.assert_any_call(
|
||||
node, source, "/etc/apt/sources.list.d/", "mos.osd_XXX.list")
|
||||
mock_write_content_to_tmp.assert_any_call(
|
||||
node, pref, "/etc/apt/preferences.d/", "mos.osd_XXX.pref")
|
||||
assert not sftp_mock.called
|
||||
|
||||
for node in nodes:
|
||||
sftp_mock.assert_any_call(node)
|
||||
sftp_mock.return_value.unlink.assert_any_call(node.source)
|
||||
sftp_mock.return_value.unlink.assert_any_call(node.preference)
|
||||
mock_get_astute.assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("priority_from, priority_to", [[100, 500]])
|
||||
def test_get_repo_highest_priority(mocker, priority_from, priority_to):
|
||||
env = mock.MagicMock()
|
||||
env.get_attributes.return_value = {
|
||||
"editable": {
|
||||
"repo_setup": {
|
||||
"repos": {
|
||||
"value": [{"priority": i}
|
||||
for i in range(priority_from, priority_to + 1)]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
assert priority_to == osd_upgrade.get_repo_highest_priority(env)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("env_id", [2])
|
||||
@pytest.mark.parametrize("user", ["user"])
|
||||
@pytest.mark.parametrize("password", ["password"])
|
||||
@pytest.mark.parametrize("nodes_count", [0, 10])
|
||||
@pytest.mark.parametrize("priority", [100, 500])
|
||||
def test_upgrade_osd(mocker, nodes_count, priority, user, password, env_id):
|
||||
env = mock.Mock()
|
||||
nodes = []
|
||||
hostnames = []
|
||||
restart_calls = []
|
||||
for idx in range(nodes_count):
|
||||
hostname = "host_{0}".format(idx)
|
||||
hostnames.append(hostname)
|
||||
node = mock.Mock(data={'hostname': hostname})
|
||||
nodes.append(node)
|
||||
restart_calls.append(mock.call(["restart", "ceph-osd-all"], node=node))
|
||||
env_get = mocker.patch("fuelclient.objects.environment.Environment",
|
||||
return_value=env)
|
||||
mocker.patch("octane.util.env.get_nodes", return_value=iter(nodes))
|
||||
mock_creds = mocker.patch(
|
||||
"octane.handlers.backup_restore.NailgunCredentialsContext")
|
||||
mock_auth_cntx = mocker.patch("octane.util.fuel_client.set_auth_context")
|
||||
mock_applied = mocker.patch("octane.commands.osd_upgrade.applied_repos")
|
||||
mock_get_priority = mocker.patch(
|
||||
"octane.commands.osd_upgrade.get_repo_highest_priority",
|
||||
return_value=priority)
|
||||
ssh_call_mock = mocker.patch("octane.util.ssh.call")
|
||||
|
||||
osd_upgrade.upgrade_osd(env_id, user, password)
|
||||
|
||||
mock_creds.assert_called_once_with(user, password)
|
||||
mock_auth_cntx.assert_called_once_with(mock_creds.return_value)
|
||||
env_get.assert_called_once_with(env_id)
|
||||
ssh_calls = []
|
||||
if nodes:
|
||||
ssh_calls.append(
|
||||
mock.call(["ceph", "osd", "set", "noout"], node=nodes[0]))
|
||||
ssh_calls.append(
|
||||
mock.call(
|
||||
['ceph-deploy', 'install', '--release', 'hammer'] + hostnames,
|
||||
node=call_node,
|
||||
stdout=ssh.PIPE,
|
||||
stderr=ssh.PIPE,
|
||||
),
|
||||
False
|
||||
))
|
||||
for node in osd_nodes:
|
||||
update_calls.append(
|
||||
(mock.call(['restart', 'ceph-osd-all'], node=node), False))
|
||||
update_calls.append((
|
||||
mock.call(["ceph", "osd", "unset", "noout"], node=call_node),
|
||||
False
|
||||
))
|
||||
update_calls.append((
|
||||
mock.call(["ceph", "osd", "stat"], node=call_node),
|
||||
False
|
||||
))
|
||||
|
||||
calls = \
|
||||
preinstall_calls + \
|
||||
update_calls + \
|
||||
rollbabk_calls + \
|
||||
dpkg_rollbabk_calls
|
||||
|
||||
ssh_calls = [i[0] for i in calls]
|
||||
mock_calls = [TestException() if i[1] else mock.DEFAULT for i in calls]
|
||||
ssh_call_mock.side_effect = mock_calls
|
||||
if exception_node is not None:
|
||||
with pytest.raises(TestException):
|
||||
osd_upgrade.upgrade_osd(env_id, user, password)
|
||||
else:
|
||||
osd_upgrade.upgrade_osd(env_id, user, password)
|
||||
ssh_call_mock.assert_has_calls(ssh_calls, any_order=True)
|
||||
assert ssh_call_mock.call_count == len(ssh_calls)
|
||||
auth_mock_client.assert_called_once_with(creds_mock.return_value)
|
||||
creds_mock.assert_called_once_with(user, password)
|
||||
if exception_node is not None and osd_node_idx:
|
||||
file_mock.write.assert_called_with(master_ip)
|
||||
node=nodes[0]))
|
||||
ssh_calls.extend(restart_calls)
|
||||
ssh_calls.append(
|
||||
mock.call(["ceph", "osd", "unset", "noout"], node=nodes[0]))
|
||||
mock_get_priority.assert_called_once_with(env)
|
||||
mock_applied.assert_called_once_with(nodes, priority + 1)
|
||||
assert ssh_calls == ssh_call_mock.mock_calls
|
||||
|
|
|
|||
Loading…
Reference in New Issue