openstack
/
senlin
Code Issues Proposed changes

Merge "Shutdown VM before destructive update action"

This commit is contained in:
Zuul 2020-11-23 10:15:10 +00:00 committed by Gerrit Code Review
parent 6133e815e3 fa2d817839
commit 028af9be62
7 changed files with 614 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
senlin/engine/actions/cluster_action.py
View File

@ -240,16 +240,17 @@ class ClusterAction(base.Action):
for node_set in plan:
child = []
nodes = list(node_set)
nodes.sort()
for node in nodes:
kwargs = {
'name': 'node_update_%s' % node[:8],
'cluster_id': self.entity.id,
'cause': consts.CAUSE_DERIVED,
'inputs': {
'new_profile_id': profile_id,
},
'inputs': self.entity.config,
}
kwargs['inputs']['new_profile_id'] = profile_id
action_id = base.Action.create(self.context, node,
consts.NODE_UPDATE, **kwargs)
child.append(action_id)
@ -299,6 +300,14 @@ class ClusterAction(base.Action):
profile_only = self.inputs.get('profile_only')
if config is not None:
# make sure config values are valid
try:
stop_timeout = config['cluster.stop_timeout_before_update']
config['cluster.stop_timeout_before_update'] = int(
stop_timeout)
except Exception as e:
return self.RES_ERROR, str(e)
self.entity.config = config
if name is not None:
self.entity.name = name

23
senlin/policies/health_policy.py
View File

@ -46,10 +46,16 @@ class HealthPolicy(base.Policy):
('BEFORE', consts.CLUSTER_DEL_NODES),
('BEFORE', consts.CLUSTER_SCALE_IN),
('BEFORE', consts.CLUSTER_RESIZE),
('BEFORE', consts.CLUSTER_UPDATE),
('BEFORE', consts.CLUSTER_RECOVER),
('BEFORE', consts.CLUSTER_REPLACE_NODES),
('BEFORE', consts.NODE_DELETE),
('AFTER', consts.CLUSTER_DEL_NODES),
('AFTER', consts.CLUSTER_SCALE_IN),
('AFTER', consts.CLUSTER_RESIZE),
('AFTER', consts.CLUSTER_UPDATE),
('AFTER', consts.CLUSTER_RECOVER),
('AFTER', consts.CLUSTER_REPLACE_NODES),
('AFTER', consts.NODE_DELETE),
]
@ -410,9 +416,10 @@ class HealthPolicy(base.Policy):
def pre_op(self, cluster_id, action, **args):
"""Hook before action execution.
One of the task for this routine is to disable health policy if the
action is a request that will shrink the cluster. The reason is that
the policy may attempt to recover nodes that are to be deleted.
Disable health policy for actions that modify cluster nodes (e.g.
scale in, delete nodes, cluster update, cluster recover and cluster
replace nodes).
For all other actions, set the health policy data in the action data.
:param cluster_id: The ID of the target cluster.
:param action: The action to be examined.
@ -421,7 +428,10 @@ class HealthPolicy(base.Policy):
"""
if action.action in (consts.CLUSTER_SCALE_IN,
consts.CLUSTER_DEL_NODES,
consts.NODE_DELETE):
consts.NODE_DELETE,
consts.CLUSTER_UPDATE,
consts.CLUSTER_RECOVER,
consts.CLUSTER_REPLACE_NODES):
health_manager.disable(cluster_id)
return True
@ -467,7 +477,10 @@ class HealthPolicy(base.Policy):
"""
if action.action in (consts.CLUSTER_SCALE_IN,
consts.CLUSTER_DEL_NODES,
consts.NODE_DELETE):
consts.NODE_DELETE,
consts.CLUSTER_UPDATE,
consts.CLUSTER_RECOVER,
consts.CLUSTER_REPLACE_NODES):
health_manager.enable(cluster_id)
return True

89
senlin/profiles/os/nova/server.py
View File

@ -331,6 +331,7 @@ class ServerProfile(base.Profile):
def __init__(self, type_name, name, **kwargs):
super(ServerProfile, self).__init__(type_name, name, **kwargs)
self.server_id = None
self.stop_timeout = cfg.CONF.default_nova_timeout
def _validate_az(self, obj, az_name, reason=None):
try:
@ -1106,7 +1107,7 @@ class ServerProfile(base.Profile):
:param obj: The node object to operate on.
:param old_flavor: The identity of the current flavor.
:param new_flavor: The identity of the new flavor.
:returns: ``None``.
:returns: Returns true if the flavor was updated or false otherwise.
:raises: `EResourceUpdate` when operation was a failure.
"""
old_flavor = self.properties[self.FLAVOR]
@ -1115,7 +1116,23 @@ class ServerProfile(base.Profile):
oldflavor = self._validate_flavor(obj, old_flavor, 'update')
newflavor = self._validate_flavor(obj, new_flavor, 'update')
if oldflavor.id == newflavor.id:
return
return False
try:
# server has to be active or stopped in order to resize
# stop server if it is active
server = cc.server_get(obj.physical_id)
if server.status == consts.VS_ACTIVE:
cc.server_stop(obj.physical_id)
cc.wait_for_server(obj.physical_id, consts.VS_SHUTOFF,
timeout=self.stop_timeout)
elif server.status != consts.VS_SHUTOFF:
raise exc.InternalError(
message='Server needs to be ACTIVE or STOPPED in order to'
' update flavor.')
except exc.InternalError as ex:
raise exc.EResourceUpdate(type='server', id=obj.physical_id,
message=str(ex))
try:
cc.server_resize(obj.physical_id, newflavor.id)
@ -1123,7 +1140,13 @@ class ServerProfile(base.Profile):
except exc.InternalError as ex:
msg = str(ex)
try:
cc.server_resize_revert(obj.physical_id)
server = cc.server_get(obj.physical_id)
if server.status == 'RESIZE':
cc.server_resize_revert(obj.physical_id)
cc.wait_for_server(obj.physical_id, consts.VS_SHUTOFF)
# start server back up in case of exception during resize
cc.server_start(obj.physical_id)
cc.wait_for_server(obj.physical_id, consts.VS_ACTIVE)
except exc.InternalError as ex1:
msg = str(ex1)
@ -1132,11 +1155,13 @@ class ServerProfile(base.Profile):
try:
cc.server_resize_confirm(obj.physical_id)
cc.wait_for_server(obj.physical_id, consts.VS_ACTIVE)
cc.wait_for_server(obj.physical_id, consts.VS_SHUTOFF)
except exc.InternalError as ex:
raise exc.EResourceUpdate(type='server', id=obj.physical_id,
message=str(ex))
return True
def _update_image(self, obj, new_profile, new_name, new_password):
"""Update image used by server node.
@ -1172,9 +1197,20 @@ class ServerProfile(base.Profile):
return False
try:
# server has to be active or stopped in order to resize
# stop server if it is active
if server.status == consts.VS_ACTIVE:
driver.server_stop(obj.physical_id)
driver.wait_for_server(obj.physical_id, consts.VS_SHUTOFF,
timeout=self.stop_timeout)
elif server.status != consts.VS_SHUTOFF:
raise exc.InternalError(
message='Server needs to be ACTIVE or STOPPED in order to'
' update image.')
driver.server_rebuild(obj.physical_id, new_image_id,
new_name, new_password)
driver.wait_for_server(obj.physical_id, consts.VS_ACTIVE)
driver.wait_for_server(obj.physical_id, consts.VS_SHUTOFF)
except exc.InternalError as ex:
raise exc.EResourceUpdate(type='server', id=obj.physical_id,
message=str(ex))
@ -1262,6 +1298,18 @@ class ServerProfile(base.Profile):
nc = self.network(obj)
internal_ports = obj.data.get('internal_ports', [])
if networks:
try:
# stop server if it is active
server = cc.server_get(obj.physical_id)
if server.status == consts.VS_ACTIVE:
cc.server_stop(obj.physical_id)
cc.wait_for_server(obj.physical_id, consts.VS_SHUTOFF,
timeout=self.stop_timeout)
except exc.InternalError as ex:
raise exc.EResourceUpdate(type='server', id=obj.physical_id,
message=str(ex))
for n in networks:
candidate_ports = self._find_port_by_net_spec(
obj, n, internal_ports)
@ -1290,7 +1338,7 @@ class ServerProfile(base.Profile):
:param obj: The node object to operate.
:param new_profile: The new profile which may contain new network
settings.
:return: ``None``
:return: Returns a tuple of booleans if network was created or deleted.
:raises: ``EResourceUpdate`` if there are driver failures.
"""
networks_current = self.properties[self.NETWORKS]
@ -1308,7 +1356,8 @@ class ServerProfile(base.Profile):
# Attach new interfaces
if networks_create:
self._update_network_add_port(obj, networks_create)
return
return networks_create, networks_delete
def do_update(self, obj, new_profile=None, **params):
"""Perform update on the server.
@ -1328,6 +1377,15 @@ class ServerProfile(base.Profile):
if not self.validate_for_update(new_profile):
return False
self.stop_timeout = params.get('cluster.stop_timeout_before_update',
cfg.CONF.default_nova_timeout)
if not isinstance(self.stop_timeout, int):
raise exc.EResourceUpdate(
type='server', id=obj.physical_id,
message='cluster.stop_timeout_before_update value must be of '
'type int.')
name_changed, new_name = self._check_server_name(obj, new_profile)
passwd_changed, new_passwd = self._check_password(obj, new_profile)
# Update server image: may have side effect of changing server name
@ -1342,13 +1400,26 @@ class ServerProfile(base.Profile):
self._update_password(obj, new_passwd)
# Update server flavor: note that flavor is a required property
self._update_flavor(obj, new_profile)
self._update_network(obj, new_profile)
flavor_changed = self._update_flavor(obj, new_profile)
network_created, network_deleted = self._update_network(
obj, new_profile)
# TODO(Yanyan Hu): Update block_device properties
# Update server metadata
self._update_metadata(obj, new_profile)
# start server if it was stopped as part of this update operation
if image_changed or flavor_changed or network_deleted:
cc = self.compute(obj)
try:
server = cc.server_get(obj.physical_id)
if server.status == consts.VS_SHUTOFF:
cc.server_start(obj.physical_id)
cc.wait_for_server(obj.physical_id, consts.VS_ACTIVE)
except exc.InternalError as ex:
raise exc.EResourceUpdate(type='server', id=obj.physical_id,
message=str(ex))
return True
def do_get_details(self, obj):

5
senlin/tests/drivers/os_test/nova_v2.py
View File

@ -15,6 +15,7 @@ import time
from oslo_utils import uuidutils
from senlin.common import consts
from senlin.drivers import base
from senlin.drivers import sdk
@ -199,7 +200,9 @@ class NovaClient(base.DriverBase):
def server_get(self, server):
return sdk.FakeResourceObject(self.fake_server_get)
def wait_for_server(self, server, timeout=None):
def wait_for_server(self, server, status=consts.VS_ACTIVE,
failures=None,
interval=2, timeout=None):
# sleep for simulated wait time if it was supplied during server_create
if server in self.simulated_waits:
time.sleep(self.simulated_waits[server])

102
senlin/tests/unit/engine/actions/test_update.py
View File

@ -110,19 +110,39 @@ class ClusterUpdateTest(base.SenlinTestCase):
cluster = mock.Mock(id='FAKE_ID', nodes=[], ACTIVE='ACTIVE')
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx)
config = {'cluster.stop_timeout_before_update': 25}
action.inputs = {'name': 'FAKE_NAME',
'metadata': {'foo': 'bar'},
'timeout': 3600,
'new_profile_id': 'FAKE_PROFILE',
'profile_only': True}
'profile_only': True,
'config': config}
res_code, res_msg = action.do_update()
self.assertEqual(action.RES_OK, res_code)
self.assertEqual('Cluster update completed.', res_msg)
self.assertEqual(action.entity.config, config)
cluster.eval_status.assert_called_once_with(
action.context, consts.CLUSTER_UPDATE, profile_id='FAKE_PROFILE',
updated_at=mock.ANY)
@mock.patch.object(ca.ClusterAction, '_update_nodes')
def test_do_update_invalid_stop_timeout(self, mock_update, mock_load):
cluster = mock.Mock(id='FAKE_ID', nodes=[], ACTIVE='ACTIVE')
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx)
config = {'cluster.stop_timeout_before_update': 'abc'}
action.inputs = {'name': 'FAKE_NAME',
'metadata': {'foo': 'bar'},
'timeout': 3600,
'new_profile_id': 'FAKE_PROFILE',
'profile_only': True,
'config': config}
res_code, res_msg = action.do_update()
self.assertEqual(action.RES_ERROR, res_code)
mock_update.assert_not_called()
def test_do_update_empty_cluster(self, mock_load):
cluster = mock.Mock(id='FAKE_ID', nodes=[], ACTIVE='ACTIVE')
mock_load.return_value = cluster
@ -149,7 +169,7 @@ class ClusterUpdateTest(base.SenlinTestCase):
node1 = mock.Mock(id='node_id1')
node2 = mock.Mock(id='node_id2')
cluster = mock.Mock(id='FAKE_ID', nodes=[node1, node2],
ACTIVE='ACTIVE')
ACTIVE='ACTIVE', config={})
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx)
@ -157,12 +177,84 @@ class ClusterUpdateTest(base.SenlinTestCase):
action.id = 'CLUSTER_ACTION_ID'
mock_wait.return_value = (action.RES_OK, 'All dependents completed')
mock_action.side_effect = ['NODE_ACTION1', 'NODE_ACTION2']
kwargs1 = {
'name': 'node_update_node_id1',
'cluster_id': cluster.id,
'cause': consts.CAUSE_DERIVED,
'inputs': {
'new_profile_id': 'FAKE_PROFILE',
},
}
kwargs2 = {
'name': 'node_update_node_id2',
'cluster_id': cluster.id,
'cause': consts.CAUSE_DERIVED,
'inputs': {
'new_profile_id': 'FAKE_PROFILE',
},
}
res_code, reason = action._update_nodes('FAKE_PROFILE',
[node1, node2])
self.assertEqual(res_code, action.RES_OK)
self.assertEqual(reason, 'Cluster update completed.')
self.assertEqual(2, mock_action.call_count)
mock_action.assert_has_calls([
mock.call(action.context, node1.id, consts.NODE_UPDATE, **kwargs1),
mock.call(action.context, node2.id, consts.NODE_UPDATE, **kwargs2),
])
self.assertEqual(1, mock_dep.call_count)
self.assertEqual(2, mock_update.call_count)
mock_start.assert_called_once_with()
cluster.eval_status.assert_called_once_with(
action.context, consts.CLUSTER_UPDATE, profile_id='FAKE_PROFILE',
updated_at=mock.ANY)
@mock.patch.object(ao.Action, 'update')
@mock.patch.object(ab.Action, 'create')
@mock.patch.object(dobj.Dependency, 'create')
@mock.patch.object(dispatcher, 'start_action')
@mock.patch.object(ca.ClusterAction, '_wait_for_dependents')
def test_update_nodes_with_config(self, mock_wait, mock_start, mock_dep,
mock_action, mock_update, mock_load):
node1 = mock.Mock(id='node_id1')
node2 = mock.Mock(id='node_id2')
cluster = mock.Mock(id='FAKE_ID', nodes=[node1, node2],
ACTIVE='ACTIVE', config={'blah': 'abc'})
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx)
action.inputs = {'new_profile_id': 'FAKE_PROFILE'}
action.id = 'CLUSTER_ACTION_ID'
mock_wait.return_value = (action.RES_OK, 'All dependents completed')
mock_action.side_effect = ['NODE_ACTION1', 'NODE_ACTION2']
kwargs1 = {
'name': 'node_update_node_id1',
'cluster_id': cluster.id,
'cause': consts.CAUSE_DERIVED,
'inputs': {
'blah': 'abc',
'new_profile_id': 'FAKE_PROFILE',
},
}
kwargs2 = {
'name': 'node_update_node_id2',
'cluster_id': cluster.id,
'cause': consts.CAUSE_DERIVED,
'inputs': {
'blah': 'abc',
'new_profile_id': 'FAKE_PROFILE',
},
}
res_code, reason = action._update_nodes('FAKE_PROFILE',
[node1, node2])
self.assertEqual(res_code, action.RES_OK)
self.assertEqual(reason, 'Cluster update completed.')
mock_action.assert_has_calls([
mock.call(action.context, node1.id, consts.NODE_UPDATE, **kwargs1),
mock.call(action.context, node2.id, consts.NODE_UPDATE, **kwargs2),
])
self.assertEqual(1, mock_dep.call_count)
self.assertEqual(2, mock_update.call_count)
mock_start.assert_called_once_with()
@ -181,7 +273,7 @@ class ClusterUpdateTest(base.SenlinTestCase):
node1 = mock.Mock(id='node_id1')
node2 = mock.Mock(id='node_id2')
cluster = mock.Mock(id='FAKE_ID', nodes=[node1, node2],
ACTIVE='ACTIVE')
ACTIVE='ACTIVE', config={})
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx)
@ -220,7 +312,7 @@ class ClusterUpdateTest(base.SenlinTestCase):
node1 = mock.Mock(id='node_id1')
node2 = mock.Mock(id='node_id2')
cluster = mock.Mock(id='FAKE_ID', nodes=[node1, node2],
ACTIVE='ACTIVE')
ACTIVE='ACTIVE', config={})
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx)

59
senlin/tests/unit/policies/test_health_policy.py
View File

@ -287,7 +287,7 @@ class TestHealthPolicy(base.SenlinTestCase):
def test_pre_op_default(self):
action = mock.Mock(context='action_context', data={},
action=consts.CLUSTER_RECOVER)
action=consts.CLUSTER_SCALE_OUT)
res = self.hp.pre_op(self.cluster.id, action)
@ -310,6 +310,36 @@ class TestHealthPolicy(base.SenlinTestCase):
self.assertTrue(res)
mock_disable.assert_called_once_with(self.cluster.id)
@mock.patch.object(health_manager, 'disable')
def test_pre_op_update(self, mock_disable):
action = mock.Mock(context='action_context', data={},
action=consts.CLUSTER_UPDATE)
res = self.hp.pre_op(self.cluster.id, action)
self.assertTrue(res)
mock_disable.assert_called_once_with(self.cluster.id)
@mock.patch.object(health_manager, 'disable')
def test_pre_op_cluster_recover(self, mock_disable):
action = mock.Mock(context='action_context', data={},
action=consts.CLUSTER_RECOVER)
res = self.hp.pre_op(self.cluster.id, action)
self.assertTrue(res)
mock_disable.assert_called_once_with(self.cluster.id)
@mock.patch.object(health_manager, 'disable')
def test_pre_op_cluster_replace_nodes(self, mock_disable):
action = mock.Mock(context='action_context', data={},
action=consts.CLUSTER_REPLACE_NODES)
res = self.hp.pre_op(self.cluster.id, action)
self.assertTrue(res)
mock_disable.assert_called_once_with(self.cluster.id)
@mock.patch.object(health_manager, 'disable')
def test_pre_op_cluster_del_nodes(self, mock_disable):
action = mock.Mock(context='action_context', data={},
@ -394,6 +424,33 @@ class TestHealthPolicy(base.SenlinTestCase):
self.assertTrue(res)
mock_enable.assert_called_once_with(self.cluster.id)
@mock.patch.object(health_manager, 'enable')
def test_post_op_update(self, mock_enable):
action = mock.Mock(action=consts.CLUSTER_UPDATE)
res = self.hp.post_op(self.cluster.id, action)
self.assertTrue(res)
mock_enable.assert_called_once_with(self.cluster.id)
@mock.patch.object(health_manager, 'enable')
def test_post_op_cluster_recover(self, mock_enable):
action = mock.Mock(action=consts.CLUSTER_RECOVER)
res = self.hp.post_op(self.cluster.id, action)
self.assertTrue(res)
mock_enable.assert_called_once_with(self.cluster.id)
@mock.patch.object(health_manager, 'enable')
def test_post_op_cluster_replace_nodes(self, mock_enable):
action = mock.Mock(action=consts.CLUSTER_REPLACE_NODES)
res = self.hp.post_op(self.cluster.id, action)
self.assertTrue(res)
mock_enable.assert_called_once_with(self.cluster.id)
@mock.patch.object(health_manager, 'enable')
def test_post_op_cluster_del_nodes(self, mock_enable):
action = mock.Mock(action=consts.CLUSTER_DEL_NODES)

371
senlin/tests/unit/profiles/test_nova_server_update.py
View File

@ -13,7 +13,7 @@
import copy
from unittest import mock
from senlin.common import consts
from senlin.common import exception as exc
from senlin.objects import node as node_obj
from senlin.profiles.os.nova import server
@ -275,6 +275,35 @@ class TestNovaServerUpdate(base.SenlinTestCase):
def test_update_flavor(self):
obj = mock.Mock(physical_id='NOVA_ID')
cc = mock.Mock()
cc.server_get.return_value = mock.Mock(status=consts.VS_ACTIVE)
profile = server.ServerProfile('t', self.spec)
profile.stop_timeout = 123
profile._computeclient = cc
x_flavors = [mock.Mock(id='123'), mock.Mock(id='456')]
mock_validate = self.patchobject(profile, '_validate_flavor',
side_effect=x_flavors)
new_spec = copy.deepcopy(self.spec)
new_spec['properties']['flavor'] = 'new_flavor'
new_profile = server.ServerProfile('t1', new_spec)
profile._update_flavor(obj, new_profile)
mock_validate.assert_has_calls([
mock.call(obj, 'FLAV', 'update'),
mock.call(obj, 'new_flavor', 'update')
])
cc.server_resize.assert_called_once_with('NOVA_ID', '456')
cc.server_resize_confirm.assert_called_once_with('NOVA_ID')
cc.wait_for_server.assert_has_calls([
mock.call('NOVA_ID', consts.VS_SHUTOFF,
timeout=profile.stop_timeout),
mock.call('NOVA_ID', 'VERIFY_RESIZE'),
mock.call('NOVA_ID', consts.VS_SHUTOFF)])
# update flavor on server that is already stopped
def test_update_flavor_stopped_server(self):
obj = mock.Mock(physical_id='NOVA_ID')
cc = mock.Mock()
cc.server_get.return_value = mock.Mock(status=consts.VS_SHUTOFF)
profile = server.ServerProfile('t', self.spec)
profile._computeclient = cc
x_flavors = [mock.Mock(id='123'), mock.Mock(id='456')]
@ -291,9 +320,9 @@ class TestNovaServerUpdate(base.SenlinTestCase):
])
cc.server_resize.assert_called_once_with('NOVA_ID', '456')
cc.server_resize_confirm.assert_called_once_with('NOVA_ID')
cc.wait_for_server.has_calls([
cc.wait_for_server.assert_has_calls([
mock.call('NOVA_ID', 'VERIFY_RESIZE'),
mock.call('NOVA_ID', 'ACTIVE')])
mock.call('NOVA_ID', consts.VS_SHUTOFF)])
def test_update_flavor_failed_validation(self):
obj = mock.Mock(physical_id='NOVA_ID')
@ -351,16 +380,76 @@ class TestNovaServerUpdate(base.SenlinTestCase):
res = profile._update_flavor(obj, new_profile)
self.assertIsNone(res)
self.assertFalse(res)
mock_validate.assert_has_calls([
mock.call(obj, 'FLAV', 'update'),
mock.call(obj, 'FLAV', 'update'),
])
self.assertEqual(0, cc.server_resize.call_count)
def test_update_flavor_server_stop_failed(self):
obj = mock.Mock(physical_id='NOVA_ID')
cc = mock.Mock()
cc.server_get.return_value = mock.Mock(status=consts.VS_ACTIVE)
cc.server_stop.side_effect = [
exc.InternalError(code=500, message='Stop failed')]
profile = server.ServerProfile('t', self.spec)
profile._computeclient = cc
new_spec = copy.deepcopy(self.spec)
new_spec['properties']['flavor'] = 'new_flavor'
new_profile = server.ServerProfile('t1', new_spec)
x_flavors = [mock.Mock(id='123'), mock.Mock(id='456')]
mock_validate = self.patchobject(profile, '_validate_flavor',
side_effect=x_flavors)
ex = self.assertRaises(exc.EResourceUpdate,
profile._update_flavor,
obj, new_profile)
mock_validate.assert_has_calls([
mock.call(obj, 'FLAV', 'update'),
mock.call(obj, 'new_flavor', 'update'),
])
cc.server_resize.assert_not_called()
cc.server_resize_revert.assert_not_called()
cc.wait_for_server.assert_not_called()
self.assertEqual("Failed in updating server 'NOVA_ID': Stop "
"failed.", str(ex))
def test_update_flavor_server_paused(self):
obj = mock.Mock(physical_id='NOVA_ID')
cc = mock.Mock()
cc.server_get.return_value = mock.Mock(status=consts.VS_PAUSED)
profile = server.ServerProfile('t', self.spec)
profile._computeclient = cc
new_spec = copy.deepcopy(self.spec)
new_spec['properties']['flavor'] = 'new_flavor'
new_profile = server.ServerProfile('t1', new_spec)
x_flavors = [mock.Mock(id='123'), mock.Mock(id='456')]
mock_validate = self.patchobject(profile, '_validate_flavor',
side_effect=x_flavors)
ex = self.assertRaises(exc.EResourceUpdate,
profile._update_flavor,
obj, new_profile)
mock_validate.assert_has_calls([
mock.call(obj, 'FLAV', 'update'),
mock.call(obj, 'new_flavor', 'update'),
])
cc.server_resize.assert_not_called()
cc.server_resize_revert.assert_not_called()
cc.wait_for_server.assert_not_called()
self.assertEqual("Failed in updating server 'NOVA_ID': Server needs "
"to be ACTIVE or STOPPED in order to update flavor.",
str(ex))
def test_update_flavor_resize_failed(self):
obj = mock.Mock(physical_id='NOVA_ID')
cc = mock.Mock()
cc.server_get.side_effect = [
mock.Mock(status=consts.VS_ACTIVE),
mock.Mock(status='RESIZE')]
cc.server_resize.side_effect = [
exc.InternalError(code=500, message='Resize failed')]
profile = server.ServerProfile('t', self.spec)
@ -382,15 +471,56 @@ class TestNovaServerUpdate(base.SenlinTestCase):
])
cc.server_resize.assert_called_once_with('NOVA_ID', '456')
cc.server_resize_revert.assert_called_once_with('NOVA_ID')
cc.wait_for_server.assert_called_once_with('NOVA_ID', 'ACTIVE')
cc.wait_for_server.assert_has_calls([
mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600),
mock.call('NOVA_ID', consts.VS_SHUTOFF),
mock.call('NOVA_ID', consts.VS_ACTIVE)
])
self.assertEqual("Failed in updating server 'NOVA_ID': Resize "
"failed.", str(ex))
def test_update_flavor_first_wait_for_server_failed(self):
obj = mock.Mock(physical_id='NOVA_ID')
cc = mock.Mock()
cc.server_get.return_value = mock.Mock(status=consts.VS_ACTIVE)
cc.wait_for_server.side_effect = [
exc.InternalError(code=500, message='TIMEOUT')
]
profile = server.ServerProfile('t', self.spec)
profile._computeclient = cc
new_spec = copy.deepcopy(self.spec)
new_spec['properties']['flavor'] = 'new_flavor'
new_profile = server.ServerProfile('t1', new_spec)
x_flavors = [mock.Mock(id='123'), mock.Mock(id='456')]
mock_validate = self.patchobject(profile, '_validate_flavor',
side_effect=x_flavors)
# do it
ex = self.assertRaises(exc.EResourceUpdate,
profile._update_flavor,
obj, new_profile)
# assertions
mock_validate.assert_has_calls([
mock.call(obj, 'FLAV', 'update'),
mock.call(obj, 'new_flavor', 'update'),
])
cc.server_resize.assert_not_called()
cc.wait_for_server.has_calls([
mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600)])
self.assertEqual("Failed in updating server 'NOVA_ID': "
"TIMEOUT.", str(ex))
def test_update_flavor_second_wait_for_server_failed(self):
obj = mock.Mock(physical_id='NOVA_ID')
cc = mock.Mock()
cc.server_get.side_effect = [
mock.Mock(status=consts.VS_ACTIVE),
mock.Mock(status='RESIZE')]
cc.wait_for_server.side_effect = [
None,
exc.InternalError(code=500, message='TIMEOUT'),
None,
None
]
@ -414,8 +544,11 @@ class TestNovaServerUpdate(base.SenlinTestCase):
])
cc.server_resize.assert_called_once_with('NOVA_ID', '456')
cc.wait_for_server.has_calls([
mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600),
mock.call('NOVA_ID', 'VERIFY_RESIZE'),
mock.call('NOVA_ID', 'ACTIVE')])
mock.call('NOVA_ID', consts.VS_SHUTOFF),
mock.call('NOVA_ID', consts.VS_ACTIVE),
])
cc.server_resize_revert.assert_called_once_with('NOVA_ID')
self.assertEqual("Failed in updating server 'NOVA_ID': "
"TIMEOUT.", str(ex))
@ -423,6 +556,9 @@ class TestNovaServerUpdate(base.SenlinTestCase):
def test_update_flavor_resize_failed_revert_failed(self):
obj = mock.Mock(physical_id='NOVA_ID')
cc = mock.Mock()
cc.server_get.side_effect = [
mock.Mock(status=consts.VS_ACTIVE),
mock.Mock(status='RESIZE')]
err_resize = exc.InternalError(code=500, message='Resize')
cc.server_resize.side_effect = err_resize
err_revert = exc.InternalError(code=500, message='Revert')
@ -448,14 +584,16 @@ class TestNovaServerUpdate(base.SenlinTestCase):
])
cc.server_resize.assert_called_once_with('NOVA_ID', '456')
cc.server_resize_revert.assert_called_once_with('NOVA_ID')
# the wait_for_server wasn't called
self.assertEqual(0, cc.wait_for_server.call_count)
cc.wait_for_server.has_calls([
mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600),
])
self.assertEqual("Failed in updating server 'NOVA_ID': "
"Revert.", str(ex))
def test_update_flavor_confirm_failed(self):
obj = mock.Mock(physical_id='NOVA_ID')
cc = mock.Mock()
cc.server_get.return_value = mock.Mock(status=consts.VS_ACTIVE)
err_confirm = exc.InternalError(code=500, message='Confirm')
cc.server_resize_confirm.side_effect = err_confirm
profile = server.ServerProfile('t', self.spec)
@ -479,13 +617,17 @@ class TestNovaServerUpdate(base.SenlinTestCase):
])
cc.server_resize.assert_called_once_with('NOVA_ID', '456')
cc.server_resize_confirm.assert_called_once_with('NOVA_ID')
cc.wait_for_server.assert_called_once_with('NOVA_ID', 'VERIFY_RESIZE')
cc.wait_for_server.has_calls([
mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600),
mock.call('NOVA_ID', 'VERIFY_RESIZE'),
])
self.assertEqual("Failed in updating server 'NOVA_ID': Confirm.",
str(ex))
def test_update_flavor_wait_confirm_failed(self):
obj = mock.Mock(physical_id='NOVA_ID')
cc = mock.Mock()
cc.server_get.return_value = mock.Mock(status=consts.VS_SHUTOFF)
err_wait = exc.InternalError(code=500, message='Wait')
cc.wait_for_server.side_effect = [None, err_wait]
profile = server.ServerProfile('t', self.spec)
@ -511,15 +653,16 @@ class TestNovaServerUpdate(base.SenlinTestCase):
cc.server_resize_confirm.assert_called_once_with('NOVA_ID')
cc.wait_for_server.assert_has_calls([
mock.call('NOVA_ID', 'VERIFY_RESIZE'),
mock.call('NOVA_ID', 'ACTIVE')
mock.call('NOVA_ID', consts.VS_SHUTOFF)
])
self.assertEqual("Failed in updating server 'NOVA_ID': Wait.",
str(ex))
def test_update_image(self):
profile = server.ServerProfile('t', self.spec)
profile.stop_timeout = 123
x_image = {'id': '123'}
x_server = mock.Mock(image=x_image)
x_server = mock.Mock(image=x_image, status=consts.VS_ACTIVE)
cc = mock.Mock()
cc.server_get.return_value = x_server
profile._computeclient = cc
@ -539,7 +682,68 @@ class TestNovaServerUpdate(base.SenlinTestCase):
])
cc.server_rebuild.assert_called_once_with(
'NOVA_ID', '456', 'new_name', 'new_pass')
cc.wait_for_server.assert_called_once_with('NOVA_ID', 'ACTIVE')
cc.wait_for_server.assert_has_calls([
mock.call('NOVA_ID', consts.VS_SHUTOFF,
timeout=profile.stop_timeout),
mock.call('NOVA_ID', consts.VS_SHUTOFF),
])
def test_update_image_server_stopped(self):
profile = server.ServerProfile('t', self.spec)
x_image = {'id': '123'}
x_server = mock.Mock(image=x_image, status=consts.VS_SHUTOFF)
cc = mock.Mock()
cc.server_get.return_value = x_server
profile._computeclient = cc
x_new_image = mock.Mock(id='456')
x_images = [x_new_image]
mock_check = self.patchobject(profile, '_validate_image',
side_effect=x_images)
obj = mock.Mock(physical_id='NOVA_ID')
new_spec = copy.deepcopy(self.spec)
new_spec['properties']['image'] = 'new_image'
new_profile = server.ServerProfile('t1', new_spec)
profile._update_image(obj, new_profile, 'new_name', 'new_pass')
mock_check.assert_has_calls([
mock.call(obj, 'new_image', reason='update'),
])
cc.server_rebuild.assert_called_once_with(
'NOVA_ID', '456', 'new_name', 'new_pass')
cc.wait_for_server.assert_has_calls([
mock.call('NOVA_ID', consts.VS_SHUTOFF),
])
def test_update_image_server_paused(self):
profile = server.ServerProfile('t', self.spec)
x_image = {'id': '123'}
x_server = mock.Mock(image=x_image, status=consts.VS_PAUSED)
cc = mock.Mock()
cc.server_get.return_value = x_server
profile._computeclient = cc
x_new_image = mock.Mock(id='456')
x_images = [x_new_image]
mock_check = self.patchobject(profile, '_validate_image',
side_effect=x_images)
obj = mock.Mock(physical_id='NOVA_ID')
new_spec = copy.deepcopy(self.spec)
new_spec['properties']['image'] = 'new_image'
new_profile = server.ServerProfile('t1', new_spec)
ex = self.assertRaises(exc.EResourceUpdate,
profile._update_image,
obj, new_profile, 'new_name', '')
msg = ("Failed in updating server 'NOVA_ID': Server needs to be ACTIVE"
" or STOPPED in order to update image.")
self.assertEqual(msg, str(ex))
mock_check.assert_has_calls([
mock.call(obj, 'new_image', reason='update'),
])
cc.server_rebuild.assert_not_called()
cc.wait_for_server.assert_not_called()
def test_update_image_new_image_is_none(self):
profile = server.ServerProfile('t', self.spec)
@ -613,7 +817,7 @@ class TestNovaServerUpdate(base.SenlinTestCase):
profile = server.ServerProfile('t', old_spec)
cc = mock.Mock()
profile._computeclient = cc
x_server = mock.Mock(image={'id': '123'})
x_server = mock.Mock(image={'id': '123'}, status=consts.VS_ACTIVE)
cc.server_get.return_value = x_server
# this is the new one
x_image = mock.Mock(id='456')
@ -631,7 +835,11 @@ class TestNovaServerUpdate(base.SenlinTestCase):
cc.server_get.assert_called_once_with('NOVA_ID')
cc.server_rebuild.assert_called_once_with(
'NOVA_ID', '456', 'new_name', 'new_pass')
cc.wait_for_server.assert_called_once_with('NOVA_ID', 'ACTIVE')
cc.wait_for_server.assert_has_calls([
# first wait is from active to shutoff and has custom timeout
mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600),
mock.call('NOVA_ID', consts.VS_SHUTOFF),
])
def test_update_image_old_image_is_none_but_failed(self):
old_spec = copy.deepcopy(self.spec)
@ -683,12 +891,42 @@ class TestNovaServerUpdate(base.SenlinTestCase):
self.assertEqual(0, cc.server_rebuild.call_count)
self.assertEqual(0, cc.wait_for_server.call_count)
def test_update_image_failed_rebuilding(self):
def test_update_image_failed_stopping(self):
profile = server.ServerProfile('t', self.spec)
x_image = {'id': '123'}
x_server = mock.Mock(image=x_image)
cc = mock.Mock()
cc.server_get.return_value = x_server
cc.server_stop.side_effect = exc.InternalError(message='FAILED')
profile._computeclient = cc
x_new_image = mock.Mock(id='456')
x_images = [x_new_image]
mock_check = self.patchobject(profile, '_validate_image',
side_effect=x_images)
obj = mock.Mock(physical_id='NOVA_ID')
new_spec = copy.deepcopy(self.spec)
new_spec['properties']['image'] = 'new_image'
new_profile = server.ServerProfile('t1', new_spec)
ex = self.assertRaises(exc.EResourceUpdate,
profile._update_image,
obj, new_profile, 'new_name', 'new_pass')
self.assertEqual("Failed in updating server 'NOVA_ID': Server needs to"
" be ACTIVE or STOPPED in order to update image.",
str(ex))
mock_check.assert_has_calls([
mock.call(obj, 'new_image', reason='update'),
])
cc.server_rebuild.assert_not_called()
cc.wait_for_server.assert_not_called()
def test_update_image_failed_rebuilding(self):
profile = server.ServerProfile('t', self.spec)
x_image = {'id': '123'}
x_server = mock.Mock(image=x_image, status=consts.VS_ACTIVE)
cc = mock.Mock()
cc.server_get.return_value = x_server
cc.server_rebuild.side_effect = exc.InternalError(message='FAILED')
profile._computeclient = cc
x_new_image = mock.Mock(id='456')
@ -711,12 +949,14 @@ class TestNovaServerUpdate(base.SenlinTestCase):
])
cc.server_rebuild.assert_called_once_with(
'NOVA_ID', '456', 'new_name', 'new_pass')
self.assertEqual(0, cc.wait_for_server.call_count)
cc.wait_for_server.assert_has_calls([
mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600),
])
def test_update_image_failed_waiting(self):
def test_update_image_failed_first_waiting(self):
profile = server.ServerProfile('t', self.spec)
x_image = {'id': '123'}
x_server = mock.Mock(image=x_image)
x_server = mock.Mock(image=x_image, status=consts.VS_ACTIVE)
cc = mock.Mock()
cc.server_get.return_value = x_server
cc.wait_for_server.side_effect = exc.InternalError(message='TIMEOUT')
@ -730,6 +970,38 @@ class TestNovaServerUpdate(base.SenlinTestCase):
new_spec['properties']['image'] = 'new_image'
new_profile = server.ServerProfile('t1', new_spec)
ex = self.assertRaises(exc.EResourceUpdate,
profile._update_image,
obj, new_profile, 'new_name', 'new_pass')
self.assertEqual("Failed in updating server 'NOVA_ID': TIMEOUT.",
str(ex))
mock_check.assert_has_calls([
mock.call(obj, 'new_image', reason='update'),
])
cc.server_rebuild.assert_not_called()
cc.wait_for_server.assert_called_once_with(
'NOVA_ID', consts.VS_SHUTOFF, timeout=600)
def test_update_image_failed_second_waiting(self):
profile = server.ServerProfile('t', self.spec)
x_image = {'id': '123'}
x_server = mock.Mock(image=x_image, status=consts.VS_ACTIVE)
cc = mock.Mock()
cc.server_get.return_value = x_server
cc.wait_for_server.side_effect = [
None,
exc.InternalError(message='TIMEOUT')]
profile._computeclient = cc
x_new_image = mock.Mock(id='456')
x_images = [x_new_image]
mock_check = self.patchobject(profile, '_validate_image',
side_effect=x_images)
obj = mock.Mock(physical_id='NOVA_ID')
new_spec = copy.deepcopy(self.spec)
new_spec['properties']['image'] = 'new_image'
new_profile = server.ServerProfile('t1', new_spec)
ex = self.assertRaises(exc.EResourceUpdate,
profile._update_image,
obj, new_profile, 'new_name', 'new_pass')
@ -741,7 +1013,9 @@ class TestNovaServerUpdate(base.SenlinTestCase):
])
cc.server_rebuild.assert_called_once_with(
'NOVA_ID', '456', 'new_name', 'new_pass')
cc.wait_for_server.assert_called_once_with('NOVA_ID', 'ACTIVE')
cc.wait_for_server.assert_has_calls([
mock.call('NOVA_ID', consts.VS_SHUTOFF, timeout=600),
mock.call('NOVA_ID', consts.VS_SHUTOFF)])
def test_create_interfaces(self):
cc = mock.Mock()
@ -847,11 +1121,13 @@ class TestNovaServerUpdate(base.SenlinTestCase):
def test_delete_interfaces(self):
cc = mock.Mock()
cc.server_get.return_value = mock.Mock(status=consts.VS_ACTIVE)
nc = mock.Mock()
net1 = mock.Mock(id='net1')
nc.network_get.return_value = net1
nc.port_find.return_value = mock.Mock(id='port3', status='DOWN')
profile = server.ServerProfile('t', self.spec)
profile.stop_timeout = 232
profile._computeclient = cc
profile._networkclient = nc
obj = mock.Mock(physical_id='NOVA_ID', data={'internal_ports': [
@ -874,6 +1150,10 @@ class TestNovaServerUpdate(base.SenlinTestCase):
nc.network_get.assert_has_calls([
mock.call('net1'), mock.call('net1')
])
cc.wait_for_server.assert_has_calls([
mock.call('NOVA_ID', consts.VS_SHUTOFF,
timeout=profile.stop_timeout),
])
cc.server_interface_delete.assert_has_calls([
mock.call('port1', 'NOVA_ID'),
mock.call('port2', 'NOVA_ID'),
@ -935,9 +1215,11 @@ class TestNovaServerUpdate(base.SenlinTestCase):
]
new_profile = server.ServerProfile('t1', new_spec)
res = profile._update_network(obj, new_profile)
networks_created, networks_deleted = profile._update_network(
obj, new_profile)
self.assertIsNone(res)
self.assertTrue(networks_created)
self.assertTrue(networks_deleted)
networks_create = [
{'floating_network': None, 'network': 'net1', 'fixed_ip': 'ip2',
@ -974,10 +1256,14 @@ class TestNovaServerUpdate(base.SenlinTestCase):
mock_check_name.return_value = True, 'NEW_NAME'
mock_check_password.return_value = True, 'NEW_PASSWORD'
mock_update_image.return_value = False
mock_update_flavor.return_value = False
mock_update_network.return_value = False, False
obj = mock.Mock(physical_id='FAKE_ID')
profile = server.ServerProfile('t', self.spec)
profile._computeclient = mock.Mock()
profile._computeclient.server_get = mock.Mock()
profile._computeclient.server_start = mock.Mock()
new_profile = server.ServerProfile('t', self.spec)
res = profile.do_update(obj, new_profile)
@ -1007,6 +1293,7 @@ class TestNovaServerUpdate(base.SenlinTestCase):
mock_update_password):
mock_check_name.return_value = False, 'NEW_NAME'
mock_check_password.return_value = False, 'OLD_PASS'
mock_update_network.return_value = False, False
obj = mock.Mock(physical_id='NOVA_ID')
profile = server.ServerProfile('t', self.spec)
@ -1083,6 +1370,9 @@ class TestNovaServerUpdate(base.SenlinTestCase):
profile = server.ServerProfile('t', self.spec)
profile._computeclient = mock.Mock()
profile._computeclient.server_get = mock.Mock()
profile._computeclient.server_get.return_value = mock.Mock(
status=consts.VS_SHUTOFF)
new_spec = copy.deepcopy(self.spec)
new_spec['properties']['image'] = 'FAKE_IMAGE_NEW'
new_profile = server.ServerProfile('t', new_spec)
@ -1094,6 +1384,10 @@ class TestNovaServerUpdate(base.SenlinTestCase):
obj, new_profile, 'OLD_NAME', 'OLD_PASS')
self.assertEqual(0, mock_update_name.call_count)
self.assertEqual(0, mock_update_password.call_count)
profile._computeclient.server_get.assert_called_once_with(
obj.physical_id)
profile._computeclient.server_start.assert_called_once_with(
obj.physical_id)
@mock.patch.object(server.ServerProfile, '_update_flavor')
@mock.patch.object(server.ServerProfile, '_update_name')
@ -1129,10 +1423,11 @@ class TestNovaServerUpdate(base.SenlinTestCase):
@mock.patch.object(server.ServerProfile, '_update_flavor')
def test_do_update_update_flavor_succeeded(self, mock_update_flavor):
mock_update_flavor.return_value = True
obj = mock.Mock(physical_id='FAKE_ID')
profile = server.ServerProfile('t', self.spec)
x_image = {'id': '123'}
x_server = mock.Mock(image=x_image)
x_server = mock.Mock(image=x_image, status=consts.VS_SHUTOFF)
cc = mock.Mock()
cc.server_get.return_value = x_server
gc = mock.Mock()
@ -1146,6 +1441,7 @@ class TestNovaServerUpdate(base.SenlinTestCase):
self.assertTrue(res)
mock_update_flavor.assert_called_with(obj, new_profile)
gc.image_find.assert_called_with('FAKE_IMAGE', False)
cc.server_start.assert_called_once_with(obj.physical_id)
@mock.patch.object(server.ServerProfile, '_update_flavor')
def test_do_update_update_flavor_failed(self, mock_update_flavor):
@ -1155,7 +1451,7 @@ class TestNovaServerUpdate(base.SenlinTestCase):
obj = mock.Mock(physical_id='NOVA_ID')
profile = server.ServerProfile('t', self.spec)
x_image = {'id': '123'}
x_server = mock.Mock(image=x_image)
x_server = mock.Mock(image=x_image, status=consts.VS_ACTIVE)
cc = mock.Mock()
cc.server_get.return_value = x_server
gc = mock.Mock()
@ -1179,10 +1475,10 @@ class TestNovaServerUpdate(base.SenlinTestCase):
@mock.patch.object(server.ServerProfile, '_update_network')
def test_do_update_update_network_succeeded(
self, mock_update_network, mock_update_flavor):
mock_update_network.return_value = True
mock_update_network.return_value = True, True
profile = server.ServerProfile('t', self.spec)
x_image = {'id': '123'}
x_server = mock.Mock(image=x_image)
x_server = mock.Mock(image=x_image, status=consts.VS_SHUTOFF)
cc = mock.Mock()
gc = mock.Mock()
cc.server_get.return_value = x_server
@ -1197,10 +1493,16 @@ class TestNovaServerUpdate(base.SenlinTestCase):
]
new_profile = server.ServerProfile('t', new_spec)
res = profile.do_update(obj, new_profile)
params = {'cluster.stop_timeout_before_update': 134}
res = profile.do_update(obj, new_profile=new_profile, **params)
self.assertTrue(res)
gc.image_find.assert_called_with('FAKE_IMAGE', False)
mock_update_network.assert_called_with(obj, new_profile)
cc.server_start.assert_called_once_with(obj.physical_id)
self.assertEqual(profile.stop_timeout,
params['cluster.stop_timeout_before_update'])
@mock.patch.object(server.ServerProfile, '_update_password')
@mock.patch.object(server.ServerProfile, '_check_password')
@ -1267,3 +1569,20 @@ class TestNovaServerUpdate(base.SenlinTestCase):
res = profile.do_update(node_obj, new_profile)
self.assertFalse(res)
def test_do_update_invalid_stop_timeout(self):
profile = server.ServerProfile('t', self.spec)
profile._computeclient = mock.Mock()
node_obj = mock.Mock(physical_id='NOVA_ID')
new_spec = copy.deepcopy(self.spec)
new_profile = server.ServerProfile('t', new_spec)
params = {'cluster.stop_timeout_before_update': '123'}
ex = self.assertRaises(exc.EResourceUpdate,
profile.do_update,
node_obj, new_profile, **params)
self.assertEqual("Failed in updating server 'NOVA_ID': "
"cluster.stop_timeout_before_update value must be of "
"type int.",
str(ex))