VMware: add support for graceful shutdown of instances
Change-Id: I40643e9d358be89c87a0311b1c1fd7718ec75361 Closes-Bug: #1377781 Co-Authored-By: David Rabel <rabel@b1-systems.de>
This commit is contained in:
Thomas Kaergel
David Rabel
parent
fefff942ef
commit
24aaf8752d
|
|
@ -13,6 +13,8 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import time
|
||||
|
||||
import mock
|
||||
from oslo_serialization import jsonutils
|
||||
from oslo_utils import units
|
||||
|
|
@ -516,6 +518,108 @@ class VMwareVMOpsTestCase(test.NoDBTestCase):
|
|||
_volumeops.detach_disk_from_vm.assert_called_once_with(
|
||||
vm_ref, self._instance, mock.ANY, destroy_disk=True)
|
||||
|
||||
@mock.patch.object(time, 'sleep')
|
||||
def _test_clean_shutdown(self, mock_sleep,
|
||||
timeout, retry_interval,
|
||||
returns_on, returns_off,
|
||||
vmware_tools_status,
|
||||
succeeds):
|
||||
"""Test the _clean_shutdown method
|
||||
|
||||
:param timeout: timeout before soft shutdown is considered a fail
|
||||
:param retry_interval: time between rechecking instance power state
|
||||
:param returns_on: how often the instance is reported as poweredOn
|
||||
:param returns_off: how often the instance is reported as poweredOff
|
||||
:param vmware_tools_status: Status of vmware tools
|
||||
:param succeeds: the expected result
|
||||
"""
|
||||
instance = self._instance
|
||||
vm_ref = mock.Mock()
|
||||
return_props = []
|
||||
expected_methods = ['get_object_properties_dict']
|
||||
props_on = {'runtime.powerState': 'poweredOn',
|
||||
'summary.guest.toolsStatus': vmware_tools_status,
|
||||
'summary.guest.toolsRunningStatus': 'guestToolsRunning'}
|
||||
props_off = {'runtime.powerState': 'poweredOff',
|
||||
'summary.guest.toolsStatus': vmware_tools_status,
|
||||
'summary.guest.toolsRunningStatus': 'guestToolsRunning'}
|
||||
|
||||
# initialize expected instance methods and returned properties
|
||||
if vmware_tools_status == "toolsOk":
|
||||
if returns_on > 0:
|
||||
expected_methods.append('ShutdownGuest')
|
||||
for x in range(returns_on + 1):
|
||||
return_props.append(props_on)
|
||||
for x in range(returns_on):
|
||||
expected_methods.append('get_object_properties_dict')
|
||||
for x in range(returns_off):
|
||||
return_props.append(props_off)
|
||||
if returns_on > 0:
|
||||
expected_methods.append('get_object_properties_dict')
|
||||
else:
|
||||
return_props.append(props_off)
|
||||
|
||||
def fake_call_method(module, method, *args, **kwargs):
|
||||
expected_method = expected_methods.pop(0)
|
||||
self.assertEqual(expected_method, method)
|
||||
if expected_method == 'get_object_properties_dict':
|
||||
props = return_props.pop(0)
|
||||
return props
|
||||
elif expected_method == 'ShutdownGuest':
|
||||
return
|
||||
|
||||
with test.nested(
|
||||
mock.patch.object(vm_util, 'get_vm_ref', return_value=vm_ref),
|
||||
mock.patch.object(self._session, '_call_method',
|
||||
side_effect=fake_call_method)
|
||||
) as (mock_get_vm_ref, mock_call_method):
|
||||
result = self._vmops._clean_shutdown(instance, timeout,
|
||||
retry_interval)
|
||||
|
||||
self.assertEqual(succeeds, result)
|
||||
mock_get_vm_ref.assert_called_once_with(self._session,
|
||||
self._instance)
|
||||
|
||||
def test_clean_shutdown_first_time(self):
|
||||
self._test_clean_shutdown(timeout=10,
|
||||
retry_interval=3,
|
||||
returns_on=1,
|
||||
returns_off=1,
|
||||
vmware_tools_status="toolsOk",
|
||||
succeeds=True)
|
||||
|
||||
def test_clean_shutdown_second_time(self):
|
||||
self._test_clean_shutdown(timeout=10,
|
||||
retry_interval=3,
|
||||
returns_on=2,
|
||||
returns_off=1,
|
||||
vmware_tools_status="toolsOk",
|
||||
succeeds=True)
|
||||
|
||||
def test_clean_shutdown_timeout(self):
|
||||
self._test_clean_shutdown(timeout=10,
|
||||
retry_interval=3,
|
||||
returns_on=4,
|
||||
returns_off=0,
|
||||
vmware_tools_status="toolsOk",
|
||||
succeeds=False)
|
||||
|
||||
def test_clean_shutdown_already_off(self):
|
||||
self._test_clean_shutdown(timeout=10,
|
||||
retry_interval=3,
|
||||
returns_on=0,
|
||||
returns_off=1,
|
||||
vmware_tools_status="toolsOk",
|
||||
succeeds=False)
|
||||
|
||||
def test_clean_shutdown_no_vwaretools(self):
|
||||
self._test_clean_shutdown(timeout=10,
|
||||
retry_interval=3,
|
||||
returns_on=1,
|
||||
returns_off=0,
|
||||
vmware_tools_status="toolsNotOk",
|
||||
succeeds=False)
|
||||
|
||||
def _test_finish_migration(self, power_on=True, resize_instance=False):
|
||||
with test.nested(
|
||||
mock.patch.object(self._vmops,
|
||||
|
|
|
|||
|
|
@ -427,8 +427,7 @@ class VMwareVCDriver(driver.ComputeDriver):
|
|||
|
||||
def power_off(self, instance, timeout=0, retry_interval=0):
|
||||
"""Power off the specified instance."""
|
||||
# TODO(PhilDay): Add support for timeout (clean shutdown)
|
||||
self._vmops.power_off(instance)
|
||||
self._vmops.power_off(instance, timeout, retry_interval)
|
||||
|
||||
def power_on(self, context, instance, network_info,
|
||||
block_device_info=None):
|
||||
|
|
|
|||
|
|
@ -1244,13 +1244,80 @@ class VMwareVMOps(object):
|
|||
if power_on:
|
||||
vm_util.power_on_instance(self._session, instance, vm_ref=vm_ref)
|
||||
|
||||
def power_off(self, instance):
|
||||
def power_off(self, instance, timeout=0, retry_interval=0):
|
||||
"""Power off the specified instance.
|
||||
|
||||
:param instance: nova.objects.instance.Instance
|
||||
:param timeout: How long to wait in seconds for the instance to
|
||||
shutdown
|
||||
:param retry_interval: Interval to check if instance is already
|
||||
shutdown in seconds.
|
||||
"""
|
||||
if timeout and self._clean_shutdown(instance,
|
||||
timeout,
|
||||
retry_interval):
|
||||
return
|
||||
|
||||
vm_util.power_off_instance(self._session, instance)
|
||||
|
||||
def _clean_shutdown(self, instance, timeout, retry_interval):
|
||||
"""Perform a soft shutdown on the VM.
|
||||
:param instance: nova.objects.instance.Instance
|
||||
:param timeout: How long to wait in seconds for the instance to
|
||||
shutdown
|
||||
:param retry_interval: Interval to check if instance is already
|
||||
shutdown in seconds.
|
||||
:return: True if the instance was shutdown within time limit,
|
||||
False otherwise.
|
||||
"""
|
||||
LOG.debug("Performing Soft shutdown on instance",
|
||||
instance=instance)
|
||||
vm_ref = vm_util.get_vm_ref(self._session, instance)
|
||||
|
||||
props = self._get_instance_props(vm_ref)
|
||||
|
||||
if props.get("runtime.powerState") != "poweredOn":
|
||||
LOG.debug("Instance not in poweredOn state.",
|
||||
instance=instance)
|
||||
return False
|
||||
|
||||
if ((props.get("summary.guest.toolsStatus") == "toolsOk") and
|
||||
(props.get("summary.guest.toolsRunningStatus") ==
|
||||
"guestToolsRunning")):
|
||||
|
||||
LOG.debug("Soft shutdown instance, timeout: %d",
|
||||
timeout, instance=instance)
|
||||
self._session._call_method(self._session.vim,
|
||||
"ShutdownGuest",
|
||||
vm_ref)
|
||||
|
||||
while timeout > 0:
|
||||
wait_time = min(retry_interval, timeout)
|
||||
props = self._get_instance_props(vm_ref)
|
||||
|
||||
if props.get("runtime.powerState") == "poweredOff":
|
||||
LOG.info("Soft shutdown succeeded.",
|
||||
instance=instance)
|
||||
return True
|
||||
|
||||
time.sleep(wait_time)
|
||||
timeout -= retry_interval
|
||||
|
||||
LOG.warning("Timed out while waiting for soft shutdown.",
|
||||
instance=instance)
|
||||
else:
|
||||
LOG.debug("VMware Tools not running", instance=instance)
|
||||
|
||||
return False
|
||||
|
||||
def _get_instance_props(self, vm_ref):
|
||||
lst_properties = ["summary.guest.toolsStatus",
|
||||
"runtime.powerState",
|
||||
"summary.guest.toolsRunningStatus"]
|
||||
return self._session._call_method(vutil,
|
||||
"get_object_properties_dict",
|
||||
vm_ref, lst_properties)
|
||||
|
||||
def power_on(self, instance):
|
||||
vm_util.power_on_instance(self._session, instance)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
---
|
||||
features:
|
||||
- |
|
||||
Add support for graceful shutdown of VMware instances. The timeout parameter of the power_off
|
||||
method is now considered by the VMware driver. If you specify a timeout greater than 0, the
|
||||
driver calls the appropriate soft shutdown method of the VMware API and only forces a hard
|
||||
shutdown if the soft shutdown did not succeed before the timeout is reached.
|
||||
Loading…
Reference in New Issue