diff --git a/nova/tests/unit/virt/vmwareapi/test_vmops.py b/nova/tests/unit/virt/vmwareapi/test_vmops.py index b3fee86f2f60..8d590e067b6b 100644 --- a/nova/tests/unit/virt/vmwareapi/test_vmops.py +++ b/nova/tests/unit/virt/vmwareapi/test_vmops.py @@ -13,6 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. +import time + import mock from oslo_serialization import jsonutils from oslo_utils import units @@ -516,6 +518,108 @@ class VMwareVMOpsTestCase(test.NoDBTestCase): _volumeops.detach_disk_from_vm.assert_called_once_with( vm_ref, self._instance, mock.ANY, destroy_disk=True) + @mock.patch.object(time, 'sleep') + def _test_clean_shutdown(self, mock_sleep, + timeout, retry_interval, + returns_on, returns_off, + vmware_tools_status, + succeeds): + """Test the _clean_shutdown method + + :param timeout: timeout before soft shutdown is considered a fail + :param retry_interval: time between rechecking instance power state + :param returns_on: how often the instance is reported as poweredOn + :param returns_off: how often the instance is reported as poweredOff + :param vmware_tools_status: Status of vmware tools + :param succeeds: the expected result + """ + instance = self._instance + vm_ref = mock.Mock() + return_props = [] + expected_methods = ['get_object_properties_dict'] + props_on = {'runtime.powerState': 'poweredOn', + 'summary.guest.toolsStatus': vmware_tools_status, + 'summary.guest.toolsRunningStatus': 'guestToolsRunning'} + props_off = {'runtime.powerState': 'poweredOff', + 'summary.guest.toolsStatus': vmware_tools_status, + 'summary.guest.toolsRunningStatus': 'guestToolsRunning'} + + # initialize expected instance methods and returned properties + if vmware_tools_status == "toolsOk": + if returns_on > 0: + expected_methods.append('ShutdownGuest') + for x in range(returns_on + 1): + return_props.append(props_on) + for x in range(returns_on): + expected_methods.append('get_object_properties_dict') + for x in range(returns_off): + return_props.append(props_off) + if returns_on > 0: + expected_methods.append('get_object_properties_dict') + else: + return_props.append(props_off) + + def fake_call_method(module, method, *args, **kwargs): + expected_method = expected_methods.pop(0) + self.assertEqual(expected_method, method) + if expected_method == 'get_object_properties_dict': + props = return_props.pop(0) + return props + elif expected_method == 'ShutdownGuest': + return + + with test.nested( + mock.patch.object(vm_util, 'get_vm_ref', return_value=vm_ref), + mock.patch.object(self._session, '_call_method', + side_effect=fake_call_method) + ) as (mock_get_vm_ref, mock_call_method): + result = self._vmops._clean_shutdown(instance, timeout, + retry_interval) + + self.assertEqual(succeeds, result) + mock_get_vm_ref.assert_called_once_with(self._session, + self._instance) + + def test_clean_shutdown_first_time(self): + self._test_clean_shutdown(timeout=10, + retry_interval=3, + returns_on=1, + returns_off=1, + vmware_tools_status="toolsOk", + succeeds=True) + + def test_clean_shutdown_second_time(self): + self._test_clean_shutdown(timeout=10, + retry_interval=3, + returns_on=2, + returns_off=1, + vmware_tools_status="toolsOk", + succeeds=True) + + def test_clean_shutdown_timeout(self): + self._test_clean_shutdown(timeout=10, + retry_interval=3, + returns_on=4, + returns_off=0, + vmware_tools_status="toolsOk", + succeeds=False) + + def test_clean_shutdown_already_off(self): + self._test_clean_shutdown(timeout=10, + retry_interval=3, + returns_on=0, + returns_off=1, + vmware_tools_status="toolsOk", + succeeds=False) + + def test_clean_shutdown_no_vwaretools(self): + self._test_clean_shutdown(timeout=10, + retry_interval=3, + returns_on=1, + returns_off=0, + vmware_tools_status="toolsNotOk", + succeeds=False) + def _test_finish_migration(self, power_on=True, resize_instance=False): with test.nested( mock.patch.object(self._vmops, diff --git a/nova/virt/vmwareapi/driver.py b/nova/virt/vmwareapi/driver.py index 9bc12ebda0c9..86cf3daa7c2c 100644 --- a/nova/virt/vmwareapi/driver.py +++ b/nova/virt/vmwareapi/driver.py @@ -427,8 +427,7 @@ class VMwareVCDriver(driver.ComputeDriver): def power_off(self, instance, timeout=0, retry_interval=0): """Power off the specified instance.""" - # TODO(PhilDay): Add support for timeout (clean shutdown) - self._vmops.power_off(instance) + self._vmops.power_off(instance, timeout, retry_interval) def power_on(self, context, instance, network_info, block_device_info=None): diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index 3d928cce3c08..352babf6d068 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -1244,13 +1244,80 @@ class VMwareVMOps(object): if power_on: vm_util.power_on_instance(self._session, instance, vm_ref=vm_ref) - def power_off(self, instance): + def power_off(self, instance, timeout=0, retry_interval=0): """Power off the specified instance. :param instance: nova.objects.instance.Instance + :param timeout: How long to wait in seconds for the instance to + shutdown + :param retry_interval: Interval to check if instance is already + shutdown in seconds. """ + if timeout and self._clean_shutdown(instance, + timeout, + retry_interval): + return + vm_util.power_off_instance(self._session, instance) + def _clean_shutdown(self, instance, timeout, retry_interval): + """Perform a soft shutdown on the VM. + :param instance: nova.objects.instance.Instance + :param timeout: How long to wait in seconds for the instance to + shutdown + :param retry_interval: Interval to check if instance is already + shutdown in seconds. + :return: True if the instance was shutdown within time limit, + False otherwise. + """ + LOG.debug("Performing Soft shutdown on instance", + instance=instance) + vm_ref = vm_util.get_vm_ref(self._session, instance) + + props = self._get_instance_props(vm_ref) + + if props.get("runtime.powerState") != "poweredOn": + LOG.debug("Instance not in poweredOn state.", + instance=instance) + return False + + if ((props.get("summary.guest.toolsStatus") == "toolsOk") and + (props.get("summary.guest.toolsRunningStatus") == + "guestToolsRunning")): + + LOG.debug("Soft shutdown instance, timeout: %d", + timeout, instance=instance) + self._session._call_method(self._session.vim, + "ShutdownGuest", + vm_ref) + + while timeout > 0: + wait_time = min(retry_interval, timeout) + props = self._get_instance_props(vm_ref) + + if props.get("runtime.powerState") == "poweredOff": + LOG.info("Soft shutdown succeeded.", + instance=instance) + return True + + time.sleep(wait_time) + timeout -= retry_interval + + LOG.warning("Timed out while waiting for soft shutdown.", + instance=instance) + else: + LOG.debug("VMware Tools not running", instance=instance) + + return False + + def _get_instance_props(self, vm_ref): + lst_properties = ["summary.guest.toolsStatus", + "runtime.powerState", + "summary.guest.toolsRunningStatus"] + return self._session._call_method(vutil, + "get_object_properties_dict", + vm_ref, lst_properties) + def power_on(self, instance): vm_util.power_on_instance(self._session, instance) diff --git a/releasenotes/notes/bug-1377781-c91d5319862bb9d8.yaml b/releasenotes/notes/bug-1377781-c91d5319862bb9d8.yaml new file mode 100644 index 000000000000..49f06ca130b9 --- /dev/null +++ b/releasenotes/notes/bug-1377781-c91d5319862bb9d8.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Add support for graceful shutdown of VMware instances. The timeout parameter of the power_off + method is now considered by the VMware driver. If you specify a timeout greater than 0, the + driver calls the appropriate soft shutdown method of the VMware API and only forces a hard + shutdown if the soft shutdown did not succeed before the timeout is reached.