Ensure power state is shutdown in power_off

In the case that vm instance received shutdown signal but not
complete shutdown, zvm driver will return vm power state as
NOSTATE, which would make user confused.

In this change, will add logic to wait vm instance power state
become to shutdown.

Change-Id: I519b5c30d38b611e52f49facc8f63d401cbfad01
(cherry picked from commit 0791ba9584)

nova/tests/unit/virt/zvm/test_zvm.py

@@ -871,20 +871,25 @@ class ZVMDriverTestCases(ZVMTestCase):
                           self.driver.reboot,
                           self.context, self.instance, {}, "SOFT")
 
-    def test_power_off(self):
-        info = ["os000001: Stopping OS000001... Done\n"]
-        self._set_fake_xcat_responses([self._generate_xcat_resp(info)])
-        self.driver.power_off(self.instance, 10, 10)
-        self.mox.VerifyAll()
+    @mock.patch('nova.virt.zvm.instance.ZVMInstance.power_off')
+    def test_power_off(self, poff):
+        self.driver.power_off(self.instance, 60, 10)
+        poff.assert_called_once_with(60, 10)
 
-    def test_power_off_inactive(self):
-        info = ["os000001: Stopping LINUX171... Failed\n"
-                "  Return Code: 200\n"
-                "  Reason Code: 12\n"
-                "  Description: Image not active\n"]
-        self._set_fake_xcat_responses([self._generate_xcat_resp(info)])
-        self.driver.power_off(self.instance)
-        self.mox.VerifyAll()
+    @mock.patch('nova.virt.zvm.instance.ZVMInstance._power_state')
+    def test_power_off_inactive(self, pst):
+        pst.side_effect = exception.ZVMXCATInternalError(
+                                        "Return Code: 200 Reason Code: 12")
+        self.driver.power_off(self.instance, 60, 10)
+        pst.assert_called_once_with("PUT", "softoff")
+
+    @mock.patch('nova.virt.zvm.instance.ZVMInstance._get_power_stat')
+    @mock.patch('nova.virt.zvm.instance.ZVMInstance._power_state')
+    def test_power_off_retry(self, pst, get_pst):
+        get_pst.side_effect = [0x00, 0x04]
+        self.driver.power_off(self.instance, 60, 10)
+        pst.assert_called_once_with("PUT", "softoff")
+        get_pst.assert_called()
 
     def test_power_off_failed(self):
         info = ["os000001: Stopping OS000001... Failed\n"]

nova/virt/zvm/driver.py

@@ -989,7 +989,7 @@ class ZVMDriver(driver.ComputeDriver):
         LOG.debug('Stopping z/VM instance %s' % instance['name'],
                   instance=instance)
         zvm_inst = ZVMInstance(instance)
-        zvm_inst.power_off()
+        zvm_inst.power_off(timeout, retry_interval)
 
     def power_on(self, context, instance, network_info,
                  block_device_info=None):

nova/virt/zvm/instance.py

@@ -15,6 +15,7 @@
 
 import binascii
 import datetime
+import time
 
 from oslo_config import cfg
 from oslo_log import log as logging
@@ -48,7 +49,7 @@ class ZVMInstance(object):
         self._volumeop = volumeop.VolumeOperator()
         self._dist_manager = dist.ListDistManager()
 
-    def power_off(self):
+    def power_off(self, timeout=0, retry_interval=10):
         """Power off z/VM instance."""
         try:
             self._power_state("PUT", "softoff")
@@ -64,6 +65,20 @@ class ZVMInstance(object):
                 LOG.error(msg)
                 raise nova_exception.InstancePowerOffFailure(reason=msg)
 
+        timeout = timeout or CONF.shutdown_timeout
+        retry_interval = retry_interval or 10
+        retry_count = timeout // retry_interval
+        while (retry_count > 0):
+            if self._get_power_stat() == power_state.SHUTDOWN:
+                # In shutdown state already
+                return
+            else:
+                time.sleep(retry_interval)
+                retry_count -= 1
+
+        LOG.warn(_LW("Failed to shutdown instance %(inst)s in %(time)d "
+                     "seconds") % {'inst': self._name, 'time': timeout})
+
     def power_on(self):
         """"Power on z/VM instance."""
         try: