Retry power on operation for Blade servers

This patch retries power on operation if it fails
to power on in definite time. This is needed
only for Blade servers. The fix is done for
Gen9 Proliant servers.

Change-Id: I088b8cf9bbde057c5536cad6368fce7d8d608f41
Closes-bug: 1725204

proliantutils/ilo/ris.py

@@ -15,6 +15,7 @@
 __author__ = 'HPE'
 
 import hashlib
+import retrying
 
 from proliantutils import exception
 from proliantutils.ilo import common
@@ -54,6 +55,9 @@ POWER_STATE = {
 CLASSCODE_FOR_GPU_DEVICES = [3]
 SUBCLASSCODE_FOR_GPU_DEVICES = [0, 1, 2, 128]
 
+MAX_RETRY_ATTEMPTS = 3  # Maximum number of attempts to be retried
+MAX_TIME_BEFORE_RETRY = 7 * 1000  # wait time in milliseconds before retry
+
 LOG = log.get_logger(__name__)
 
 
@@ -854,6 +858,28 @@ class RISOperations(rest.RestConnectorBase, operations.IloOperations):
         """
         self._press_pwr_btn(pushType="PressAndHold")
 
+    @retrying.retry(
+        stop_max_attempt_number=MAX_RETRY_ATTEMPTS,
+        retry_on_result=lambda state: state != 'ON',
+        wait_fixed=MAX_TIME_BEFORE_RETRY
+    )
+    def _retry_until_powered_on(self, power):
+        """This method retries power on operation.
+
+        :param: power : target power state
+        """
+        # If the system is in the same power state as
+        # requested by the user, it gives the error
+        # InvalidOperationForSystemState. To avoid this error
+        # the power state is checked before power on
+        # operation is performed.
+        status = self.get_host_power_status()
+        if (status != power):
+            self._perform_power_op(POWER_STATE[power])
+            return self.get_host_power_status()
+        else:
+            return status
+
     def set_host_power(self, power):
         """Toggle the power button of server.
 
@@ -874,8 +900,10 @@ class RISOperations(rest.RestConnectorBase, operations.IloOperations):
             LOG.debug(self._("Node is already in '%(power)s' power state."),
                       {'power': power})
             return
-
-        self._perform_power_op(POWER_STATE[power])
+        if power == 'ON' and 'Proliant BL' in self.get_product_name():
+            self._retry_until_powered_on(power)
+        else:
+            self._perform_power_op(POWER_STATE[power])
 
     def get_http_boot_url(self):
         """Request the http boot url from system in uefi boot mode.

proliantutils/tests/ilo/test_ris.py

@@ -1161,6 +1161,34 @@ class IloRisTestCase(testtools.TestCase):
         host_power_status_mock.assert_called_once_with()
         perform_power_op_mock.assert_called_once_with('ForceOff')
 
+    @mock.patch.object(ris.RISOperations, '_perform_power_op')
+    @mock.patch.object(ris.RISOperations, 'get_host_power_status')
+    @mock.patch.object(ris.RISOperations, 'get_product_name')
+    def test_set_host_power_change_on(self, product_mock,
+                                      host_power_status_mock,
+                                      perform_power_op_mock):
+        host_power_status_mock.return_value = 'OFF'
+        self.client.set_host_power('On')
+        product_mock.return_value = 'BL460'
+        host_power_status_mock.assert_called_once_with()
+        perform_power_op_mock.assert_called_once_with('On')
+
+    @mock.patch.object(ris.RISOperations, '_perform_power_op')
+    @mock.patch.object(ris.RISOperations, 'get_host_power_status')
+    def test_retry_until_powered_on_3times(self, host_power_status_mock,
+                                           perform_power_mock):
+        host_power_status_mock.side_effect = ['OFF', 'OFF', 'ON']
+        self.client._retry_until_powered_on('ON')
+        self.assertEqual(3, host_power_status_mock.call_count)
+
+    @mock.patch.object(ris.RISOperations, '_perform_power_op')
+    @mock.patch.object(ris.RISOperations, 'get_host_power_status')
+    def test_retry_until_powered_on(self, host_power_status_mock,
+                                    perform_power_mock):
+        host_power_status_mock.return_value = 'ON'
+        self.client._retry_until_powered_on('ON')
+        self.assertEqual(1, host_power_status_mock.call_count)
+
     @mock.patch.object(ris.RISOperations, '_perform_power_op')
     def test_reset_server(self, mock_perform_power):
         self.client.reset_server()