From 060e42b7fd185bf3b827b7aa5c9ab6131f6a21d7 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 17 May 2019 14:58:59 +0100 Subject: [PATCH] [ironic] Don't remove instance info twice in destroy During teardown, at the end of the ironic virt driver's destroy method, we call _cleanup_deploy, which since https://review.opendev.org/#/c/563722/ removes instance_info from the ironic node. Given that we're destroying the node, the instance_info will have been cleared from the node anyway, so we don't need to do this. Further, it can cause tear down to fail if automated cleaning is enabled in ironic. This happens because ironic prevents updates to nodes that are undergoing a state transition, as is the case during cleaning. The virt driver will retry this request by default every 2 seconds with 60 attempts. Typically this is not long enough for cleaning to complete, so tear down fails with the following error: Conflict: Node a00696d5-32ba-475e-9528-59bf11cffea6 can not be updated while a state transition is in progress. (HTTP 409) This change skips the instance info update in _cleanup_deploy in the case of tear down. Change-Id: Iea337f73c231db2cb9d9f639b92475daaede6793 Closes-Bug: #1815799 --- nova/tests/unit/virt/ironic/test_driver.py | 27 +++++++++++++++++++--- nova/virt/ironic/driver.py | 13 ++++++++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/nova/tests/unit/virt/ironic/test_driver.py b/nova/tests/unit/virt/ironic/test_driver.py index 6a422a71c0dd..c3f4692c7250 100644 --- a/nova/tests/unit/virt/ironic/test_driver.py +++ b/nova/tests/unit/virt/ironic/test_driver.py @@ -1666,7 +1666,8 @@ class IronicDriverTestCase(test.NoDBTestCase): mock_node.get_by_instance_uuid.assert_called_with( instance.uuid, fields=ironic_driver._NODE_FIELDS) - mock_cleanup_deploy.assert_called_with(node, instance, network_info) + mock_cleanup_deploy.assert_called_with(node, instance, network_info, + remove_instance_info=False) # For states that makes sense check if set_provision_state has # been called @@ -1699,7 +1700,8 @@ class IronicDriverTestCase(test.NoDBTestCase): mock_sps.side_effect = exception.NovaException() self.assertRaises(exception.NovaException, self.driver.destroy, self.ctx, instance, None, None) - mock_clean.assert_called_once_with(node, instance, None) + mock_clean.assert_called_once_with(node, instance, None, + remove_instance_info=False) @mock.patch.object(FAKE_CLIENT.node, 'update') @mock.patch.object(ironic_driver.IronicDriver, @@ -1718,7 +1720,8 @@ class IronicDriverTestCase(test.NoDBTestCase): mock_update.side_effect = SystemError('unexpected error') self.assertRaises(SystemError, self.driver.destroy, self.ctx, instance, None, None) - mock_clean.assert_called_once_with(node, instance, None) + mock_clean.assert_called_once_with(node, instance, None, + remove_instance_info=False) @mock.patch.object(FAKE_CLIENT.node, 'set_provision_state') @mock.patch.object(ironic_driver.IronicDriver, @@ -2549,6 +2552,24 @@ class IronicDriverTestCase(test.NoDBTestCase): mock_call.has_calls( [mock.call('node.update', node.uuid, expected_patch)]) + @mock.patch.object(ironic_driver.IronicDriver, '_stop_firewall') + @mock.patch.object(ironic_driver.IronicDriver, '_unplug_vifs') + @mock.patch.object(ironic_driver.IronicDriver, + '_cleanup_volume_target_info') + @mock.patch.object(cw.IronicClientWrapper, 'call') + def test__cleanup_deploy_no_remove_ii(self, mock_call, mock_vol, + mock_unvif, mock_stop_fw): + # TODO(TheJulia): This REALLY should be updated to cover all of the + # calls that take place. + node = ironic_utils.get_test_node(driver='fake') + instance = fake_instance.fake_instance_obj(self.ctx, + node=node.uuid) + self.driver._cleanup_deploy(node, instance, remove_instance_info=False) + mock_vol.assert_called_once_with(instance) + mock_unvif.assert_called_once_with(node, instance, None) + mock_stop_fw.assert_called_once_with(instance, None) + self.assertFalse(mock_call.called) + class IronicDriverSyncTestCase(IronicDriverTestCase): diff --git a/nova/virt/ironic/driver.py b/nova/virt/ironic/driver.py index a7051a68a991..77a4c3d1fc63 100644 --- a/nova/virt/ironic/driver.py +++ b/nova/virt/ironic/driver.py @@ -486,11 +486,13 @@ class IronicDriver(virt_driver.ComputeDriver): 'reason': e}, instance=instance) - def _cleanup_deploy(self, node, instance, network_info=None): + def _cleanup_deploy(self, node, instance, network_info=None, + remove_instance_info=True): self._cleanup_volume_target_info(instance) self._unplug_vifs(node, instance, network_info) self._stop_firewall(instance, network_info) - self._remove_instance_info_from_node(node, instance) + if remove_instance_info: + self._remove_instance_info_from_node(node, instance) def _wait_for_active(self, instance): """Wait for the node to be marked as ACTIVE in Ironic.""" @@ -1336,7 +1338,12 @@ class IronicDriver(virt_driver.ComputeDriver): # removed from ironic node. self._remove_instance_info_from_node(node, instance) finally: - self._cleanup_deploy(node, instance, network_info) + # NOTE(mgoddard): We don't need to remove instance info at this + # point since we will have already done it. The destroy will only + # succeed if this method returns without error, so we will end up + # removing the instance info eventually. + self._cleanup_deploy(node, instance, network_info, + remove_instance_info=False) LOG.info('Successfully unprovisioned Ironic node %s', node.uuid, instance=instance)