From 35f222c55deaa13c599de80eb9eaf0e4cbf8165e Mon Sep 17 00:00:00 2001 From: Christopher Dearborn Date: Thu, 18 May 2017 15:35:04 -0400 Subject: [PATCH] Wait until iDRAC is ready before out-of-band cleaning When out-of-band cleaning is initiated, the node is PXE booted and the ramdisk is loaded. After in-band cleaning completes, the node is rebooted. At that point, the iDRAC automatically creates and runs an "Export Configuration" job. Out-of-band cleaning then starts: either RAID configuration creation or deletion. If the export job has not finished by the time the RAID deletion or creation job is attempted to be created, then the RAID job creation fails. This patch causes RAID configuration creation and deletion to wait until the iDRAC declares itself to be ready before proceeding with out-of-band cleaning. This ensures that the export job has completed before creating another job. Change-Id: I79faba2206b86288ae636c46468a8b2dc321f979 Closes-Bug: 1691808 Depends-On: I929deada3dda7b09a6f29033fff89d9b0382aef8 --- driver-requirements.txt | 2 +- ironic/drivers/modules/drac/raid.py | 10 ++++++++++ .../tests/unit/drivers/modules/drac/test_raid.py | 16 ++++++++++++++++ .../drac-fix-oob-cleaning-b4b717895e243c9b.yaml | 7 +++++++ 4 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/drac-fix-oob-cleaning-b4b717895e243c9b.yaml diff --git a/driver-requirements.txt b/driver-requirements.txt index 6855f2b3d8..c4a53c5201 100644 --- a/driver-requirements.txt +++ b/driver-requirements.txt @@ -10,7 +10,7 @@ python-ironic-inspector-client>=1.5.0 python-oneviewclient<3.0.0,>=2.5.2 python-scciclient>=0.4.0 UcsSdk==0.8.2.2 -python-dracclient>=0.1.0 +python-dracclient>=1.2.0 # The CIMC drivers use the Cisco IMC SDK version 0.7.2 or greater ImcSdk>=0.7.2 diff --git a/ironic/drivers/modules/drac/raid.py b/ironic/drivers/modules/drac/raid.py index d1ea6de5df..626121c5c4 100644 --- a/ironic/drivers/modules/drac/raid.py +++ b/ironic/drivers/modules/drac/raid.py @@ -705,6 +705,11 @@ class DracRAID(base.RAIDInterface): """ node = task.node + # The node is rebooting at this point, so wait for the iDRAC to + # enter the ready state before proceeding with cleaning + client = drac_common.get_drac_client(node) + client.wait_until_idrac_is_ready() + logical_disks = node.target_raid_config['logical_disks'] for disk in logical_disks: if (disk['size_gb'] == 'MAX' and 'physical_disks' not in disk): @@ -756,6 +761,11 @@ class DracRAID(base.RAIDInterface): """ node = task.node + # The node is rebooting at this point, so wait for the iDRAC to + # enter the ready state before proceeding with cleaning + client = drac_common.get_drac_client(node) + client.wait_until_idrac_is_ready() + controllers = set() for disk in list_virtual_disks(node): controllers.add(disk.controller) diff --git a/ironic/tests/unit/drivers/modules/drac/test_raid.py b/ironic/tests/unit/drivers/modules/drac/test_raid.py index 28eff4b79f..ec9f57ae18 100644 --- a/ironic/tests/unit/drivers/modules/drac/test_raid.py +++ b/ironic/tests/unit/drivers/modules/drac/test_raid.py @@ -615,6 +615,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): task.driver.raid.create_configuration( task, create_root_volume=True, create_nonroot_volumes=False) + mock_client.wait_until_idrac_is_ready.assert_called_once() mock_client.create_virtual_disk.assert_called_once_with( 'RAID.Integrated.1-1', ['Disk.Bay.0:Enclosure.Internal.0-1:RAID.Integrated.1-1', @@ -647,6 +648,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): return_value = task.driver.raid.create_configuration( task, create_root_volume=False, create_nonroot_volumes=False) + mock_client.wait_until_idrac_is_ready.assert_called_once() self.assertEqual(0, mock_client.create_virtual_disk.call_count) self.assertEqual(0, mock_commit_config.call_count) @@ -688,6 +690,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): task.driver.raid.create_configuration( task, create_root_volume=True, create_nonroot_volumes=True) + mock_client.wait_until_idrac_is_ready.assert_called_once() mock_client.create_virtual_disk.assert_called_once_with( 'RAID.Integrated.1-1', ['Disk.Bay.0:Enclosure.Internal.0-1:RAID.Integrated.1-1', @@ -729,6 +732,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): task.driver.raid.create_configuration( task, create_root_volume=True, create_nonroot_volumes=True) + mock_client.wait_until_idrac_is_ready.assert_called_once() mock_client.create_virtual_disk.assert_has_calls( [mock.call( 'controller-2', @@ -800,6 +804,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): task.driver.raid.create_configuration( task, create_root_volume=True, create_nonroot_volumes=True) + mock_client.wait_until_idrac_is_ready.assert_called_once() mock_client.create_virtual_disk.assert_has_calls( [mock.call( 'RAID.Integrated.1-1', @@ -859,6 +864,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): task.driver.raid.create_configuration( task, create_root_volume=True, create_nonroot_volumes=True) + mock_client.wait_until_idrac_is_ready.assert_called_once() mock_client.create_virtual_disk.assert_has_calls( [mock.call( 'RAID.Integrated.1-1', @@ -919,6 +925,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): task.driver.raid.create_configuration( task, create_root_volume=True, create_nonroot_volumes=True) + mock_client.wait_until_idrac_is_ready.assert_called_once() mock_client.create_virtual_disk.assert_has_calls( [mock.call( 'RAID.Integrated.1-1', @@ -975,6 +982,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): exception.InvalidParameterValue, task.driver.raid.create_configuration, task) + mock_client.wait_until_idrac_is_ready.assert_called_once() @mock.patch.object(drac_common, 'get_drac_client', spec_set=True, autospec=True) @@ -1007,6 +1015,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): task.driver.raid.create_configuration( task, create_root_volume=True, create_nonroot_volumes=True) + mock_client.wait_until_idrac_is_ready.assert_called_once() mock_client.create_virtual_disk.assert_has_calls( [mock.call( 'RAID.Integrated.1-1', @@ -1061,6 +1070,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): exception.DracOperationError, task.driver.raid.create_configuration, task, create_root_volume=True, create_nonroot_volumes=True) + mock_client.wait_until_idrac_is_ready.assert_called_once() @mock.patch.object(drac_common, 'get_drac_client', spec_set=True, autospec=True) @@ -1098,6 +1108,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): task.driver.raid.create_configuration( task, create_root_volume=True, create_nonroot_volumes=True) + mock_client.wait_until_idrac_is_ready.assert_called_once() mock_client.create_virtual_disk.assert_has_calls( [mock.call( 'RAID.Integrated.1-1', @@ -1163,6 +1174,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): exception.DracOperationError, task.driver.raid.create_configuration, task, create_root_volume=True, create_nonroot_volumes=True) + mock_client.wait_until_idrac_is_ready.assert_called_once() @mock.patch.object(drac_common, 'get_drac_client', spec_set=True, autospec=True) @@ -1198,6 +1210,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): exception.DracOperationError, task.driver.raid.create_configuration, task, create_root_volume=True, create_nonroot_volumes=True) + mock_client.wait_until_idrac_is_ready.assert_called_once() @mock.patch.object(drac_common, 'get_drac_client', spec_set=True, autospec=True) @@ -1235,6 +1248,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): exception.DracOperationError, task.driver.raid.create_configuration, task, create_root_volume=True, create_nonroot_volumes=True) + mock_client.wait_until_idrac_is_ready.assert_called_once() @mock.patch.object(drac_common, 'get_drac_client', spec_set=True, autospec=True) @@ -1269,6 +1283,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): shared=False) as task: return_value = task.driver.raid.delete_configuration(task) + mock_client.wait_until_idrac_is_ready.assert_called_once() mock_client.delete_virtual_disk.assert_called_once_with( 'Disk.Virtual.0:RAID.Integrated.1-1') mock_commit_config.assert_called_once_with( @@ -1298,6 +1313,7 @@ class DracRaidInterfaceTestCase(db_base.DbTestCase): shared=False) as task: return_value = task.driver.raid.delete_configuration(task) + mock_client.wait_until_idrac_is_ready.assert_called_once() self.assertEqual(0, mock_client.delete_virtual_disk.call_count) self.assertEqual(0, mock_commit_config.call_count) diff --git a/releasenotes/notes/drac-fix-oob-cleaning-b4b717895e243c9b.yaml b/releasenotes/notes/drac-fix-oob-cleaning-b4b717895e243c9b.yaml new file mode 100644 index 0000000000..6e64ef819a --- /dev/null +++ b/releasenotes/notes/drac-fix-oob-cleaning-b4b717895e243c9b.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - Fixes issue where RAID creation/deletion done as part of out-of-band + cleaning would frequently fail when using the iDRAC driver due to + Export Configuration job running. For more information, see + https://bugs.launchpad.net/ironic/+bug/1691808. Note that this fix + requires python-dracclient version 1.2.0 or higher.