From 24f976cd13d884e2c8604a5ace7ed4ed05334107 Mon Sep 17 00:00:00 2001 From: Xing Yang Date: Mon, 6 Jan 2014 17:16:05 -0500 Subject: [PATCH] Fixed a problem in iSCSI multipath Multipathing during copy image to volume and copy volume to image operations doesn't work properly if there are different targets associated with different portals for a mulitpath device. Change-Id: I65c93f3788020c944db0d3a55063a6415554ff11 Closes-Bug: #1266048 (cherry picked from commit 3e4f554f614c8cb6d5f014c72f3c635184a4dec2) --- cinder/brick/initiator/connector.py | 57 +++++++++++++++++----- cinder/tests/brick/test_brick_connector.py | 43 +++++++++++----- 2 files changed, 74 insertions(+), 26 deletions(-) diff --git a/cinder/brick/initiator/connector.py b/cinder/brick/initiator/connector.py index 0f03fababd2..ef8bea74fb3 100644 --- a/cinder/brick/initiator/connector.py +++ b/cinder/brick/initiator/connector.py @@ -202,9 +202,10 @@ class ISCSIConnector(InitiatorConnector): check_exit_code=[0, 255])[0] \ or "" - for ip in self._get_target_portals_from_iscsiadm_output(out): + for ip, iqn in self._get_target_portals_from_iscsiadm_output(out): props = connection_properties.copy() props['target_portal'] = ip + props['target_iqn'] = iqn self._connect_to_iscsi_portal(props) self._rescan_iscsi() @@ -256,12 +257,20 @@ class ISCSIConnector(InitiatorConnector): target_iqn - iSCSI Qualified Name target_lun - LUN id of the volume """ + # Moved _rescan_iscsi and _rescan_multipath + # from _disconnect_volume_multipath_iscsi to here. + # Otherwise, if we do rescan after _linuxscsi.remove_multipath_device + # but before logging out, the removed devices under /dev/disk/by-path + # will reappear after rescan. + self._rescan_iscsi() host_device = self._get_device_path(connection_properties) multipath_device = None if self.use_multipath: + self._rescan_multipath() multipath_device = self._get_multipath_device_name(host_device) if multipath_device: - self._linuxscsi.remove_multipath_device(multipath_device) + device_realpath = os.path.realpath(host_device) + self._linuxscsi.remove_multipath_device(device_realpath) return self._disconnect_volume_multipath_iscsi( connection_properties, multipath_device) @@ -326,14 +335,13 @@ class ISCSIConnector(InitiatorConnector): **kwargs) def _get_target_portals_from_iscsiadm_output(self, output): - return [line.split()[0] for line in output.splitlines()] + # return both portals and iqns + return [line.split() for line in output.splitlines()] def _disconnect_volume_multipath_iscsi(self, connection_properties, multipath_name): """This removes a multipath device and it's LUNs.""" LOG.debug("Disconnect multipath device %s" % multipath_name) - self._rescan_iscsi() - self._rescan_multipath() block_devices = self.driver.get_all_block_devices() devices = [] for dev in block_devices: @@ -344,17 +352,42 @@ class ISCSIConnector(InitiatorConnector): if mpdev: devices.append(mpdev) + # Do a discovery to find all targets. + # Targets for multiple paths for the same multipath device + # may not be the same. + out = self._run_iscsiadm_bare(['-m', + 'discovery', + '-t', + 'sendtargets', + '-p', + connection_properties['target_portal']], + check_exit_code=[0, 255])[0] \ + or "" + + ips_iqns = self._get_target_portals_from_iscsiadm_output(out) + if not devices: # disconnect if no other multipath devices - self._disconnect_mpath(connection_properties) + self._disconnect_mpath(connection_properties, ips_iqns) return + # Get a target for all other multipath devices other_iqns = [self._get_multipath_iqn(device) for device in devices] + # Get all the targets for the current multipath device + current_iqns = [iqn for ip, iqn in ips_iqns] - if connection_properties['target_iqn'] not in other_iqns: + in_use = False + for current in current_iqns: + if current in other_iqns: + in_use = True + break + + # If no other multipath device attached has the same iqn + # as the current device + if not in_use: # disconnect if no other multipath devices with same iqn - self._disconnect_mpath(connection_properties) + self._disconnect_mpath(connection_properties, ips_iqns) return # else do not disconnect iscsi portals, @@ -449,13 +482,11 @@ class ISCSIConnector(InitiatorConnector): return [] return [entry for entry in devices if entry.startswith("ip-")] - def _disconnect_mpath(self, connection_properties): - entries = self._get_iscsi_devices() - ips = [ip.split("-")[1] for ip in entries - if connection_properties['target_iqn'] in ip] - for ip in ips: + def _disconnect_mpath(self, connection_properties, ips_iqns): + for ip, iqn in ips_iqns: props = connection_properties.copy() props['target_portal'] = ip + props['target_iqn'] = iqn self._disconnect_from_iscsi_portal(props) self._rescan_multipath() diff --git a/cinder/tests/brick/test_brick_connector.py b/cinder/tests/brick/test_brick_connector.py index 1088c798e80..1d5d06f4496 100644 --- a/cinder/tests/brick/test_brick_connector.py +++ b/cinder/tests/brick/test_brick_connector.py @@ -186,6 +186,8 @@ class ISCSIConnectorTestCase(ConnectorTestCase): ('iscsiadm -m node -T %s -p %s --op update' ' -n node.startup -v automatic' % (iqn, location)), + ('iscsiadm -m node --rescan'), + ('iscsiadm -m session --rescan'), ('tee -a /sys/block/sdb/device/delete'), ('iscsiadm -m node -T %s -p %s --op update' ' -n node.startup -v manual' % (iqn, location)), @@ -199,7 +201,6 @@ class ISCSIConnectorTestCase(ConnectorTestCase): self.assertEqual(expected_commands, self.cmds) def test_connect_volume_with_multipath(self): - location = '10.0.2.15:3260' name = 'volume-00000001' iqn = 'iqn.2010-10.org.openstack:%s' % name @@ -213,7 +214,7 @@ class ISCSIConnectorTestCase(ConnectorTestCase): lambda *args, **kwargs: "%s %s" % (location, iqn)) self.stubs.Set(self.connector_with_multipath, '_get_target_portals_from_iscsiadm_output', - lambda x: [location]) + lambda x: [[location, iqn]]) self.stubs.Set(self.connector_with_multipath, '_connect_to_iscsi_portal', lambda x: None) @@ -250,7 +251,9 @@ class ISCSIConnectorTestCase(ConnectorTestCase): test_output = '''10.15.84.19:3260 iqn.1992-08.com.netapp:sn.33615311 10.15.85.19:3260 iqn.1992-08.com.netapp:sn.33615311''' res = connector._get_target_portals_from_iscsiadm_output(test_output) - expected = ['10.15.84.19:3260', '10.15.85.19:3260'] + ip_iqn1 = ['10.15.84.19:3260', 'iqn.1992-08.com.netapp:sn.33615311'] + ip_iqn2 = ['10.15.85.19:3260', 'iqn.1992-08.com.netapp:sn.33615311'] + expected = [ip_iqn1, ip_iqn2] self.assertEqual(expected, res) def test_get_multipath_device_name(self): @@ -288,12 +291,16 @@ class ISCSIConnectorTestCase(ConnectorTestCase): def test_disconnect_volume_multipath_iscsi(self): result = [] - def fake_disconnect_mpath(properties): + def fake_disconnect_from_iscsi_portal(properties): result.append(properties) iqn1 = 'iqn.2013-01.ro.com.netapp:node.netapp01' iqn2 = 'iqn.2013-01.ro.com.netapp:node.netapp02' iqns = [iqn1, iqn2] - dev = ('ip-10.0.0.1:3260-iscsi-%s-lun-0' % iqn1) + portal = '10.0.0.1:3260' + dev = ('ip-%s-iscsi-%s-lun-0' % (portal, iqn1)) + self.stubs.Set(self.connector, + '_get_target_portals_from_iscsiadm_output', + lambda x: [[portal, iqn1]]) self.stubs.Set(self.connector, '_rescan_iscsi', lambda: None) self.stubs.Set(self.connector, '_rescan_multipath', lambda: None) self.stubs.Set(self.connector.driver, 'get_all_block_devices', @@ -302,27 +309,37 @@ class ISCSIConnectorTestCase(ConnectorTestCase): lambda x: '/dev/mapper/md-3') self.stubs.Set(self.connector, '_get_multipath_iqn', lambda x: iqns.pop()) - self.stubs.Set(self.connector, '_disconnect_mpath', - fake_disconnect_mpath) - fake_property = {'target_iqn': "You'll-never-find-this-iqn"} + self.stubs.Set(self.connector, '_disconnect_from_iscsi_portal', + fake_disconnect_from_iscsi_portal) + fake_property = {'target_portal': portal, + 'target_iqn': iqn1} self.connector._disconnect_volume_multipath_iscsi(fake_property, 'fake/multipath') - self.assertEqual([fake_property], result) + # Target in use by other mp devices, don't disconnect + self.assertEqual([], result) def test_disconnect_volume_multipath_iscsi_without_other_mp_devices(self): result = [] - def fake_disconnect_mpath(properties): + def fake_disconnect_from_iscsi_portal(properties): result.append(properties) + portal = '10.0.2.15:3260' + name = 'volume-00000001' + iqn = 'iqn.2010-10.org.openstack:%s' % name + self.stubs.Set(self.connector, + '_get_target_portals_from_iscsiadm_output', + lambda x: [[portal, iqn]]) self.stubs.Set(self.connector, '_rescan_iscsi', lambda: None) self.stubs.Set(self.connector, '_rescan_multipath', lambda: None) self.stubs.Set(self.connector.driver, 'get_all_block_devices', lambda: []) - self.stubs.Set(self.connector, '_disconnect_mpath', - fake_disconnect_mpath) - fake_property = {'target_iqn': "You'll-never-find-this-iqn"} + self.stubs.Set(self.connector, '_disconnect_from_iscsi_portal', + fake_disconnect_from_iscsi_portal) + fake_property = {'target_portal': portal, + 'target_iqn': iqn} self.connector._disconnect_volume_multipath_iscsi(fake_property, 'fake/multipath') + # Target not in use by other mp devices, disconnect self.assertEqual([fake_property], result)