libvirt: check for interface when detach_interface fails

When using Neutron and deleting an instance, we race against
deleting the domain and Neutron sending a vif-deleted event
which triggers a call to detach_interface. If the network
device is not found when we go to detach it from the config,
libvirt raises an error like:

libvirtError: operation failed: no matching network device was found

Unfortunately libvirt does not have a unique error code for this
and the error message is translatable, so we can't key off of it
to check if the failure is just due to the device not being found.

This change adds a method to the guest object to lookup the interface
device config by MAC address and if not found, we simply log a warning
rather than tracing an error for a case that we can expect when using
Neutron.

Closes-Bug: #1536671

Change-Id: I8ae352ff3eeb760c97d1a6fa9d7a59e881d7aea1

nova/tests/unit/virt/libvirt/test_driver.py

@@ -14467,6 +14467,30 @@ class LibvirtDriverTestCase(test.NoDBTestCase):
             'detach_interface', power_state.SHUTDOWN,
             expected_flags=(fakelibvirt.VIR_DOMAIN_AFFECT_CONFIG))
 
+    @mock.patch('nova.virt.libvirt.driver.LOG')
+    def test_detach_interface_device_not_found(self, mock_log):
+        # Asserts that we don't log an error when the interface device is not
+        # found on the guest after a libvirt error during detach.
+        instance = self._create_instance()
+        vif = _fake_network_info(self, 1)[0]
+        guest = mock.Mock(spec='nova.virt.libvirt.guest.Guest')
+        guest.get_power_state = mock.Mock()
+        self.drvr._host.get_guest = mock.Mock(return_value=guest)
+        self.drvr.vif_driver = mock.Mock()
+        error = fakelibvirt.libvirtError(
+            'no matching network device was found')
+        error.err = (fakelibvirt.VIR_ERR_OPERATION_FAILED,)
+        guest.detach_device = mock.Mock(side_effect=error)
+        # mock out that get_interface_by_mac doesn't find the interface
+        guest.get_interface_by_mac = mock.Mock(return_value=None)
+        self.drvr.detach_interface(instance, vif)
+        guest.get_interface_by_mac.assert_called_once_with(vif['address'])
+        # an error shouldn't be logged, but a warning should be logged
+        self.assertFalse(mock_log.error.called)
+        self.assertEqual(1, mock_log.warning.call_count)
+        self.assertIn('the device is no longer found on the guest',
+                      six.text_type(mock_log.warning.call_args[0]))
+
     def test_rescue(self):
         instance = self._create_instance({'config_drive': None})
         dummyxml = ("<domain type='kvm'><name>instance-0000000a</name>"

nova/tests/unit/virt/libvirt/test_guest.py

@@ -411,6 +411,10 @@ class GuestTestCase(test.NoDBTestCase):
         self.assertEqual(1, len(devs))
         self.assertIsInstance(devs[0], vconfig.LibvirtConfigGuestInterface)
 
+        self.assertIsNotNone(
+            self.guest.get_interface_by_mac('fa:16:3e:f9:af:ae'))
+        self.assertIsNone(self.guest.get_interface_by_mac(None))
+
     def test_get_info(self):
         self.domain.info.return_value = (1, 2, 3, 4, 5)
         self.domain.ID.return_value = 6

nova/virt/libvirt/driver.py

@@ -1511,10 +1511,28 @@ class LibvirtDriver(driver.ComputeDriver):
                              "instance disappeared."),
                          instance=instance)
             else:
-                LOG.error(_LE('detaching network adapter failed.'),
-                         instance=instance, exc_info=True)
-                raise exception.InterfaceDetachFailed(
-                        instance_uuid=instance.uuid)
+                # NOTE(mriedem): When deleting an instance and using Neutron,
+                # we can be racing against Neutron deleting the port and
+                # sending the vif-deleted event which then triggers a call to
+                # detach the interface, so we might have failed because the
+                # network device no longer exists. Libvirt will fail with
+                # "operation failed: no matching network device was found"
+                # which unfortunately does not have a unique error code so we
+                # need to look up the interface by MAC and if it's not found
+                # then we can just log it as a warning rather than tracing an
+                # error.
+                mac = vif.get('address')
+                interface = guest.get_interface_by_mac(mac)
+                if interface:
+                    LOG.error(_LE('detaching network adapter failed.'),
+                             instance=instance, exc_info=True)
+                    raise exception.InterfaceDetachFailed(
+                            instance_uuid=instance.uuid)
+
+                # The interface is gone so just log it as a warning.
+                LOG.warning(_LW('Detaching interface %(mac)s failed  because '
+                                'the device is no longer found on the guest.'),
+                            {'mac': mac}, instance=instance)
 
     def _create_snapshot_metadata(self, image_meta, instance,
                                   img_fmt, snp_name):

nova/virt/libvirt/guest.py

@@ -186,6 +186,22 @@ class Guest(object):
 
         return interfaces
 
+    def get_interface_by_mac(self, mac):
+        """Lookup a LibvirtConfigGuestInterface by the MAC address.
+
+        :param mac: MAC address of the guest interface.
+        :type mac: str
+        :returns: nova.virt.libvirt.config.LibvirtConfigGuestInterface instance
+            if found, else None
+        """
+
+        if mac:
+            interfaces = self.get_all_devices(
+                vconfig.LibvirtConfigGuestInterface)
+            for interface in interfaces:
+                if interface.mac_addr == mac:
+                    return interface
+
     def get_vcpus_info(self):
         """Returns virtual cpus information of guest.