Wait 2 seconds between gratuitous ARP updates instead of 1 second

An unfortunate scenario in Linux kernel may end up with no gratuitous
ARP update being processed by network peers, resulting in connectivity
recovery slowdown when moving an IP address between devices.

Change-Id: Iefd0d01d12d06ce6398c4c5634c634991a78bbe9
Closes-Bug: #1690165
(cherry picked from commit 96c5dd6a2b)

neutron/agent/linux/ip_lib.py

@@ -15,6 +15,7 @@
 
 import os
 import re
+import time
 
 import eventlet
 import netaddr
@@ -1035,25 +1036,41 @@ def iproute_arg_supported(command, arg):
 
 
 def _arping(ns_name, iface_name, address, count, log_exception):
-    # Pass -w to set timeout to ensure exit if interface removed while running
-    arping_cmd = ['arping', '-A', '-I', iface_name, '-c', count,
-                  '-w', 1.5 * count, address]
-    try:
-        ip_wrapper = IPWrapper(namespace=ns_name)
-        # Since arping is used to send gratuitous ARP, a response is not
-        # expected. In some cases (no response) and with some platforms
-        # (>=Ubuntu 14.04), arping exit code can be 1.
-        ip_wrapper.netns.execute(arping_cmd, extra_ok_codes=[1])
-    except Exception as exc:
-        msg = _("Failed sending gratuitous ARP "
-                "to %(addr)s on %(iface)s in namespace %(ns)s: %(err)s")
-        logger_method = LOG.exception
-        if not log_exception:
-            logger_method = LOG.warning
-        logger_method(msg, {'addr': address,
-                            'iface': iface_name,
-                            'ns': ns_name,
-                            'err': exc})
+    # Due to a Linux kernel bug*, it's advised to spread gratuitous updates
+    # more, injecting an interval between consequent packets that is longer
+    # than 1s which is currently hardcoded** in arping. To achieve that, we
+    # call arping tool the 'count' number of times, each issuing a single ARP
+    # update, and wait between iterations.
+    #
+    # *  https://patchwork.ozlabs.org/patch/760372/
+    # ** https://github.com/iputils/iputils/pull/86
+    first = True
+    for i in range(count):
+        if not first:
+            # hopefully enough for kernel to get out of locktime loop
+            time.sleep(2)
+        first = False
+
+        # Pass -w to set timeout to ensure exit if interface removed while
+        # running
+        arping_cmd = ['arping', '-A', '-I', iface_name, '-c', 1,
+                      '-w', 1.5, address]
+        try:
+            ip_wrapper = IPWrapper(namespace=ns_name)
+            # Since arping is used to send gratuitous ARP, a response is not
+            # expected. In some cases (no response) and with some platforms
+            # (>=Ubuntu 14.04), arping exit code can be 1.
+            ip_wrapper.netns.execute(arping_cmd, extra_ok_codes=[1])
+        except Exception as exc:
+            msg = _("Failed sending gratuitous ARP "
+                    "to %(addr)s on %(iface)s in namespace %(ns)s: %(err)s")
+            logger_method = LOG.exception
+            if not log_exception:
+                logger_method = LOG.warning
+            logger_method(msg, {'addr': address,
+                                'iface': iface_name,
+                                'ns': ns_name,
+                                'err': exc})
 
 
 def send_ip_addr_adv_notif(

neutron/tests/unit/agent/linux/test_ip_lib.py

@@ -1399,14 +1399,17 @@ class TestArpPing(TestIPCmdBase):
                                       config)
 
         self.assertTrue(spawn_n.called)
-        mIPWrapper.assert_called_once_with(namespace=mock.sentinel.ns_name)
+        mIPWrapper.assert_has_calls([
+            mock.call(namespace=mock.sentinel.ns_name),
+            mock.call().netns.execute(mock.ANY, extra_ok_codes=mock.ANY)
+        ] * ARPING_COUNT)
 
         ip_wrapper = mIPWrapper(namespace=mock.sentinel.ns_name)
 
         # Just test that arping is called with the right arguments
         arping_cmd = ['arping', '-A',
                       '-I', mock.sentinel.iface_name,
-                      '-c', ARPING_COUNT,
+                      '-c', 1,
                       '-w', mock.ANY,
                       address]
         ip_wrapper.netns.execute.assert_any_call(arping_cmd,