From 2051ffb822dd89112cf41681659a2d434bb8534d Mon Sep 17 00:00:00 2001 From: armando-migliaccio Date: Mon, 7 Sep 2015 04:53:50 -0700 Subject: [PATCH] Retry metadata request on connection refused error This testcase may fail intermittently on 'Connection refused' error. This could be due to the fact that the metadata proxy setup is not exactly complete at the time the request is issued; in fact there is no synchronization between the router being up and the metadata request being issued, and clearly this may be the reason of accidental but seldom failures. In order to rule out this possibility and stabilize the test, let's retry on connection refused only. If we continue to fail, then the next step would be to dump the content of iptables to figure out why the error occurs. Closes-bug: #1461172 Conflicts: neutron/tests/functional/agent/test_l3_agent.py Change-Id: I65a5bf4fbbcad6ba93a46d36cabe7844ff528d8d (cherry picked from commit 4b2e6842f320405cd963f560bc06849b4b7bb1eb) --- .../tests/functional/agent/test_l3_agent.py | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/neutron/tests/functional/agent/test_l3_agent.py b/neutron/tests/functional/agent/test_l3_agent.py index 70db50e263a..f4b67dd5237 100755 --- a/neutron/tests/functional/agent/test_l3_agent.py +++ b/neutron/tests/functional/agent/test_l3_agent.py @@ -16,6 +16,7 @@ import copy import functools import os.path +import time import mock import netaddr @@ -53,6 +54,7 @@ LOG = logging.getLogger(__name__) _uuid = uuidutils.generate_uuid METADATA_REQUEST_TIMEOUT = 60 +METADATA_REQUEST_SLEEP = 5 def get_ovs_bridge(br_name): @@ -887,6 +889,28 @@ class MetadataL3AgentTestCase(L3AgentTestFramework): self.agent.conf.metadata_proxy_socket, workers=0, backlog=4096, mode=self.SOCKET_MODE) + def _query_metadata_proxy(self, client_ns): + url = 'http://%(host)s:%(port)s' % {'host': dhcp.METADATA_DEFAULT_IP, + 'port': dhcp.METADATA_PORT} + cmd = 'curl', '--max-time', METADATA_REQUEST_TIMEOUT, '-D-', url + i = 0 + CONNECTION_REFUSED_TIMEOUT = METADATA_REQUEST_TIMEOUT // 2 + while i <= CONNECTION_REFUSED_TIMEOUT: + try: + raw_headers = client_ns.netns.execute(cmd) + break + except RuntimeError as e: + if 'Connection refused' in str(e): + time.sleep(METADATA_REQUEST_SLEEP) + i += METADATA_REQUEST_SLEEP + else: + self.fail('metadata proxy unreachable ' + 'on %s before timeout' % url) + + if i > CONNECTION_REFUSED_TIMEOUT: + self.fail('Timed out waiting metadata proxy to become available') + return raw_headers.splitlines()[0] + def test_access_to_metadata_proxy(self): """Test access to the l3-agent metadata proxy. @@ -921,16 +945,9 @@ class MetadataL3AgentTestCase(L3AgentTestFramework): router_ip_cidr.partition('/')[0]) # Query metadata proxy - url = 'http://%(host)s:%(port)s' % {'host': dhcp.METADATA_DEFAULT_IP, - 'port': dhcp.METADATA_PORT} - cmd = 'curl', '--max-time', METADATA_REQUEST_TIMEOUT, '-D-', url - try: - raw_headers = client_ns.netns.execute(cmd) - except RuntimeError: - self.fail('metadata proxy unreachable on %s before timeout' % url) + firstline = self._query_metadata_proxy(client_ns) # Check status code - firstline = raw_headers.splitlines()[0] self.assertIn(str(webob.exc.HTTPOk.code), firstline.split())