Retry metadata request on connection refused error

This testcase may fail intermittently on 'Connection refused' error.
This could be due to the fact that the metadata proxy setup is not exactly
complete at the time the request is issued; in fact there is no
synchronization between the router being up and the metadata request being
issued, and clearly this may be the reason of accidental but seldom failures.

In order to rule out this possibility and stabilize the test, let's retry
on connection refused only. If we continue to fail, then the next step would
be to dump the content of iptables to figure out why the error occurs.

Closes-bug: #1461172

Conflicts:
	neutron/tests/functional/agent/test_l3_agent.py

Change-Id: I65a5bf4fbbcad6ba93a46d36cabe7844ff528d8d
(cherry picked from commit 4b2e6842f3)
This commit is contained in:
armando-migliaccio 2015-09-07 04:53:50 -07:00 committed by Ihar Hrachyshka
parent d9398261ce
commit 2051ffb822
1 changed files with 25 additions and 8 deletions

View File

@ -16,6 +16,7 @@
import copy
import functools
import os.path
import time
import mock
import netaddr
@ -53,6 +54,7 @@ LOG = logging.getLogger(__name__)
_uuid = uuidutils.generate_uuid
METADATA_REQUEST_TIMEOUT = 60
METADATA_REQUEST_SLEEP = 5
def get_ovs_bridge(br_name):
@ -887,6 +889,28 @@ class MetadataL3AgentTestCase(L3AgentTestFramework):
self.agent.conf.metadata_proxy_socket,
workers=0, backlog=4096, mode=self.SOCKET_MODE)
def _query_metadata_proxy(self, client_ns):
url = 'http://%(host)s:%(port)s' % {'host': dhcp.METADATA_DEFAULT_IP,
'port': dhcp.METADATA_PORT}
cmd = 'curl', '--max-time', METADATA_REQUEST_TIMEOUT, '-D-', url
i = 0
CONNECTION_REFUSED_TIMEOUT = METADATA_REQUEST_TIMEOUT // 2
while i <= CONNECTION_REFUSED_TIMEOUT:
try:
raw_headers = client_ns.netns.execute(cmd)
break
except RuntimeError as e:
if 'Connection refused' in str(e):
time.sleep(METADATA_REQUEST_SLEEP)
i += METADATA_REQUEST_SLEEP
else:
self.fail('metadata proxy unreachable '
'on %s before timeout' % url)
if i > CONNECTION_REFUSED_TIMEOUT:
self.fail('Timed out waiting metadata proxy to become available')
return raw_headers.splitlines()[0]
def test_access_to_metadata_proxy(self):
"""Test access to the l3-agent metadata proxy.
@ -921,16 +945,9 @@ class MetadataL3AgentTestCase(L3AgentTestFramework):
router_ip_cidr.partition('/')[0])
# Query metadata proxy
url = 'http://%(host)s:%(port)s' % {'host': dhcp.METADATA_DEFAULT_IP,
'port': dhcp.METADATA_PORT}
cmd = 'curl', '--max-time', METADATA_REQUEST_TIMEOUT, '-D-', url
try:
raw_headers = client_ns.netns.execute(cmd)
except RuntimeError:
self.fail('metadata proxy unreachable on %s before timeout' % url)
firstline = self._query_metadata_proxy(client_ns)
# Check status code
firstline = raw_headers.splitlines()[0]
self.assertIn(str(webob.exc.HTTPOk.code), firstline.split())