Add connection timeout to the nechecker requests

Sometimes tests hang while performing requests to the
netchecker. To prevent that 5 seconds connection timeout
is added. Also @retry decorator is used for re-trying
connection if it fails.

Change-Id: Iea90a04fc713304fb98807488021be9971273280

fuel_ccp_tests/helpers/utils.py

@@ -155,3 +155,27 @@ def generate_keys():
 
 def clean_dir(dirpath):
     shutil.rmtree(dirpath)
+
+
+def retry(tries_number=3, exception=Exception):
+    def _retry(func):
+        assert tries_number >= 1, 'ERROR! @retry is called with no tries!'
+
+        def wrapper(*args, **kwargs):
+            iter_number = 1
+            while True:
+                try:
+                    LOG.debug('Calling function "{0}" with args "{1}" and '
+                              'kwargs "{2}". Try # {3}.'.format(func.__name__,
+                                                                args,
+                                                                kwargs,
+                                                                iter_number))
+                    return func(*args, **kwargs)
+                except exception as e:
+                    if iter_number > tries_number:
+                        LOG.debug('Failed to execute function "{0}" with {1} '
+                                  'tries!'.format(func.__name__, tries_number))
+                        raise e
+                iter_number += 1
+        return wrapper
+    return _retry

fuel_ccp_tests/tests/system/test_netchecker.py

@@ -22,6 +22,7 @@ from k8sclient.client import rest
 
 import base_test
 from fuel_ccp_tests.helpers import ext
+from fuel_ccp_tests.helpers import utils
 from fuel_ccp_tests import logger
 from fuel_ccp_tests import settings
 
@@ -94,10 +95,11 @@ class TestFuelCCPNetCheckerMixin:
                 k8s.wait_ds_ready(dsname=daemon_set_spec['metadata']['name'])
 
     @staticmethod
+    @utils.retry(3, requests.exceptions.RequestException)
     def get_netchecker_status(kube_host_ip, netchecker_pod_port=31081):
         net_status_url = 'http://{0}:{1}/api/v1/connectivity_check'.format(
             kube_host_ip, netchecker_pod_port)
-        return requests.get(net_status_url)
+        return requests.get(net_status_url, timeout=5)
 
     @staticmethod
     def wait_netchecker_running(kube_host_ip, timeout=120, interval=5):