common/sender: Deal with ReadTimeout in send()

If there's a ReadTimeout when sending a HTTP request, the
plugin now retries before exiting.

Change-Id: I0218bd412419079ec9544810a6ab863f3de1a915

collectd_ceilometer/common/sender.py

@@ -21,6 +21,7 @@ import threading
 
 import requests
 import six
+import time
 
 from collectd_ceilometer.common.keystone_light import ClientV3
 from collectd_ceilometer.common.keystone_light import KeystoneException
@@ -123,8 +124,12 @@ class Sender(object):
     def _handle_http_error(self, exc, metername, payload, auth_token):
         raise exc
 
-    def send(self, metername, payload, **kwargs):
-        """Send the payload to Gnocchi/Aodh"""
+    def send(self, metername, payload, retry=0, **kwargs):
+        """Send the payload to Gnocchi/Aodh
+
+        :param retry: The number of times to attempt sending when the request
+            times out. Default is 0, which does not retry.
+        """
 
         # get the auth_token
         auth_token = self._authenticate()
@@ -174,6 +179,15 @@ class Sender(object):
                 self._handle_http_error(exc, metername, payload,
                                         auth_token, **kwargs)
 
+        except requests.exceptions.ReadTimeout as rto:
+            if retry > 0:
+                LOGGER.debug("ReadTimeout Error.. trying again...")
+                time.sleep(1)
+                self.send(metername, payload, retry=(retry - 1), **kwargs)
+            else:
+                LOGGER.error("Too many timeouts trying to send\n", rto)
+                raise rto
+
     @classmethod
     def _perform_request(cls, url, payload, auth_token, req_type="post"):
         """Perform the POST/PUT request."""

collectd_ceilometer/tests/common/test_sender.py

@@ -31,6 +31,93 @@ class TestSender(unittest.TestCase):
         super(TestSender, self).setUp()
         self.sender = common_sender.Sender()
 
+        self.sender._url_base = \
+            "http://my-gnocchi-endpoint/v1/action"
+
+    @mock.patch('time.sleep')
+    @mock.patch.object(common_sender.Sender, '_create_request_url')
+    @mock.patch.object(common_sender, 'LOGGER')
+    @mock.patch.object(common_sender.Sender, '_perform_request')
+    def test_send_readtimeout_error(self, sender_perf_req,
+                                    logger, sender_create_request_url,
+                                    time_sleep):
+        """Tests the behaviour of send when a timeout error occurs.
+
+        Set-up: There is a ReadTimeout side effect from _perform_request
+        Test: Call send
+        Expected behaviour: Send will be called a second time.
+        """
+        response_created = requests.Response()
+        response_created.status_code = 201
+
+        self.sender._auth_token = "my-auth-token"
+
+        sender_perf_req.side_effect = [requests.exceptions.ReadTimeout,
+                                       response_created]
+
+        self.sender.send(metername="my-metername",
+                         payload="my-payload",
+                         retry=1
+                         )
+
+        logger.debug.assert_called_with("ReadTimeout Error.. trying again...")
+
+    @mock.patch('time.sleep')
+    @mock.patch.object(common_sender.Sender, '_create_request_url')
+    @mock.patch.object(common_sender, 'LOGGER')
+    @mock.patch.object(common_sender.Sender, '_perform_request')
+    def test_send_readtimeout_error_retry_0(self, sender_perf_req, logger,
+                                            sender_create_request_url,
+                                            time_sleep):
+        """Tests that ReadTimeout error is raised when retry=0.
+
+        Set-up: There is a ReadTimeout side effect from _perform_request.
+        Test: Call send(.... retry=0)
+        Expected behaviour:
+          * ReadTimeout is raised from send()
+          * there are no re-send attempts (perform_request is only called once)
+        """
+        self.sender._auth_token = "my-auth-token"
+
+        sender_perf_req.side_effect = requests.exceptions.ReadTimeout
+
+        with self.assertRaises(requests.exceptions.ReadTimeout):
+            self.sender.send(metername="my-metername",
+                             payload="my-payload",
+                             retry=0
+                             )
+        logger.debug.assert_not_called()
+        sender_perf_req.called_once()
+        logger.error.assert_called_with("Too many timeouts trying to send\n",
+                                        mock.ANY)
+
+    @mock.patch('time.sleep')
+    @mock.patch.object(common_sender.Sender, '_create_request_url')
+    @mock.patch.object(common_sender, 'LOGGER')
+    @mock.patch.object(common_sender.Sender, '_perform_request')
+    def test_send_timeout_recurrent(self, sender_perf_req, logger,
+                                    sender_create_request_url,
+                                    time_sleep):
+        """Test that resending is attempted when there's a ReadTimout error.
+
+        Set-up:
+        Test: Call send() with retry=<some_number>
+        Expected behaviour:
+          * TimeoutError will be re-raised after attempting multiple times
+        """
+        self.sender._auth_token = "my-auth-token"
+        sender_perf_req.side_effect = requests.exceptions.ReadTimeout
+
+        with self.assertRaises(requests.exceptions.ReadTimeout):
+            self.sender.send(metername="my-metername",
+                             payload="my-payload",
+                             retry=5
+                             )
+
+        logger.debug.assert_called_with("ReadTimeout Error.. trying again...")
+        sender_perf_req.assert_called()
+        time_sleep.assert_has_calls([mock.call(1)] * 5)
+
     @mock.patch.object(requests, 'post')
     @mock.patch.object(requests, 'get')
     def test_perform_request_req_type_get(self, get, post):