diff --git a/collectd_ceilometer/common/sender.py b/collectd_ceilometer/common/sender.py index f3112d9..6b8a399 100644 --- a/collectd_ceilometer/common/sender.py +++ b/collectd_ceilometer/common/sender.py @@ -21,6 +21,7 @@ import threading import requests import six +import time from collectd_ceilometer.common.keystone_light import ClientV3 from collectd_ceilometer.common.keystone_light import KeystoneException @@ -123,8 +124,12 @@ class Sender(object): def _handle_http_error(self, exc, metername, payload, auth_token): raise exc - def send(self, metername, payload, **kwargs): - """Send the payload to Gnocchi/Aodh""" + def send(self, metername, payload, retry=0, **kwargs): + """Send the payload to Gnocchi/Aodh + + :param retry: The number of times to attempt sending when the request + times out. Default is 0, which does not retry. + """ # get the auth_token auth_token = self._authenticate() @@ -174,6 +179,15 @@ class Sender(object): self._handle_http_error(exc, metername, payload, auth_token, **kwargs) + except requests.exceptions.ReadTimeout as rto: + if retry > 0: + LOGGER.debug("ReadTimeout Error.. trying again...") + time.sleep(1) + self.send(metername, payload, retry=(retry - 1), **kwargs) + else: + LOGGER.error("Too many timeouts trying to send\n", rto) + raise rto + @classmethod def _perform_request(cls, url, payload, auth_token, req_type="post"): """Perform the POST/PUT request.""" diff --git a/collectd_ceilometer/tests/common/test_sender.py b/collectd_ceilometer/tests/common/test_sender.py index a2a6001..5f9ec58 100644 --- a/collectd_ceilometer/tests/common/test_sender.py +++ b/collectd_ceilometer/tests/common/test_sender.py @@ -31,6 +31,93 @@ class TestSender(unittest.TestCase): super(TestSender, self).setUp() self.sender = common_sender.Sender() + self.sender._url_base = \ + "http://my-gnocchi-endpoint/v1/action" + + @mock.patch('time.sleep') + @mock.patch.object(common_sender.Sender, '_create_request_url') + @mock.patch.object(common_sender, 'LOGGER') + @mock.patch.object(common_sender.Sender, '_perform_request') + def test_send_readtimeout_error(self, sender_perf_req, + logger, sender_create_request_url, + time_sleep): + """Tests the behaviour of send when a timeout error occurs. + + Set-up: There is a ReadTimeout side effect from _perform_request + Test: Call send + Expected behaviour: Send will be called a second time. + """ + response_created = requests.Response() + response_created.status_code = 201 + + self.sender._auth_token = "my-auth-token" + + sender_perf_req.side_effect = [requests.exceptions.ReadTimeout, + response_created] + + self.sender.send(metername="my-metername", + payload="my-payload", + retry=1 + ) + + logger.debug.assert_called_with("ReadTimeout Error.. trying again...") + + @mock.patch('time.sleep') + @mock.patch.object(common_sender.Sender, '_create_request_url') + @mock.patch.object(common_sender, 'LOGGER') + @mock.patch.object(common_sender.Sender, '_perform_request') + def test_send_readtimeout_error_retry_0(self, sender_perf_req, logger, + sender_create_request_url, + time_sleep): + """Tests that ReadTimeout error is raised when retry=0. + + Set-up: There is a ReadTimeout side effect from _perform_request. + Test: Call send(.... retry=0) + Expected behaviour: + * ReadTimeout is raised from send() + * there are no re-send attempts (perform_request is only called once) + """ + self.sender._auth_token = "my-auth-token" + + sender_perf_req.side_effect = requests.exceptions.ReadTimeout + + with self.assertRaises(requests.exceptions.ReadTimeout): + self.sender.send(metername="my-metername", + payload="my-payload", + retry=0 + ) + logger.debug.assert_not_called() + sender_perf_req.called_once() + logger.error.assert_called_with("Too many timeouts trying to send\n", + mock.ANY) + + @mock.patch('time.sleep') + @mock.patch.object(common_sender.Sender, '_create_request_url') + @mock.patch.object(common_sender, 'LOGGER') + @mock.patch.object(common_sender.Sender, '_perform_request') + def test_send_timeout_recurrent(self, sender_perf_req, logger, + sender_create_request_url, + time_sleep): + """Test that resending is attempted when there's a ReadTimout error. + + Set-up: + Test: Call send() with retry= + Expected behaviour: + * TimeoutError will be re-raised after attempting multiple times + """ + self.sender._auth_token = "my-auth-token" + sender_perf_req.side_effect = requests.exceptions.ReadTimeout + + with self.assertRaises(requests.exceptions.ReadTimeout): + self.sender.send(metername="my-metername", + payload="my-payload", + retry=5 + ) + + logger.debug.assert_called_with("ReadTimeout Error.. trying again...") + sender_perf_req.assert_called() + time_sleep.assert_has_calls([mock.call(1)] * 5) + @mock.patch.object(requests, 'post') @mock.patch.object(requests, 'get') def test_perform_request_req_type_get(self, get, post):