Merge "Make multiple attempts to obtain gnocchiclient" into stable/zed

ceilometer/declarative.py

@@ -195,5 +195,5 @@ def load_definitions(conf, defaults, config_file, fallback_file=None):
                   "Using default config.")
         definition_cfg = defaults
 
-    LOG.info("Definitions: %s", definition_cfg)
+    LOG.debug("Definitions: %s", definition_cfg)
     return definition_cfg

ceilometer/publisher/gnocchi.py

@@ -18,6 +18,7 @@ import itertools
 import json
 import operator
 import pkg_resources
+import tenacity
 import threading
 
 from gnocchiclient import exceptions as gnocchi_exc
@@ -213,13 +214,28 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
         self._gnocchi_project_id_lock = threading.Lock()
         self._gnocchi_resource_lock = LockedDefaultDict(threading.Lock)
 
-        self._gnocchi = gnocchi_client.get_gnocchiclient(
-            conf, request_timeout=timeout)
+        try:
+            self._gnocchi = self._get_gnocchi_client(conf, timeout)
+        except tenacity.RetryError as e:
+            raise e.last_attempt._exception from None
+
         self._already_logged_event_types = set()
         self._already_logged_metric_names = set()
 
         self._already_configured_archive_policies = False
 
+    @tenacity.retry(
+        stop=tenacity.stop_after_attempt(10),
+        wait=tenacity.wait_fixed(5),
+        retry=(
+            tenacity.retry_if_exception_type(ka_exceptions.ServiceUnavailable)
+            | tenacity.retry_if_exception_type(ka_exceptions.DiscoveryFailure)
+            | tenacity.retry_if_exception_type(ka_exceptions.ConnectTimeout)
+        ),
+        reraise=False)
+    def _get_gnocchi_client(self, conf, timeout):
+        return gnocchi_client.get_gnocchiclient(conf, request_timeout=timeout)
+
     @staticmethod
     def _load_definitions(conf, archive_policy_override,
                           resources_definition_file):

ceilometer/tests/unit/publisher/test_gnocchi.py

@@ -343,6 +343,15 @@ class PublisherTest(base.BaseTestCase):
             'Filtered project [service] not found in keystone, ignoring the '
             'filter_project option')
 
+    @mock.patch('ceilometer.publisher.gnocchi.GnocchiPublisher'
+                '._get_gnocchi_client')
+    def test_get_gnocchi_client(self, gnocchi_cli):
+        url = netutils.urlsplit("gnocchi://")
+        gnocchi_cli.side_effect = ka_exceptions.DiscoveryFailure
+        cfg = self.conf.conf
+        publisher = gnocchi.GnocchiPublisher
+        self.assertRaises(ka_exceptions.DiscoveryFailure, publisher, cfg, url)
+
     def test_activity_filter_match_swift_event(self):
         self.samples[0].name = 'storage.objects.outgoing.bytes'
         self.samples[0].resource_id = 'a2d42c23-d518-46b6-96ab-3fba2e146859'