Reset client session when placement endpoint not found

If the report client is able to access keystone to get a service
catalog, but that catalog does not include a placement service (because
it hasn't been added to the catalog yet), creating resource providers
and other placement entities will fail. This is expected.

What's not expected is that creating entities will continue to fail for
quite some time, even if placement is added to the catalog. This is
because the keystone session caches the service catalog for some amount
of time.

Therefore we need to create a new client session when EndpointNotFound
happens. This has been added in this change by extracting creation of
the report client's _client to its own method that we can call from the
exception handler. The resource provider and aggregate maps are wiped at
this time, to make sure we are starting from a clean slate. While this
isn't likely to cause a problem in real life scenarios, in the manual
testing I was doing it created issues.

I've made the _client method synchronized so that in the unlikely event
that the resource tracker is trying to do its update job while some
other thing is happening, we won't waste the client. This may not be
necessary, but probably doesn't harm anything.

Change-Id: I02ac615dc26a4a0d1fd28a638f622777e41d14e4
Co-Authored-By: zhangzhenzhong <zzzhang0118@gmail.com>
Closes-Bug: #1697825

nova/scheduler/client/report.py

@@ -30,6 +30,7 @@ from nova.i18n import _LE, _LI, _LW
 from nova import objects
 from nova.objects import fields
 from nova.scheduler import utils as scheduler_utils
+from nova import utils
 
 CONF = nova.conf.CONF
 LOG = logging.getLogger(__name__)
@@ -39,6 +40,7 @@ DISK_GB = fields.ResourceClass.DISK_GB
 _RE_INV_IN_USE = re.compile("Inventory for (.+) on resource provider "
                             "(.+) in use")
 WARN_EVERY = 10
+PLACEMENT_CLIENT_SEMAPHORE = 'placement_client'
 
 
 def warn_limit(self, msg):
@@ -60,6 +62,9 @@ def safe_connect(f):
                 _LW('The placement API endpoint not found. Placement is '
                     'optional in Newton, but required in Ocata. Please '
                     'enable the placement service before upgrading.'))
+            # Reset client session so there is a new catalog, which
+            # gets cached when keystone is first successfully contacted.
+            self._client = self._create_client()
         except ks_exc.MissingAuthPlugin:
             warn_limit(
                 self,
@@ -234,21 +239,26 @@ class SchedulerReportClient(object):
         # A dict, keyed by resource provider UUID, of sets of aggregate UUIDs
         # the provider is associated with
         self._provider_aggregate_map = {}
-        auth_plugin = keystone.load_auth_from_conf_options(
-            CONF, 'placement')
-        # Set content-type and accept on every request to ensure we notify
-        # placement service of our request and response body media type
-        # preferences.
-        self._client = keystone.load_session_from_conf_options(
-            CONF, 'placement', auth=auth_plugin,
-            additional_headers={'accept': 'application/json'})
-
+        self._client = self._create_client()
         # NOTE(danms): Keep track of how naggy we've been
         self._warn_count = 0
         self.ks_filter = {'service_type': 'placement',
                           'region_name': CONF.placement.os_region_name,
                           'interface': CONF.placement.os_interface}
 
+    @utils.synchronized(PLACEMENT_CLIENT_SEMAPHORE)
+    def _create_client(self):
+        """Create the HTTP session accessing the placement service."""
+        # Flush _resource_providers and aggregates so we start from a
+        # clean slate.
+        self._resource_providers = {}
+        self._provider_aggregate_map = {}
+        auth_plugin = keystone.load_auth_from_conf_options(
+            CONF, 'placement')
+        return keystone.load_session_from_conf_options(
+            CONF, 'placement', auth=auth_plugin,
+            additional_headers={'accept': 'application/json'})
+
     def get(self, url, version=None):
         kwargs = {}
         if version is not None:

nova/tests/unit/scheduler/client/test_report.py

@@ -57,6 +57,21 @@ class SafeConnectedTestCase(test.NoDBTestCase):
         self.client._get_resource_provider("fake")
         self.assertTrue(req.called)
 
+    @mock.patch('nova.scheduler.client.report.SchedulerReportClient.'
+                '_create_client')
+    @mock.patch('keystoneauth1.session.Session.request')
+    def test_missing_endpoint_create_client(self, req, create_client):
+        """Test EndpointNotFound retry behavior.
+
+        A missing endpoint should cause _create_client to be called.
+        """
+        req.side_effect = ks_exc.EndpointNotFound()
+        self.client._get_resource_provider("fake")
+
+        # This is the second time _create_client is called, but the first since
+        # the mock was created.
+        self.assertTrue(create_client.called)
+
     @mock.patch('keystoneauth1.session.Session.request')
     def test_missing_auth(self, req):
         """Test Missing Auth handled correctly.