diff --git a/keystonemiddleware/auth_token/__init__.py b/keystonemiddleware/auth_token/__init__.py
index 4ac73886..003f14e6 100644
--- a/keystonemiddleware/auth_token/__init__.py
+++ b/keystonemiddleware/auth_token/__init__.py
@@ -768,6 +768,11 @@ class AuthProtocol(BaseAuthProtocol):
                 ksm_exceptions.RevocationListError,
                 ksm_exceptions.ServiceError) as e:
             self.log.critical('Unable to validate token: %s', e)
+            if self._delay_auth_decision:
+                self.log.debug('Keystone unavailable; marking token as '
+                               'invalid and deferring auth decision.')
+                raise ksm_exceptions.InvalidToken(
+                    'Keystone unavailable: %s' % e)
             raise webob.exc.HTTPServiceUnavailable(
                 'The Keystone service is temporarily unavailable.')
         except ksm_exceptions.InvalidToken:
diff --git a/keystonemiddleware/tests/unit/auth_token/test_auth_token_middleware.py b/keystonemiddleware/tests/unit/auth_token/test_auth_token_middleware.py
index f9916ccd..9051d718 100644
--- a/keystonemiddleware/tests/unit/auth_token/test_auth_token_middleware.py
+++ b/keystonemiddleware/tests/unit/auth_token/test_auth_token_middleware.py
@@ -2043,6 +2043,18 @@ class v3AuthTokenMiddlewareTest(BaseAuthTokenMiddlewareTest,
 
 class DelayedAuthTests(BaseAuthTokenMiddlewareTest):
 
+    def token_response(self, request, context):
+        auth_id = request.headers.get('X-Auth-Token')
+        self.assertEqual(auth_id, FAKE_ADMIN_TOKEN_ID)
+
+        if request.headers.get('X-Subject-Token') == ERROR_TOKEN:
+            msg = 'Network connection refused.'
+            raise ksc_exceptions.ConnectionRefused(msg)
+
+        # All others just fail
+        context.status_code = 404
+        return ''
+
     def test_header_in_401(self):
         body = uuid.uuid4().hex
         www_authenticate_uri = 'http://local.test'
@@ -2101,6 +2113,60 @@ class DelayedAuthTests(BaseAuthTokenMiddlewareTest):
         self.assertFalse(token_auth.has_service_token)
         self.assertIsNone(token_auth.service)
 
+    def test_auth_plugin_with_token(self):
+        self.requests_mock.get('%s/v3/auth/tokens' % BASE_URI,
+                               text=self.token_response,
+                               headers={'X-Subject-Token': uuid.uuid4().hex})
+
+        body = uuid.uuid4().hex
+        www_authenticate_uri = 'http://local.test'
+        conf = {
+            'delay_auth_decision': 'True',
+            'www_authenticate_uri': www_authenticate_uri,
+            'auth_type': 'admin_token',
+            'endpoint': '%s/v3' % BASE_URI,
+            'token': FAKE_ADMIN_TOKEN_ID,
+        }
+
+        middleware = self.create_simple_middleware(body=body, conf=conf)
+        resp = self.call(middleware, headers={
+            'X-Auth-Token': 'non-keystone-token'})
+        self.assertEqual(six.b(body), resp.body)
+
+        token_auth = resp.request.environ['keystone.token_auth']
+
+        self.assertFalse(token_auth.has_user_token)
+        self.assertIsNone(token_auth.user)
+        self.assertFalse(token_auth.has_service_token)
+        self.assertIsNone(token_auth.service)
+
+    def test_auth_plugin_with_token_keystone_down(self):
+        self.requests_mock.get('%s/v3/auth/tokens' % BASE_URI,
+                               text=self.token_response,
+                               headers={'X-Subject-Token': ERROR_TOKEN})
+
+        body = uuid.uuid4().hex
+        www_authenticate_uri = 'http://local.test'
+        conf = {
+            'delay_auth_decision': 'True',
+            'www_authenticate_uri': www_authenticate_uri,
+            'auth_type': 'admin_token',
+            'endpoint': '%s/v3' % BASE_URI,
+            'token': FAKE_ADMIN_TOKEN_ID,
+            'http_request_max_retries': '0'
+        }
+
+        middleware = self.create_simple_middleware(body=body, conf=conf)
+        resp = self.call(middleware, headers={'X-Auth-Token': ERROR_TOKEN})
+        self.assertEqual(six.b(body), resp.body)
+
+        token_auth = resp.request.environ['keystone.token_auth']
+
+        self.assertFalse(token_auth.has_user_token)
+        self.assertIsNone(token_auth.user)
+        self.assertFalse(token_auth.has_service_token)
+        self.assertIsNone(token_auth.service)
+
 
 class CommonCompositeAuthTests(object):
     """Test Composite authentication.
diff --git a/releasenotes/notes/delay_auth_instead_of_503-f9b46bf4fbc11455.yaml b/releasenotes/notes/delay_auth_instead_of_503-f9b46bf4fbc11455.yaml
new file mode 100644
index 00000000..11ce28aa
--- /dev/null
+++ b/releasenotes/notes/delay_auth_instead_of_503-f9b46bf4fbc11455.yaml
@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    When ``delay_auth_decision`` is enabled and a Keystone failure prevents
+    a final decision about whether a token is valid or invalid, it will be
+    marked invalid and the application will be responsible for a final auth
+    decision. This is similar to what happens when a token is confirmed *not*
+    valid. This allows a Keystone outage to only affect Keystone users in a
+    multi-auth system.