Allow fetching an expired token

A service user from auth_token middleware should be able to fetch a
token that has expired within a certain window so that long running
operations can finish.

Implements bp: allow-expired
Change-Id: I784f719be88481048f5aa7a79d34a54907438cf3

api-ref/source/v3/authenticate-v3.inc

@@ -382,6 +382,7 @@ Request
    - X-Auth-Token: X-Auth-Token
    - X-Subject-Token: X-Subject-Token
    - nocatalog: nocatalog
+   - allow_expired: allow_expired
 
 Response Parameters
 -------------------
@@ -441,6 +442,7 @@ Request
 
    - X-Auth-Token: X-Auth-Token
    - X-Subject-Token: X-Subject-Token
+   - allow_expired: allow_expired
 
 
 Revoke token

api-ref/source/v3/index.rst

@@ -23,6 +23,11 @@ For information about Identity API protection, see
 <http://docs.openstack.org/admin-guide/identity_service_api_protection.html>`_
 in the OpenStack Cloud Administrator Guide.
 
+What's New in Version 3.8
+=========================
+
+- Allow a service user to fetch a token that has expired.
+
 What's New in Version 3.7
 =========================
 

api-ref/source/v3/parameters.yaml

@@ -130,6 +130,13 @@ user_id_path:
   type: string
 
 # variables in query
+allow_expired:
+  description: |
+    (Since v3.8) Allow fetching a token that has expired. By default expired
+    tokens return a 404 exception.
+  in: query
+  required: false
+  type: bool
 domain_enabled_query:
   description: |
     If set to true, then only domains that are enabled will be returned, if set

doc/source/configuration.rst

@@ -691,7 +691,10 @@ still be used for validating tokens. Excess secondary keys (beyond
 deleted.
 
 Rotating keys too frequently, or with ``[fernet_tokens] max_active_keys`` set
-too low, will cause tokens to become invalid prior to their expiration.
+too low, will cause tokens to become invalid prior to their expiration. As
+tokens may be fetched beyond there initial expiration period keys should not be
+fully rotated within the period of ``[token] expiration`` + ``[token]
+allow_expired_window`` seconds to prevent the tokens becoming unavailable.
 
 Caching Layer
 =============

keystone/auth/controllers.py

@@ -540,8 +540,9 @@ class Auth(controller.V3Controller):
     @controller.protected()
     def check_token(self, request):
         token_id = request.context_dict.get('subject_token_id')
+        window_seconds = self._token_validation_window(request)
         token_data = self.token_provider_api.validate_token(
-            token_id)
+            token_id, window_seconds=window_seconds)
         # NOTE(morganfainberg): The code in
         # ``keystone.common.wsgi.render_response`` will remove the content
         # body.
@@ -555,9 +556,10 @@ class Auth(controller.V3Controller):
     @controller.protected()
     def validate_token(self, request):
         token_id = request.context_dict.get('subject_token_id')
+        window_seconds = self._token_validation_window(request)
         include_catalog = 'nocatalog' not in request.params
         token_data = self.token_provider_api.validate_token(
-            token_id)
+            token_id, window_seconds=window_seconds)
         if not include_catalog and 'catalog' in token_data['token']:
             del token_data['token']['catalog']
         return render_token_data_response(token_id, token_data)

keystone/common/controller.py

@@ -131,10 +131,12 @@ def protected(callback=None):
                 # TODO(henry-nash): Move this entire code to a member
                 # method inside v3 Auth
                 if request.context_dict.get('subject_token_id') is not None:
+                    window_seconds = self._token_validation_window(request)
                     token_ref = token_model.KeystoneToken(
                         token_id=request.context_dict['subject_token_id'],
                         token_data=self.token_provider_api.validate_token(
-                            request.context_dict['subject_token_id']))
+                            request.context_dict['subject_token_id'],
+                            window_seconds=window_seconds))
                     policy_dict.setdefault('target', {})
                     policy_dict['target'].setdefault(self.member_name, {})
                     policy_dict['target'][self.member_name]['user_id'] = (
@@ -812,3 +814,11 @@ class V3Controller(wsgi.Application):
         for blocked_param in blocked_keys:
             del ref[blocked_param]
         return ref
+
+    def _token_validation_window(self, request):
+        # NOTE(jamielennox): it's dumb that i have to put this here. We should
+        # only validate subject token in one place.
+        allow_expired = request.params.get('allow_expired')
+        allow_expired = strutils.bool_from_string(allow_expired, default=False)
+
+        return CONF.token.allow_expired_window if allow_expired else 0

keystone/common/request.py

@@ -122,3 +122,4 @@ class Request(webob.Request):
     auth_type = environ_getter('AUTH_TYPE', None)
     remote_domain = environ_getter('REMOTE_DOMAIN', None)
     context = environ_getter(context.REQUEST_CONTEXT_ENV, None)
+    token_auth = environ_getter('keystone.token_auth', None)

keystone/conf/token.py

@@ -140,6 +140,16 @@ Enable storing issued token data to token validation cache so that first token
 validation doesn't actually cause full validation cycle.
 """))
 
+allow_expired_window = cfg.IntOpt(
+    'allow_expired_window',
+    default=48 * 60 * 60,
+    help=utils.fmt("""
+This controls the number of seconds that a token can be retrieved for beyond
+the built-in expiry time. This allows long running operations to succeed.
+Defaults to two days.
+"""))
+
+
 GROUP_NAME = __name__.split('.')[-1]
 ALL_OPTS = [
     bind,
@@ -153,6 +163,7 @@ ALL_OPTS = [
     allow_rescope_scoped_token,
     infer_roles,
     cache_on_issue,
+    allow_expired_window,
 ]
 
 

keystone/tests/unit/test_backend_sql.py

@@ -13,9 +13,11 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import datetime
 import functools
 import uuid
 
+import freezegun
 import mock
 from oslo_db import exception as db_exception
 from oslo_db import options
@@ -735,6 +737,30 @@ class SqlToken(SqlTests, token_tests.TokenTests):
         self.assertEqual(token_sql._expiry_range_batched, mysql_strategy.func)
         self.assertEqual({'batch_size': 1000}, mysql_strategy.keywords)
 
+    def test_expiry_range_with_allow_expired(self):
+        window_secs = 200
+        self.config_fixture.config(group='token',
+                                   allow_expired_window=window_secs)
+
+        tok = token_sql.Token()
+        time = datetime.datetime.utcnow()
+
+        with freezegun.freeze_time(time):
+            # unknown strategy just ensures we are getting the dumbest strategy
+            # that will remove everything in one go
+            strategy = tok._expiry_range_strategy('unkown')
+            upper_bound_func = token_sql._expiry_upper_bound_func
+
+            # session is ignored for dumb strategy
+            expiry_times = list(strategy(session=None,
+                                         upper_bound_func=upper_bound_func))
+
+            # basically just ensure that we are removing things in the past
+            delta = datetime.timedelta(seconds=window_secs)
+            previous_time = datetime.datetime.utcnow() - delta
+
+        self.assertEqual([previous_time], expiry_times)
+
 
 class SqlCatalog(SqlTests, catalog_tests.CatalogTests):
 

keystone/tests/unit/test_v3_auth.py

@@ -202,9 +202,15 @@ class TokenAPITests(object):
         trust = self.assertValidTrustResponse(r)
         return (trustee_user, trust)
 
-    def _validate_token(self, token, expected_status=http_client.OK):
+    def _validate_token(self, token,
+                        expected_status=http_client.OK, allow_expired=False):
+        path = '/v3/auth/tokens'
+
+        if allow_expired:
+            path += '?allow_expired=1'
+
         return self.admin_request(
-            path='/v3/auth/tokens/',
+            path=path,
             headers={'X-Auth-Token': self.get_admin_token(),
                      'X-Subject-Token': token},
             method='GET',
@@ -2257,6 +2263,38 @@ class TokenAPITests(object):
         self.assertDictEqual(v2_token_data['access']['token']['bind'],
                              token_data['token']['bind'])
 
+    def test_fetch_expired_allow_expired(self):
+        self.config_fixture.config(group='token',
+                                   expiration=10,
+                                   allow_expired_window=20)
+        time = datetime.datetime.utcnow()
+        with freezegun.freeze_time(time) as frozen_datetime:
+            token = self._get_project_scoped_token()
+
+            # initially it validates because it's within time
+            frozen_datetime.tick(delta=datetime.timedelta(seconds=2))
+            self._validate_token(token)
+
+            # after passing expiry time validation fails
+            frozen_datetime.tick(delta=datetime.timedelta(seconds=12))
+            self._validate_token(token, expected_status=http_client.NOT_FOUND)
+
+            # flush the tokens, this will only have an effect on sql
+            try:
+                self.token_provider_api._persistence.flush_expired_tokens()
+            except exception.NotImplemented:
+                pass
+
+            # but if we pass allow_expired it validates
+            self._validate_token(token, allow_expired=True)
+
+            # and then if we're passed the allow_expired_window it will fail
+            # anyway raises expired when now > expiration + window
+            frozen_datetime.tick(delta=datetime.timedelta(seconds=22))
+            self._validate_token(token,
+                                 allow_expired=True,
+                                 expected_status=http_client.NOT_FOUND)
+
 
 class TokenDataTests(object):
     """Test the data in specific token types."""

keystone/tests/unit/token/test_backends.py

@@ -285,7 +285,8 @@ class TokenTests(object):
 
     def test_flush_expired_token(self):
         token_id = uuid.uuid4().hex
-        expire_time = timeutils.utcnow() - datetime.timedelta(minutes=1)
+        window = self.config_fixture.conf.token.allow_expired_window + 5
+        expire_time = timeutils.utcnow() - datetime.timedelta(minutes=window)
         data = {'id_hash': token_id, 'id': token_id, 'a': 'b',
                 'expires': expire_time,
                 'trust_id': None,
@@ -296,7 +297,7 @@ class TokenTests(object):
         self.assertDictEqual(data, data_ref)
 
         token_id = uuid.uuid4().hex
-        expire_time = timeutils.utcnow() + datetime.timedelta(minutes=1)
+        expire_time = timeutils.utcnow() + datetime.timedelta(minutes=window)
         data = {'id_hash': token_id, 'id': token_id, 'a': 'b',
                 'expires': expire_time,
                 'trust_id': None,

keystone/token/persistence/backends/sql.py

@@ -13,6 +13,7 @@
 # under the License.
 
 import copy
+import datetime
 import functools
 
 from oslo_log import log
@@ -47,6 +48,12 @@ class TokenModel(sql.ModelBase, sql.DictBase):
     )
 
 
+def _expiry_upper_bound_func():
+    # don't flush anything within the grace window
+    sec = datetime.timedelta(seconds=CONF.token.allow_expired_window)
+    return timeutils.utcnow() - sec
+
+
 def _expiry_range_batched(session, upper_bound_func, batch_size):
     """Return the stop point of the next batch for expiration.
 
@@ -274,7 +281,7 @@ class Token(token.persistence.TokenDriverBase):
             expiry_range_func = self._expiry_range_strategy(dialect)
             query = session.query(TokenModel.expires)
             total_removed = 0
-            upper_bound_func = timeutils.utcnow
+            upper_bound_func = _expiry_upper_bound_func
             for expiry_time in expiry_range_func(session, upper_bound_func):
                 delete_query = query.filter(TokenModel.expires <=
                                             expiry_time)

keystone/token/provider.py

@@ -14,6 +14,7 @@
 
 """Token provider interface."""
 
+import datetime
 import sys
 
 from oslo_log import log
@@ -155,7 +156,7 @@ class Manager(manager.Manager):
         else:
             return self.check_revocation_v3(token)
 
-    def validate_token(self, token_id):
+    def validate_token(self, token_id, window_seconds=0):
         if not token_id:
             raise exception.TokenNotFound(_('No token in the request'))
 
@@ -170,7 +171,7 @@ class Manager(manager.Manager):
                 # instead.
                 token_id = token_ref
             token_ref = self._validate_token(token_id)
-            self._is_valid_token(token_ref)
+            self._is_valid_token(token_ref, window_seconds=window_seconds)
             return token_ref
         except exception.Unauthorized as e:
             LOG.debug('Unable to validate token: %s', e)
@@ -180,7 +181,7 @@ class Manager(manager.Manager):
     def _validate_token(self, token_id):
         return self.driver.validate_token(token_id)
 
-    def _is_valid_token(self, token):
+    def _is_valid_token(self, token, window_seconds=0):
         """Verify the token is valid format and has not expired."""
         current_time = timeutils.normalize_time(timeutils.utcnow())
 
@@ -192,8 +193,13 @@ class Manager(manager.Manager):
                                         token_data.get('expires'))
             if not expires_at:
                 expires_at = token_data['token']['expires']
-            expiry = timeutils.normalize_time(
-                timeutils.parse_isotime(expires_at))
+
+            expiry = timeutils.parse_isotime(expires_at)
+            expiry = timeutils.normalize_time(expiry)
+
+            # add a window in which you can fetch a token beyond expiry
+            expiry += datetime.timedelta(seconds=window_seconds)
+
         except Exception:
             LOG.exception(_LE('Unexpected error or malformed token '
                               'determining token expiry: %s'), token)

releasenotes/notes/fetch-expired-token-f5d845b9601bc1ef.yaml

@@ -0,0 +1,18 @@
+---
+features:
+  - >
+    [`blueprint allow-expired <https://blueprints.launchpad.net/keystone/+spec/allow-expired>`_]
+    An `allow_expired` flag is added to the token validation call
+    (``GET/HEAD  /v3/auth/tokens``) that allows fetching a token that has
+    expired. This allows for validating tokens in long running operations.
+upgrade:
+  - >
+    [`blueprint allow-expired <https://blueprints.launchpad.net/keystone/+spec/allow-expired>`_]
+    To allow long running operations to complete services must be able to fetch
+    expired tokens via the ``allow_expired`` flag. The length of time a token is
+    retrievable for beyond its traditional expiry is managed by the
+    ``[token] allow_expired_window`` option and so the data must be retrievable
+    for this about of time. When using fernet tokens this means that the key
+    rotation period must exceed this time so that older tokens are still
+    decrytable. Ensure that you do not rotate fernet keys faster than
+    ``[token] expiration`` + ``[token] allow_expired_window`` seconds.