Return 404 instead of 401 for tokens w/o roles

If a scoped-token was validated and the user didn't have any role assignment
on a project, keystone would return a 401 Unauthorized. This was the
case when the fernet token provider was enabled because the reference is
rebuilt on every request. The uuid token provider has a different behavior - if
the token isn't found in the backend a 404 Not Found is returned. Furthermore,
for persisted tokens, any validation error will result in 404, such as in the
case where user no longer have any roles assigned for the given scope.

These two behaviors should be consistent regardless of the token provider.

Conflicts:
	keystone/tests/unit/test_v3_auth.py
	keystone/token/provider.py

Closes-Bug: 1541621
Change-Id: If9fd6060ed13a7c03ab8d70ebed1adecafef9160
(cherry picked from commit f1792f4089)

keystone/tests/unit/test_v3_auth.py

@@ -419,6 +419,44 @@ class TokenAPITests(object):
             headers={'X-Subject-Token': v3_token})
         self.assertValidProjectScopedTokenResponse(r, require_catalog=False)
 
+    def test_remove_all_roles_from_scope_result_in_404(self):
+        # create a new user
+        new_user_password = uuid.uuid4().hex
+        new_user = {
+            'name': uuid.uuid4().hex,
+            'domain_id': self.domain['id'],
+            'password': new_user_password,
+            'email': uuid.uuid4().hex,
+        }
+        new_user = self.identity_api.create_user(new_user)
+
+        # give the new user a role on a project
+        path = '/projects/%s/users/%s/roles/%s' % (
+            self.project['id'], new_user['id'], self.role['id'])
+        self.put(path=path)
+
+        # authenticate as the new user and get a project-scoped token
+        auth_data = self.build_authentication_request(
+            user_id=new_user['id'],
+            password=new_user_password,
+            project_id=self.project['id'])
+        subject_token_id = self.v3_authenticate_token(auth_data).headers.get(
+            'X-Subject-Token')
+
+        # make sure the project-scoped token is valid
+        headers = {'X-Subject-Token': subject_token_id}
+        r = self.get('/auth/tokens', headers=headers)
+        self.assertValidProjectScopedTokenResponse(r)
+
+        # remove the roles from the user for the given scope
+        path = '/projects/%s/users/%s/roles/%s' % (
+            self.project['id'], new_user['id'], self.role['id'])
+        self.delete(path=path)
+
+        # token validation should now result in 404
+        self.get('/auth/tokens', headers=headers,
+                 expected_status=http_client.NOT_FOUND)
+
 
 class AllowRescopeScopedTokenDisabledTests(test_v3.RestfulTestCase):
     def config_overrides(self):

keystone/token/provider.py

@@ -233,21 +233,25 @@ class Manager(manager.Manager):
         if not token_id:
             raise exception.TokenNotFound(_('No token in the request'))
 
-        unique_id = utils.generate_unique_id(token_id)
-        # NOTE(lbragstad): Only go to persistent storage if we have a token to
-        # fetch from the backend. If the Fernet token provider is being used
-        # this step isn't necessary. The Fernet token reference is persisted in
-        # the token_id, so in this case set the token_ref as the identifier of
-        # the token.
-        if not self._needs_persistence:
-            token_ref = token_id
-        else:
-            # NOTE(morganfainberg): Ensure we never use the long-form token_id
-            # (PKI) as part of the cache_key.
-            token_ref = self._persistence.get_token(unique_id)
-        token = self._validate_v3_token(token_ref)
-        self._is_valid_token(token)
-        return token
+        try:
+            unique_id = utils.generate_unique_id(token_id)
+            # NOTE(lbragstad): Only go to persistent storage if we have a
+            # token to fetch from the backend. If the Fernet token provider is
+            # being used this step isn't necessary. The Fernet token reference
+            # is persisted in the token_id, so in this case set the token_ref
+            # as the identifier of the token.
+            if not self._needs_persistence:
+                token_ref = token_id
+            else:
+                # NOTE(morganfainberg): Ensure we never use the long-form
+                # token_id (PKI) as part of the cache_key.
+                token_ref = self._persistence.get_token(unique_id)
+            token = self._validate_v3_token(token_ref)
+            self._is_valid_token(token)
+            return token
+        except exception.Unauthorized as e:
+            LOG.debug('Unable to validate token: %s', e)
+            raise exception.TokenNotFound(token_id=token_id)
 
     @MEMOIZE
     def _validate_token(self, token_id):
@@ -259,7 +263,11 @@ class Manager(manager.Manager):
         token_ref = self._persistence.get_token(token_id)
         version = self.driver.get_token_version(token_ref)
         if version == self.V3:
-            return self.driver.validate_v3_token(token_ref)
+            try:
+                return self.driver.validate_v3_token(token_ref)
+            except exception.Unauthorized as e:
+                LOG.debug('Unable to validate token: %s', e)
+                raise exception.TokenNotFound(token_id=token_id)
         elif version == self.V2:
             return self.driver.validate_v2_token(token_ref)
         raise exception.UnsupportedTokenVersionException()