Merge "Rework keystone auth_token middleware configs"

doc/source/devref/development.environment.rst

@@ -59,7 +59,10 @@ On Fedora-based distributions (e.g., Fedora/RHEL/CentOS/Scientific Linux):
     $ cp ./etc/sahara/sahara.conf.sample-basic ./etc/sahara/sahara.conf
 
 5. Look through the sahara.conf and change parameters which default values do
-not suite you. Set ``os_auth_host`` to the address of OpenStack keystone.
+not suite you. Set ``[keystone_authtoken]/auth_uri`` and
+``[keystone_authtoken]/identity_uri`` to complete public Identity API endpoint
+(like ``http://127.0.0.1:5000/v2.0/``) and to unversioned complete admin
+Identity API endpoint (like ``https://localhost:35357/``) correspondingly.
 
 If you are using Neutron instead of Nova Network add ``use_neutron = True`` to
 config.  If the linux kernel you're utilizing support network namespaces then

doc/source/userdoc/upgrade.guide.rst

@@ -12,3 +12,20 @@ Main binary renamed to sahara-all
 
 Please, note that you should use `sahara-all` instead of `sahara-api` to start
 the All-In-One Sahara.
+
+sahara.conf upgrade
++++++++++++++++++++
+
+We've migrated from custom auth_token middleware config options to the common
+config options. To update your config file you should replace the following
+old config opts with the new ones.
+
+* ``os_auth_protocol``, ``os_auth_host``, ``os_auth_port``
+  -> ``[keystone_authtoken]/auth_uri`` and ``[keystone_authtoken]/identity_uri``;
+  it should be the full uri, for example: ``http://127.0.0.1:5000/v2.0/``
+* ``os_admin_username`` -> ``[keystone_authtoken]/admin_user``
+* ``os_admin_password`` -> ``[keystone_authtoken]/admin_password``
+* ``os_admin_tenant_name`` -> ``[keystone_authtoken]/admin_tenant_name``
+
+You can find more info about config file options in sahara repository in file
+``etc/sahara/sahara.conf.sample``.

etc/sahara/sahara.conf.sample

@@ -44,28 +44,6 @@
 # Options defined in sahara.main
 #
 
-# Protocol used to access OpenStack Identity service. (string
-# value)
-#os_auth_protocol=http
-
-# IP or hostname of machine on which OpenStack Identity
-# service is located. (string value)
-#os_auth_host=127.0.0.1
-
-# Port of OpenStack Identity service. (string value)
-#os_auth_port=5000
-
-# This OpenStack user is used to verify provided tokens. The
-# user must have admin role in <os_admin_tenant_name> tenant.
-# (string value)
-#os_admin_username=admin
-
-# Password of the admin user. (string value)
-#os_admin_password=nova
-
-# Name of tenant where the user is admin. (string value)
-#os_admin_tenant_name=admin
-
 # Region name used to get services endpoints. (string value)
 #os_region_name=<None>
 
@@ -404,3 +382,158 @@
 #db_max_retries=20
 
 
+[keystone_authtoken]
+
+#
+# Options defined in keystoneclient.middleware.auth_token
+#
+
+# Prefix to prepend at the beginning of the path. Deprecated,
+# use identity_uri. (string value)
+#auth_admin_prefix=
+
+# Host providing the admin Identity API endpoint. Deprecated,
+# use identity_uri. (string value)
+#auth_host=127.0.0.1
+
+# Port of the admin Identity API endpoint. Deprecated, use
+# identity_uri. (integer value)
+#auth_port=35357
+
+# Protocol of the admin Identity API endpoint (http or https).
+# Deprecated, use identity_uri. (string value)
+#auth_protocol=https
+
+# Complete public Identity API endpoint (string value)
+#auth_uri=<None>
+
+# Complete admin Identity API endpoint. This should specify
+# the unversioned root endpoint e.g. https://localhost:35357/
+# (string value)
+#identity_uri=<None>
+
+# API version of the admin Identity API endpoint (string
+# value)
+#auth_version=<None>
+
+# Do not handle authorization requests within the middleware,
+# but delegate the authorization decision to downstream WSGI
+# components (boolean value)
+#delay_auth_decision=false
+
+# Request timeout value for communicating with Identity API
+# server. (boolean value)
+#http_connect_timeout=<None>
+
+# How many times are we trying to reconnect when communicating
+# with Identity API Server. (integer value)
+#http_request_max_retries=3
+
+# This option is deprecated and may be removed in a future
+# release. Single shared secret with the Keystone
+# configuration used for bootstrapping a Keystone
+# installation, or otherwise bypassing the normal
+# authentication process. This option should not be used, use
+# `admin_user` and `admin_password` instead. (string value)
+#admin_token=<None>
+
+# Keystone account username (string value)
+#admin_user=<None>
+
+# Keystone account password (string value)
+#admin_password=<None>
+
+# Keystone service account tenant name to validate user tokens
+# (string value)
+#admin_tenant_name=admin
+
+# Env key for the swift cache (string value)
+#cache=<None>
+
+# Required if Keystone server requires client certificate
+# (string value)
+#certfile=<None>
+
+# Required if Keystone server requires client certificate
+# (string value)
+#keyfile=<None>
+
+# A PEM encoded Certificate Authority to use when verifying
+# HTTPs connections. Defaults to system CAs. (string value)
+#cafile=<None>
+
+# Verify HTTPS connections. (boolean value)
+#insecure=false
+
+# Directory used to cache files related to PKI tokens (string
+# value)
+#signing_dir=<None>
+
+# Optionally specify a list of memcached server(s) to use for
+# caching. If left undefined, tokens will instead be cached
+# in-process. (list value)
+# Deprecated group/name - [DEFAULT]/memcache_servers
+#memcached_servers=<None>
+
+# In order to prevent excessive effort spent validating
+# tokens, the middleware caches previously-seen tokens for a
+# configurable duration (in seconds). Set to -1 to disable
+# caching completely. (integer value)
+#token_cache_time=300
+
+# Determines the frequency at which the list of revoked tokens
+# is retrieved from the Identity service (in seconds). A high
+# number of revocation events combined with a low cache
+# duration may significantly reduce performance. (integer
+# value)
+#revocation_cache_time=10
+
+# (optional) if defined, indicate whether token data should be
+# authenticated or authenticated and encrypted. Acceptable
+# values are MAC or ENCRYPT.  If MAC, token data is
+# authenticated (with HMAC) in the cache. If ENCRYPT, token
+# data is encrypted and authenticated in the cache. If the
+# value is not one of these options or empty, auth_token will
+# raise an exception on initialization. (string value)
+#memcache_security_strategy=<None>
+
+# (optional, mandatory if memcache_security_strategy is
+# defined) this string is used for key derivation. (string
+# value)
+#memcache_secret_key=<None>
+
+# (optional) indicate whether to set the X-Service-Catalog
+# header. If False, middleware will not ask for service
+# catalog on token validation and will not set the X-Service-
+# Catalog header. (boolean value)
+#include_service_catalog=true
+
+# Used to control the use and type of token binding. Can be
+# set to: "disabled" to not check token binding. "permissive"
+# (default) to validate binding information if the bind type
+# is of a form known to the server and ignore it if not.
+# "strict" like "permissive" but if the bind type is unknown
+# the token will be rejected. "required" any form of token
+# binding is needed to be allowed. Finally the name of a
+# binding method that must be present in tokens. (string
+# value)
+#enforce_token_bind=permissive
+
+# If true, the revocation list will be checked for cached
+# tokens. This requires that PKI tokens are configured on the
+# Keystone server. (boolean value)
+#check_revocations_for_cached=false
+
+# Hash algorithms to use for hashing PKI tokens. This may be a
+# single algorithm or multiple. The algorithms are those
+# supported by Python standard hashlib.new(). The hashes will
+# be tried in the order given, so put the preferred one first
+# for performance. The result of the first hash will be stored
+# in the cache. This will typically be set to multiple values
+# only while migrating from a less secure algorithm to a more
+# secure one. Once all the old tokens are expired this option
+# should be set to a single value for better performance.
+# (list value)
+#hash_algorithms=md5
+
+

etc/sahara/sahara.conf.sample-basic

@@ -7,17 +7,6 @@
 # Port that will be used to listen on (integer value)
 #port=8386
 
-
-
-# Address and credentials that will be used to check auth tokens
-#os_auth_host=127.0.0.1
-#os_auth_port=5000
-#os_admin_username=admin
-#os_admin_password=nova
-#os_admin_tenant_name=admin
-
-
-
 # If set to True, Sahara will use floating IPs to communicate
 # with instances. To make sure that all instances have
 # floating IPs assigned in Nova Network set
@@ -112,3 +101,21 @@
 # database (string value)
 #connection=<None>
 
+[keystone_authtoken]
+# Complete public Identity API endpoint (string value)
+#auth_uri=http://127.0.0.1:5000/v2.0/
+
+# Complete admin Identity API endpoint. This should specify
+# the unversioned root endpoint eg. https://localhost:35357/
+# (string value)
+#identity_uri=http://127.0.0.1:35357/
+
+# Keystone account username (string value)
+#admin_user=demo
+
+# Keystone account password (string value)
+#admin_password=nova
+
+# Keystone service account tenant name to validate user tokens
+# (string value)
+#admin_tenant_name=demo

sahara/api/acl.py

@@ -0,0 +1,50 @@
+# Copyright (c) 2014 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from keystoneclient.middleware import auth_token
+from oslo.config import cfg
+
+from sahara.openstack.common import log
+
+LOG = log.getLogger(__name__)
+CONF = cfg.CONF
+
+AUTH_OPT_GROUP_NAME = 'keystone_authtoken'
+
+# Keystone auth uri that could be used in other places in Sahara
+AUTH_URI = None
+
+
+def register_auth_opts(conf):
+    """Register keystoneclient auth_token middleware options."""
+
+    conf.register_opts(auth_token.opts, group=AUTH_OPT_GROUP_NAME)
+    auth_token.CONF = conf
+
+
+register_auth_opts(CONF)
+
+
+def wrap(app, conf):
+    """Wrap wsgi application with ACL check."""
+
+    auth_cfg = dict(conf.get(AUTH_OPT_GROUP_NAME))
+    auth_protocol = auth_token.AuthProtocol(app, conf=auth_cfg)
+
+    # store auth uri in global var to be able to use it in runtime
+    global AUTH_URI
+    AUTH_URI = auth_protocol.auth_uri
+
+    return auth_protocol

sahara/context.py

@@ -20,6 +20,7 @@ from eventlet import greenpool
 from eventlet import semaphore
 from oslo.config import cfg
 
+from sahara.api import acl
 from sahara import exceptions as ex
 from sahara.openstack.common import log as logging
 
@@ -40,6 +41,7 @@ class Context(object):
                  roles=None,
                  is_admin=None,
                  remote_semaphore=None,
+                 auth_uri=None,
                  **kwargs):
         if kwargs:
             LOG.warn('Arguments dropped when creating context: %s', kwargs)
@@ -53,6 +55,7 @@ class Context(object):
         self.remote_semaphore = remote_semaphore or semaphore.Semaphore(
             CONF.cluster_remote_threshold)
         self.roles = roles
+        self.auth_uri = auth_uri or acl.AUTH_URI
 
     def clone(self):
         return Context(
@@ -76,6 +79,7 @@ class Context(object):
             'tenant_name': self.tenant_name,
             'is_admin': self.is_admin,
             'roles': self.roles,
+            'auth_uri': self.auth_uri,
         }
 
     def is_auth_capable(self):

sahara/main.py

@@ -16,12 +16,12 @@
 import os
 
 import flask
-from keystoneclient.middleware import auth_token
 from oslo.config import cfg
 import six
 import stevedore
 from werkzeug import exceptions as werkzeug_exceptions
 
+from sahara.api import acl
 from sahara.api import v10 as api_v10
 from sahara.api import v11 as api_v11
 from sahara import config
@@ -42,27 +42,6 @@ LOG = log.getLogger(__name__)
 
 
 opts = [
-    cfg.StrOpt('os_auth_protocol',
-               default='http',
-               help='Protocol used to access OpenStack Identity service.'),
-    cfg.StrOpt('os_auth_host',
-               default='127.0.0.1',
-               help='IP or hostname of machine on which OpenStack Identity '
-                    'service is located.'),
-    cfg.StrOpt('os_auth_port',
-               default='5000',
-               help='Port of OpenStack Identity service.'),
-    cfg.StrOpt('os_admin_username',
-               default='admin',
-               help='This OpenStack user is used to verify provided tokens. '
-                    'The user must have admin role in <os_admin_tenant_name> '
-                    'tenant.'),
-    cfg.StrOpt('os_admin_password',
-               default='nova',
-               help='Password of the admin user.'),
-    cfg.StrOpt('os_admin_tenant_name',
-               default='admin',
-               help='Name of tenant where the user is admin.'),
     cfg.StrOpt('os_region_name',
                help='Region name used to get services endpoints.'),
     cfg.StrOpt('infrastructure_engine',
@@ -160,20 +139,10 @@ def make_app():
                   ' flag --log-exchange')
 
     if CONF.log_exchange:
-        cfg = app.config
+        app.wsgi_app = log_exchange.LogExchange.factory(CONF)(app.wsgi_app)
-        app.wsgi_app = log_exchange.LogExchange.factory(cfg)(app.wsgi_app)
 
-    app.wsgi_app = auth_valid.filter_factory(app.config)(app.wsgi_app)
+    app.wsgi_app = auth_valid.wrap(app.wsgi_app)
-
+    app.wsgi_app = acl.wrap(app.wsgi_app, CONF)
-    app.wsgi_app = auth_token.filter_factory(
-        app.config,
-        auth_host=CONF.os_auth_host,
-        auth_port=CONF.os_auth_port,
-        auth_protocol=CONF.os_auth_protocol,
-        admin_user=CONF.os_admin_username,
-        admin_password=CONF.os_admin_password,
-        admin_tenant_name=CONF.os_admin_tenant_name
-    )(app.wsgi_app)
 
     return app
 

sahara/middleware/auth_valid.py

@@ -25,9 +25,8 @@ LOG = logging.getLogger(__name__)
 class AuthValidator:
     """Handles token auth results and tenants."""
 
-    def __init__(self, app, conf):
+    def __init__(self, app):
         self.app = app
-        self.conf = conf
 
     def __call__(self, env, start_response):
         """Ensures that tenants in url and token are equal.
@@ -61,11 +60,7 @@ class AuthValidator:
         return self.app(env, start_response)
 
 
-def filter_factory(global_conf, **local_conf):
+def wrap(app):
-    conf = global_conf.copy()
+    """Wrap wsgi application with auth validator check."""
-    conf.update(local_conf)
 
-    def auth_filter(app):
+    return AuthValidator(app)
-        return AuthValidator(app, conf)
-
-    return auth_filter

sahara/service/trusts.py

@@ -46,7 +46,7 @@ def create_trust(cluster):
 def use_os_admin_auth_token(cluster):
     if cluster.trust_id:
         ctx = context.current()
-        ctx.username = CONF.os_admin_username
+        ctx.username = CONF.keystone_authtoken.admin_user
         ctx.tenant_id = cluster.tenant_id
         client = keystone.client_for_trusts(cluster.trust_id)
         ctx.token = client.auth_token

sahara/swift/utils.py

@@ -13,12 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import re
-
 from oslo.config import cfg
+from six.moves.urllib import parse as urlparse
 
 from sahara import context
-from sahara.utils.openstack import base
 
 
 CONF = cfg.CONF
@@ -30,19 +28,10 @@ SWIFT_URL_SUFFIX_START = '.'
 SWIFT_URL_SUFFIX = SWIFT_URL_SUFFIX_START + 'sahara'
 
 
-def _get_service_address(service_type):
-    ctx = context.current()
-    identity_url = base.url_for(ctx.service_catalog, service_type)
-    address_regexp = r"^\w+://(.+?)/"
-    identity_host = re.search(address_regexp, identity_url).group(1)
-    return identity_host
-
-
 def retrieve_auth_url():
-    """This function return auth url v2 api. Hadoop swift library doesn't
+    """This function return auth url v2.0 api. Hadoop Swift library doesn't
     support keystone v3 api.
     """
-    protocol = CONF.os_auth_protocol
+    info = urlparse.urlparse(context.current().auth_uri)
-    host = _get_service_address('identity')
 
-    return "%s://%s/v2.0/" % (protocol, host)
+    return "%s://%s:%s/%s/" % (info.scheme, info.hostname, info.port, 'v2.0')

sahara/tests/unit/swift/test_swift_helper.py

@@ -13,8 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import mock
-
 from sahara.swift import swift_helper as h
 from sahara.tests.unit import base
 
@@ -34,13 +32,10 @@ SERVICE_SPECIFIC = ["auth.url", "tenant",
 
 class SwiftIntegrationTestCase(base.SaharaTestCase):
 
-    @mock.patch('sahara.utils.openstack.base.url_for')
+    def test_get_swift_configs(self):
-    def test_get_swift_configs(self, authUrlConfig):
+        self.setup_context(tenant_name='test_tenant',
-        self.setup_context(tenant_name='test_tenant')
+                           auth_uri='http://localhost:8080/v2.0/')
-        self.override_config("os_auth_protocol", 'http')
-        self.override_config("os_auth_port", '8080')
         self.override_config("os_region_name", 'regionOne')
-        authUrlConfig.return_value = "http://localhost:8080/v2.0/"
 
         result = h.get_swift_configs()
         self.assertEqual(8, len(result))

sahara/tests/unit/swift/test_utils.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sahara.swift import utils
+from sahara.tests.unit import base as testbase
+
+
+class SwiftUtilsTest(testbase.SaharaTestCase):
+
+    def setUp(self):
+        self.override_config('use_identity_api_v3', True)
+
+    def test_retrieve_auth_url(self):
+        correct = "https://127.0.0.1:8080/v2.0/"
+
+        def _assert(uri):
+            self.setup_context(auth_uri=uri)
+            self.assertEqual(correct, utils.retrieve_auth_url())
+
+        _assert("%s/" % correct)
+        _assert("https://127.0.0.1:8080")
+        _assert("https://127.0.0.1:8080/")
+        _assert("https://127.0.0.1:8080/v2.0")
+        _assert("https://127.0.0.1:8080/v2.0/")
+        _assert("https://127.0.0.1:8080/v42/")
+        _assert("https://127.0.0.1:8080/foo")

sahara/tests/unit/utils/openstack/test_base.py

@@ -13,11 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from sahara.tests.unit import base as ub
+from sahara.tests.unit import base as testbase
-from sahara.utils.openstack import base as b
+from sahara.utils.openstack import base
 
 
-class TestBase(ub.SaharaTestCase):
+class TestBase(testbase.SaharaTestCase):
 
     def test_url_for_regions(self):
         service_catalog = (
@@ -38,8 +38,47 @@ class TestBase(ub.SaharaTestCase):
 
         self.override_config("os_region_name", "RegionOne")
         self.assertEqual("http://172.18.184.5:8774/v2",
-                         b.url_for(service_catalog, "compute"))
+                         base.url_for(service_catalog, "compute"))
 
         self.override_config("os_region_name", "RegionTwo")
         self.assertEqual("http://172.18.184.6:8774/v2",
-                         b.url_for(service_catalog, "compute"))
+                         base.url_for(service_catalog, "compute"))
+
+
+class AuthUrlTest(testbase.SaharaTestCase):
+
+    def test_retrieve_auth_url_api_v3(self):
+        self.override_config('use_identity_api_v3', True)
+        correct = "https://127.0.0.1:8080/v3/"
+
+        def _assert(uri):
+            self.setup_context(auth_uri=uri)
+            self.assertEqual(correct, base.retrieve_auth_url())
+
+        _assert("%s/" % correct)
+        _assert("https://127.0.0.1:8080")
+        _assert("https://127.0.0.1:8080/")
+        _assert("https://127.0.0.1:8080/v2.0")
+        _assert("https://127.0.0.1:8080/v2.0/")
+        _assert("https://127.0.0.1:8080/v3")
+        _assert("https://127.0.0.1:8080/v3/")
+        _assert("https://127.0.0.1:8080/v42")
+        _assert("https://127.0.0.1:8080/v42/")
+
+    def test_retrieve_auth_url_api_v20(self):
+        self.override_config('use_identity_api_v3', False)
+        correct = "https://127.0.0.1:8080/v2.0/"
+
+        def _assert(uri):
+            self.setup_context(auth_uri=uri)
+            self.assertEqual(correct, base.retrieve_auth_url())
+
+        _assert("%s/" % correct)
+        _assert("https://127.0.0.1:8080")
+        _assert("https://127.0.0.1:8080/")
+        _assert("https://127.0.0.1:8080/v2.0")
+        _assert("https://127.0.0.1:8080/v2.0/")
+        _assert("https://127.0.0.1:8080/v3")
+        _assert("https://127.0.0.1:8080/v3/")
+        _assert("https://127.0.0.1:8080/v42")
+        _assert("https://127.0.0.1:8080/v42/")

sahara/utils/openstack/base.py

@@ -16,8 +16,10 @@
 import json
 
 from oslo.config import cfg
-from sahara import exceptions as ex
+from six.moves.urllib import parse as urlparse
 
+from sahara import context
+from sahara import exceptions as ex
 
 CONF = cfg.CONF
 
@@ -76,9 +78,7 @@ def _get_case_insensitive(dictionary, key):
 
 
 def retrieve_auth_url():
-    protocol = CONF.os_auth_protocol
+    info = urlparse.urlparse(context.current().auth_uri)
-    host = CONF.os_auth_host
+    version = 'v3' if CONF.use_identity_api_v3 else 'v2.0'
-    port = CONF.os_auth_port
 
-    return "%s://%s:%s/%s/" % (protocol, host, port,
+    return "%s://%s:%s/%s/" % (info.scheme, info.hostname, info.port, version)
-                               'v3' if CONF.use_identity_api_v3 else 'v2.0')

sahara/utils/openstack/keystone.py

@@ -58,8 +58,10 @@ def _admin_client(project_name=None, trust_id=None):
                         ' less than v3')
 
     auth_url = base.retrieve_auth_url()
-    keystone = keystone_client_v3.Client(username=CONF.os_admin_username,
+    username = CONF.keystone_authtoken.admin_user
-                                         password=CONF.os_admin_password,
+    password = CONF.keystone_authtoken.admin_password
+    keystone = keystone_client_v3.Client(username=username,
+                                         password=password,
                                          project_name=project_name,
                                          auth_url=auth_url,
                                          trust_id=trust_id)
@@ -68,7 +70,8 @@ def _admin_client(project_name=None, trust_id=None):
 
 
 def client_for_admin():
-    return _admin_client(project_name=CONF.os_admin_tenant_name)
+    project_name = CONF.keystone_authtoken.admin_tenant_name
+    return _admin_client(project_name=project_name)
 
 
 def client_for_trusts(trust_id):

tools/config/oslo.config.generator.rc

@@ -0,0 +1 @@
+export SAHARA_CONFIG_GENERATOR_EXTRA_MODULES="keystoneclient.middleware.auth_token"