rework certs stuff

remove ca_certs using for incoming requests
do small refactoring for keystone client using

Change-Id: I83dbb71248835cfc361eca691647deaa99023c8a

ec2api/api/__init__.py

@@ -35,10 +35,10 @@ import webob.exc
 from ec2api.api import apirequest
 from ec2api.api import ec2utils
 from ec2api.api import faults
+from ec2api import clients
 from ec2api import context
 from ec2api import exception
 from ec2api.i18n import _
-from ec2api import utils
 from ec2api import wsgi
 
 
@@ -61,14 +61,6 @@ CONF.register_opts(ec2_opts)
 CONF.import_opt('use_forwarded_for', 'ec2api.api.auth')
 
 
-EMPTY_SHA256_HASH = (
-    'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
-# This is the buffer size used when calculating sha256 checksums.
-# Experimenting with various buffer sizes showed that this value generally
-# gave the best result (in terms of performance).
-PAYLOAD_BUFFER = 1024 * 1024
-
-
 # Fault Wrapper around all EC2 requests #
 class FaultWrapper(wsgi.Middleware):
 
@@ -119,12 +111,6 @@ class RequestLogging(wsgi.Middleware):
             context=ctxt)
 
 
-class InvalidCredentialsException(Exception):
-    def __init__(self, msg):
-        super(Exception, self).__init__()
-        self.msg = msg
-
-
 class EC2KeystoneAuth(wsgi.Middleware):
 
     """Authenticate an EC2 request with keystone and convert to context."""
@@ -211,7 +197,7 @@ class EC2KeystoneAuth(wsgi.Middleware):
             'verb': req.method,
             'path': req.path,
             'params': params,
-            # python3 takes only keys fo json from headers object
+            # python3 takes only keys for json from headers object
             'headers': {k: req.headers[k] for k in req.headers},
             'body_hash': body_hash
         }
@@ -224,7 +210,7 @@ class EC2KeystoneAuth(wsgi.Middleware):
         creds_json = jsonutils.dumps(creds)
         headers = {'Content-Type': 'application/json'}
         params = {'data': creds_json, 'headers': headers}
-        utils.update_request_params_with_ssl(params)
+        clients.update_request_params_with_ssl(params)
         response = requests.request('POST', token_url, **params)
         status_code = response.status_code
         if status_code != 200:
@@ -242,7 +228,7 @@ class EC2KeystoneAuth(wsgi.Middleware):
                                              status=400)
         auth = keystone_identity_access.AccessInfoPlugin(auth_ref)
         params = {'auth': auth}
-        utils.update_request_params_with_ssl(params)
+        clients.update_request_params_with_ssl(params)
         session = keystone_session.Session(**params)
         remote_address = req.remote_addr
         if CONF.use_forwarded_for:

ec2api/api/address.py

@@ -19,10 +19,10 @@ except ImportError:
 from novaclient import exceptions as nova_exception
 from oslo_config import cfg
 
-from ec2api.api import clients
 from ec2api.api import common
 from ec2api.api import ec2utils
 from ec2api.api import internet_gateway as internet_gateway_api
+from ec2api import clients
 from ec2api.db import api as db_api
 from ec2api import exception
 from ec2api.i18n import _

ec2api/api/availability_zone.py

@@ -16,8 +16,8 @@ from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_utils import netutils
 
-from ec2api.api import clients
 from ec2api.api import common
+from ec2api import clients
 from ec2api import exception
 
 

ec2api/api/dhcp_options.py

@@ -17,9 +17,9 @@ import netaddr
 from oslo_config import cfg
 from oslo_log import log as logging
 
-from ec2api.api import clients
 from ec2api.api import common
 from ec2api.api import ec2utils
+from ec2api import clients
 from ec2api.db import api as db_api
 from ec2api import exception
 from ec2api.i18n import _

ec2api/api/ec2utils.py

@@ -24,7 +24,7 @@ from oslo_log import log as logging
 from oslo_utils import timeutils
 import six
 
-from ec2api.api import clients
+from ec2api import clients
 from ec2api.db import api as db_api
 from ec2api import exception
 from ec2api.i18n import _, _LE

ec2api/api/image.py

@@ -31,10 +31,10 @@ from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_utils import timeutils
 
-from ec2api.api import clients
 from ec2api.api import common
 from ec2api.api import ec2utils
 from ec2api.api import instance as instance_api
+from ec2api import clients
 from ec2api.db import api as db_api
 from ec2api import exception
 from ec2api.i18n import _, _LE, _LI, _LW

ec2api/api/instance.py

@@ -25,11 +25,11 @@ from oslo_log import log as logging
 from oslo_utils import timeutils
 import six
 
-from ec2api.api import clients
 from ec2api.api import common
 from ec2api.api import ec2utils
 from ec2api.api import network_interface as network_interface_api
 from ec2api.api import security_group as security_group_api
+from ec2api import clients
 from ec2api import context as ec2_context
 from ec2api.db import api as db_api
 from ec2api import exception

ec2api/api/internet_gateway.py

@@ -21,9 +21,9 @@ datastore.
 from neutronclient.common import exceptions as neutron_exception
 from oslo_log import log as logging
 
-from ec2api.api import clients
 from ec2api.api import common
 from ec2api.api import ec2utils
+from ec2api import clients
 from ec2api.db import api as db_api
 from ec2api import exception
 from ec2api.i18n import _

ec2api/api/key_pair.py

@@ -18,8 +18,8 @@ from novaclient import exceptions as nova_exception
 from oslo_config import cfg
 from oslo_log import log as logging
 
-from ec2api.api import clients
 from ec2api.api import common
+from ec2api import clients
 from ec2api import exception
 from ec2api.i18n import _
 

ec2api/api/network_interface.py

@@ -22,11 +22,11 @@ from oslo_log import log as logging
 from oslo_utils import timeutils
 
 from ec2api.api import address as address_api
-from ec2api.api import clients
 from ec2api.api import common
 from ec2api.api import dhcp_options
 from ec2api.api import ec2utils
 from ec2api.api import security_group as security_group_api
+from ec2api import clients
 from ec2api.db import api as db_api
 from ec2api import exception
 from ec2api.i18n import _

ec2api/api/opts.py

@@ -15,7 +15,6 @@ import itertools
 import ec2api.api
 import ec2api.api.auth
 import ec2api.api.availability_zone
-import ec2api.api.clients
 import ec2api.api.common
 import ec2api.api.dhcp_options
 import ec2api.api.ec2utils
@@ -30,7 +29,6 @@ def list_opts():
              ec2api.api.ec2_opts,
              ec2api.api.auth.auth_opts,
              ec2api.api.availability_zone.availability_zone_opts,
-             ec2api.api.clients.ec2_opts,
              ec2api.api.common.ec2_opts,
              ec2api.api.dhcp_options.ec2_opts,
              ec2api.api.ec2utils.ec2_opts,

ec2api/api/route_table.py

@@ -19,10 +19,10 @@ import netaddr
 from novaclient import exceptions as nova_exception
 import six
 
-from ec2api.api import clients
 from ec2api.api import common
 from ec2api.api import ec2utils
 from ec2api.api import vpn_connection as vpn_connection_api
+from ec2api import clients
 from ec2api.db import api as db_api
 from ec2api import exception
 from ec2api.i18n import _

ec2api/api/security_group.py

@@ -23,10 +23,10 @@ from novaclient import exceptions as nova_exception
 from oslo_config import cfg
 from oslo_log import log as logging
 
-from ec2api.api import clients
 from ec2api.api import common
 from ec2api.api import ec2utils
 from ec2api.api import validator
+from ec2api import clients
 from ec2api.db import api as db_api
 from ec2api import exception
 from ec2api.i18n import _

ec2api/api/snapshot.py

@@ -14,9 +14,9 @@
 
 from cinderclient import exceptions as cinder_exception
 
-from ec2api.api import clients
 from ec2api.api import common
 from ec2api.api import ec2utils
+from ec2api import clients
 from ec2api.db import api as db_api
 from ec2api import exception
 from ec2api.i18n import _

ec2api/api/subnet.py

@@ -17,12 +17,12 @@ from neutronclient.common import exceptions as neutron_exception
 from oslo_config import cfg
 from oslo_log import log as logging
 
-from ec2api.api import clients
 from ec2api.api import common
 from ec2api.api import ec2utils
 from ec2api.api import network_interface as network_interface_api
 from ec2api.api import route_table as route_table_api
 from ec2api.api import vpn_gateway as vpn_gateway_api
+from ec2api import clients
 from ec2api.db import api as db_api
 from ec2api import exception
 from ec2api.i18n import _

ec2api/api/volume.py

@@ -16,9 +16,9 @@ from cinderclient import exceptions as cinder_exception
 from novaclient import exceptions as nova_exception
 from oslo_log import log as logging
 
-from ec2api.api import clients
 from ec2api.api import common
 from ec2api.api import ec2utils
+from ec2api import clients
 from ec2api import context as ec2_context
 from ec2api.db import api as db_api
 from ec2api import exception

ec2api/api/vpc.py

@@ -17,7 +17,6 @@ from neutronclient.common import exceptions as neutron_exception
 from oslo_config import cfg
 from oslo_log import log as logging
 
-from ec2api.api import clients
 from ec2api.api import common
 from ec2api.api import ec2utils
 from ec2api.api import internet_gateway as internet_gateway_api
@@ -25,6 +24,7 @@ from ec2api.api import route_table as route_table_api
 from ec2api.api import security_group as security_group_api
 from ec2api.api import subnet as subnet_api
 from ec2api.api import vpn_gateway as vpn_gateway_api
+from ec2api import clients
 from ec2api.db import api as db_api
 from ec2api import exception
 from ec2api.i18n import _

ec2api/api/vpn_connection.py

@@ -21,9 +21,9 @@ from neutronclient.common import exceptions as neutron_exception
 from oslo_log import log as logging
 import six
 
-from ec2api.api import clients
 from ec2api.api import common
 from ec2api.api import ec2utils
+from ec2api import clients
 from ec2api.db import api as db_api
 from ec2api import exception
 from ec2api.i18n import _

ec2api/api/vpn_gateway.py

@@ -15,10 +15,10 @@
 from neutronclient.common import exceptions as neutron_exception
 from oslo_log import log as logging
 
-from ec2api.api import clients
 from ec2api.api import common
 from ec2api.api import ec2utils
 from ec2api.api import vpn_connection as vpn_connection_api
+from ec2api import clients
 from ec2api.db import api as db_api
 from ec2api import exception
 from ec2api.i18n import _

ec2api/clients.py

@@ -14,6 +14,9 @@
 
 from cinderclient import client as cinderclient
 from glanceclient import client as glanceclient
+from keystoneclient.auth.identity.generic import password as keystone_auth
+from keystoneclient import client as keystoneclient
+from keystoneclient import session as keystone_session
 from neutronclient.v2_0 import client as neutronclient
 from novaclient import api_versions as nova_api_versions
 from novaclient import client as novaclient
@@ -21,18 +24,32 @@ from oslo_config import cfg
 from oslo_log import log as logging
 import oslo_messaging as messaging
 
-from ec2api import context as ec2_context
-from ec2api.i18n import _LI, _LW
+from ec2api.i18n import _, _LI, _LW
 
 logger = logging.getLogger(__name__)
 
 ec2_opts = [
+    cfg.BoolOpt('ssl_insecure',
+                default=False,
+                help="Verify HTTPS connections."),
+    cfg.StrOpt('ssl_ca_file',
+               help="CA certificate file to use to verify "
+                    "connecting clients"),
     cfg.StrOpt('nova_service_type',
                default='compute',
                help='Service type of Compute API, registered in Keystone '
                     'catalog. Should be v2.1 with microversion support. '
                     'If it is obsolete v2, a lot of useful EC2 compliant '
-                    'instance properties will be unavailable.')
+                    'instance properties will be unavailable.'),
+    # TODO(andrey-mp): keystone v3 allows to pass domain_name
+    # or domain_id to auth. This code should support this feature.
+    cfg.StrOpt('admin_user',
+               help=_("Admin user to access specific cloud resourses")),
+    cfg.StrOpt('admin_password',
+               help=_("Admin password"),
+               secret=True),
+    cfg.StrOpt('admin_tenant_name',
+               help=_("Admin tenant name")),
 ]
 
 CONF = cfg.CONF
@@ -89,8 +106,8 @@ def cinder(context):
 
 
 def keystone(context):
-    keystone_client_class = ec2_context.get_keystone_client_class()
-    return keystone_client_class(session=context.session)
+    return keystoneclient.Client(auth_url=CONF.keystone_url,
+                                 session=context.session)
 
 
 def nova_cert(context):
@@ -180,5 +197,31 @@ class _rpc_RequestContextSerializer(messaging.NoOpSerializer):
     def serialize_context(self, context):
         return context.to_dict()
 
-    def deserialize_context(self, context):
-        return ec2_context.RequestContext.from_dict(context)
+
+_admin_session = None
+
+
+def get_os_admin_session():
+    """Create a context to interact with OpenStack as an administrator."""
+    # NOTE(ft): this is a singletone because keystone's session looks thread
+    # safe for both regular and token renewal requests
+    global _admin_session
+    if not _admin_session:
+        auth = keystone_auth.Password(
+            username=CONF.admin_user,
+            password=CONF.admin_password,
+            project_name=CONF.admin_tenant_name,
+            tenant_name=CONF.admin_tenant_name,
+            auth_url=CONF.keystone_url,
+        )
+        params = {'auth': auth}
+        update_request_params_with_ssl(params)
+        _admin_session = keystone_session.Session(**params)
+
+    return _admin_session
+
+
+def update_request_params_with_ssl(params):
+    verify = CONF.ssl_ca_file or not CONF.ssl_insecure
+    if verify is not True:
+        params['verify'] = verify

ec2api/context.py

@@ -14,37 +14,18 @@
 
 """RequestContext: context for requests that persist through all of ec2."""
 
-from keystoneclient.auth.identity.generic import password as keystone_auth
-from keystoneclient import client as keystone_client
-from keystoneclient import session as keystone_session
-from keystoneclient.v2_0 import client as keystone_client_v2
-from keystoneclient.v3 import client as keystone_client_v3
 from oslo_config import cfg
 from oslo_context import context
 from oslo_log import log as logging
 from oslo_utils import timeutils
 import six
 
+from ec2api import clients
 from ec2api import exception
-from ec2api.i18n import _, _LW
-from ec2api import utils
+from ec2api.i18n import _LW
 
 
-ec2_opts = [
-    cfg.StrOpt('admin_user',
-               help=_("Admin user")),
-    cfg.StrOpt('admin_password',
-               help=_("Admin password"),
-               secret=True),
-    cfg.StrOpt('admin_tenant_name',
-               help=_("Admin tenant name")),
-    # TODO(andrey-mp): keystone v3 allows to pass domain_name
-    # or domain_id to auth. This code should support this feature.
-]
-
 CONF = cfg.CONF
-CONF.register_opts(ec2_opts)
-
 LOG = logging.getLogger(__name__)
 
 
@@ -145,49 +126,6 @@ def is_user_context(context):
     return True
 
 
-_keystone_client_class = None
-
-
-def get_keystone_client_class():
-    global _keystone_client_class
-    if _keystone_client_class is None:
-        keystone = keystone_client.Client(auth_url=CONF.keystone_url)
-        if isinstance(keystone, keystone_client_v2.Client):
-            _keystone_client_class = keystone_client_v2.Client
-        elif isinstance(keystone, keystone_client_v3.Client):
-            _keystone_client_class = keystone_client_v3.Client
-        else:
-            raise exception.EC2KeystoneDiscoverFailure()
-    return _keystone_client_class
-
-
-_admin_session = None
-
-
-def get_os_admin_context():
-    """Create a context to interact with OpenStack as an administrator."""
-    # NOTE(ft): this is a singletone because keystone's session looks thread
-    # safe for both regular and token renewal requests
-    global _admin_session
-    if not _admin_session:
-        auth = keystone_auth.Password(
-            username=CONF.admin_user,
-            password=CONF.admin_password,
-            project_name=CONF.admin_tenant_name,
-            tenant_name=CONF.admin_tenant_name,
-            auth_url=CONF.keystone_url,
-        )
-        params = {'auth': auth}
-        utils.update_request_params_with_ssl(params)
-        _admin_session = keystone_session.Session(**params)
-
-    return RequestContext(
-            None, None,
-            session=_admin_session,
-            is_os_admin=True,
-            overwrite=False)
-
-
 def require_context(ctxt):
     """Raise exception.AuthFailure()
 
@@ -195,3 +133,13 @@ def require_context(ctxt):
     """
     if not ctxt.is_os_admin and not is_user_context(ctxt):
         raise exception.AuthFailure()
+
+
+def get_os_admin_context():
+    """Create a context to interact with OpenStack as an administrator."""
+    admin_session = clients.get_os_admin_session()
+    return RequestContext(
+        None, None,
+        session=admin_session,
+        is_os_admin=True,
+        overwrite=False)

ec2api/opts.py

@@ -12,7 +12,7 @@
 
 import itertools
 
-import ec2api.context
+import ec2api.clients
 import ec2api.db.api
 import ec2api.exception
 import ec2api.paths
@@ -25,7 +25,7 @@ def list_opts():
     return [
         ('DEFAULT',
          itertools.chain(
-             ec2api.context.ec2_opts,
+             ec2api.clients.ec2_opts,
              ec2api.db.api.tpool_opts,
              ec2api.exception.exc_log_opts,
              ec2api.paths.path_opts,

ec2api/tests/unit/test_clients.py

@@ -119,11 +119,10 @@ class ClientsTestCase(base.BaseTestCase):
         cinder.assert_called_with('1', service_type='volume',
                                   session=mock.sentinel.session)
 
-    @mock.patch('ec2api.context.get_keystone_client_class',
-                return_value=mock.Mock(return_value=mock.Mock()))
-    def test_keystone(self, keystone_client_class):
+    @mock.patch('keystoneclient.client.Client')
+    def test_keystone(self, keystone):
         context = mock.NonCallableMock(session=mock.sentinel.session)
         res = clients.keystone(context)
-        self.assertEqual(keystone_client_class.return_value.return_value, res)
-        keystone_client_class.return_value.assert_called_with(
-            session=mock.sentinel.session)
+        self.assertEqual(keystone.return_value, res)
+        keystone.assert_called_with(auth_url='http://localhost:5000/v2.0',
+                                    session=mock.sentinel.session)

ec2api/tests/unit/test_context.py

@@ -14,8 +14,6 @@
 
 import imp
 
-from keystoneclient.v2_0 import client as keystone_client_v2
-from keystoneclient.v3 import client as keystone_client_v3
 import mock
 from oslo_config import cfg
 from oslo_config import fixture as config_fixture
@@ -23,7 +21,6 @@ from oslo_context import context
 from oslotest import base as test_base
 
 from ec2api import context as ec2_context
-from ec2api import exception
 
 cfg.CONF.import_opt('keystone_url', 'ec2api.api')
 
@@ -67,24 +64,3 @@ class ContextTestCase(test_base.BaseTestCase):
         password_plugin.reset_mock()
         ec2_context.get_os_admin_context()
         self.assertFalse(password_plugin.called)
-
-    @mock.patch('keystoneclient.client.Client')
-    def test_get_keystone_client_class(self, client):
-        client.return_value = mock.MagicMock(spec=keystone_client_v2.Client)
-        ec2_context._keystone_client_class = None
-        client_class = ec2_context.get_keystone_client_class()
-        client.assert_called_once_with(auth_url='http://localhost:5000/v2.0')
-        self.assertEqual(keystone_client_v2.Client, client_class)
-        client.reset_mock()
-
-        client.return_value = mock.MagicMock(spec=keystone_client_v3.Client)
-        ec2_context._keystone_client_class = None
-        client_class = ec2_context.get_keystone_client_class()
-        client.assert_called_once_with(auth_url='http://localhost:5000/v2.0')
-        self.assertEqual(keystone_client_v3.Client, client_class)
-        client.reset_mock()
-
-        client.return_value = mock.MagicMock()
-        ec2_context._keystone_client_class = None
-        self.assertRaises(exception.EC2KeystoneDiscoverFailure,
-                          ec2_context.get_keystone_client_class)

ec2api/tests/unit/test_instance.py

@@ -23,8 +23,8 @@ from novaclient import exceptions as nova_exception
 from oslotest import base as test_base
 import six
 
-import ec2api.api.clients
 from ec2api.api import instance as instance_api
+import ec2api.clients
 from ec2api import exception
 from ec2api.tests.unit import base
 from ec2api.tests.unit import fakes
@@ -1990,7 +1990,7 @@ class InstancePrivateTestCase(test_base.BaseTestCase):
             search_opts={'all_tenants': True,
                          'project_id': context.project_id})
 
-    @mock.patch('ec2api.api.clients.nova', wraps=ec2api.api.clients.nova)
+    @mock.patch('ec2api.clients.nova', wraps=ec2api.clients.nova)
     @mock.patch('ec2api.context.get_os_admin_context')
     @mock.patch('cinderclient.client.Client')
     @mock.patch('novaclient.client.Client')

ec2api/utils.py

@@ -79,9 +79,3 @@ def xhtml_escape(value):
 
     """
     return saxutils.escape(value, {'"': '"', "'": '''})
-
-
-def update_request_params_with_ssl(params):
-    verify = CONF.ssl_ca_file or not CONF.ssl_insecure
-    if verify is not True:
-        params['verify'] = verify

ec2api/wsgi.py

@@ -47,12 +47,6 @@ wsgi_opts = [
              'generate log lines. The following values can be formatted '
              'into it: client_ip, date_time, request_line, status_code, '
              'body_length, wall_seconds.'),
-    cfg.BoolOpt('ssl_insecure',
-                default=False,
-                help="Verify HTTPS connections."),
-    cfg.StrOpt('ssl_ca_file',
-               help="CA certificate file to use to verify "
-                    "connecting clients"),
     cfg.StrOpt('ssl_cert_file',
                help="SSL certificate of API server"),
     cfg.StrOpt('ssl_key_file',
@@ -163,7 +157,6 @@ class Server(object):
 
         if self._use_ssl:
             try:
-                ca_file = CONF.ssl_ca_file
                 cert_file = CONF.ssl_cert_file
                 key_file = CONF.ssl_key_file
 
@@ -171,10 +164,6 @@ class Server(object):
                     raise RuntimeError(
                           _("Unable to find cert_file : %s") % cert_file)
 
-                if ca_file and not os.path.exists(ca_file):
-                    raise RuntimeError(
-                          _("Unable to find ca_file : %s") % ca_file)
-
                 if key_file and not os.path.exists(key_file):
                     raise RuntimeError(
                           _("Unable to find key_file : %s") % key_file)
@@ -191,10 +180,6 @@ class Server(object):
                     'cert_reqs': ssl.CERT_NONE,
                 }
 
-                if CONF.ssl_ca_file:
-                    ssl_kwargs['ca_certs'] = ca_file
-                    ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED
-
                 dup_socket = eventlet.wrap_ssl(dup_socket,
                                                **ssl_kwargs)
             except Exception: