diff --git a/ec2api/cmd/api.py b/ec2api/cmd/api.py index 11307ea1..9274d44d 100644 --- a/ec2api/cmd/api.py +++ b/ec2api/cmd/api.py @@ -25,17 +25,13 @@ from ec2api import config from ec2api.openstack.common import log as logging from ec2api import service -CONF = cfg.CONF -CONF.import_opt('use_ssl', 'ec2api.service') - def main(): config.parse_args(sys.argv) logging.setup('ec2api') clients.rpc_init(cfg.CONF) - server = service.WSGIService( - 'ec2api', use_ssl=CONF.use_ssl, max_url_len=16384) + server = service.WSGIService('ec2api', max_url_len=16384) service.serve(server) service.wait() diff --git a/ec2api/cmd/api_metadata.py b/ec2api/cmd/api_metadata.py new file mode 100644 index 00000000..5e090cec --- /dev/null +++ b/ec2api/cmd/api_metadata.py @@ -0,0 +1,35 @@ +# Copyright 2014 +# The Cloudscaling Group, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +EC2api API Metadata Server +""" + +import sys + +from ec2api import config +from ec2api.openstack.common import log as logging +from ec2api import service + + +def main(): + config.parse_args(sys.argv) + logging.setup("ec2api") + + server = service.WSGIService('metadata') + service.serve(server, workers=server.workers) + service.wait() + +if __name__ == '__main__': + main() diff --git a/ec2api/metadata/__init__.py b/ec2api/metadata/__init__.py new file mode 100644 index 00000000..3dcb4238 --- /dev/null +++ b/ec2api/metadata/__init__.py @@ -0,0 +1,185 @@ +# Copyright 2014 +# The Cloudscaling Group, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import hmac +import urlparse + +import httplib2 +from keystoneclient.v2_0 import client as keystone_client +from oslo.config import cfg +import webob + +from ec2api import context as ec2context +from ec2api.metadata import api +from ec2api.openstack.common import gettextutils as textutils +from ec2api.openstack.common.gettextutils import _ +from ec2api.openstack.common import log as logging +from ec2api import wsgi + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF +CONF.import_opt('use_forwarded_for', 'ec2api.api.auth') + +metadata_opts = [ + cfg.StrOpt('nova_metadata_ip', + default='127.0.0.1', + help=_("IP address used by Nova metadata server.")), + cfg.IntOpt('nova_metadata_port', + default=8775, + help=_("TCP Port used by Nova metadata server.")), + cfg.StrOpt('nova_metadata_protocol', + default='http', + choices=['http', 'https'], + help=_("Protocol to access nova metadata, http or https")), + cfg.BoolOpt('nova_metadata_insecure', + default=False, + help=_("Allow to perform insecure SSL (https) requests to " + "nova metadata")), + cfg.StrOpt('auth_ca_cert', + help=_("Certificate Authority public key (CA cert) " + "file for ssl")), + cfg.StrOpt('nova_client_cert', + default='', + help=_("Client certificate for nova metadata api server.")), + cfg.StrOpt('nova_client_priv_key', + default='', + help=_("Private key of client certificate.")), + cfg.StrOpt('admin_user', + help=_("Admin user")), + cfg.StrOpt('admin_password', + help=_("Admin password"), + secret=True), + cfg.StrOpt('admin_tenant_name', + help=_("Admin tenant name")), + cfg.StrOpt('metadata_proxy_shared_secret', + default='', + help=_('Shared secret to sign instance-id request'), + secret=True), +] + +CONF.register_opts(metadata_opts, group='metadata') + + +class MetadataRequestHandler(wsgi.Application): + """Serve metadata.""" + + @webob.dec.wsgify(RequestClass=wsgi.Request) + def __call__(self, req): + try: + LOG.debug("Request: %s", req) + + return self._proxy_request(req) + except Exception: + LOG.exception(textutils._LE("Unexpected error.")) + msg = _('An unknown error has occurred. ' + 'Please try your request again.') + return webob.exc.HTTPInternalServerError(explanation=unicode(msg)) + + def _proxy_request(self, req): + headers = self._build_proxy_request_headers(req) + if not headers: + return webob.exc.HTTPNotFound() + nova_ip_port = '%s:%s' % (CONF.metadata.nova_metadata_ip, + CONF.metadata.nova_metadata_port) + url = urlparse.urlunsplit(( + CONF.metadata.nova_metadata_protocol, + nova_ip_port, + req.path_info, + req.query_string, + '')) + + h = httplib2.Http( + ca_certs=CONF.metadata.auth_ca_cert, + disable_ssl_certificate_validation=( + CONF.metadata.nova_metadata_insecure) + ) + if (CONF.metadata.nova_client_cert and + CONF.metadata.nova_client_priv_key): + h.add_certificate(CONF.metadata.nova_client_priv_key, + CONF.metadata.nova_client_cert, + nova_ip_port) + resp, content = h.request(url, method=req.method, headers=headers, + body=req.body) + + if resp.status == 200: + LOG.debug(str(resp)) + req.response.content_type = resp['content-type'] + req.response.body = content + return req.response + elif resp.status == 403: + LOG.warn(textutils._LW( + 'The remote metadata server responded with Forbidden. This ' + 'response usually occurs when shared secrets do not match.' + )) + return webob.exc.HTTPForbidden() + elif resp.status == 400: + return webob.exc.HTTPBadRequest() + elif resp.status == 404: + return webob.exc.HTTPNotFound() + elif resp.status == 409: + return webob.exc.HTTPConflict() + elif resp.status == 500: + msg = _( + 'Remote metadata server experienced an internal server error.' + ) + LOG.warn(msg) + return webob.exc.HTTPInternalServerError(explanation=unicode(msg)) + else: + raise Exception(_('Unexpected response code: %s') % resp.status) + + def _build_proxy_request_headers(self, req): + if req.headers.get('X-Instance-ID'): + return req.headers + + instance_ip = self._get_instance_ip(req) + context = self._get_context() + instance_id, project_id = api.get_instance_and_project_id(context, + instance_ip) + if not instance_id: + return None + + return { + 'X-Forwarded-For': instance_ip, + 'X-Instance-ID': instance_id, + 'X-Tenant-ID': project_id, + 'X-Instance-ID-Signature': self._sign_instance_id(instance_id), + } + + def _get_instance_ip(self, req): + instance_ip = req.remote_addr + if CONF.use_forwarded_for: + instance_ip = req.headers.get('X-Forwarded-For', instance_ip) + return instance_ip + + def _get_context(self): + # TODO(ft): make authentification token reusable + keystone = keystone_client.Client( + username=CONF.metadata.admin_user, + password=CONF.metadata.admin_password, + tenant_name=CONF.metadata.admin_tenant_name, + auth_url=CONF.keystone_url, + ) + service_catalog = keystone.service_catalog.get_data() + return ec2context.RequestContext( + keystone.auth_user_id, + keystone.auth_tenant_id, + None, None, + auth_token=keystone.auth_token, + service_catalog=service_catalog) + + def _sign_instance_id(self, instance_id): + return hmac.new(CONF.metadata.metadata_proxy_shared_secret, + instance_id, + hashlib.sha256).hexdigest() diff --git a/ec2api/metadata/api.py b/ec2api/metadata/api.py new file mode 100644 index 00000000..f98d8eb2 --- /dev/null +++ b/ec2api/metadata/api.py @@ -0,0 +1,40 @@ +# Copyright 2014 +# The Cloudscaling Group, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools + +from novaclient import exceptions as nova_exception + +from ec2api.api import clients + + +def get_instance_and_project_id(context, fixed_ip): + nova = clients.nova(context) + try: + os_address = nova.fixed_ips.get(fixed_ip) + except nova_exception.NotFound: + return None, None + if not os_address.hostname: + return None, None + + os_instances = nova.servers.list( + search_opts={'hostname': os_address.hostname, + 'all_tenants': True}) + for os_instance in os_instances: + if any(addr['addr'] == fixed_ip and addr['OS-EXT-IPS:type'] == 'fixed' + for addr in itertools.chain( + *os_instance.addresses.itervalues())): + return os_instance.id, os_instance.tenant_id + + return None, None diff --git a/ec2api/service.py b/ec2api/service.py index 34d5164e..c8823062 100644 --- a/ec2api/service.py +++ b/ec2api/service.py @@ -28,22 +28,34 @@ from ec2api import wsgi LOG = logging.getLogger(__name__) service_opts = [ - cfg.BoolOpt('use_ssl', - default=False, - help='Enable ssl connections or not'), cfg.StrOpt('ec2api_listen', default="0.0.0.0", help='The IP address on which the EC2 API will listen.'), cfg.IntOpt('ec2api_listen_port', default=8788, help='The port on which the EC2 API will listen.'), + cfg.BoolOpt('ec2api_use_ssl', + default=False, + help='Enable ssl connections or not for EC2 API'), cfg.IntOpt('ec2api_workers', help='Number of workers for EC2 API service. The default will ' 'be equal to the number of CPUs available.'), + cfg.StrOpt('metadata_listen', + default="0.0.0.0", + help='The IP address on which the metadata API will listen.'), + cfg.IntOpt('metadata_listen_port', + default=8789, + help='The port on which the metadata API will listen.'), + cfg.BoolOpt('metadata_use_ssl', + default=False, + help='Enable ssl connections or not for EC2 API Metadata'), + cfg.IntOpt('metadata_workers', + help='Number of workers for metadata service. The default will ' + 'be the number of CPUs available.'), cfg.IntOpt('service_down_time', default=60, help='Maximum time since last check-in for up service'), - ] +] CONF = cfg.CONF CONF.register_opts(service_opts) @@ -52,7 +64,7 @@ CONF.register_opts(service_opts) class WSGIService(object): """Provides ability to launch API from a 'paste' configuration.""" - def __init__(self, name, loader=None, use_ssl=False, max_url_len=None): + def __init__(self, name, loader=None, max_url_len=None): """Initialize, but do not start the WSGI server. :param name: The name of the WSGI server given to the loader. @@ -64,9 +76,10 @@ class WSGIService(object): self.manager = self._get_manager() self.loader = loader or wsgi.Loader() self.app = self.loader.load_app(name) - self.host = getattr(CONF, 'ec2api_listen', "0.0.0.0") - self.port = getattr(CONF, 'ec2api_listen_port', 0) - self.workers = (getattr(CONF, 'ec2api_workers', None) or + self.host = getattr(CONF, '%s_listen' % name, "0.0.0.0") + self.port = getattr(CONF, '%s_listen_port' % name, 0) + self.use_ssl = getattr(CONF, '%s_use_ssl' % name, False) + self.workers = (getattr(CONF, '%s_workers' % name, None) or self.cpu_count()) if self.workers and self.workers < 1: worker_name = '%s_workers' % name @@ -75,7 +88,6 @@ class WSGIService(object): {'worker_name': worker_name, 'workers': str(self.workers)}) raise exception.InvalidInput(msg) - self.use_ssl = use_ssl self.server = wsgi.Server(name, self.app, host=self.host, diff --git a/ec2api/tests/base.py b/ec2api/tests/base.py index 4185a0c9..cde1dbe0 100644 --- a/ec2api/tests/base.py +++ b/ec2api/tests/base.py @@ -45,6 +45,7 @@ class ApiTestCase(test_base.BaseTestCase): self.nova_servers = nova_mock.return_value.servers self.nova_flavors = nova_mock.return_value.flavors self.nova_floating_ips = nova_mock.return_value.floating_ips + self.nova_fixed_ips = nova_mock.return_value.fixed_ips self.nova_key_pairs = nova_mock.return_value.keypairs self.nova_security_groups = nova_mock.return_value.security_groups self.nova_security_group_rules = ( diff --git a/ec2api/tests/test_metadata.py b/ec2api/tests/test_metadata.py new file mode 100644 index 00000000..33f72340 --- /dev/null +++ b/ec2api/tests/test_metadata.py @@ -0,0 +1,225 @@ +# Copyright 2014 +# The Cloudscaling Group, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +from oslo.config import cfg +from oslotest import base as test_base +import testtools +import webob + +from ec2api import metadata +from ec2api.tests import matchers + + +class ProxyTestCase(test_base.BaseTestCase): + + def setUp(self): + super(ProxyTestCase, self).setUp() + self.handler = metadata.MetadataRequestHandler() + conf = cfg.CONF + self.addCleanup(conf.reset) + conf.set_override('nova_metadata_ip', '9.9.9.9', group='metadata') + conf.set_override('nova_metadata_port', 8775, group='metadata') + conf.set_override('nova_metadata_protocol', 'http', group='metadata') + conf.set_override('nova_metadata_insecure', True, group='metadata') + conf.set_override('auth_ca_cert', None, group='metadata') + conf.set_override('nova_client_cert', 'nova_cert', group='metadata') + conf.set_override('nova_client_priv_key', 'nova_priv_key', + group='metadata') + conf.set_override('admin_user', 'admin', group='metadata') + conf.set_override('admin_password', 'password', group='metadata') + conf.set_override('admin_tenant_name', 'service', group='metadata') + conf.set_override('metadata_proxy_shared_secret', 'secret', + group='metadata') + + @mock.patch.object(metadata.MetadataRequestHandler, '_proxy_request') + def test_call(self, proxy): + req = mock.Mock() + proxy.return_value = 'value' + + retval = self.handler(req) + self.assertEqual(retval, 'value') + + @mock.patch.object(metadata, 'LOG') + @mock.patch.object(metadata.MetadataRequestHandler, '_proxy_request') + def test_call_internal_server_error(self, proxy, log): + req = mock.Mock() + proxy.side_effect = Exception + retval = self.handler(req) + self.assertIsInstance(retval, webob.exc.HTTPInternalServerError) + self.assertEqual(len(log.mock_calls), 2) + + @mock.patch.object(metadata.MetadataRequestHandler, + '_build_proxy_request_headers') + def _proxy_request_test_helper(self, build_headers, + response_code=200, method='GET'): + hdrs = {'X-Forwarded-For': '8.8.8.8'} + body = 'body' + + req = mock.Mock(path_info='/the_path', query_string='', headers=hdrs, + method=method, body=body) + resp = mock.MagicMock(status=response_code) + req.response = resp + build_headers.return_value = hdrs + with mock.patch('httplib2.Http') as mock_http: + resp.__getitem__.return_value = "text/plain" + mock_http.return_value.request.return_value = (resp, 'content') + + retval = self.handler._proxy_request(req) + mock_http.assert_called_once_with( + ca_certs=None, disable_ssl_certificate_validation=True) + mock_http.assert_has_calls([ + mock.call().add_certificate( + cfg.CONF.metadata.nova_client_priv_key, + cfg.CONF.metadata.nova_client_cert, + "%s:%s" % (cfg.CONF.metadata.nova_metadata_ip, + cfg.CONF.metadata.nova_metadata_port) + ), + mock.call().request( + 'http://9.9.9.9:8775/the_path', + method=method, + headers={ + 'X-Forwarded-For': '8.8.8.8', + }, + body=body + )] + ) + build_headers.assert_called_once_with(req) + + return retval + + def test_proxy_request_post(self): + response = self._proxy_request_test_helper(method='POST') + self.assertEqual(response.content_type, "text/plain") + self.assertEqual(response.body, 'content') + + def test_proxy_request_200(self): + response = self._proxy_request_test_helper(response_code=200) + self.assertEqual(response.content_type, "text/plain") + self.assertEqual(response.body, 'content') + + def test_proxy_request_400(self): + self.assertIsInstance( + self._proxy_request_test_helper(response_code=400), + webob.exc.HTTPBadRequest) + + def test_proxy_request_403(self): + self.assertIsInstance( + self._proxy_request_test_helper(response_code=403), + webob.exc.HTTPForbidden) + + def test_proxy_request_404(self): + self.assertIsInstance( + self._proxy_request_test_helper(response_code=404), + webob.exc.HTTPNotFound) + + def test_proxy_request_409(self): + self.assertIsInstance( + self._proxy_request_test_helper(response_code=409), + webob.exc.HTTPConflict) + + def test_proxy_request_500(self): + self.assertIsInstance( + self._proxy_request_test_helper(response_code=500), + webob.exc.HTTPInternalServerError) + + def test_proxy_request_other_code(self): + with testtools.ExpectedException(Exception): + self._proxy_request_test_helper(response_code=302) + + @mock.patch.object(metadata.MetadataRequestHandler, + '_build_proxy_request_headers') + def test_proxy_request_no_headers(self, build_headers): + build_headers.return_value = None + self.assertIsInstance( + self.handler._proxy_request('fake_request'), + webob.exc.HTTPNotFound) + build_headers.assert_called_once_with('fake_request') + + @mock.patch.object(metadata.MetadataRequestHandler, '_sign_instance_id') + @mock.patch.object(metadata.MetadataRequestHandler, '_get_context') + @mock.patch.object(metadata.MetadataRequestHandler, '_get_instance_ip') + def test_build_proxy_request_headers(self, get_instance_ip, get_context, + sign_instance_id): + req = mock.Mock(headers={}) + + req.headers = {'X-Instance-ID': 'fake_instance_id', + 'fake_key': 'fake_value'} + + self.assertThat(self.handler._build_proxy_request_headers(req), + matchers.DictMatches(req.headers)) + + req.headers = {'fake_key': 'fake_value'} + get_instance_ip.return_value = 'fake_instance_ip' + get_context.return_value = 'fake_context' + sign_instance_id.return_value = 'signed' + + with mock.patch('ec2api.metadata.api.' + 'get_instance_and_project_id') as get_ids: + + get_ids.return_value = None, None + self.assertIsNone(self.handler._build_proxy_request_headers(req)) + + get_ids.return_value = ('fake_instance_id', 'fake_project_id') + self.assertThat(self.handler._build_proxy_request_headers(req), + matchers.DictMatches( + {'X-Forwarded-For': 'fake_instance_ip', + 'X-Instance-ID': 'fake_instance_id', + 'X-Tenant-ID': 'fake_project_id', + 'X-Instance-ID-Signature': 'signed'})) + get_instance_ip.assert_called_with(req) + get_context.assert_called_with() + sign_instance_id.assert_called_with('fake_instance_id') + get_ids.assert_called_with('fake_context', 'fake_instance_ip') + + def test_sign_instance_id(self): + self.assertEqual( + self.handler._sign_instance_id('foo'), + '773ba44693c7553d6ee20f61ea5d2757a9a4f4a44d2841ae4e95b52e4cd62db4' + ) + + def test_get_instance_ip(self): + req = mock.Mock(remote_addr='fake_addr', headers={}) + + self.assertEqual('fake_addr', self.handler._get_instance_ip(req)) + + cfg.CONF.set_override('use_forwarded_for', True) + self.assertEqual('fake_addr', self.handler._get_instance_ip(req)) + + req.headers['X-Forwarded-For'] = 'fake_forwarded_for' + self.assertEqual('fake_forwarded_for', + self.handler._get_instance_ip(req)) + + cfg.CONF.set_override('use_forwarded_for', False) + self.assertEqual('fake_addr', self.handler._get_instance_ip(req)) + + @mock.patch('keystoneclient.v2_0.client.Client') + def test_get_context(self, keystone): + service_catalog = mock.MagicMock() + service_catalog.get_data.return_value = 'fake_service_catalog' + keystone.return_value = mock.Mock(auth_user_id='fake_user_id', + auth_tenant_id='fake_project_id', + auth_token='fake_token', + service_catalog=service_catalog) + context = self.handler._get_context() + self.assertEqual('fake_user_id', context.user_id) + self.assertEqual('fake_project_id', context.project_id) + self.assertEqual('fake_token', context.auth_token) + self.assertEqual('fake_service_catalog', context.service_catalog) + conf = cfg.CONF + keystone.assert_called_with( + username=conf.metadata.admin_user, + password=conf.metadata.admin_password, + tenant_name=conf.metadata.admin_tenant_name, + auth_url=conf.keystone_url) diff --git a/ec2api/tests/test_metadata_api.py b/ec2api/tests/test_metadata_api.py new file mode 100644 index 00000000..08b3727f --- /dev/null +++ b/ec2api/tests/test_metadata_api.py @@ -0,0 +1,56 @@ +# Copyright 2014 +# The Cloudscaling Group, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +from novaclient import exceptions as nova_exception + +from ec2api.metadata import api +from ec2api.tests import base +from ec2api.tests import fakes + + +class MetadataApiTestCase(base.ApiTestCase): + # TODO(ft): 'execute' feature isn't used here, but some mocks and + # fake context are. ApiTestCase should be split to some classes to use + # its feature optimally + + def test_get_instance_and_project_id(self): + fake_context = self._create_context() + + def check_none_result(): + self.assertEqual((None, None), + api.get_instance_and_project_id( + fake_context, + fakes.IP_NETWORK_INTERFACE_2)) + + self.nova_fixed_ips.get.return_value = mock.Mock(hostname=None) + check_none_result() + + self.nova_servers.list.return_value = [fakes.OS_INSTANCE_2] + check_none_result() + + self.nova_servers.list.return_value = [fakes.OS_INSTANCE_1, + fakes.OS_INSTANCE_2] + self.nova_fixed_ips.get.return_value = mock.Mock(hostname='fake_name') + self.assertEqual((fakes.ID_OS_INSTANCE_1, fakes.ID_OS_PROJECT), + api.get_instance_and_project_id( + fake_context, fakes.IP_NETWORK_INTERFACE_2)) + self.nova_fixed_ips.get.assert_called_with( + fakes.IP_NETWORK_INTERFACE_2) + self.nova_servers.list.assert_called_with( + search_opts={'hostname': 'fake_name', + 'all_tenants': True}) + + self.nova_fixed_ips.get.side_effect = nova_exception.NotFound('fake') + check_none_result() diff --git a/etc/ec2api/api-paste.ini b/etc/ec2api/api-paste.ini index 4f5011d3..0388923c 100644 --- a/etc/ec2api/api-paste.ini +++ b/etc/ec2api/api-paste.ini @@ -28,6 +28,19 @@ paste.filter_factory = ec2api.api:Validator.factory [app:ec2apiexecutor] paste.app_factory = ec2api.api:Executor.factory +############ +# Metadata # +############ +[composite:metadata] +use = egg:Paste#urlmap +/: meta + +[pipeline:meta] +pipeline = ec2apifaultwrap logrequest metaapp + +[app:metaapp] +paste.app_factory = ec2api.metadata:MetadataRequestHandler.factory + ########## # Shared # ##########