From 4140eb40048ab8f7ebbc684b03c64a15bb39aed9 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Mon, 4 Jan 2016 12:52:34 -0500 Subject: [PATCH] Remove Deprecated EC2 and ObjectStore impl/tests In Id7936be290b6febd18deb4c2db8ea4d678d4d9b1, we removed entries from api-paste.ini for EC2 API service. In this review we remove all the unnecessary code, docs and tests associated with objectstore and ec2 service. Note that this does not cleanup the Instance object or change any of the versioned objects. We just drop any code associated with testing the REST endpoint(s) that are no longer needed. Also added shims such that the api-paste.ini from liberty will still work (grenade job) and added logs and response messages for prompting administrators to cleanup their old api-paste.ini and switching to the stand alone EC2 API project for their needs. Change-Id: I8bf7cbaa7015bb61656ab90ccc8f944aaeebb095 --- doc/source/conf.py | 4 - doc/source/man/index.rst | 2 - doc/source/man/nova-api-ec2.rst | 48 - doc/source/man/nova-objectstore.rst | 55 - nova/api/ec2/__init__.py | 641 +--- nova/api/ec2/apirequest.py | 142 - nova/api/ec2/cloud.py | 1985 +--------- nova/api/ec2/faults.py | 73 - nova/api/ec2/inst_state.py | 56 - nova/api/opts.py | 6 - nova/cloudpipe/pipelib.py | 6 +- nova/cmd/all.py | 3 +- nova/cmd/api.py | 6 +- nova/cmd/objectstore.py | 41 - nova/objectstore/__init__.py | 24 - nova/objectstore/s3server.py | 383 -- nova/opts.py | 4 - nova/service.py | 11 - nova/tests/fixtures.py | 2 - nova/tests/unit/api/ec2/__init__.py | 0 .../unit/api/ec2/public_key/dummy.fingerprint | 1 - nova/tests/unit/api/ec2/public_key/dummy.pub | 1 - nova/tests/unit/api/ec2/test_api.py | 635 ---- nova/tests/unit/api/ec2/test_apirequest.py | 94 - nova/tests/unit/api/ec2/test_cinder_cloud.py | 1119 ------ nova/tests/unit/api/ec2/test_cloud.py | 3320 ----------------- nova/tests/unit/api/ec2/test_ec2_validate.py | 276 -- nova/tests/unit/api/ec2/test_ec2utils.py | 61 - .../tests/unit/api/ec2/test_error_response.py | 132 - nova/tests/unit/api/ec2/test_faults.py | 46 - nova/tests/unit/api/ec2/test_middleware.py | 215 -- nova/tests/unit/api/test_validator.py | 18 - nova/tests/unit/image/test_glance.py | 2 + nova/tests/unit/image/test_s3.py | 267 -- nova/tests/unit/test_bdm.py | 248 -- nova/tests/unit/test_objectstore.py | 155 - nova/utils.py | 1 - ..._and_objectstore_api-4ccb539db1d171fa.yaml | 7 + setup.cfg | 1 - tests-py3.txt | 11 - 40 files changed, 36 insertions(+), 10066 deletions(-) delete mode 100644 doc/source/man/nova-api-ec2.rst delete mode 100644 doc/source/man/nova-objectstore.rst delete mode 100644 nova/api/ec2/apirequest.py delete mode 100644 nova/api/ec2/faults.py delete mode 100644 nova/api/ec2/inst_state.py delete mode 100644 nova/cmd/objectstore.py delete mode 100644 nova/objectstore/__init__.py delete mode 100644 nova/objectstore/s3server.py delete mode 100644 nova/tests/unit/api/ec2/__init__.py delete mode 100644 nova/tests/unit/api/ec2/public_key/dummy.fingerprint delete mode 100644 nova/tests/unit/api/ec2/public_key/dummy.pub delete mode 100644 nova/tests/unit/api/ec2/test_api.py delete mode 100644 nova/tests/unit/api/ec2/test_apirequest.py delete mode 100644 nova/tests/unit/api/ec2/test_cinder_cloud.py delete mode 100644 nova/tests/unit/api/ec2/test_cloud.py delete mode 100644 nova/tests/unit/api/ec2/test_ec2_validate.py delete mode 100644 nova/tests/unit/api/ec2/test_ec2utils.py delete mode 100644 nova/tests/unit/api/ec2/test_error_response.py delete mode 100644 nova/tests/unit/api/ec2/test_faults.py delete mode 100644 nova/tests/unit/api/ec2/test_middleware.py delete mode 100644 nova/tests/unit/image/test_s3.py delete mode 100644 nova/tests/unit/test_bdm.py delete mode 100644 nova/tests/unit/test_objectstore.py create mode 100644 releasenotes/notes/remove_ec2_and_objectstore_api-4ccb539db1d171fa.yaml diff --git a/doc/source/conf.py b/doc/source/conf.py index c691fcbd63ce..36875ce0cfd1 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -111,8 +111,6 @@ modindex_common_prefix = ['nova.'] man_pages = [ ('man/nova-all', 'nova-all', u'Cloud controller fabric', [u'OpenStack'], 1), - ('man/nova-api-ec2', 'nova-api-ec2', u'Cloud controller fabric', - [u'OpenStack'], 1), ('man/nova-api-metadata', 'nova-api-metadata', u'Cloud controller fabric', [u'OpenStack'], 1), ('man/nova-api-os-compute', 'nova-api-os-compute', @@ -143,8 +141,6 @@ man_pages = [ [u'OpenStack'], 1), ('man/nova-serialproxy', 'nova-serialproxy', u'Cloud controller fabric', [u'OpenStack'], 1), - ('man/nova-objectstore', 'nova-objectstore', u'Cloud controller fabric', - [u'OpenStack'], 1), ('man/nova-rootwrap', 'nova-rootwrap', u'Cloud controller fabric', [u'OpenStack'], 1), ('man/nova-scheduler', 'nova-scheduler', u'Cloud controller fabric', diff --git a/doc/source/man/index.rst b/doc/source/man/index.rst index a9191cccb319..6b9a7bec9bab 100644 --- a/doc/source/man/index.rst +++ b/doc/source/man/index.rst @@ -26,7 +26,6 @@ Reference :maxdepth: 1 nova-all - nova-api-ec2 nova-api-metadata nova-api-os-compute nova-api @@ -41,7 +40,6 @@ Reference nova-manage nova-network nova-novncproxy - nova-objectstore nova-rootwrap nova-scheduler nova-spicehtml5proxy diff --git a/doc/source/man/nova-api-ec2.rst b/doc/source/man/nova-api-ec2.rst deleted file mode 100644 index 8d357017f3d0..000000000000 --- a/doc/source/man/nova-api-ec2.rst +++ /dev/null @@ -1,48 +0,0 @@ -============ -nova-api-ec2 -============ - ----------------------------- -Server for the Nova EC2 API ----------------------------- - -:Author: openstack@lists.openstack.org -:Date: 2012-09-27 -:Copyright: OpenStack Foundation -:Version: 2012.1 -:Manual section: 1 -:Manual group: cloud computing - -SYNOPSIS -======== - - nova-api-ec2 [options] - -DESCRIPTION -=========== - -nova-api-ec2 is a server daemon that serves the Nova EC2 API - -OPTIONS -======= - - **General options** - -FILES -======== - -* /etc/nova/nova.conf -* /etc/nova/api-paste.ini -* /etc/nova/policy.json -* /etc/nova/rootwrap.conf -* /etc/nova/rootwrap.d/ - -SEE ALSO -======== - -* `OpenStack Nova `__ - -BUGS -==== - -* Nova bugs are managed at Launchpad `Bugs : Nova `__ diff --git a/doc/source/man/nova-objectstore.rst b/doc/source/man/nova-objectstore.rst deleted file mode 100644 index 0048342656b4..000000000000 --- a/doc/source/man/nova-objectstore.rst +++ /dev/null @@ -1,55 +0,0 @@ -================ -nova-objectstore -================ - ------------------------------ -Nova Objectstore Server ------------------------------ - -:Author: openstack@lists.openstack.org -:Date: 2012-09-27 -:Copyright: OpenStack Foundation -:Version: 2012.1 -:Manual section: 1 -:Manual group: cloud computing - -SYNOPSIS -======== - - nova-objectstore [options] - -DESCRIPTION -=========== - -Implementation of an S3-like storage server based on local files. - -Useful to test features that will eventually run on S3, or if you want to -run something locally that was once running on S3. - -We don't support all the features of S3, but it does work with the -standard S3 client for the most basic semantics. - -Used for testing when do not have OpenStack Swift installed. - -OPTIONS -======= - - **General options** - -FILES -======== - -* /etc/nova/nova.conf -* /etc/nova/policy.json -* /etc/nova/rootwrap.conf -* /etc/nova/rootwrap.d/ - -SEE ALSO -======== - -* `OpenStack Nova `__ - -BUGS -==== - -* Nova bugs are managed at Launchpad `Bugs : Nova `__ diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 2cee009c81dc..95792796fa56 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -13,640 +13,37 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -""" -Starting point for routing EC2 requests. -""" - -import hashlib - -from oslo_config import cfg -from oslo_context import context as common_context -from oslo_log import log as logging -from oslo_serialization import jsonutils -from oslo_service import sslutils -from oslo_utils import importutils -from oslo_utils import netutils -from oslo_utils import timeutils -import requests -import six -import webob import webob.dec import webob.exc -from nova.api.ec2 import apirequest -from nova.api.ec2 import ec2utils -from nova.api.ec2 import faults -from nova.api import validator -from nova import context -from nova import exception -from nova.i18n import _ -from nova.i18n import _LE -from nova.i18n import _LI -from nova.i18n import _LW -from nova.openstack.common import memorycache from nova import wsgi - -LOG = logging.getLogger(__name__) - -ec2_opts = [ - cfg.IntOpt('lockout_attempts', - default=5, - help='Number of failed auths before lockout.'), - cfg.IntOpt('lockout_minutes', - default=15, - help='Number of minutes to lockout if triggered.'), - cfg.IntOpt('lockout_window', - default=15, - help='Number of minutes for lockout window.'), - cfg.StrOpt('keystone_ec2_url', - default='http://localhost:5000/v2.0/ec2tokens', - help='URL to get token from ec2 request.'), - cfg.BoolOpt('ec2_private_dns_show_ip', - default=False, - help='Return the IP address as private dns hostname in ' - 'describe instances'), - cfg.BoolOpt('ec2_strict_validation', - default=True, - help='Validate security group names' - ' according to EC2 specification'), - cfg.IntOpt('ec2_timestamp_expiry', - default=300, - help='Time in seconds before ec2 timestamp expires'), - cfg.BoolOpt('keystone_ec2_insecure', default=False, help='Disable SSL ' - 'certificate verification.'), - ] - -CONF = cfg.CONF -CONF.register_opts(ec2_opts) -CONF.import_opt('use_forwarded_for', 'nova.api.auth') -sslutils.is_enabled(CONF) +_DEPRECATION_MESSAGE = ('The in tree EC2 API has been removed in Mitaka. ' + 'Please remove entries from api-paste.ini') -# Fault Wrapper around all EC2 requests -class FaultWrapper(wsgi.Middleware): - """Calls the middleware stack, captures any exceptions into faults.""" +class DeprecatedMiddleware(wsgi.Middleware): + def __init__(self, *args, **kwargs): + super(DeprecatedMiddleware, self).__init__(args[0]) @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): - try: - return req.get_response(self.application) - except Exception: - LOG.exception(_LE("FaultWrapper error")) - return faults.Fault(webob.exc.HTTPInternalServerError()) + return webob.exc.HTTPException(message=_DEPRECATION_MESSAGE) -class RequestLogging(wsgi.Middleware): - """Access-Log akin logging for all EC2 API requests.""" - +class DeprecatedApplication(wsgi.Application): @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): - start = timeutils.utcnow() - rv = req.get_response(self.application) - self.log_request_completion(rv, req, start) - return rv - - def log_request_completion(self, response, request, start): - apireq = request.environ.get('ec2.request', None) - if apireq: - controller = apireq.controller - action = apireq.action - else: - controller = None - action = None - ctxt = request.environ.get('nova.context', None) - delta = timeutils.utcnow() - start - seconds = delta.seconds - microseconds = delta.microseconds - LOG.info( - "%s.%ss %s %s %s %s:%s %s [%s] %s %s", - seconds, - microseconds, - request.remote_addr, - request.method, - "%s%s" % (request.script_name, request.path_info), - controller, - action, - response.status_int, - request.user_agent, - request.content_type, - response.content_type, - context=ctxt) # noqa - - -class Lockout(wsgi.Middleware): - """Lockout for x minutes on y failed auths in a z minute period. - - x = lockout_timeout flag - y = lockout_window flag - z = lockout_attempts flag - - Uses memcached if lockout_memcached_servers flag is set, otherwise it - uses a very simple in-process cache. Due to the simplicity of - the implementation, the timeout window is started with the first - failed request, so it will block if there are x failed logins within - that period. - - There is a possible race condition where simultaneous requests could - sneak in before the lockout hits, but this is extremely rare and would - only result in a couple of extra failed attempts. - """ - - def __init__(self, application): - """middleware can use fake for testing.""" - self.mc = memorycache.get_client() - super(Lockout, self).__init__(application) - - @webob.dec.wsgify(RequestClass=wsgi.Request) - def __call__(self, req): - access_key = str(req.params['AWSAccessKeyId']) - failures_key = "authfailures-%s" % access_key - failures = int(self.mc.get(failures_key) or 0) - if failures >= CONF.lockout_attempts: - detail = _("Too many failed authentications.") - raise webob.exc.HTTPForbidden(explanation=detail) - res = req.get_response(self.application) - if res.status_int == 403: - failures = self.mc.incr(failures_key) - if failures is None: - # NOTE(vish): To use incr, failures has to be a string. - self.mc.set(failures_key, '1', time=CONF.lockout_window * 60) - elif failures >= CONF.lockout_attempts: - LOG.warning(_LW('Access key %(access_key)s has had ' - '%(failures)d failed authentications and ' - 'will be locked out for %(lock_mins)d ' - 'minutes.'), - {'access_key': access_key, - 'failures': failures, - 'lock_mins': CONF.lockout_minutes}) - self.mc.set(failures_key, str(failures), - time=CONF.lockout_minutes * 60) - return res - - -class EC2KeystoneAuth(wsgi.Middleware): - """Authenticate an EC2 request with keystone and convert to context.""" - - def _get_signature(self, req): - """Extract the signature from the request. - - This can be a get/post variable or for version 4 also in a header - called 'Authorization'. - - params['Signature'] == version 0,1,2,3 - - params['X-Amz-Signature'] == version 4 - - header 'Authorization' == version 4 - """ - sig = req.params.get('Signature') or req.params.get('X-Amz-Signature') - if sig is None and 'Authorization' in req.headers: - auth_str = req.headers['Authorization'] - sig = auth_str.partition("Signature=")[2].split(',')[0] - - return sig - - def _get_access(self, req): - """Extract the access key identifier. - - For version 0/1/2/3 this is passed as the AccessKeyId parameter, for - version 4 it is either an X-Amz-Credential parameter or a Credential= - field in the 'Authorization' header string. - """ - access = req.params.get('AWSAccessKeyId') - if access is None: - cred_param = req.params.get('X-Amz-Credential') - if cred_param: - access = cred_param.split("/")[0] - - if access is None and 'Authorization' in req.headers: - auth_str = req.headers['Authorization'] - cred_str = auth_str.partition("Credential=")[2].split(',')[0] - access = cred_str.split("/")[0] - - return access - - @webob.dec.wsgify(RequestClass=wsgi.Request) - def __call__(self, req): - # NOTE(alevine) We need to calculate the hash here because - # subsequent access to request modifies the req.body so the hash - # calculation will yield invalid results. - body_hash = hashlib.sha256(req.body).hexdigest() - - request_id = common_context.generate_request_id() - signature = self._get_signature(req) - if not signature: - msg = _("Signature not provided") - return faults.ec2_error_response(request_id, "AuthFailure", msg, - status=400) - access = self._get_access(req) - if not access: - msg = _("Access key not provided") - return faults.ec2_error_response(request_id, "AuthFailure", msg, - status=400) - - if 'X-Amz-Signature' in req.params or 'Authorization' in req.headers: - auth_params = {} - else: - # Make a copy of args for authentication and signature verification - auth_params = dict(req.params) - # Not part of authentication args - auth_params.pop('Signature', None) - - cred_dict = { - 'access': access, - 'signature': signature, - 'host': req.host, - 'verb': req.method, - 'path': req.path, - 'params': auth_params, - 'headers': req.headers, - 'body_hash': body_hash - } - if "ec2" in CONF.keystone_ec2_url: - creds = {'ec2Credentials': cred_dict} - else: - creds = {'auth': {'OS-KSEC2:ec2Credentials': cred_dict}} - creds_json = jsonutils.dumps(creds) - headers = {'Content-Type': 'application/json'} - - verify = not CONF.keystone_ec2_insecure - if verify and CONF.ssl.ca_file: - verify = CONF.ssl.ca_file - - cert = None - if CONF.ssl.cert_file and CONF.ssl.key_file: - cert = (CONF.ssl.cert_file, CONF.ssl.key_file) - elif CONF.ssl.cert_file: - cert = CONF.ssl.cert_file - - response = requests.request('POST', CONF.keystone_ec2_url, - data=creds_json, headers=headers, - verify=verify, cert=cert) - status_code = response.status_code - if status_code != 200: - msg = response.reason - return faults.ec2_error_response(request_id, "AuthFailure", msg, - status=status_code) - result = response.json() - - try: - token_id = result['access']['token']['id'] - user_id = result['access']['user']['id'] - project_id = result['access']['token']['tenant']['id'] - user_name = result['access']['user'].get('name') - project_name = result['access']['token']['tenant'].get('name') - roles = [role['name'] for role - in result['access']['user']['roles']] - except (AttributeError, KeyError) as e: - LOG.error(_LE("Keystone failure: %s"), e) - msg = _("Failure parsing response from keystone: %s") % e - return faults.ec2_error_response(request_id, "AuthFailure", msg, - status=400) - - remote_address = req.remote_addr - if CONF.use_forwarded_for: - remote_address = req.headers.get('X-Forwarded-For', - remote_address) - - catalog = result['access']['serviceCatalog'] - ctxt = context.RequestContext(user_id, - project_id, - user_name=user_name, - project_name=project_name, - roles=roles, - auth_token=token_id, - remote_address=remote_address, - service_catalog=catalog) - - req.environ['nova.context'] = ctxt - - return self.application - - -class NoAuth(wsgi.Middleware): - """Add user:project as 'nova.context' to WSGI environ.""" - - @webob.dec.wsgify(RequestClass=wsgi.Request) - def __call__(self, req): - if 'AWSAccessKeyId' not in req.params: - raise webob.exc.HTTPBadRequest() - user_id, _sep, project_id = req.params['AWSAccessKeyId'].partition(':') - project_id = project_id or user_id - remote_address = req.remote_addr - if CONF.use_forwarded_for: - remote_address = req.headers.get('X-Forwarded-For', remote_address) - ctx = context.RequestContext(user_id, - project_id, - is_admin=True, - remote_address=remote_address) - - req.environ['nova.context'] = ctx - return self.application - - -class Requestify(wsgi.Middleware): - - def __init__(self, app, controller): - super(Requestify, self).__init__(app) - self.controller = importutils.import_object(controller) - - @webob.dec.wsgify(RequestClass=wsgi.Request) - def __call__(self, req): - # Not all arguments are mandatory with v4 signatures, as some data is - # passed in the header, not query arguments. - required_args = ['Action', 'Version'] - non_args = ['Action', 'Signature', 'AWSAccessKeyId', 'SignatureMethod', - 'SignatureVersion', 'Version', 'Timestamp'] - args = dict(req.params) - try: - expired = ec2utils.is_ec2_timestamp_expired(req.params, - expires=CONF.ec2_timestamp_expiry) - if expired: - msg = _("Timestamp failed validation.") - LOG.debug("Timestamp failed validation") - raise webob.exc.HTTPForbidden(explanation=msg) - - # Raise KeyError if omitted - action = req.params['Action'] - # Fix bug lp:720157 for older (version 1) clients - # If not present assume v4 - version = req.params.get('SignatureVersion', 4) - if int(version) == 1: - non_args.remove('SignatureMethod') - if 'SignatureMethod' in args: - args.pop('SignatureMethod') - for non_arg in non_args: - if non_arg in required_args: - # Remove, but raise KeyError if omitted - args.pop(non_arg) - else: - args.pop(non_arg, None) - except KeyError: - raise webob.exc.HTTPBadRequest() - except exception.InvalidRequest as err: - raise webob.exc.HTTPBadRequest(explanation=six.text_type(err)) - - LOG.debug('action: %s', action) - for key, value in args.items(): - LOG.debug('arg: %(key)s\t\tval: %(value)s', - {'key': key, 'value': value}) - - # Success! - api_request = apirequest.APIRequest(self.controller, action, - req.params['Version'], args) - req.environ['ec2.request'] = api_request - return self.application - - -class Authorizer(wsgi.Middleware): - - """Authorize an EC2 API request. - - Return a 401 if ec2.controller and ec2.action in WSGI environ may not be - executed in nova.context. - """ - - def __init__(self, application): - super(Authorizer, self).__init__(application) - self.action_roles = { - 'CloudController': { - 'DescribeAvailabilityZones': ['all'], - 'DescribeRegions': ['all'], - 'DescribeSnapshots': ['all'], - 'DescribeKeyPairs': ['all'], - 'CreateKeyPair': ['all'], - 'DeleteKeyPair': ['all'], - 'DescribeSecurityGroups': ['all'], - 'ImportKeyPair': ['all'], - 'AuthorizeSecurityGroupIngress': ['netadmin'], - 'RevokeSecurityGroupIngress': ['netadmin'], - 'CreateSecurityGroup': ['netadmin'], - 'DeleteSecurityGroup': ['netadmin'], - 'GetConsoleOutput': ['projectmanager', 'sysadmin'], - 'DescribeVolumes': ['projectmanager', 'sysadmin'], - 'CreateVolume': ['projectmanager', 'sysadmin'], - 'AttachVolume': ['projectmanager', 'sysadmin'], - 'DetachVolume': ['projectmanager', 'sysadmin'], - 'DescribeInstances': ['all'], - 'DescribeAddresses': ['all'], - 'AllocateAddress': ['netadmin'], - 'ReleaseAddress': ['netadmin'], - 'AssociateAddress': ['netadmin'], - 'DisassociateAddress': ['netadmin'], - 'RunInstances': ['projectmanager', 'sysadmin'], - 'TerminateInstances': ['projectmanager', 'sysadmin'], - 'RebootInstances': ['projectmanager', 'sysadmin'], - 'UpdateInstance': ['projectmanager', 'sysadmin'], - 'StartInstances': ['projectmanager', 'sysadmin'], - 'StopInstances': ['projectmanager', 'sysadmin'], - 'DeleteVolume': ['projectmanager', 'sysadmin'], - 'DescribeImages': ['all'], - 'DeregisterImage': ['projectmanager', 'sysadmin'], - 'RegisterImage': ['projectmanager', 'sysadmin'], - 'DescribeImageAttribute': ['all'], - 'ModifyImageAttribute': ['projectmanager', 'sysadmin'], - 'UpdateImage': ['projectmanager', 'sysadmin'], - 'CreateImage': ['projectmanager', 'sysadmin'], - }, - 'AdminController': { - # All actions have the same permission: ['none'] (the default) - # superusers will be allowed to run them - # all others will get HTTPUnauthorized. - }, - } - - @webob.dec.wsgify(RequestClass=wsgi.Request) - def __call__(self, req): - context = req.environ['nova.context'] - controller = req.environ['ec2.request'].controller.__class__.__name__ - action = req.environ['ec2.request'].action - allowed_roles = self.action_roles[controller].get(action, ['none']) - if self._matches_any_role(context, allowed_roles): - return self.application - else: - LOG.info(_LI('Unauthorized request for controller=%(controller)s ' - 'and action=%(action)s'), - {'controller': controller, 'action': action}, - context=context) - raise webob.exc.HTTPUnauthorized() - - def _matches_any_role(self, context, roles): - """Return True if any role in roles is allowed in context.""" - if context.is_admin: - return True - if 'all' in roles: - return True - if 'none' in roles: - return False - return any(role in context.roles for role in roles) - - -class Validator(wsgi.Middleware): - - def validate_ec2_id(val): - if not validator.validate_str()(val): - return False - try: - ec2utils.ec2_id_to_id(val) - except exception.InvalidEc2Id: - return False - return True - - validator.validate_ec2_id = validate_ec2_id - - validator.DEFAULT_VALIDATOR = { - 'instance_id': validator.validate_ec2_id, - 'volume_id': validator.validate_ec2_id, - 'image_id': validator.validate_ec2_id, - 'attribute': validator.validate_str(), - 'image_location': validator.validate_image_path, - 'public_ip': netutils.is_valid_ipv4, - 'region_name': validator.validate_str(), - 'group_name': validator.validate_str(max_length=255), - 'group_description': validator.validate_str(max_length=255), - 'size': validator.validate_int(), - 'user_data': validator.validate_user_data - } - - def __init__(self, application): - super(Validator, self).__init__(application) - - @webob.dec.wsgify(RequestClass=wsgi.Request) - def __call__(self, req): - if validator.validate(req.environ['ec2.request'].args, - validator.DEFAULT_VALIDATOR): - return self.application - else: - raise webob.exc.HTTPBadRequest() - - -def exception_to_ec2code(ex): - """Helper to extract EC2 error code from exception. - - For other than EC2 exceptions (those without ec2_code attribute), - use exception name. - """ - if hasattr(ex, 'ec2_code'): - code = ex.ec2_code - else: - code = type(ex).__name__ - return code - - -def ec2_error_ex(ex, req, code=None, message=None, unexpected=False): - """Return an EC2 error response based on passed exception and log - the exception on an appropriate log level: - - * DEBUG: expected errors - * ERROR: unexpected errors - - All expected errors are treated as client errors and 4xx HTTP - status codes are always returned for them. - - Unexpected 5xx errors may contain sensitive information, - suppress their messages for security. - """ - if not code: - code = exception_to_ec2code(ex) - status = getattr(ex, 'code', None) - if not status: - status = 500 - - if unexpected: - log_fun = LOG.error - log_msg = _LE("Unexpected %(ex_name)s raised: %(ex_str)s") - else: - log_fun = LOG.debug - log_msg = "%(ex_name)s raised: %(ex_str)s" - # NOTE(jruzicka): For compatibility with EC2 API, treat expected - # exceptions as client (4xx) errors. The exception error code is 500 - # by default and most exceptions inherit this from NovaException even - # though they are actually client errors in most cases. - if status >= 500: - status = 400 - - context = req.environ['nova.context'] - request_id = context.request_id - log_msg_args = { - 'ex_name': type(ex).__name__, - 'ex_str': ex - } - log_fun(log_msg, log_msg_args, context=context) - - if ex.args and not message and (not unexpected or status < 500): - message = six.text_type(ex.args[0]) - if unexpected: - # Log filtered environment for unexpected errors. - env = req.environ.copy() - for k in list(env.keys()): - if not isinstance(env[k], six.string_types): - env.pop(k) - log_fun(_LE('Environment: %s'), jsonutils.dumps(env)) - if not message: - message = _('Unknown error occurred.') - return faults.ec2_error_response(request_id, code, message, status=status) - - -class Executor(wsgi.Application): - - """Execute an EC2 API request. - - Executes 'ec2.action' upon 'ec2.controller', passing 'nova.context' and - 'ec2.action_args' (all variables in WSGI environ.) Returns an XML - response, or a 400 upon failure. - """ - - @webob.dec.wsgify(RequestClass=wsgi.Request) - def __call__(self, req): - context = req.environ['nova.context'] - api_request = req.environ['ec2.request'] - try: - result = api_request.invoke(context) - except exception.InstanceNotFound as ex: - ec2_id = ec2utils.id_to_ec2_inst_id(ex.kwargs['instance_id']) - message = ex.msg_fmt % {'instance_id': ec2_id} - return ec2_error_ex(ex, req, message=message) - except exception.VolumeNotFound as ex: - ec2_id = ec2utils.id_to_ec2_vol_id(ex.kwargs['volume_id']) - message = ex.msg_fmt % {'volume_id': ec2_id} - return ec2_error_ex(ex, req, message=message) - except exception.SnapshotNotFound as ex: - ec2_id = ec2utils.id_to_ec2_snap_id(ex.kwargs['snapshot_id']) - message = ex.msg_fmt % {'snapshot_id': ec2_id} - return ec2_error_ex(ex, req, message=message) - except (exception.CannotDisassociateAutoAssignedFloatingIP, - exception.FloatingIpAssociated, - exception.FloatingIpNotFound, - exception.FloatingIpBadRequest, - exception.ImageNotActive, - exception.InvalidInstanceIDMalformed, - exception.InvalidVolumeIDMalformed, - exception.InvalidKeypair, - exception.InvalidParameterValue, - exception.InvalidPortRange, - exception.InvalidVolume, - exception.KeyPairExists, - exception.KeypairNotFound, - exception.MissingParameter, - exception.NoFloatingIpInterface, - exception.NoMoreFixedIps, - exception.Forbidden, - exception.QuotaError, - exception.SecurityGroupExists, - exception.SecurityGroupLimitExceeded, - exception.SecurityGroupRuleExists, - exception.VolumeUnattached, - # Following aren't translated to valid EC2 errors. - exception.ImageNotFound, - exception.ImageNotFoundEC2, - exception.InvalidAttribute, - exception.InvalidRequest, - exception.NotFound) as ex: - return ec2_error_ex(ex, req) - except Exception as ex: - return ec2_error_ex(ex, req, unexpected=True) - else: - resp = webob.Response() - resp.status = 200 - resp.headers['Content-Type'] = 'text/xml' - resp.body = str(result) - return resp + return webob.exc.HTTPException(message=_DEPRECATION_MESSAGE) + + +FaultWrapper = DeprecatedMiddleware +RequestLogging = DeprecatedMiddleware +Lockout = DeprecatedMiddleware +EC2KeystoneAuth = DeprecatedMiddleware +NoAuth = DeprecatedMiddleware +Requestify = DeprecatedMiddleware +Authorizer = DeprecatedMiddleware +Validator = DeprecatedMiddleware +Executor = DeprecatedApplication diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py deleted file mode 100644 index 3fd505e37371..000000000000 --- a/nova/api/ec2/apirequest.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -APIRequest class -""" - -import datetime -# TODO(termie): replace minidom with etree -from xml.dom import minidom - -from lxml import etree -from oslo_log import log as logging -from oslo_utils import encodeutils -import six - -from nova.api.ec2 import ec2utils -from nova import exception - -LOG = logging.getLogger(__name__) - - -def _underscore_to_camelcase(str): - return ''.join([x[:1].upper() + x[1:] for x in str.split('_')]) - - -def _underscore_to_xmlcase(str): - res = _underscore_to_camelcase(str) - return res[:1].lower() + res[1:] - - -def _database_to_isoformat(datetimeobj): - """Return a xs:dateTime parsable string from datatime.""" - return datetimeobj.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + 'Z' - - -class APIRequest(object): - def __init__(self, controller, action, version, args): - self.controller = controller - self.action = action - self.version = version - self.args = args - - def invoke(self, context): - try: - method = getattr(self.controller, - ec2utils.camelcase_to_underscore(self.action)) - except AttributeError: - LOG.debug('Unsupported API request: controller = ' - '%(controller)s, action = %(action)s', - {'controller': self.controller, - 'action': self.action}) - # TODO(gundlach): Raise custom exception, trap in apiserver, - # and reraise as 400 error. - raise exception.InvalidRequest() - - args = ec2utils.dict_from_dotted_str(self.args.items()) - - for key in args.keys(): - # NOTE(vish): Turn numeric dict keys into lists - if isinstance(args[key], dict): - if args[key] != {} and list(args[key].keys())[0].isdigit(): - s = args[key].items() - s.sort() - args[key] = [v for k, v in s] - - result = method(context, **args) - return self._render_response(result, context.request_id) - - def _render_response(self, response_data, request_id): - xml = minidom.Document() - - response_el = xml.createElement(self.action + 'Response') - response_el.setAttribute('xmlns', - 'http://ec2.amazonaws.com/doc/%s/' % self.version) - request_id_el = xml.createElement('requestId') - request_id_el.appendChild(xml.createTextNode(request_id)) - response_el.appendChild(request_id_el) - if response_data is True: - self._render_dict(xml, response_el, {'return': 'true'}) - else: - self._render_dict(xml, response_el, response_data) - - xml.appendChild(response_el) - - response = xml.toxml() - root = etree.fromstring(response) - response = etree.tostring(root, pretty_print=True) - - xml.unlink() - - # Don't write private key to log - if self.action != "CreateKeyPair": - LOG.debug(response) - else: - LOG.debug("CreateKeyPair: Return Private Key") - - return response - - def _render_dict(self, xml, el, data): - try: - for key in data.keys(): - val = data[key] - el.appendChild(self._render_data(xml, key, val)) - except Exception: - LOG.debug(data) - raise - - def _render_data(self, xml, el_name, data): - el_name = _underscore_to_xmlcase(el_name) - data_el = xml.createElement(el_name) - - if isinstance(data, list): - for item in data: - data_el.appendChild(self._render_data(xml, 'item', item)) - elif isinstance(data, dict): - self._render_dict(xml, data_el, data) - elif hasattr(data, '__dict__'): - self._render_dict(xml, data_el, data.__dict__) - elif isinstance(data, bool): - data_el.appendChild(xml.createTextNode(str(data).lower())) - elif isinstance(data, datetime.datetime): - data_el.appendChild( - xml.createTextNode(_database_to_isoformat(data))) - elif data is not None: - data_el.appendChild(xml.createTextNode( - encodeutils.safe_encode(six.text_type(data)))) - - return data_el diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index f7360609ceab..893227e030ef 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -14,1995 +14,20 @@ # License for the specific language governing permissions and limitations # under the License. -""" -Cloud Controller: Implementation of EC2 REST API calls, which are -dispatched to other nodes via AMQP RPC. State is via distributed -datastore. -""" - -import base64 -import time - -from oslo_config import cfg from oslo_log import log as logging from oslo_log import versionutils -from oslo_utils import timeutils -import six -from nova.api.ec2 import ec2utils -from nova.api.ec2 import inst_state -from nova.api.metadata import password -from nova.api.openstack import extensions -from nova.api import validator -from nova import availability_zones -from nova import block_device -from nova.cloudpipe import pipelib -from nova import compute -from nova.compute import api as compute_api -from nova.compute import vm_states -from nova import exception -from nova.i18n import _ -from nova.i18n import _LI from nova.i18n import _LW -from nova.image import s3 -from nova import network -from nova.network.security_group import neutron_driver -from nova.network.security_group import openstack_driver -from nova import objects -from nova import quota -from nova import servicegroup -from nova import utils -from nova import volume - -ec2_opts = [ - cfg.StrOpt('ec2_host', - default='$my_ip', - help='The IP address of the EC2 API server'), - cfg.StrOpt('ec2_dmz_host', - default='$my_ip', - help='The internal IP address of the EC2 API server'), - cfg.IntOpt('ec2_port', - default=8773, - min=1, - max=65535, - help='The port of the EC2 API server'), - cfg.StrOpt('ec2_scheme', - default='http', - choices=('http', 'https'), - help='The protocol to use when connecting to the EC2 API ' - 'server'), - cfg.StrOpt('ec2_path', - default='/', - help='The path prefix used to call the ec2 API server'), - cfg.ListOpt('region_list', - default=[], - help='List of region=fqdn pairs separated by commas'), -] - -CONF = cfg.CONF -CONF.register_opts(ec2_opts) -CONF.import_opt('my_ip', 'nova.netconf') -CONF.import_opt('vpn_key_suffix', 'nova.cloudpipe.pipelib') -CONF.import_opt('internal_service_availability_zone', - 'nova.availability_zones') LOG = logging.getLogger(__name__) -QUOTAS = quota.QUOTAS - - -# EC2 ID can return the following error codes: -# http://docs.aws.amazon.com/AWSEC2/latest/APIReference/api-error-codes.html -# Validate methods are split to return valid EC2 error codes for different -# resource types -def _validate_ec2_id(val): - if not validator.validate_str()(val): - raise exception.InvalidEc2Id(ec2_id=val) - ec2utils.ec2_id_to_id(val) - - -def validate_volume_id(volume_id): - try: - _validate_ec2_id(volume_id) - except exception.InvalidEc2Id: - raise exception.InvalidVolumeIDMalformed(volume_id=volume_id) - - -def validate_instance_id(instance_id): - try: - _validate_ec2_id(instance_id) - except exception.InvalidEc2Id: - raise exception.InvalidInstanceIDMalformed(instance_id=instance_id) - - -# EC2 API can return the following values as documented in the EC2 API -# http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ -# ApiReference-ItemType-InstanceStateType.html -# pending 0 | running 16 | shutting-down 32 | terminated 48 | stopping 64 | -# stopped 80 -_STATE_DESCRIPTION_MAP = { - None: inst_state.PENDING, - vm_states.ACTIVE: inst_state.RUNNING, - vm_states.BUILDING: inst_state.PENDING, - vm_states.DELETED: inst_state.TERMINATED, - vm_states.SOFT_DELETED: inst_state.TERMINATED, - vm_states.STOPPED: inst_state.STOPPED, - vm_states.PAUSED: inst_state.PAUSE, - vm_states.SUSPENDED: inst_state.SUSPEND, - vm_states.RESCUED: inst_state.RESCUE, - vm_states.RESIZED: inst_state.RESIZE, -} - - -def _state_description(vm_state, _shutdown_terminate): - """Map the vm state to the server status string.""" - # Note(maoy): We do not provide EC2 compatibility - # in shutdown_terminate flag behavior. So we ignore - # it here. - name = _STATE_DESCRIPTION_MAP.get(vm_state, vm_state) - - return {'code': inst_state.name_to_code(name), - 'name': name} - - -def _parse_block_device_mapping(bdm): - """Parse BlockDeviceMappingItemType into flat hash - BlockDevicedMapping..DeviceName - BlockDevicedMapping..Ebs.SnapshotId - BlockDevicedMapping..Ebs.VolumeSize - BlockDevicedMapping..Ebs.DeleteOnTermination - BlockDevicedMapping..Ebs.NoDevice - BlockDevicedMapping..VirtualName - => remove .Ebs and allow volume id in SnapshotId - """ - ebs = bdm.pop('ebs', None) - if ebs: - ec2_id = ebs.pop('snapshot_id', None) - if ec2_id: - if ec2_id.startswith('snap-'): - bdm['snapshot_id'] = ec2utils.ec2_snap_id_to_uuid(ec2_id) - elif ec2_id.startswith('vol-'): - bdm['volume_id'] = ec2utils.ec2_vol_id_to_uuid(ec2_id) - ebs.setdefault('delete_on_termination', True) - bdm.update(ebs) - return bdm - - -def _properties_get_mappings(properties): - return block_device.mappings_prepend_dev(properties.get('mappings', [])) - - -def _format_block_device_mapping(bdm): - """Construct BlockDeviceMappingItemType - {'device_name': '...', 'snapshot_id': , ...} - => BlockDeviceMappingItemType - """ - keys = (('deviceName', 'device_name'), - ('virtualName', 'virtual_name')) - item = {} - for name, k in keys: - if k in bdm: - item[name] = bdm[k] - if bdm.get('no_device'): - item['noDevice'] = True - if ('snapshot_id' in bdm) or ('volume_id' in bdm): - ebs_keys = (('snapshotId', 'snapshot_id'), - ('snapshotId', 'volume_id'), # snapshotId is abused - ('volumeSize', 'volume_size'), - ('deleteOnTermination', 'delete_on_termination')) - ebs = {} - for name, k in ebs_keys: - if bdm.get(k) is not None: - if k == 'snapshot_id': - ebs[name] = ec2utils.id_to_ec2_snap_id(bdm[k]) - elif k == 'volume_id': - ebs[name] = ec2utils.id_to_ec2_vol_id(bdm[k]) - else: - ebs[name] = bdm[k] - assert 'snapshotId' in ebs - item['ebs'] = ebs - return item - - -def _format_mappings(properties, result): - """Format multiple BlockDeviceMappingItemType.""" - mappings = [{'virtualName': m['virtual'], 'deviceName': m['device']} - for m in _properties_get_mappings(properties) - if block_device.is_swap_or_ephemeral(m['virtual'])] - - block_device_mapping = [_format_block_device_mapping(bdm) for bdm in - properties.get('block_device_mapping', [])] - - # NOTE(yamahata): overwrite mappings with block_device_mapping - for bdm in block_device_mapping: - for i in range(len(mappings)): - if bdm.get('deviceName') == mappings[i].get('deviceName'): - del mappings[i] - break - mappings.append(bdm) - - # NOTE(yamahata): trim ebs.no_device == true. Is this necessary? - mappings = [bdm for bdm in mappings if not (bdm.get('noDevice', False))] - - if mappings: - result['blockDeviceMapping'] = mappings - class CloudController(object): - """CloudController provides the critical dispatch between - inbound API calls through the endpoint and messages - sent to the other nodes. -""" def __init__(self): versionutils.report_deprecated_feature( - LOG, - _LW('The in tree EC2 API is deprecated as of Kilo release and may ' - 'be removed in a future release. The openstack ec2-api ' - 'project http://git.openstack.org/cgit/openstack/ec2-api/ ' - 'is the target replacement for this functionality.') + LOG, + _LW('The in tree EC2 API has been removed in Mitaka. ' + 'Please remove entries from api-paste.ini and use ' + 'the OpenStack ec2-api project ' + 'http://git.openstack.org/cgit/openstack/ec2-api/') ) - self.image_service = s3.S3ImageService() - self.network_api = network.API() - self.volume_api = volume.API() - self.security_group_api = get_cloud_security_group_api() - self.compute_api = compute.API(network_api=self.network_api, - volume_api=self.volume_api, - security_group_api=self.security_group_api) - self.keypair_api = compute_api.KeypairAPI() - self.servicegroup_api = servicegroup.API() - - def __str__(self): - return 'CloudController' - - def _enforce_valid_instance_ids(self, context, instance_ids): - # NOTE(mikal): Amazon's implementation of the EC2 API requires that - # _all_ instance ids passed in be valid. - instances = {} - if instance_ids: - for ec2_id in instance_ids: - instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, ec2_id) - instance = self.compute_api.get(context, instance_uuid) - instances[ec2_id] = instance - return instances - - def _get_image_state(self, image): - # NOTE(vish): fallback status if image_state isn't set - state = image.get('status') - if state == 'active': - state = 'available' - return image['properties'].get('image_state', state) - - def describe_availability_zones(self, context, **kwargs): - if ('zone_name' in kwargs and - 'verbose' in kwargs['zone_name'] and - context.is_admin): - return self._describe_availability_zones_verbose(context, - **kwargs) - else: - return self._describe_availability_zones(context, **kwargs) - - def _describe_availability_zones(self, context, **kwargs): - ctxt = context.elevated() - available_zones, not_available_zones = \ - availability_zones.get_availability_zones(ctxt) - - result = [] - for zone in available_zones: - # Hide internal_service_availability_zone - if zone == CONF.internal_service_availability_zone: - continue - result.append({'zoneName': zone, - 'zoneState': "available"}) - for zone in not_available_zones: - result.append({'zoneName': zone, - 'zoneState': "not available"}) - return {'availabilityZoneInfo': result} - - def _describe_availability_zones_verbose(self, context, **kwargs): - ctxt = context.elevated() - available_zones, not_available_zones = \ - availability_zones.get_availability_zones(ctxt) - - # Available services - enabled_services = objects.ServiceList.get_all(context, - disabled=False, set_zones=True) - zone_hosts = {} - host_services = {} - for service in enabled_services: - zone_hosts.setdefault(service.availability_zone, []) - if service.host not in zone_hosts[service.availability_zone]: - zone_hosts[service.availability_zone].append( - service.host) - - host_services.setdefault(service.availability_zone + - service.host, []) - host_services[service.availability_zone + service.host].\ - append(service) - - result = [] - for zone in available_zones: - result.append({'zoneName': zone, - 'zoneState': "available"}) - for host in zone_hosts[zone]: - result.append({'zoneName': '|- %s' % host, - 'zoneState': ''}) - - for service in host_services[zone + host]: - alive = self.servicegroup_api.service_is_up(service) - art = (alive and ":-)") or "XXX" - active = 'enabled' - if service.disabled: - active = 'disabled' - result.append({'zoneName': '| |- %s' % service.binary, - 'zoneState': ('%s %s %s' - % (active, art, - service.updated_at))}) - - for zone in not_available_zones: - result.append({'zoneName': zone, - 'zoneState': "not available"}) - - return {'availabilityZoneInfo': result} - - def describe_regions(self, context, region_name=None, **kwargs): - if CONF.region_list: - regions = [] - for region in CONF.region_list: - name, _sep, host = region.partition('=') - endpoint = '%s://%s:%s%s' % (CONF.ec2_scheme, - host, - CONF.ec2_port, - CONF.ec2_path) - regions.append({'regionName': name, - 'regionEndpoint': endpoint}) - else: - regions = [{'regionName': 'nova', - 'regionEndpoint': '%s://%s:%s%s' % (CONF.ec2_scheme, - CONF.ec2_host, - CONF.ec2_port, - CONF.ec2_path)}] - return {'regionInfo': regions} - - def describe_snapshots(self, - context, - snapshot_id=None, - owner=None, - restorable_by=None, - **kwargs): - if snapshot_id: - snapshots = [] - for ec2_id in snapshot_id: - internal_id = ec2utils.ec2_snap_id_to_uuid(ec2_id) - snapshot = self.volume_api.get_snapshot( - context, - snapshot_id=internal_id) - snapshots.append(snapshot) - else: - snapshots = self.volume_api.get_all_snapshots(context) - - formatted_snapshots = [] - for s in snapshots: - formatted = self._format_snapshot(context, s) - if formatted: - formatted_snapshots.append(formatted) - return {'snapshotSet': formatted_snapshots} - - def _format_snapshot(self, context, snapshot): - # NOTE(mikal): this is just a set of strings in cinder. If they - # implement an enum, then we should move this code to use it. The - # valid ec2 statuses are "pending", "completed", and "error". - status_map = {'new': 'pending', - 'creating': 'pending', - 'available': 'completed', - 'active': 'completed', - 'deleting': 'pending', - 'deleted': None, - 'error': 'error'} - - mapped_status = status_map.get(snapshot['status'], snapshot['status']) - if not mapped_status: - return None - - s = {} - s['snapshotId'] = ec2utils.id_to_ec2_snap_id(snapshot['id']) - s['volumeId'] = ec2utils.id_to_ec2_vol_id(snapshot['volume_id']) - s['status'] = mapped_status - s['startTime'] = snapshot['created_at'] - s['progress'] = snapshot['progress'] - s['ownerId'] = snapshot['project_id'] - s['volumeSize'] = snapshot['volume_size'] - s['description'] = snapshot['display_description'] - return s - - def create_snapshot(self, context, volume_id, **kwargs): - validate_volume_id(volume_id) - LOG.info(_LI("Create snapshot of volume %s"), volume_id, - context=context) - volume_id = ec2utils.ec2_vol_id_to_uuid(volume_id) - args = (context, volume_id, kwargs.get('name'), - kwargs.get('description')) - if kwargs.get('force', False): - snapshot = self.volume_api.create_snapshot_force(*args) - else: - snapshot = self.volume_api.create_snapshot(*args) - - smap = objects.EC2SnapshotMapping(context, uuid=snapshot['id']) - smap.create() - - return self._format_snapshot(context, snapshot) - - def delete_snapshot(self, context, snapshot_id, **kwargs): - snapshot_id = ec2utils.ec2_snap_id_to_uuid(snapshot_id) - self.volume_api.delete_snapshot(context, snapshot_id) - return True - - def describe_key_pairs(self, context, key_name=None, **kwargs): - key_pairs = self.keypair_api.get_key_pairs(context, context.user_id) - if key_name is not None: - key_pairs = [x for x in key_pairs if x['name'] in key_name] - - # If looking for non existent key pair - if key_name is not None and not key_pairs: - msg = _('Could not find key pair(s): %s') % ','.join(key_name) - raise exception.KeypairNotFound(message=msg) - - result = [] - for key_pair in key_pairs: - # filter out the vpn keys - suffix = CONF.vpn_key_suffix - if context.is_admin or not key_pair['name'].endswith(suffix): - result.append({ - 'keyName': key_pair['name'], - 'keyFingerprint': key_pair['fingerprint'], - }) - - return {'keySet': result} - - def create_key_pair(self, context, key_name, **kwargs): - LOG.info(_LI("Create key pair %s"), key_name, context=context) - - keypair, private_key = self.keypair_api.create_key_pair( - context, context.user_id, key_name) - - return {'keyName': key_name, - 'keyFingerprint': keypair['fingerprint'], - 'keyMaterial': private_key} - # TODO(vish): when context is no longer an object, pass it here - - def import_key_pair(self, context, key_name, public_key_material, - **kwargs): - LOG.info(_LI("Import key %s"), key_name, context=context) - - public_key = base64.b64decode(public_key_material) - - keypair = self.keypair_api.import_key_pair(context, - context.user_id, - key_name, - public_key) - - return {'keyName': key_name, - 'keyFingerprint': keypair['fingerprint']} - - def delete_key_pair(self, context, key_name, **kwargs): - LOG.info(_LI("Delete key pair %s"), key_name, context=context) - try: - self.keypair_api.delete_key_pair(context, context.user_id, - key_name) - except exception.NotFound: - # aws returns true even if the key doesn't exist - pass - return True - - def describe_security_groups(self, context, group_name=None, group_id=None, - **kwargs): - search_opts = ec2utils.search_opts_from_filters(kwargs.get('filter')) - - raw_groups = self.security_group_api.list(context, - group_name, - group_id, - context.project_id, - search_opts=search_opts) - - groups = [self._format_security_group(context, g) for g in raw_groups] - - return {'securityGroupInfo': - list(sorted(groups, - key=lambda k: (k['ownerId'], k['groupName'])))} - - def _format_security_group(self, context, group): - g = {} - g['groupDescription'] = group['description'] - g['groupName'] = group['name'] - g['ownerId'] = group['project_id'] - g['ipPermissions'] = [] - for rule in group['rules']: - r = {} - r['groups'] = [] - r['ipRanges'] = [] - if rule['group_id']: - if rule.get('grantee_group'): - source_group = rule['grantee_group'] - r['groups'] += [{'groupName': source_group['name'], - 'userId': source_group['project_id']}] - else: - # rule is not always joined with grantee_group - # for example when using neutron driver. - source_group = self.security_group_api.get( - context, id=rule['group_id']) - r['groups'] += [{'groupName': source_group.get('name'), - 'userId': source_group.get('project_id')}] - if rule['protocol']: - r['ipProtocol'] = rule['protocol'].lower() - r['fromPort'] = rule['from_port'] - r['toPort'] = rule['to_port'] - g['ipPermissions'] += [dict(r)] - else: - for protocol, min_port, max_port in (('icmp', -1, -1), - ('tcp', 1, 65535), - ('udp', 1, 65535)): - r['ipProtocol'] = protocol - r['fromPort'] = min_port - r['toPort'] = max_port - g['ipPermissions'] += [dict(r)] - else: - r['ipProtocol'] = rule['protocol'] - r['fromPort'] = rule['from_port'] - r['toPort'] = rule['to_port'] - r['ipRanges'] += [{'cidrIp': rule['cidr']}] - g['ipPermissions'] += [r] - return g - - def _rule_args_to_dict(self, context, kwargs): - rules = [] - if 'groups' not in kwargs and 'ip_ranges' not in kwargs: - rule = self._rule_dict_last_step(context, **kwargs) - if rule: - rules.append(rule) - return rules - if 'ip_ranges' in kwargs: - rules = self._cidr_args_split(kwargs) - else: - rules = [kwargs] - finalset = [] - for rule in rules: - if 'groups' in rule: - groups_values = self._groups_args_split(rule) - for groups_value in groups_values: - final = self._rule_dict_last_step(context, **groups_value) - finalset.append(final) - else: - final = self._rule_dict_last_step(context, **rule) - finalset.append(final) - return finalset - - def _cidr_args_split(self, kwargs): - cidr_args_split = [] - cidrs = kwargs['ip_ranges'] - for key, cidr in six.iteritems(cidrs): - mykwargs = kwargs.copy() - del mykwargs['ip_ranges'] - mykwargs['cidr_ip'] = cidr['cidr_ip'] - cidr_args_split.append(mykwargs) - return cidr_args_split - - def _groups_args_split(self, kwargs): - groups_args_split = [] - groups = kwargs['groups'] - for key, group in six.iteritems(groups): - mykwargs = kwargs.copy() - del mykwargs['groups'] - if 'group_name' in group: - mykwargs['source_security_group_name'] = group['group_name'] - if 'user_id' in group: - mykwargs['source_security_group_owner_id'] = group['user_id'] - if 'group_id' in group: - mykwargs['source_security_group_id'] = group['group_id'] - groups_args_split.append(mykwargs) - return groups_args_split - - def _rule_dict_last_step(self, context, to_port=None, from_port=None, - ip_protocol=None, cidr_ip=None, user_id=None, - source_security_group_name=None, - source_security_group_owner_id=None): - - if source_security_group_name: - source_project_id = self._get_source_project_id(context, - source_security_group_owner_id) - - source_security_group = objects.SecurityGroup.get_by_name( - context.elevated(), - source_project_id, - source_security_group_name) - notfound = exception.SecurityGroupNotFound - if not source_security_group: - raise notfound(security_group_id=source_security_group_name) - group_id = source_security_group.id - return self.security_group_api.new_group_ingress_rule( - group_id, ip_protocol, from_port, to_port) - else: - cidr = self.security_group_api.parse_cidr(cidr_ip) - return self.security_group_api.new_cidr_ingress_rule( - cidr, ip_protocol, from_port, to_port) - - def _validate_group_identifier(self, group_name, group_id): - if not group_name and not group_id: - err = _("need group_name or group_id") - raise exception.MissingParameter(reason=err) - - def _validate_rulevalues(self, rulesvalues): - if not rulesvalues: - err = _("can't build a valid rule") - raise exception.MissingParameter(reason=err) - - def _validate_security_group_protocol(self, values): - validprotocols = ['tcp', 'udp', 'icmp', '6', '17', '1'] - if 'ip_protocol' in values and \ - values['ip_protocol'] not in validprotocols: - protocol = values['ip_protocol'] - err = _("Invalid IP protocol %(protocol)s") % \ - {'protocol': protocol} - raise exception.InvalidParameterValue(message=err) - - def revoke_security_group_ingress(self, context, group_name=None, - group_id=None, **kwargs): - self._validate_group_identifier(group_name, group_id) - - security_group = self.security_group_api.get(context, group_name, - group_id) - - extensions.check_compute_policy(context, 'security_groups', - security_group, 'compute_extension') - - prevalues = kwargs.get('ip_permissions', [kwargs]) - - rule_ids = [] - for values in prevalues: - rulesvalues = self._rule_args_to_dict(context, values) - self._validate_rulevalues(rulesvalues) - for values_for_rule in rulesvalues: - values_for_rule['parent_group_id'] = security_group['id'] - - rule_ids.append(self.security_group_api.rule_exists( - security_group, values_for_rule)) - - rule_ids = [id for id in rule_ids if id] - - if rule_ids: - self.security_group_api.remove_rules(context, security_group, - rule_ids) - - return True - - msg = _("No rule for the specified parameters.") - raise exception.InvalidParameterValue(message=msg) - - # TODO(soren): This has only been tested with Boto as the client. - # Unfortunately, it seems Boto is using an old API - # for these operations, so support for newer API versions - # is sketchy. - def authorize_security_group_ingress(self, context, group_name=None, - group_id=None, **kwargs): - self._validate_group_identifier(group_name, group_id) - - security_group = self.security_group_api.get(context, group_name, - group_id) - - extensions.check_compute_policy(context, 'security_groups', - security_group, 'compute_extension') - - prevalues = kwargs.get('ip_permissions', [kwargs]) - postvalues = [] - for values in prevalues: - self._validate_security_group_protocol(values) - rulesvalues = self._rule_args_to_dict(context, values) - self._validate_rulevalues(rulesvalues) - for values_for_rule in rulesvalues: - values_for_rule['parent_group_id'] = security_group['id'] - if self.security_group_api.rule_exists(security_group, - values_for_rule): - raise exception.SecurityGroupRuleExists( - rule=values_for_rule) - postvalues.append(values_for_rule) - - if postvalues: - self.security_group_api.add_rules(context, security_group['id'], - security_group['name'], postvalues) - return True - - msg = _("No rule for the specified parameters.") - raise exception.InvalidParameterValue(message=msg) - - def _get_source_project_id(self, context, source_security_group_owner_id): - if source_security_group_owner_id: - # Parse user:project for source group. - source_parts = source_security_group_owner_id.split(':') - - # If no project name specified, assume it's same as user name. - # Since we're looking up by project name, the user name is not - # used here. It's only read for EC2 API compatibility. - if len(source_parts) == 2: - source_project_id = source_parts[1] - else: - source_project_id = source_parts[0] - else: - source_project_id = context.project_id - - return source_project_id - - def create_security_group(self, context, group_name, group_description): - if isinstance(group_name, six.text_type): - group_name = utils.utf8(group_name) - if CONF.ec2_strict_validation: - # EC2 specification gives constraints for name and description: - # Accepts alphanumeric characters, spaces, dashes, and underscores - allowed = '^[a-zA-Z0-9_\- ]+$' - self.security_group_api.validate_property(group_name, 'name', - allowed) - self.security_group_api.validate_property(group_description, - 'description', allowed) - else: - # Amazon accepts more symbols. - # So, allow POSIX [:print:] characters. - allowed = r'^[\x20-\x7E]+$' - self.security_group_api.validate_property(group_name, 'name', - allowed) - - group_ref = self.security_group_api.create_security_group( - context, group_name, group_description) - - return {'securityGroupSet': [self._format_security_group(context, - group_ref)]} - - def delete_security_group(self, context, group_name=None, group_id=None, - **kwargs): - if not group_name and not group_id: - err = _("need group_name or group_id") - raise exception.MissingParameter(reason=err) - - security_group = self.security_group_api.get(context, group_name, - group_id) - - extensions.check_compute_policy(context, 'security_groups', - security_group, 'compute_extension') - - self.security_group_api.destroy(context, security_group) - - return True - - def get_password_data(self, context, instance_id, **kwargs): - # instance_id may be passed in as a list of instances - if isinstance(instance_id, list): - ec2_id = instance_id[0] - else: - ec2_id = instance_id - validate_instance_id(ec2_id) - instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, ec2_id) - instance = self.compute_api.get(context, instance_uuid) - output = password.extract_password(instance) - # NOTE(vish): this should be timestamp from the metadata fields - # but it isn't important enough to implement properly - now = timeutils.utcnow() - return {"InstanceId": ec2_id, - "Timestamp": now, - "passwordData": output} - - def get_console_output(self, context, instance_id, **kwargs): - LOG.info(_LI("Get console output for instance %s"), instance_id, - context=context) - # instance_id may be passed in as a list of instances - if isinstance(instance_id, list): - ec2_id = instance_id[0] - else: - ec2_id = instance_id - validate_instance_id(ec2_id) - instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, ec2_id) - instance = self.compute_api.get(context, instance_uuid, - want_objects=True) - output = self.compute_api.get_console_output(context, instance) - now = timeutils.utcnow() - return {"InstanceId": ec2_id, - "Timestamp": now, - "output": base64.b64encode(output)} - - def describe_volumes(self, context, volume_id=None, **kwargs): - if volume_id: - volumes = [] - for ec2_id in volume_id: - validate_volume_id(ec2_id) - internal_id = ec2utils.ec2_vol_id_to_uuid(ec2_id) - volume = self.volume_api.get(context, internal_id) - volumes.append(volume) - else: - volumes = self.volume_api.get_all(context) - volumes = [self._format_volume(context, v) for v in volumes] - return {'volumeSet': volumes} - - def _format_volume(self, context, volume): - valid_ec2_api_volume_status_map = { - 'attaching': 'in-use', - 'detaching': 'in-use'} - - instance_ec2_id = None - - if volume.get('instance_uuid', None): - instance_uuid = volume['instance_uuid'] - # Make sure instance exists - objects.Instance.get_by_uuid(context.elevated(), instance_uuid) - - instance_ec2_id = ec2utils.id_to_ec2_inst_id(instance_uuid) - - v = {} - v['volumeId'] = ec2utils.id_to_ec2_vol_id(volume['id']) - v['status'] = valid_ec2_api_volume_status_map.get(volume['status'], - volume['status']) - v['size'] = volume['size'] - v['availabilityZone'] = volume['availability_zone'] - v['createTime'] = volume['created_at'] - if v['status'] == 'in-use': - v['attachmentSet'] = [{'attachTime': volume.get('attach_time'), - 'deleteOnTermination': False, - 'device': volume['mountpoint'], - 'instanceId': instance_ec2_id, - 'status': self._get_volume_attach_status( - volume), - 'volumeId': v['volumeId']}] - else: - v['attachmentSet'] = [{}] - if volume.get('snapshot_id') is not None: - v['snapshotId'] = ec2utils.id_to_ec2_snap_id(volume['snapshot_id']) - else: - v['snapshotId'] = None - - return v - - def create_volume(self, context, **kwargs): - snapshot_ec2id = kwargs.get('snapshot_id', None) - if snapshot_ec2id is not None: - snapshot_id = ec2utils.ec2_snap_id_to_uuid(kwargs['snapshot_id']) - snapshot = self.volume_api.get_snapshot(context, snapshot_id) - LOG.info(_LI("Create volume from snapshot %s"), snapshot_ec2id, - context=context) - else: - snapshot = None - LOG.info(_LI("Create volume of %s GB"), - kwargs.get('size'), - context=context) - - create_kwargs = dict(snapshot=snapshot, - volume_type=kwargs.get('volume_type'), - metadata=kwargs.get('metadata'), - availability_zone=kwargs.get('availability_zone')) - - volume = self.volume_api.create(context, - kwargs.get('size'), - kwargs.get('name'), - kwargs.get('description'), - **create_kwargs) - - vmap = objects.EC2VolumeMapping(context) - vmap.uuid = volume['id'] - vmap.create() - - # TODO(vish): Instance should be None at db layer instead of - # trying to lazy load, but for now we turn it into - # a dict to avoid an error. - return self._format_volume(context, dict(volume)) - - def delete_volume(self, context, volume_id, **kwargs): - validate_volume_id(volume_id) - volume_id = ec2utils.ec2_vol_id_to_uuid(volume_id) - self.volume_api.delete(context, volume_id) - return True - - def attach_volume(self, context, - volume_id, - instance_id, - device, **kwargs): - validate_instance_id(instance_id) - validate_volume_id(volume_id) - volume_id = ec2utils.ec2_vol_id_to_uuid(volume_id) - instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, instance_id) - instance = self.compute_api.get(context, instance_uuid, - want_objects=True) - LOG.info(_LI('Attach volume %(volume_id)s to instance %(instance_id)s ' - 'at %(device)s'), - {'volume_id': volume_id, - 'instance_id': instance_id, - 'device': device}, - context=context) - - self.compute_api.attach_volume(context, instance, volume_id, device) - volume = self.volume_api.get(context, volume_id) - ec2_attach_status = ec2utils.status_to_ec2_attach_status(volume) - - return {'attachTime': volume['attach_time'], - 'device': volume['mountpoint'], - 'instanceId': ec2utils.id_to_ec2_inst_id(instance_uuid), - 'requestId': context.request_id, - 'status': ec2_attach_status, - 'volumeId': ec2utils.id_to_ec2_vol_id(volume_id)} - - def _get_instance_from_volume(self, context, volume): - if volume.get('instance_uuid'): - try: - inst_uuid = volume['instance_uuid'] - return objects.Instance.get_by_uuid(context, inst_uuid) - except exception.InstanceNotFound: - pass - raise exception.VolumeUnattached(volume_id=volume['id']) - - def detach_volume(self, context, volume_id, **kwargs): - validate_volume_id(volume_id) - volume_id = ec2utils.ec2_vol_id_to_uuid(volume_id) - LOG.info(_LI("Detach volume %s"), volume_id, context=context) - volume = self.volume_api.get(context, volume_id) - instance = self._get_instance_from_volume(context, volume) - - self.compute_api.detach_volume(context, instance, volume) - resp_volume = self.volume_api.get(context, volume_id) - ec2_attach_status = ec2utils.status_to_ec2_attach_status(resp_volume) - - return {'attachTime': volume['attach_time'], - 'device': volume['mountpoint'], - 'instanceId': ec2utils.id_to_ec2_inst_id( - volume['instance_uuid']), - 'requestId': context.request_id, - 'status': ec2_attach_status, - 'volumeId': ec2utils.id_to_ec2_vol_id(volume_id)} - - def _format_kernel_id(self, context, instance_ref, result, key): - kernel_uuid = instance_ref['kernel_id'] - if kernel_uuid is None or kernel_uuid == '': - return - result[key] = ec2utils.glance_id_to_ec2_id(context, kernel_uuid, 'aki') - - def _format_ramdisk_id(self, context, instance_ref, result, key): - ramdisk_uuid = instance_ref['ramdisk_id'] - if ramdisk_uuid is None or ramdisk_uuid == '': - return - result[key] = ec2utils.glance_id_to_ec2_id(context, ramdisk_uuid, - 'ari') - - def describe_instance_attribute(self, context, instance_id, attribute, - **kwargs): - def _unsupported_attribute(instance, result): - raise exception.InvalidAttribute(attr=attribute) - - def _format_attr_block_device_mapping(instance, result): - tmp = {} - self._format_instance_root_device_name(instance, tmp) - self._format_instance_bdm(context, instance.uuid, - tmp['rootDeviceName'], result) - - def _format_attr_disable_api_termination(instance, result): - result['disableApiTermination'] = instance.disable_terminate - - def _format_attr_group_set(instance, result): - CloudController._format_group_set(instance, result) - - def _format_attr_instance_initiated_shutdown_behavior(instance, - result): - if instance.shutdown_terminate: - result['instanceInitiatedShutdownBehavior'] = 'terminate' - else: - result['instanceInitiatedShutdownBehavior'] = 'stop' - - def _format_attr_instance_type(instance, result): - self._format_instance_type(instance, result) - - def _format_attr_kernel(instance, result): - self._format_kernel_id(context, instance, result, 'kernel') - - def _format_attr_ramdisk(instance, result): - self._format_ramdisk_id(context, instance, result, 'ramdisk') - - def _format_attr_root_device_name(instance, result): - self._format_instance_root_device_name(instance, result) - - def _format_attr_source_dest_check(instance, result): - _unsupported_attribute(instance, result) - - def _format_attr_user_data(instance, result): - result['userData'] = base64.b64decode(instance.user_data) - - attribute_formatter = { - 'blockDeviceMapping': _format_attr_block_device_mapping, - 'disableApiTermination': _format_attr_disable_api_termination, - 'groupSet': _format_attr_group_set, - 'instanceInitiatedShutdownBehavior': - _format_attr_instance_initiated_shutdown_behavior, - 'instanceType': _format_attr_instance_type, - 'kernel': _format_attr_kernel, - 'ramdisk': _format_attr_ramdisk, - 'rootDeviceName': _format_attr_root_device_name, - 'sourceDestCheck': _format_attr_source_dest_check, - 'userData': _format_attr_user_data, - } - - fn = attribute_formatter.get(attribute) - if fn is None: - raise exception.InvalidAttribute(attr=attribute) - - validate_instance_id(instance_id) - instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, instance_id) - instance = self.compute_api.get(context, instance_uuid, - want_objects=True) - result = {'instance_id': instance_id} - fn(instance, result) - return result - - def describe_instances(self, context, **kwargs): - # Optional DescribeInstances argument - instance_id = kwargs.get('instance_id', None) - filters = kwargs.get('filter', None) - instances = self._enforce_valid_instance_ids(context, instance_id) - return self._format_describe_instances(context, - instance_id=instance_id, - instance_cache=instances, - filter=filters) - - def describe_instances_v6(self, context, **kwargs): - # Optional DescribeInstancesV6 argument - instance_id = kwargs.get('instance_id', None) - filters = kwargs.get('filter', None) - instances = self._enforce_valid_instance_ids(context, instance_id) - return self._format_describe_instances(context, - instance_id=instance_id, - instance_cache=instances, - filter=filters, - use_v6=True) - - def _format_describe_instances(self, context, **kwargs): - return {'reservationSet': self._format_instances(context, **kwargs)} - - def _format_run_instances(self, context, reservation_id): - i = self._format_instances(context, reservation_id=reservation_id) - assert len(i) == 1 - return i[0] - - def _format_terminate_instances(self, context, instance_id, - previous_states): - instances_set = [] - for (ec2_id, previous_state) in zip(instance_id, previous_states): - i = {} - i['instanceId'] = ec2_id - i['previousState'] = _state_description(previous_state['vm_state'], - previous_state['shutdown_terminate']) - try: - instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, ec2_id) - instance = self.compute_api.get(context, instance_uuid, - want_objects=True) - i['currentState'] = _state_description(instance.vm_state, - instance.shutdown_terminate) - except exception.NotFound: - i['currentState'] = _state_description( - inst_state.SHUTTING_DOWN, True) - instances_set.append(i) - return {'instancesSet': instances_set} - - def _format_stop_instances(self, context, instance_ids, previous_states): - instances_set = [] - for (ec2_id, previous_state) in zip(instance_ids, previous_states): - i = {} - i['instanceId'] = ec2_id - i['previousState'] = _state_description(previous_state['vm_state'], - previous_state['shutdown_terminate']) - i['currentState'] = _state_description(inst_state.STOPPING, True) - instances_set.append(i) - return {'instancesSet': instances_set} - - def _format_start_instances(self, context, instance_id, previous_states): - instances_set = [] - for (ec2_id, previous_state) in zip(instance_id, previous_states): - i = {} - i['instanceId'] = ec2_id - i['previousState'] = _state_description(previous_state['vm_state'], - previous_state['shutdown_terminate']) - i['currentState'] = _state_description(None, True) - instances_set.append(i) - return {'instancesSet': instances_set} - - def _format_instance_bdm(self, context, instance_uuid, root_device_name, - result): - """Format InstanceBlockDeviceMappingResponseItemType.""" - root_device_type = 'instance-store' - root_device_short_name = block_device.strip_dev(root_device_name) - if root_device_name == root_device_short_name: - root_device_name = block_device.prepend_dev(root_device_name) - mapping = [] - bdms = objects.BlockDeviceMappingList.get_by_instance_uuid( - context, instance_uuid) - for bdm in bdms: - volume_id = bdm.volume_id - if volume_id is None or bdm.no_device: - continue - - if (bdm.is_volume and - (bdm.device_name == root_device_name or - bdm.device_name == root_device_short_name)): - root_device_type = 'ebs' - - vol = self.volume_api.get(context, volume_id) - LOG.debug("vol = %s\n", vol) - # TODO(yamahata): volume attach time - ebs = {'volumeId': ec2utils.id_to_ec2_vol_id(volume_id), - 'deleteOnTermination': bdm.delete_on_termination, - 'attachTime': vol['attach_time'] or '', - 'status': self._get_volume_attach_status(vol), } - res = {'deviceName': bdm.device_name, - 'ebs': ebs, } - mapping.append(res) - - if mapping: - result['blockDeviceMapping'] = mapping - result['rootDeviceType'] = root_device_type - - @staticmethod - def _get_volume_attach_status(volume): - return (volume['status'] - if volume['status'] in ('attaching', 'detaching') else - volume['attach_status']) - - @staticmethod - def _format_instance_root_device_name(instance, result): - result['rootDeviceName'] = (instance.get('root_device_name') or - block_device.DEFAULT_ROOT_DEV_NAME) - - @staticmethod - def _format_instance_type(instance, result): - flavor = instance.get_flavor() - result['instanceType'] = flavor.name - - @staticmethod - def _format_group_set(instance, result): - security_group_names = [] - if instance.get('security_groups'): - for security_group in instance.security_groups: - security_group_names.append(security_group['name']) - result['groupSet'] = utils.convert_to_list_dict( - security_group_names, 'groupId') - - def _format_instances(self, context, instance_id=None, use_v6=False, - instances_cache=None, **search_opts): - # TODO(termie): this method is poorly named as its name does not imply - # that it will be making a variety of database calls - # rather than simply formatting a bunch of instances that - # were handed to it - reservations = {} - - if not instances_cache: - instances_cache = {} - - # NOTE(vish): instance_id is an optional list of ids to filter by - if instance_id: - instances = [] - for ec2_id in instance_id: - if ec2_id in instances_cache: - instances.append(instances_cache[ec2_id]) - else: - try: - instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, - ec2_id) - instance = self.compute_api.get(context, instance_uuid, - want_objects=True) - except exception.NotFound: - continue - instances.append(instance) - else: - try: - # always filter out deleted instances - search_opts['deleted'] = False - instances = self.compute_api.get_all(context, - search_opts=search_opts, - sort_keys=['created_at'], - sort_dirs=['asc'], - want_objects=True) - except exception.NotFound: - instances = [] - - for instance in instances: - if not context.is_admin: - if pipelib.is_vpn_image(instance.image_ref): - continue - i = {} - instance_uuid = instance.uuid - ec2_id = ec2utils.id_to_ec2_inst_id(instance_uuid) - i['instanceId'] = ec2_id - image_uuid = instance.image_ref - i['imageId'] = ec2utils.glance_id_to_ec2_id(context, image_uuid) - self._format_kernel_id(context, instance, i, 'kernelId') - self._format_ramdisk_id(context, instance, i, 'ramdiskId') - i['instanceState'] = _state_description( - instance.vm_state, instance.shutdown_terminate) - - fixed_ip = None - floating_ip = None - ip_info = ec2utils.get_ip_info_for_instance(context, instance) - if ip_info['fixed_ips']: - fixed_ip = ip_info['fixed_ips'][0] - if ip_info['floating_ips']: - floating_ip = ip_info['floating_ips'][0] - if ip_info['fixed_ip6s']: - i['dnsNameV6'] = ip_info['fixed_ip6s'][0] - if CONF.ec2_private_dns_show_ip: - i['privateDnsName'] = fixed_ip - else: - i['privateDnsName'] = instance.hostname - i['privateIpAddress'] = fixed_ip - if floating_ip is not None: - i['ipAddress'] = floating_ip - i['dnsName'] = floating_ip - i['keyName'] = instance.key_name - i['tagSet'] = [] - - for k, v in six.iteritems(utils.instance_meta(instance)): - i['tagSet'].append({'key': k, 'value': v}) - - client_token = self._get_client_token(context, instance_uuid) - if client_token: - i['clientToken'] = client_token - - if context.is_admin: - i['keyName'] = '%s (%s, %s)' % (i['keyName'], - instance.project_id, - instance.host) - i['productCodesSet'] = utils.convert_to_list_dict([], - 'product_codes') - self._format_instance_type(instance, i) - i['launchTime'] = instance.created_at - i['amiLaunchIndex'] = instance.launch_index - self._format_instance_root_device_name(instance, i) - self._format_instance_bdm(context, instance.uuid, - i['rootDeviceName'], i) - zone = availability_zones.get_instance_availability_zone(context, - instance) - i['placement'] = {'availabilityZone': zone} - if instance.reservation_id not in reservations: - r = {} - r['reservationId'] = instance.reservation_id - r['ownerId'] = instance.project_id - self._format_group_set(instance, r) - r['instancesSet'] = [] - reservations[instance.reservation_id] = r - reservations[instance.reservation_id]['instancesSet'].append(i) - - return list(reservations.values()) - - def describe_addresses(self, context, public_ip=None, **kwargs): - if public_ip: - floatings = [] - for address in public_ip: - floating = self.network_api.get_floating_ip_by_address(context, - address) - floatings.append(floating) - else: - floatings = self.network_api.get_floating_ips_by_project(context) - addresses = [self._format_address(context, f) for f in floatings] - return {'addressesSet': addresses} - - def _format_address(self, context, floating_ip): - ec2_id = None - if floating_ip['fixed_ip_id']: - if utils.is_neutron(): - fixed_vm_uuid = floating_ip['instance']['uuid'] - if fixed_vm_uuid is not None: - ec2_id = ec2utils.id_to_ec2_inst_id(fixed_vm_uuid) - else: - fixed_id = floating_ip['fixed_ip_id'] - fixed = self.network_api.get_fixed_ip(context, fixed_id) - if fixed['instance_uuid'] is not None: - ec2_id = ec2utils.id_to_ec2_inst_id(fixed['instance_uuid']) - address = {'public_ip': floating_ip['address'], - 'instance_id': ec2_id} - if context.is_admin: - details = "%s (%s)" % (address['instance_id'], - floating_ip['project_id']) - address['instance_id'] = details - return address - - def allocate_address(self, context, **kwargs): - LOG.info(_LI("Allocate address"), context=context) - public_ip = self.network_api.allocate_floating_ip(context) - return {'publicIp': public_ip} - - def release_address(self, context, public_ip, **kwargs): - LOG.info(_LI('Release address %s'), public_ip, context=context) - self.network_api.release_floating_ip(context, address=public_ip) - return {'return': "true"} - - def associate_address(self, context, instance_id, public_ip, **kwargs): - LOG.info(_LI("Associate address %(public_ip)s to instance " - "%(instance_id)s"), - {'public_ip': public_ip, 'instance_id': instance_id}, - context=context) - instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, instance_id) - instance = self.compute_api.get(context, instance_uuid, - want_objects=True) - - cached_ipinfo = ec2utils.get_ip_info_for_instance(context, instance) - fixed_ips = cached_ipinfo['fixed_ips'] + cached_ipinfo['fixed_ip6s'] - if not fixed_ips: - msg = _('Unable to associate IP Address, no fixed_ips.') - raise exception.NoMoreFixedIps(message=msg) - - # TODO(tr3buchet): this will associate the floating IP with the - # first fixed_ip an instance has. This should be - # changed to support specifying a particular fixed_ip if - # multiple exist but this may not apply to ec2.. - if len(fixed_ips) > 1: - LOG.warning(_LW('multiple fixed_ips exist, using the first: %s'), - fixed_ips[0]) - - self.network_api.associate_floating_ip(context, instance, - floating_address=public_ip, - fixed_address=fixed_ips[0]) - return {'return': 'true'} - - def disassociate_address(self, context, public_ip, **kwargs): - instance_id = self.network_api.get_instance_id_by_floating_address( - context, public_ip) - if instance_id: - instance = self.compute_api.get(context, instance_id, - want_objects=True) - LOG.info(_LI("Disassociate address %s"), - public_ip, context=context) - self.network_api.disassociate_floating_ip(context, instance, - address=public_ip) - else: - msg = _('Floating ip is not associated.') - raise exception.InvalidAssociation(message=msg) - return {'return': "true"} - - def run_instances(self, context, **kwargs): - min_count = int(kwargs.get('min_count', 1)) - max_count = int(kwargs.get('max_count', min_count)) - try: - min_count = utils.validate_integer( - min_count, "min_count", min_value=1) - max_count = utils.validate_integer( - max_count, "max_count", min_value=1) - except exception.InvalidInput as e: - raise exception.InvalidInput(message=e.format_message()) - - if min_count > max_count: - msg = _('min_count must be <= max_count') - raise exception.InvalidInput(message=msg) - - client_token = kwargs.get('client_token') - if client_token: - resv_id = self._resv_id_from_token(context, client_token) - if resv_id: - # since this client_token already corresponds to a reservation - # id, this returns a proper response without creating a new - # instance - return self._format_run_instances(context, resv_id) - - if kwargs.get('kernel_id'): - kernel = self._get_image(context, kwargs['kernel_id']) - kwargs['kernel_id'] = ec2utils.id_to_glance_id(context, - kernel['id']) - if kwargs.get('ramdisk_id'): - ramdisk = self._get_image(context, kwargs['ramdisk_id']) - kwargs['ramdisk_id'] = ec2utils.id_to_glance_id(context, - ramdisk['id']) - for bdm in kwargs.get('block_device_mapping', []): - _parse_block_device_mapping(bdm) - - image = self._get_image(context, kwargs['image_id']) - image_uuid = ec2utils.id_to_glance_id(context, image['id']) - - if image: - image_state = self._get_image_state(image) - else: - raise exception.ImageNotFoundEC2(image_id=kwargs['image_id']) - - if image_state != 'available': - msg = _('Image must be available') - raise exception.ImageNotActive(message=msg) - - iisb = kwargs.get('instance_initiated_shutdown_behavior', 'stop') - shutdown_terminate = (iisb == 'terminate') - - flavor = objects.Flavor.get_by_name(context, - kwargs.get('instance_type', None)) - - (instances, resv_id) = self.compute_api.create(context, - instance_type=flavor, - image_href=image_uuid, - max_count=int(kwargs.get('max_count', min_count)), - min_count=min_count, - kernel_id=kwargs.get('kernel_id'), - ramdisk_id=kwargs.get('ramdisk_id'), - key_name=kwargs.get('key_name'), - user_data=kwargs.get('user_data'), - security_group=kwargs.get('security_group'), - availability_zone=kwargs.get('placement', {}).get( - 'availability_zone'), - block_device_mapping=kwargs.get('block_device_mapping', {}), - shutdown_terminate=shutdown_terminate) - - instances = self._format_run_instances(context, resv_id) - if instances: - instance_ids = [i['instanceId'] for i in instances['instancesSet']] - self._add_client_token(context, client_token, instance_ids) - return instances - - def _add_client_token(self, context, client_token, instance_ids): - """Add client token to reservation ID mapping.""" - if client_token: - for ec2_id in instance_ids: - instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, ec2_id) - instance = objects.Instance.get_by_uuid(context, - instance_uuid, expected_attrs=['system_metadata']) - instance.system_metadata.update( - {'EC2_client_token': client_token}) - instance.save() - - def _get_client_token(self, context, instance_uuid): - """Get client token for a given instance.""" - instance = objects.Instance.get_by_uuid(context, - instance_uuid, expected_attrs=['system_metadata']) - return instance.system_metadata.get('EC2_client_token') - - def _remove_client_token(self, context, instance_ids): - """Remove client token to reservation ID mapping.""" - - for ec2_id in instance_ids: - instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, ec2_id) - instance = objects.Instance.get_by_uuid(context, - instance_uuid, expected_attrs=['system_metadata']) - instance.system_metadata.pop('EC2_client_token', None) - instance.save() - - def _resv_id_from_token(self, context, client_token): - """Get reservation ID from db.""" - resv_id = None - sys_metas = self.compute_api.get_all_system_metadata( - context, search_filts=[{'key': ['EC2_client_token']}, - {'value': [client_token]}]) - - for sys_meta in sys_metas: - if sys_meta and sys_meta.get('value') == client_token: - instance = objects.Instance.get_by_uuid( - context, sys_meta['instance_id'], expected_attrs=None) - resv_id = instance.get('reservation_id') - break - return resv_id - - def _ec2_ids_to_instances(self, context, instance_id): - """Get all instances first, to prevent partial executions.""" - instances = [] - extra = ['system_metadata', 'metadata', 'info_cache'] - for ec2_id in instance_id: - validate_instance_id(ec2_id) - instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, ec2_id) - instance = objects.Instance.get_by_uuid( - context, instance_uuid, expected_attrs=extra) - instances.append(instance) - return instances - - def terminate_instances(self, context, instance_id, **kwargs): - """Terminate each instance in instance_id, which is a list of ec2 ids. - instance_id is a kwarg so its name cannot be modified. - """ - previous_states = self._ec2_ids_to_instances(context, instance_id) - self._remove_client_token(context, instance_id) - LOG.debug("Going to start terminating instances") - for instance in previous_states: - self.compute_api.delete(context, instance) - return self._format_terminate_instances(context, - instance_id, - previous_states) - - def reboot_instances(self, context, instance_id, **kwargs): - """instance_id is a list of instance ids.""" - instances = self._ec2_ids_to_instances(context, instance_id) - LOG.info(_LI("Reboot instance %r"), instance_id, context=context) - for instance in instances: - self.compute_api.reboot(context, instance, 'HARD') - return True - - def stop_instances(self, context, instance_id, **kwargs): - """Stop each instances in instance_id. - Here instance_id is a list of instance ids - """ - instances = self._ec2_ids_to_instances(context, instance_id) - LOG.debug("Going to stop instances") - for instance in instances: - extensions.check_compute_policy(context, 'stop', instance) - self.compute_api.stop(context, instance) - return self._format_stop_instances(context, instance_id, - instances) - - def start_instances(self, context, instance_id, **kwargs): - """Start each instances in instance_id. - Here instance_id is a list of instance ids - """ - instances = self._ec2_ids_to_instances(context, instance_id) - LOG.debug("Going to start instances") - for instance in instances: - extensions.check_compute_policy(context, 'start', instance) - self.compute_api.start(context, instance) - return self._format_start_instances(context, instance_id, - instances) - - def _get_image(self, context, ec2_id): - try: - internal_id = ec2utils.ec2_id_to_id(ec2_id) - image = self.image_service.show(context, internal_id) - except (exception.InvalidEc2Id, exception.ImageNotFound): - filters = {'name': ec2_id} - images = self.image_service.detail(context, filters=filters) - try: - return images[0] - except IndexError: - raise exception.ImageNotFound(image_id=ec2_id) - image_type = ec2_id.split('-')[0] - if ec2utils.image_type(image.get('container_format')) != image_type: - raise exception.ImageNotFound(image_id=ec2_id) - return image - - def _format_image(self, image): - """Convert from format defined by GlanceImageService to S3 format.""" - i = {} - image_type = ec2utils.image_type(image.get('container_format')) - ec2_id = ec2utils.image_ec2_id(image.get('id'), image_type) - name = image.get('name') - i['imageId'] = ec2_id - kernel_id = image['properties'].get('kernel_id') - if kernel_id: - i['kernelId'] = ec2utils.image_ec2_id(kernel_id, 'aki') - ramdisk_id = image['properties'].get('ramdisk_id') - if ramdisk_id: - i['ramdiskId'] = ec2utils.image_ec2_id(ramdisk_id, 'ari') - i['imageOwnerId'] = image.get('owner') - - img_loc = image['properties'].get('image_location') - if img_loc: - i['imageLocation'] = img_loc - else: - i['imageLocation'] = "%s (%s)" % (img_loc, name) - - i['name'] = name - if not name and img_loc: - # This should only occur for images registered with ec2 api - # prior to that api populating the glance name - i['name'] = img_loc - - i['imageState'] = self._get_image_state(image) - i['description'] = image.get('description') - display_mapping = {'aki': 'kernel', - 'ari': 'ramdisk', - 'ami': 'machine'} - i['imageType'] = display_mapping.get(image_type) - i['isPublic'] = not not image.get('is_public') - i['architecture'] = image['properties'].get('architecture') - - properties = image['properties'] - root_device_name = block_device.properties_root_device_name(properties) - root_device_type = 'instance-store' - - for bdm in properties.get('block_device_mapping', []): - if (block_device.strip_dev(bdm.get('device_name')) == - block_device.strip_dev(root_device_name) and - ('snapshot_id' in bdm or 'volume_id' in bdm) and - not bdm.get('no_device')): - root_device_type = 'ebs' - i['rootDeviceName'] = (root_device_name or - block_device.DEFAULT_ROOT_DEV_NAME) - i['rootDeviceType'] = root_device_type - - _format_mappings(properties, i) - - return i - - def describe_images(self, context, image_id=None, **kwargs): - # NOTE: image_id is a list! - if image_id: - images = [] - for ec2_id in image_id: - try: - image = self._get_image(context, ec2_id) - except exception.NotFound: - raise exception.ImageNotFound(image_id=ec2_id) - images.append(image) - else: - images = self.image_service.detail(context) - images = [self._format_image(i) for i in images] - return {'imagesSet': images} - - def deregister_image(self, context, image_id, **kwargs): - LOG.info(_LI("De-registering image %s"), image_id, context=context) - image = self._get_image(context, image_id) - internal_id = image['id'] - self.image_service.delete(context, internal_id) - return True - - def _register_image(self, context, metadata): - image = self.image_service.create(context, metadata) - image_type = ec2utils.image_type(image.get('container_format')) - image_id = ec2utils.image_ec2_id(image['id'], image_type) - return image_id - - def register_image(self, context, image_location=None, **kwargs): - if image_location is None and kwargs.get('name'): - image_location = kwargs['name'] - if image_location is None: - msg = _('imageLocation is required') - raise exception.MissingParameter(reason=msg) - - metadata = {'properties': {'image_location': image_location}} - - if kwargs.get('name'): - metadata['name'] = kwargs['name'] - else: - metadata['name'] = image_location - - if 'root_device_name' in kwargs: - metadata['properties']['root_device_name'] = kwargs.get( - 'root_device_name') - - mappings = [_parse_block_device_mapping(bdm) for bdm in - kwargs.get('block_device_mapping', [])] - if mappings: - metadata['properties']['block_device_mapping'] = mappings - - image_id = self._register_image(context, metadata) - LOG.info(_LI('Registered image %(image_location)s with id ' - '%(image_id)s'), - {'image_location': image_location, 'image_id': image_id}, - context=context) - return {'imageId': image_id} - - def describe_image_attribute(self, context, image_id, attribute, **kwargs): - def _block_device_mapping_attribute(image, result): - _format_mappings(image['properties'], result) - - def _launch_permission_attribute(image, result): - result['launchPermission'] = [] - if image['is_public']: - result['launchPermission'].append({'group': 'all'}) - - def _root_device_name_attribute(image, result): - _prop_root_dev_name = block_device.properties_root_device_name - result['rootDeviceName'] = _prop_root_dev_name(image['properties']) - if result['rootDeviceName'] is None: - result['rootDeviceName'] = block_device.DEFAULT_ROOT_DEV_NAME - - def _kernel_attribute(image, result): - kernel_id = image['properties'].get('kernel_id') - if kernel_id: - result['kernel'] = { - 'value': ec2utils.image_ec2_id(kernel_id, 'aki') - } - - def _ramdisk_attribute(image, result): - ramdisk_id = image['properties'].get('ramdisk_id') - if ramdisk_id: - result['ramdisk'] = { - 'value': ec2utils.image_ec2_id(ramdisk_id, 'ari') - } - - supported_attributes = { - 'blockDeviceMapping': _block_device_mapping_attribute, - 'launchPermission': _launch_permission_attribute, - 'rootDeviceName': _root_device_name_attribute, - 'kernel': _kernel_attribute, - 'ramdisk': _ramdisk_attribute, - } - - fn = supported_attributes.get(attribute) - if fn is None: - raise exception.InvalidAttribute(attr=attribute) - try: - image = self._get_image(context, image_id) - except exception.NotFound: - raise exception.ImageNotFound(image_id=image_id) - - result = {'imageId': image_id} - fn(image, result) - return result - - def modify_image_attribute(self, context, image_id, attribute, - operation_type, **kwargs): - # TODO(devcamcar): Support users and groups other than 'all'. - if attribute != 'launchPermission': - raise exception.InvalidAttribute(attr=attribute) - if 'user_group' not in kwargs: - msg = _('user or group not specified') - raise exception.MissingParameter(reason=msg) - if len(kwargs['user_group']) != 1 and kwargs['user_group'][0] != 'all': - msg = _('only group "all" is supported') - raise exception.InvalidParameterValue(message=msg) - if operation_type not in ['add', 'remove']: - msg = _('operation_type must be add or remove') - raise exception.InvalidParameterValue(message=msg) - LOG.info(_LI("Updating image %s publicity"), image_id, context=context) - - try: - image = self._get_image(context, image_id) - except exception.NotFound: - raise exception.ImageNotFound(image_id=image_id) - internal_id = image['id'] - del(image['id']) - - image['is_public'] = (operation_type == 'add') - try: - return self.image_service.update(context, internal_id, image) - except exception.ImageNotAuthorized: - msg = _('Not allowed to modify attributes for image %s') % image_id - raise exception.Forbidden(message=msg) - - def update_image(self, context, image_id, **kwargs): - internal_id = ec2utils.ec2_id_to_id(image_id) - result = self.image_service.update(context, internal_id, dict(kwargs)) - return result - - # TODO(yamahata): race condition - # At the moment there is no way to prevent others from - # manipulating instances/volumes/snapshots. - # As other code doesn't take it into consideration, here we don't - # care of it for now. Ostrich algorithm - # TODO(mriedem): Consider auto-locking the instance when stopping it and - # doing the snapshot, then unlock it when that is done. Locking the - # instance in the database would prevent other APIs from changing the state - # of the instance during this operation for non-admin users. - def create_image(self, context, instance_id, **kwargs): - # NOTE(yamahata): name/description are ignored by register_image(), - # do so here - no_reboot = kwargs.get('no_reboot', False) - name = kwargs.get('name') - validate_instance_id(instance_id) - ec2_instance_id = instance_id - instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, ec2_instance_id) - instance = self.compute_api.get(context, instance_uuid, - want_objects=True) - - # CreateImage only supported for the analogue of EBS-backed instances - if not self.compute_api.is_volume_backed_instance(context, instance): - msg = _("Invalid value '%(ec2_instance_id)s' for instanceId. " - "Instance does not have a volume attached at root " - "(%(root)s)") % {'root': instance.root_device_name, - 'ec2_instance_id': ec2_instance_id} - raise exception.InvalidParameterValue(err=msg) - - # stop the instance if necessary - restart_instance = False - if not no_reboot: - vm_state = instance.vm_state - - # if the instance is in subtle state, refuse to proceed. - if vm_state not in (vm_states.ACTIVE, vm_states.STOPPED): - raise exception.InstanceNotRunning(instance_id=ec2_instance_id) - - if vm_state == vm_states.ACTIVE: - restart_instance = True - # NOTE(mriedem): We do a call here so that we're sure the - # stop request is complete before we begin polling the state. - self.compute_api.stop(context, instance, do_cast=False) - - # wait instance for really stopped (and not transitioning tasks) - start_time = time.time() - while (vm_state != vm_states.STOPPED and - instance.task_state is not None): - time.sleep(1) - instance.refresh() - vm_state = instance.vm_state - # NOTE(yamahata): timeout and error. 1 hour for now for safety. - # Is it too short/long? - # Or is there any better way? - timeout = 1 * 60 * 60 - if time.time() > start_time + timeout: - err = (_("Couldn't stop instance %(instance)s within " - "1 hour. Current vm_state: %(vm_state)s, " - "current task_state: %(task_state)s") % - {'instance': instance_uuid, - 'vm_state': vm_state, - 'task_state': instance.task_state}) - raise exception.InternalError(message=err) - - # meaningful image name - name_map = dict(instance=instance_uuid, now=utils.isotime()) - name = name or _('image of %(instance)s at %(now)s') % name_map - - new_image = self.compute_api.snapshot_volume_backed(context, - instance, - name) - - ec2_id = ec2utils.glance_id_to_ec2_id(context, new_image['id']) - - if restart_instance: - self.compute_api.start(context, instance) - - return {'imageId': ec2_id} - - def create_tags(self, context, **kwargs): - """Add tags to a resource - - Returns True on success, error on failure. - - :param context: context under which the method is called - """ - resources = kwargs.get('resource_id', None) - tags = kwargs.get('tag', None) - - if resources is None or tags is None: - msg = _('resource_id and tag are required') - raise exception.MissingParameter(reason=msg) - - if not isinstance(resources, (tuple, list, set)): - msg = _('Expecting a list of resources') - raise exception.InvalidParameterValue(message=msg) - - for r in resources: - if ec2utils.resource_type_from_id(context, r) != 'instance': - msg = _('Only instances implemented') - raise exception.InvalidParameterValue(message=msg) - - if not isinstance(tags, (tuple, list, set)): - msg = _('Expecting a list of tagSets') - raise exception.InvalidParameterValue(message=msg) - - metadata = {} - for tag in tags: - if not isinstance(tag, dict): - err = _('Expecting tagSet to be key/value pairs') - raise exception.InvalidParameterValue(message=err) - - key = tag.get('key', None) - val = tag.get('value', None) - - if key is None or val is None: - err = _('Expecting both key and value to be set') - raise exception.InvalidParameterValue(message=err) - - metadata[key] = val - - for ec2_id in resources: - instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, ec2_id) - instance = self.compute_api.get(context, instance_uuid, - want_objects=True) - self.compute_api.update_instance_metadata(context, - instance, metadata) - - return True - - def delete_tags(self, context, **kwargs): - """Delete tags - - Returns True on success, error on failure. - - :param context: context under which the method is called - """ - resources = kwargs.get('resource_id', None) - tags = kwargs.get('tag', None) - if resources is None or tags is None: - msg = _('resource_id and tag are required') - raise exception.MissingParameter(reason=msg) - - if not isinstance(resources, (tuple, list, set)): - msg = _('Expecting a list of resources') - raise exception.InvalidParameterValue(message=msg) - - for r in resources: - if ec2utils.resource_type_from_id(context, r) != 'instance': - msg = _('Only instances implemented') - raise exception.InvalidParameterValue(message=msg) - - if not isinstance(tags, (tuple, list, set)): - msg = _('Expecting a list of tagSets') - raise exception.InvalidParameterValue(message=msg) - - for ec2_id in resources: - instance_uuid = ec2utils.ec2_inst_id_to_uuid(context, ec2_id) - instance = self.compute_api.get(context, instance_uuid, - want_objects=True) - for tag in tags: - if not isinstance(tag, dict): - msg = _('Expecting tagSet to be key/value pairs') - raise exception.InvalidParameterValue(message=msg) - - key = tag.get('key', None) - if key is None: - msg = _('Expecting key to be set') - raise exception.InvalidParameterValue(message=msg) - - self.compute_api.delete_instance_metadata(context, - instance, key) - - return True - - def describe_tags(self, context, **kwargs): - """List tags - - Returns a dict with a single key 'tagSet' on success, error on failure. - - :param context: context under which the method is called - """ - filters = kwargs.get('filter', None) - search_filts = [] - if filters: - for filter_block in filters: - key_name = filter_block.get('name', None) - val = filter_block.get('value', None) - if val: - if isinstance(val, dict): - val = val.values() - if not isinstance(val, (tuple, list, set)): - val = (val,) - if key_name: - search_block = {} - if key_name in ('resource_id', 'resource-id'): - search_block['resource_id'] = [] - for res_id in val: - search_block['resource_id'].append( - ec2utils.ec2_inst_id_to_uuid(context, res_id)) - elif key_name in ['key', 'value']: - search_block[key_name] = \ - [ec2utils.regex_from_ec2_regex(v) for v in val] - elif key_name in ('resource_type', 'resource-type'): - for res_type in val: - if res_type != 'instance': - raise exception.InvalidParameterValue( - message=_('Only instances implemented')) - search_block[key_name] = 'instance' - if len(search_block.keys()) > 0: - search_filts.append(search_block) - ts = [] - for tag in self.compute_api.get_all_instance_metadata(context, - search_filts): - ts.append({ - 'resource_id': ec2utils.id_to_ec2_inst_id(tag['instance_id']), - 'resource_type': 'instance', - 'key': tag['key'], - 'value': tag['value'] - }) - return {"tagSet": ts} - - -class EC2SecurityGroupExceptions(object): - @staticmethod - def raise_invalid_property(msg): - raise exception.InvalidParameterValue(message=msg) - - @staticmethod - def raise_group_already_exists(msg): - raise exception.SecurityGroupExists(message=msg) - - @staticmethod - def raise_invalid_group(msg): - raise exception.InvalidGroup(reason=msg) - - @staticmethod - def raise_invalid_cidr(cidr, decoding_exception=None): - if decoding_exception: - raise decoding_exception - else: - raise exception.InvalidParameterValue(message=_("Invalid CIDR")) - - @staticmethod - def raise_over_quota(msg): - raise exception.SecurityGroupLimitExceeded(msg) - - @staticmethod - def raise_not_found(msg): - pass - - -class CloudSecurityGroupNovaAPI(EC2SecurityGroupExceptions, - compute_api.SecurityGroupAPI): - pass - - -class CloudSecurityGroupNeutronAPI(EC2SecurityGroupExceptions, - neutron_driver.SecurityGroupAPI): - pass - - -def get_cloud_security_group_api(): - if cfg.CONF.security_group_api.lower() == 'nova': - return CloudSecurityGroupNovaAPI() - elif openstack_driver.is_neutron_security_groups(): - return CloudSecurityGroupNeutronAPI() - else: - raise NotImplementedError() diff --git a/nova/api/ec2/faults.py b/nova/api/ec2/faults.py deleted file mode 100644 index c9ca2a4ecf29..000000000000 --- a/nova/api/ec2/faults.py +++ /dev/null @@ -1,73 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_config import cfg -from oslo_log import log as logging -import webob.dec -import webob.exc - -import nova.api.ec2 -from nova import context -from nova import utils - -CONF = cfg.CONF -LOG = logging.getLogger(__name__) - - -def ec2_error_response(request_id, code, message, status=500): - """Helper to construct an EC2 compatible error response.""" - LOG.debug('EC2 error response: %(code)s: %(message)s', - {'code': code, 'message': message}) - resp = webob.Response() - resp.status = status - resp.headers['Content-Type'] = 'text/xml' - resp.body = str('\n' - '%s' - '%s' - '%s' % - (utils.xhtml_escape(utils.utf8(code)), - utils.xhtml_escape(utils.utf8(message)), - utils.xhtml_escape(utils.utf8(request_id)))) - return resp - - -class Fault(webob.exc.HTTPException): - """Captures exception and return REST Response.""" - - def __init__(self, exception): - """Create a response for the given webob.exc.exception.""" - self.wrapped_exc = exception - - @webob.dec.wsgify - def __call__(self, req): - """Generate a WSGI response based on the exception passed to ctor.""" - code = nova.api.ec2.exception_to_ec2code(self.wrapped_exc) - status = self.wrapped_exc.status_int - message = self.wrapped_exc.explanation - - if status == 501: - message = "The requested function is not supported" - - if 'AWSAccessKeyId' not in req.params: - raise webob.exc.HTTPBadRequest() - user_id, _sep, project_id = req.params['AWSAccessKeyId'].partition(':') - project_id = project_id or user_id - remote_address = getattr(req, 'remote_address', '127.0.0.1') - if CONF.use_forwarded_for: - remote_address = req.headers.get('X-Forwarded-For', remote_address) - - ctxt = context.RequestContext(user_id, - project_id, - remote_address=remote_address) - resp = ec2_error_response(ctxt.request_id, code, - message=message, status=status) - return resp diff --git a/nova/api/ec2/inst_state.py b/nova/api/ec2/inst_state.py deleted file mode 100644 index ca8282b84db0..000000000000 --- a/nova/api/ec2/inst_state.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2011 Isaku Yamahata -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -PENDING_CODE = 0 -RUNNING_CODE = 16 -SHUTTING_DOWN_CODE = 32 -TERMINATED_CODE = 48 -STOPPING_CODE = 64 -STOPPED_CODE = 80 - -PENDING = 'pending' -RUNNING = 'running' -SHUTTING_DOWN = 'shutting-down' -TERMINATED = 'terminated' -STOPPING = 'stopping' -STOPPED = 'stopped' - -# non-ec2 value -MIGRATE = 'migrate' -RESIZE = 'resize' -PAUSE = 'pause' -SUSPEND = 'suspend' -RESCUE = 'rescue' - -# EC2 API instance status code -_NAME_TO_CODE = { - PENDING: PENDING_CODE, - RUNNING: RUNNING_CODE, - SHUTTING_DOWN: SHUTTING_DOWN_CODE, - TERMINATED: TERMINATED_CODE, - STOPPING: STOPPING_CODE, - STOPPED: STOPPED_CODE, - - # approximation - MIGRATE: RUNNING_CODE, - RESIZE: RUNNING_CODE, - PAUSE: STOPPED_CODE, - SUSPEND: STOPPED_CODE, - RESCUE: RUNNING_CODE, -} - - -def name_to_code(name): - return _NAME_TO_CODE.get(name, PENDING_CODE) diff --git a/nova/api/opts.py b/nova/api/opts.py index 18505a92b0c6..492b58637a18 100644 --- a/nova/api/opts.py +++ b/nova/api/opts.py @@ -13,8 +13,6 @@ import itertools import nova.api.auth -import nova.api.ec2 -import nova.api.ec2.cloud import nova.api.metadata.base import nova.api.metadata.handler import nova.api.metadata.vendordata_json @@ -68,7 +66,6 @@ import nova.db.sqlalchemy.api import nova.exception import nova.image.download.file import nova.image.glance -import nova.image.s3 import nova.ipv6.api import nova.keymgr import nova.keymgr.barbican @@ -85,7 +82,6 @@ import nova.network.rpcapi import nova.network.security_group.openstack_driver import nova.notifications import nova.objects.network -import nova.objectstore.s3server import nova.paths import nova.pci.request import nova.pci.whitelist @@ -129,8 +125,6 @@ def list_opts(): [nova.api.metadata.vendordata_json.file_opt], [nova.api.openstack.compute.allow_instance_snapshots_opt], nova.api.auth.auth_opts, - nova.api.ec2.cloud.ec2_opts, - nova.api.ec2.ec2_opts, nova.api.metadata.base.metadata_opts, nova.api.metadata.handler.metadata_opts, nova.api.openstack.common.osapi_opts, diff --git a/nova/cloudpipe/pipelib.py b/nova/cloudpipe/pipelib.py index 4e69efba1eed..a18d7430d2fc 100644 --- a/nova/cloudpipe/pipelib.py +++ b/nova/cloudpipe/pipelib.py @@ -74,13 +74,9 @@ def _load_boot_script(): with open(CONF.boot_script_template, "r") as shellfile: s = string.Template(shellfile.read()) - CONF.import_opt('ec2_dmz_host', 'nova.api.ec2.cloud') - CONF.import_opt('ec2_port', 'nova.api.ec2.cloud') CONF.import_opt('cnt_vpn_clients', 'nova.network.manager') - return s.substitute(cc_dmz=CONF.ec2_dmz_host, - cc_port=CONF.ec2_port, - dmz_net=CONF.dmz_net, + return s.substitute(dmz_net=CONF.dmz_net, dmz_mask=CONF.dmz_mask, num_vpn=CONF.cnt_vpn_clients) diff --git a/nova/cmd/all.py b/nova/cmd/all.py index d65bd0282c21..0764698e867e 100644 --- a/nova/cmd/all.py +++ b/nova/cmd/all.py @@ -32,7 +32,6 @@ from oslo_log import log as logging from nova import config from nova.i18n import _LE from nova import objects -from nova.objectstore import s3server from nova import service from nova import utils from nova.vnc import xvp_proxy @@ -62,7 +61,7 @@ def main(): except (Exception, SystemExit): LOG.exception(_LE('Failed to load %s-api'), api) - for mod in [s3server, xvp_proxy]: + for mod in [xvp_proxy]: try: launcher.launch_service(mod.get_wsgi_server()) except (Exception, SystemExit): diff --git a/nova/cmd/api.py b/nova/cmd/api.py index b025f7fce97e..a63871b565f6 100644 --- a/nova/cmd/api.py +++ b/nova/cmd/api.py @@ -48,10 +48,6 @@ def main(): launcher = service.process_launcher() for api in CONF.enabled_apis: should_use_ssl = api in CONF.enabled_ssl_apis - if api == 'ec2': - server = service.WSGIService(api, use_ssl=should_use_ssl, - max_url_len=16384) - else: - server = service.WSGIService(api, use_ssl=should_use_ssl) + server = service.WSGIService(api, use_ssl=should_use_ssl) launcher.launch_service(server, workers=server.workers or 1) launcher.wait() diff --git a/nova/cmd/objectstore.py b/nova/cmd/objectstore.py deleted file mode 100644 index e15d84409969..000000000000 --- a/nova/cmd/objectstore.py +++ /dev/null @@ -1,41 +0,0 @@ - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Daemon for nova objectstore. Supports S3 API.""" - -import sys - -from oslo_log import log as logging -from oslo_reports import guru_meditation_report as gmr - -from nova import config -from nova.objectstore import s3server -from nova import service -from nova import utils -from nova import version - - -def main(): - config.parse_args(sys.argv) - logging.setup(config.CONF, "nova") - utils.monkey_patch() - - gmr.TextGuruMeditation.setup_autorun(version) - - server = s3server.get_wsgi_server() - service.serve(server) - service.wait() diff --git a/nova/objectstore/__init__.py b/nova/objectstore/__init__.py deleted file mode 100644 index e771027ef55c..000000000000 --- a/nova/objectstore/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -:mod:`nova.objectstore` -- S3-type object store -===================================================== - -.. automodule:: nova.objectstore - :platform: Unix - :synopsis: Currently a trivial file-based system, getting extended w/ swift. -""" diff --git a/nova/objectstore/s3server.py b/nova/objectstore/s3server.py deleted file mode 100644 index b0332abbd630..000000000000 --- a/nova/objectstore/s3server.py +++ /dev/null @@ -1,383 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2010 OpenStack Foundation -# Copyright 2009 Facebook -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Implementation of an S3-like storage server based on local files. - -Useful to test features that will eventually run on S3, or if you want to -run something locally that was once running on S3. - -We don't support all the features of S3, but it does work with the -standard S3 client for the most basic semantics. To use the standard -S3 client with this module:: - - c = S3.AWSAuthConnection("", "", server="localhost", port=8888, - is_secure=False) - c.create_bucket("mybucket") - c.put("mybucket", "mykey", "a value") - print c.get("mybucket", "mykey").body - -""" - -import bisect -import datetime -import os -import os.path -import urllib - -from oslo_config import cfg -from oslo_log import log as logging -from oslo_log import versionutils -from oslo_utils import fileutils -import routes -import six -import webob - -from nova.i18n import _LW -from nova import paths -from nova import utils -from nova import wsgi - - -LOG = logging.getLogger(__name__) - - -s3_opts = [ - cfg.StrOpt('buckets_path', - default=paths.state_path_def('buckets'), - help='Path to S3 buckets'), - cfg.StrOpt('s3_listen', - default="0.0.0.0", - help='IP address for S3 API to listen'), - cfg.IntOpt('s3_listen_port', - default=3333, - min=1, - max=65535, - help='Port for S3 API to listen'), -] - -CONF = cfg.CONF -CONF.register_opts(s3_opts) - - -def get_wsgi_server(): - return wsgi.Server("S3 Objectstore", - S3Application(CONF.buckets_path), - port=CONF.s3_listen_port, - host=CONF.s3_listen) - - -class S3Application(wsgi.Router): - """Implementation of an S3-like storage server based on local files. - - If bucket depth is given, we break files up into multiple directories - to prevent hitting file system limits for number of files in each - directories. 1 means one level of directories, 2 means 2, etc. - - """ - - def __init__(self, root_directory, bucket_depth=0, mapper=None): - versionutils.report_deprecated_feature( - LOG, - _LW('The in tree EC2 API is deprecated as of Kilo release and may ' - 'be removed in a future release. The openstack ec2-api ' - 'project http://git.openstack.org/cgit/openstack/ec2-api/ ' - 'is the target replacement for this functionality.') - ) - if mapper is None: - mapper = routes.Mapper() - - mapper.connect('/', - controller=lambda *a, **kw: RootHandler(self)(*a, **kw)) - mapper.connect('/{bucket}/{object_name}', - controller=lambda *a, **kw: ObjectHandler(self)(*a, **kw)) - mapper.connect('/{bucket_name}/', - controller=lambda *a, **kw: BucketHandler(self)(*a, **kw)) - self.directory = os.path.abspath(root_directory) - fileutils.ensure_tree(self.directory) - self.bucket_depth = bucket_depth - super(S3Application, self).__init__(mapper) - - -class BaseRequestHandler(object): - """Base class emulating Tornado's web framework pattern in WSGI. - - This is a direct port of Tornado's implementation, so some key decisions - about how the code interacts have already been chosen. - - The two most common ways of designing web frameworks can be - classified as async object-oriented and sync functional. - - Tornado's is on the OO side because a response is built up in and using - the shared state of an object and one of the object's methods will - eventually trigger the "finishing" of the response asynchronously. - - Most WSGI stuff is in the functional side, we pass a request object to - every call down a chain and the eventual return value will be a response. - - Part of the function of the routing code in S3Application as well as the - code in BaseRequestHandler's __call__ method is to merge those two styles - together enough that the Tornado code can work without extensive - modifications. - - To do that it needs to give the Tornado-style code clean objects that it - can modify the state of for each request that is processed, so we use a - very simple factory lambda to create new state for each request, that's - the stuff in the router, and when we let the Tornado code modify that - object to handle the request, then we return the response it generated. - This wouldn't work the same if Tornado was being more async'y and doing - other callbacks throughout the process, but since Tornado is being - relatively simple here we can be satisfied that the response will be - complete by the end of the get/post method. - - """ - - def __init__(self, application): - self.application = application - - @webob.dec.wsgify - def __call__(self, request): - method = request.method.lower() - f = getattr(self, method, self.invalid) - self.request = request - self.response = webob.Response() - params = request.environ['wsgiorg.routing_args'][1] - del params['controller'] - f(**params) - return self.response - - def get_argument(self, arg, default): - return self.request.params.get(arg, default) - - def set_header(self, header, value): - self.response.headers[header] = value - - def set_status(self, status_code): - self.response.status = status_code - - def set_404(self): - self.render_xml({"Error": { - "Code": "NoSuchKey", - "Message": "The resource you requested does not exist" - }}) - self.set_status(404) - - def finish(self, body=''): - self.response.body = utils.utf8(body) - - def invalid(self, **kwargs): - pass - - def render_xml(self, value): - assert isinstance(value, dict) and len(value) == 1 - self.set_header("Content-Type", "application/xml; charset=UTF-8") - name = list(value.keys())[0] - parts = [] - parts.append('<' + utils.utf8(name) + - ' xmlns="http://doc.s3.amazonaws.com/2006-03-01">') - self._render_parts(list(value.values())[0], parts) - parts.append('') - self.finish('\n' + - ''.join(parts)) - - def _render_parts(self, value, parts=None): - if not parts: - parts = [] - - if isinstance(value, six.string_types): - parts.append(utils.xhtml_escape(value)) - elif type(value) in six.integer_types: - parts.append(str(value)) - elif isinstance(value, bool): - parts.append(str(value)) - elif isinstance(value, datetime.datetime): - parts.append(value.strftime("%Y-%m-%dT%H:%M:%S.000Z")) - elif isinstance(value, dict): - for name, subvalue in six.iteritems(value): - if not isinstance(subvalue, list): - subvalue = [subvalue] - for subsubvalue in subvalue: - parts.append('<' + utils.utf8(name) + '>') - self._render_parts(subsubvalue, parts) - parts.append('') - else: - raise Exception("Unknown S3 value type %r", value) - - def _object_path(self, bucket, object_name): - if self.application.bucket_depth < 1: - return os.path.abspath(os.path.join( - self.application.directory, bucket, object_name)) - hash = utils.get_hash_str(object_name) - path = os.path.abspath(os.path.join( - self.application.directory, bucket)) - for i in range(self.application.bucket_depth): - path = os.path.join(path, hash[:2 * (i + 1)]) - return os.path.join(path, object_name) - - -class RootHandler(BaseRequestHandler): - def get(self): - names = os.listdir(self.application.directory) - buckets = [] - for name in names: - path = os.path.join(self.application.directory, name) - info = os.stat(path) - buckets.append({ - "Name": name, - "CreationDate": datetime.datetime.utcfromtimestamp( - info.st_ctime), - }) - self.render_xml({"ListAllMyBucketsResult": { - "Buckets": {"Bucket": buckets}, - }}) - - -class BucketHandler(BaseRequestHandler): - def get(self, bucket_name): - prefix = self.get_argument("prefix", u"") - marker = self.get_argument("marker", u"") - max_keys = int(self.get_argument("max-keys", 50000)) - path = os.path.abspath(os.path.join(self.application.directory, - bucket_name)) - terse = int(self.get_argument("terse", 0)) - if (not path.startswith(self.application.directory) or - not os.path.isdir(path)): - self.set_404() - return - object_names = [] - for root, dirs, files in os.walk(path): - for file_name in files: - object_names.append(os.path.join(root, file_name)) - skip = len(path) + 1 - for i in range(self.application.bucket_depth): - skip += 2 * (i + 1) + 1 - object_names = [n[skip:] for n in object_names] - object_names.sort() - contents = [] - - start_pos = 0 - if marker: - start_pos = bisect.bisect_right(object_names, marker, start_pos) - if prefix: - start_pos = bisect.bisect_left(object_names, prefix, start_pos) - - truncated = False - for object_name in object_names[start_pos:]: - if not object_name.startswith(prefix): - break - if len(contents) >= max_keys: - truncated = True - break - object_path = self._object_path(bucket_name, object_name) - c = {"Key": object_name} - if not terse: - info = os.stat(object_path) - c.update({ - "LastModified": datetime.datetime.utcfromtimestamp( - info.st_mtime), - "Size": info.st_size, - }) - contents.append(c) - marker = object_name - self.render_xml({"ListBucketResult": { - "Name": bucket_name, - "Prefix": prefix, - "Marker": marker, - "MaxKeys": max_keys, - "IsTruncated": truncated, - "Contents": contents, - }}) - - def put(self, bucket_name): - path = os.path.abspath(os.path.join( - self.application.directory, bucket_name)) - if (not path.startswith(self.application.directory) or - os.path.exists(path)): - self.set_status(403) - return - fileutils.ensure_tree(path) - self.finish() - - def delete(self, bucket_name): - path = os.path.abspath(os.path.join( - self.application.directory, bucket_name)) - if (not path.startswith(self.application.directory) or - not os.path.isdir(path)): - self.set_404() - return - if len(os.listdir(path)) > 0: - self.set_status(403) - return - os.rmdir(path) - self.set_status(204) - self.finish() - - def head(self, bucket_name): - path = os.path.abspath(os.path.join(self.application.directory, - bucket_name)) - if (not path.startswith(self.application.directory) or - not os.path.isdir(path)): - self.set_404() - return - self.set_status(200) - self.finish() - - -class ObjectHandler(BaseRequestHandler): - def get(self, bucket, object_name): - object_name = urllib.unquote(object_name) - path = self._object_path(bucket, object_name) - if (not path.startswith(self.application.directory) or - not os.path.isfile(path)): - self.set_404() - return - info = os.stat(path) - self.set_header("Content-Type", "application/unknown") - self.set_header("Last-Modified", datetime.datetime.utcfromtimestamp( - info.st_mtime)) - with open(path, "r") as object_file: - self.finish(object_file.read()) - - def put(self, bucket, object_name): - object_name = urllib.unquote(object_name) - bucket_dir = os.path.abspath(os.path.join( - self.application.directory, bucket)) - if (not bucket_dir.startswith(self.application.directory) or - not os.path.isdir(bucket_dir)): - self.set_404() - return - path = self._object_path(bucket, object_name) - if not path.startswith(bucket_dir) or os.path.isdir(path): - self.set_status(403) - return - directory = os.path.dirname(path) - fileutils.ensure_tree(directory) - with open(path, "w") as object_file: - object_file.write(self.request.body) - self.set_header('ETag', - '"%s"' % utils.get_hash_str(self.request.body)) - self.finish() - - def delete(self, bucket, object_name): - object_name = urllib.unquote(object_name) - path = self._object_path(bucket, object_name) - if (not path.startswith(self.application.directory) or - not os.path.isfile(path)): - self.set_404() - return - os.unlink(path) - self.set_status(204) - self.finish() diff --git a/nova/opts.py b/nova/opts.py index e1f6bb3f2cc7..4c4317a5ca46 100644 --- a/nova/opts.py +++ b/nova/opts.py @@ -38,7 +38,6 @@ import nova.db.sqlalchemy.api import nova.exception import nova.image.download.file import nova.image.glance -import nova.image.s3 import nova.ipv6.api import nova.keymgr import nova.keymgr.barbican @@ -46,7 +45,6 @@ import nova.keymgr.conf_key_mgr import nova.netconf import nova.notifications import nova.objects.network -import nova.objectstore.s3server import nova.paths import nova.pci.request import nova.pci.whitelist @@ -87,11 +85,9 @@ def list_opts(): nova.db.api.db_opts, nova.db.sqlalchemy.api.db_opts, nova.exception.exc_log_opts, - nova.image.s3.s3_opts, nova.netconf.netconf_opts, nova.notifications.notify_opts, nova.objects.network.network_opts, - nova.objectstore.s3server.s3_opts, nova.paths.path_opts, nova.pci.request.pci_alias_opts, nova.pci.whitelist.pci_opts, diff --git a/nova/service.py b/nova/service.py index ee79735bf008..095fe21dce2e 100644 --- a/nova/service.py +++ b/nova/service.py @@ -63,17 +63,6 @@ service_opts = [ cfg.ListOpt('enabled_ssl_apis', default=[], help='A list of APIs with enabled SSL'), - cfg.StrOpt('ec2_listen', - default="0.0.0.0", - help='The IP address on which the EC2 API will listen.'), - cfg.IntOpt('ec2_listen_port', - default=8773, - min=1, - max=65535, - help='The port on which the EC2 API will listen.'), - cfg.IntOpt('ec2_workers', - help='Number of workers for EC2 API service. The default will ' - 'be equal to the number of CPUs available.'), cfg.StrOpt('osapi_compute_listen', default="0.0.0.0", help='The IP address on which the OpenStack API will listen.'), diff --git a/nova/tests/fixtures.py b/nova/tests/fixtures.py index 9b2d3276b4f0..39467cccca32 100644 --- a/nova/tests/fixtures.py +++ b/nova/tests/fixtures.py @@ -342,10 +342,8 @@ class OSAPIFixture(fixtures.Fixture): # in order to run these in tests we need to bind only to local # host, and dynamically allocate ports conf_overrides = { - 'ec2_listen': '127.0.0.1', 'osapi_compute_listen': '127.0.0.1', 'metadata_listen': '127.0.0.1', - 'ec2_listen_port': 0, 'osapi_compute_listen_port': 0, 'metadata_listen_port': 0, 'verbose': True, diff --git a/nova/tests/unit/api/ec2/__init__.py b/nova/tests/unit/api/ec2/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/nova/tests/unit/api/ec2/public_key/dummy.fingerprint b/nova/tests/unit/api/ec2/public_key/dummy.fingerprint deleted file mode 100644 index 715bca27a27f..000000000000 --- a/nova/tests/unit/api/ec2/public_key/dummy.fingerprint +++ /dev/null @@ -1 +0,0 @@ -1c:87:d1:d9:32:fd:62:3c:78:2b:c0:ad:c0:15:88:df diff --git a/nova/tests/unit/api/ec2/public_key/dummy.pub b/nova/tests/unit/api/ec2/public_key/dummy.pub deleted file mode 100644 index d4cf2bc0d857..000000000000 --- a/nova/tests/unit/api/ec2/public_key/dummy.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-dss AAAAB3NzaC1kc3MAAACBAMGJlY9XEIm2X234pdO5yFWMp2JuOQx8U0E815IVXhmKxYCBK9ZakgZOIQmPbXoGYyV+mziDPp6HJ0wKYLQxkwLEFr51fAZjWQvRss0SinURRuLkockDfGFtD4pYJthekr/rlqMKlBSDUSpGq8jUWW60UJ18FGooFpxR7ESqQRx/AAAAFQC96LRglaUeeP+E8U/yblEJocuiWwAAAIA3XiMR8Skiz/0aBm5K50SeQznQuMJTyzt9S9uaz5QZWiFu69hOyGSFGw8fqgxEkXFJIuHobQQpGYQubLW0NdaYRqyE/Vud3JUJUb8Texld6dz8vGemyB5d1YvtSeHIo8/BGv2msOqR3u5AZTaGCBD9DhpSGOKHEdNjTtvpPd8S8gAAAIBociGZ5jf09iHLVENhyXujJbxfGRPsyNTyARJfCOGl0oFV6hEzcQyw8U/ePwjgvjc2UizMWLl8tsb2FXKHRdc2v+ND3Us+XqKQ33X3ADP4FZ/+Oj213gMyhCmvFTP0u5FmHog9My4CB7YcIWRuUR42WlhQ2IfPvKwUoTk3R+T6Og== www-data@mk diff --git a/nova/tests/unit/api/ec2/test_api.py b/nova/tests/unit/api/ec2/test_api.py deleted file mode 100644 index c4c7139e160b..000000000000 --- a/nova/tests/unit/api/ec2/test_api.py +++ /dev/null @@ -1,635 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Unit tests for the API endpoint.""" - -import random -import re -from six.moves import StringIO - -import boto -import boto.connection -from boto.ec2 import regioninfo -from boto import exception as boto_exc -# newer versions of boto use their own wrapper on top of httplib.HTTPResponse -if hasattr(boto.connection, 'HTTPResponse'): - httplib = boto.connection -else: - from six.moves import http_client as httplib -import fixtures -from oslo_utils import encodeutils -from oslo_utils import versionutils -import webob - -from nova.api import auth -from nova.api import ec2 -from nova.api.ec2 import ec2utils -from nova import block_device -from nova import context -from nova import exception -from nova import test -from nova.tests.unit import matchers - - -class FakeHttplibSocket(object): - """a fake socket implementation for httplib.HTTPResponse, trivial.""" - def __init__(self, response_string): - self.response_string = response_string - self._buffer = StringIO(response_string) - - def makefile(self, _mode, _other): - """Returns the socket's internal buffer.""" - return self._buffer - - -class FakeHttplibConnection(object): - """A fake httplib.HTTPConnection for boto to use - - requests made via this connection actually get translated and routed into - our WSGI app, we then wait for the response and turn it back into - the HTTPResponse that boto expects. - """ - def __init__(self, app, host, is_secure=False): - self.app = app - self.host = host - - def request(self, method, path, data, headers): - req = webob.Request.blank(path) - req.method = method - req.body = encodeutils.safe_encode(data) - req.headers = headers - req.headers['Accept'] = 'text/html' - req.host = self.host - # Call the WSGI app, get the HTTP response - resp = str(req.get_response(self.app)) - # For some reason, the response doesn't have "HTTP/1.0 " prepended; I - # guess that's a function the web server usually provides. - resp = "HTTP/1.0 %s" % resp - self.sock = FakeHttplibSocket(resp) - self.http_response = httplib.HTTPResponse(self.sock) - # NOTE(vish): boto is accessing private variables for some reason - self._HTTPConnection__response = self.http_response - self.http_response.begin() - - def getresponse(self): - return self.http_response - - def getresponsebody(self): - return self.sock.response_string - - def close(self): - """Required for compatibility with boto/tornado.""" - pass - - -class XmlConversionTestCase(test.NoDBTestCase): - """Unit test api xml conversion.""" - def test_number_conversion(self): - conv = ec2utils._try_convert - self.assertIsNone(conv('None')) - self.assertEqual(conv('True'), True) - self.assertEqual(conv('TRUE'), True) - self.assertEqual(conv('true'), True) - self.assertEqual(conv('False'), False) - self.assertEqual(conv('FALSE'), False) - self.assertEqual(conv('false'), False) - self.assertEqual(conv('0'), 0) - self.assertEqual(conv('42'), 42) - self.assertEqual(conv('3.14'), 3.14) - self.assertEqual(conv('-57.12'), -57.12) - self.assertEqual(conv('0x57'), 0x57) - self.assertEqual(conv('-0x57'), -0x57) - self.assertEqual(conv('-'), '-') - self.assertEqual(conv('-0'), 0) - self.assertEqual(conv('0.0'), 0.0) - self.assertEqual(conv('1e-8'), 0.0) - self.assertEqual(conv('-1e-8'), 0.0) - self.assertEqual(conv('0xDD8G'), '0xDD8G') - self.assertEqual(conv('0XDD8G'), '0XDD8G') - self.assertEqual(conv('-stringy'), '-stringy') - self.assertEqual(conv('stringy'), 'stringy') - self.assertEqual(conv('add'), 'add') - self.assertEqual(conv('remove'), 'remove') - self.assertEqual(conv(''), '') - - -class Ec2utilsTestCase(test.NoDBTestCase): - def test_ec2_id_to_id(self): - self.assertEqual(ec2utils.ec2_id_to_id('i-0000001e'), 30) - self.assertEqual(ec2utils.ec2_id_to_id('ami-1d'), 29) - self.assertEqual(ec2utils.ec2_id_to_id('snap-0000001c'), 28) - self.assertEqual(ec2utils.ec2_id_to_id('vol-0000001b'), 27) - - def test_bad_ec2_id(self): - self.assertRaises(exception.InvalidEc2Id, - ec2utils.ec2_id_to_id, - 'badone') - - def test_id_to_ec2_id(self): - self.assertEqual(ec2utils.id_to_ec2_id(30), 'i-0000001e') - self.assertEqual(ec2utils.id_to_ec2_id(29, 'ami-%08x'), 'ami-0000001d') - self.assertEqual(ec2utils.id_to_ec2_snap_id(28), 'snap-0000001c') - self.assertEqual(ec2utils.id_to_ec2_vol_id(27), 'vol-0000001b') - - def test_dict_from_dotted_str(self): - in_str = [('BlockDeviceMapping.1.DeviceName', '/dev/sda1'), - ('BlockDeviceMapping.1.Ebs.SnapshotId', 'snap-0000001c'), - ('BlockDeviceMapping.1.Ebs.VolumeSize', '80'), - ('BlockDeviceMapping.1.Ebs.DeleteOnTermination', 'false'), - ('BlockDeviceMapping.2.DeviceName', '/dev/sdc'), - ('BlockDeviceMapping.2.VirtualName', 'ephemeral0')] - expected_dict = { - 'block_device_mapping': { - '1': {'device_name': '/dev/sda1', - 'ebs': {'snapshot_id': 'snap-0000001c', - 'volume_size': 80, - 'delete_on_termination': False}}, - '2': {'device_name': '/dev/sdc', - 'virtual_name': 'ephemeral0'}}} - out_dict = ec2utils.dict_from_dotted_str(in_str) - - self.assertThat(out_dict, matchers.DictMatches(expected_dict)) - - def test_properties_root_device_name(self): - mappings = [{"device": "/dev/sda1", "virtual": "root"}] - properties0 = {'mappings': mappings} - properties1 = {'root_device_name': '/dev/sdb', 'mappings': mappings} - - root_device_name = block_device.properties_root_device_name( - properties0) - self.assertEqual(root_device_name, '/dev/sda1') - - root_device_name = block_device.properties_root_device_name( - properties1) - self.assertEqual(root_device_name, '/dev/sdb') - - def test_regex_from_ec2_regex(self): - def _test_re(ec2_regex, expected, literal, match=True): - regex = ec2utils.regex_from_ec2_regex(ec2_regex) - self.assertEqual(regex, expected) - if match: - self.assertIsNotNone(re.match(regex, literal)) - else: - self.assertIsNone(re.match(regex, literal)) - - # wildcards - _test_re('foo', '\Afoo\Z(?s)', 'foo') - _test_re('foo', '\Afoo\Z(?s)', 'baz', match=False) - _test_re('foo?bar', '\Afoo.bar\Z(?s)', 'foo bar') - _test_re('foo?bar', '\Afoo.bar\Z(?s)', 'foo bar', match=False) - _test_re('foo*bar', '\Afoo.*bar\Z(?s)', 'foo QUUX bar') - - # backslashes and escaped wildcards - _test_re('foo\\', '\Afoo\\\\\Z(?s)', 'foo\\') - _test_re('foo*bar', '\Afoo.*bar\Z(?s)', 'zork QUUX bar', match=False) - _test_re('foo\\?bar', '\Afoo[?]bar\Z(?s)', 'foo?bar') - _test_re('foo\\?bar', '\Afoo[?]bar\Z(?s)', 'foo bar', match=False) - _test_re('foo\\*bar', '\Afoo[*]bar\Z(?s)', 'foo*bar') - _test_re('foo\\*bar', '\Afoo[*]bar\Z(?s)', 'foo bar', match=False) - - # analog to the example given in the EC2 API docs - ec2_regex = '\*nova\?\\end' - expected = r'\A[*]nova[?]\\end\Z(?s)' - literal = r'*nova?\end' - _test_re(ec2_regex, expected, literal) - - def test_mapping_prepend_dev(self): - mappings = [ - {'virtual': 'ami', - 'device': 'sda1'}, - {'virtual': 'root', - 'device': '/dev/sda1'}, - - {'virtual': 'swap', - 'device': 'sdb1'}, - {'virtual': 'swap', - 'device': '/dev/sdb2'}, - - {'virtual': 'ephemeral0', - 'device': 'sdc1'}, - {'virtual': 'ephemeral1', - 'device': '/dev/sdc1'}] - expected_result = [ - {'virtual': 'ami', - 'device': 'sda1'}, - {'virtual': 'root', - 'device': '/dev/sda1'}, - - {'virtual': 'swap', - 'device': '/dev/sdb1'}, - {'virtual': 'swap', - 'device': '/dev/sdb2'}, - - {'virtual': 'ephemeral0', - 'device': '/dev/sdc1'}, - {'virtual': 'ephemeral1', - 'device': '/dev/sdc1'}] - self.assertThat(block_device.mappings_prepend_dev(mappings), - matchers.DictListMatches(expected_result)) - - -class ApiEc2TestCase(test.TestCase): - """Unit test for the cloud controller on an EC2 API.""" - def setUp(self): - super(ApiEc2TestCase, self).setUp() - self.host = '127.0.0.1' - # NOTE(vish): skipping the Authorizer - roles = ['sysadmin', 'netadmin'] - ctxt = context.RequestContext('fake', 'fake', roles=roles) - self.app = auth.InjectContext(ctxt, ec2.FaultWrapper( - ec2.RequestLogging(ec2.Requestify(ec2.Authorizer(ec2.Executor() - ), 'nova.api.ec2.cloud.CloudController')))) - self.useFixture(fixtures.FakeLogger('boto')) - - def expect_http(self, host=None, is_secure=False, api_version=None): - """Returns a new EC2 connection.""" - self.ec2 = boto.connect_ec2( - aws_access_key_id='fake', - aws_secret_access_key='fake', - is_secure=False, - region=regioninfo.RegionInfo(None, 'test', self.host), - port=8773, - path='/services/Cloud') - if api_version: - self.ec2.APIVersion = api_version - - self.mox.StubOutWithMock(self.ec2, 'new_http_connection') - self.http = FakeHttplibConnection( - self.app, '%s:8773' % (self.host), False) - if versionutils.is_compatible('2.14', boto.Version, same_major=False): - self.ec2.new_http_connection(host or self.host, 8773, - is_secure).AndReturn(self.http) - elif versionutils.is_compatible('2', boto.Version, same_major=False): - self.ec2.new_http_connection(host or '%s:8773' % (self.host), - is_secure).AndReturn(self.http) - else: - self.ec2.new_http_connection(host, is_secure).AndReturn(self.http) - return self.http - - def test_xmlns_version_matches_request_version(self): - self.expect_http(api_version='2010-10-30') - self.mox.ReplayAll() - - # Any request should be fine - self.ec2.get_all_instances() - self.assertIn(self.ec2.APIVersion, self.http.getresponsebody(), - 'The version in the xmlns of the response does ' - 'not match the API version given in the request.') - - def test_describe_instances(self): - """Test that, after creating a user and a project, the describe - instances call to the API works properly. - """ - self.expect_http() - self.mox.ReplayAll() - self.assertEqual(self.ec2.get_all_instances(), []) - - def test_terminate_invalid_instance(self): - # Attempt to terminate an invalid instance. - self.expect_http() - self.mox.ReplayAll() - self.assertRaises(boto_exc.EC2ResponseError, - self.ec2.terminate_instances, "i-00000005") - - def test_get_all_key_pairs(self): - """Test that, after creating a user and project and generating - a key pair, that the API call to list key pairs works properly. - """ - keyname = "".join(random.choice("sdiuisudfsdcnpaqwertasd") - for x in range(random.randint(4, 8))) - self.expect_http() - self.mox.ReplayAll() - self.ec2.create_key_pair(keyname) - rv = self.ec2.get_all_key_pairs() - results = [k for k in rv if k.name == keyname] - self.assertEqual(len(results), 1) - - def test_create_duplicate_key_pair(self): - """Test that, after successfully generating a keypair, - requesting a second keypair with the same name fails sanely. - """ - self.expect_http() - self.mox.ReplayAll() - self.ec2.create_key_pair('test') - - try: - self.ec2.create_key_pair('test') - except boto_exc.EC2ResponseError as e: - if e.code == 'InvalidKeyPair.Duplicate': - pass - else: - self.assertEqual('InvalidKeyPair.Duplicate', e.code) - else: - self.fail('Exception not raised.') - - def test_get_all_security_groups(self): - # Test that we can retrieve security groups. - self.expect_http() - self.mox.ReplayAll() - - rv = self.ec2.get_all_security_groups() - - self.assertEqual(len(rv), 1) - self.assertEqual(rv[0].name, 'default') - - def test_create_delete_security_group(self): - # Test that we can create a security group. - self.expect_http() - self.mox.ReplayAll() - - security_group_name = "".join(random.choice("sdiuisudfsdcnpaqwertasd") - for x in range(random.randint(4, 8))) - - self.ec2.create_security_group(security_group_name, 'test group') - - self.expect_http() - self.mox.ReplayAll() - - rv = self.ec2.get_all_security_groups() - self.assertEqual(len(rv), 2) - self.assertIn(security_group_name, [group.name for group in rv]) - - self.expect_http() - self.mox.ReplayAll() - - self.ec2.delete_security_group(security_group_name) - - def test_group_name_valid_chars_security_group(self): - """Test that we sanely handle invalid security group names. - - EC2 API Spec states we should only accept alphanumeric characters, - spaces, dashes, and underscores. Amazon implementation - accepts more characters - so, [:print:] is ok. - """ - bad_strict_ec2 = "aa \t\x01\x02\x7f" - bad_amazon_ec2 = "aa #^% -=99" - test_raise = [ - (True, bad_amazon_ec2, "test desc"), - (True, "test name", bad_amazon_ec2), - (False, bad_strict_ec2, "test desc"), - ] - for t in test_raise: - self.expect_http() - self.mox.ReplayAll() - self.flags(ec2_strict_validation=t[0]) - self.assertRaises(boto_exc.EC2ResponseError, - self.ec2.create_security_group, - t[1], - t[2]) - test_accept = [ - (False, bad_amazon_ec2, "test desc"), - (False, "test name", bad_amazon_ec2), - ] - for t in test_accept: - self.expect_http() - self.mox.ReplayAll() - self.flags(ec2_strict_validation=t[0]) - self.ec2.create_security_group(t[1], t[2]) - self.expect_http() - self.mox.ReplayAll() - self.ec2.delete_security_group(t[1]) - - def test_group_name_valid_length_security_group(self): - """Test that we sanely handle invalid security group names. - - API Spec states that the length should not exceed 255 char. - """ - self.expect_http() - self.mox.ReplayAll() - - # Test block group_name > 255 chars - security_group_name = "".join(random.choice("poiuytrewqasdfghjklmnbvc") - for x in range(random.randint(256, 266))) - - self.assertRaises(boto_exc.EC2ResponseError, - self.ec2.create_security_group, - security_group_name, - 'test group') - - def test_authorize_revoke_security_group_cidr(self): - """Test that we can add and remove CIDR based rules - to a security group - """ - self.expect_http() - self.mox.ReplayAll() - - security_group_name = "".join(random.choice("sdiuisudfsdcnpaqwertasd") - for x in range(random.randint(4, 8))) - - group = self.ec2.create_security_group(security_group_name, - 'test group') - - self.expect_http() - self.mox.ReplayAll() - group.connection = self.ec2 - - group.authorize('tcp', 80, 81, '0.0.0.0/0') - group.authorize('icmp', -1, -1, '0.0.0.0/0') - group.authorize('udp', 80, 81, '0.0.0.0/0') - group.authorize('tcp', 1, 65535, '0.0.0.0/0') - group.authorize('udp', 1, 65535, '0.0.0.0/0') - group.authorize('icmp', 1, 0, '0.0.0.0/0') - group.authorize('icmp', 0, 1, '0.0.0.0/0') - group.authorize('icmp', 0, 0, '0.0.0.0/0') - - def _assert(message, *args): - try: - group.authorize(*args) - except boto_exc.EC2ResponseError as e: - self.assertEqual(e.status, 400, 'Expected status to be 400') - self.assertIn(message, e.error_message) - else: - raise self.failureException('EC2ResponseError not raised') - - # Invalid CIDR address - _assert('Invalid CIDR', 'tcp', 80, 81, '0.0.0.0/0444') - # Missing ports - _assert('Not enough parameters', 'tcp', '0.0.0.0/0') - # from port cannot be greater than to port - _assert('Invalid port range', 'tcp', 100, 1, '0.0.0.0/0') - # For tcp, negative values are not allowed - _assert('Invalid port range', 'tcp', -1, 1, '0.0.0.0/0') - # For tcp, valid port range 1-65535 - _assert('Invalid port range', 'tcp', 1, 65599, '0.0.0.0/0') - # Invalid Cidr for ICMP type - _assert('Invalid CIDR', 'icmp', -1, -1, '0.0.444.0/4') - # Invalid protocol - _assert('Invalid IP protocol', 'xyz', 1, 14, '0.0.0.0/0') - # Invalid port - _assert('Invalid input received: To and From ports must be integers', - 'tcp', " ", "81", '0.0.0.0/0') - # Invalid icmp port - _assert('Invalid input received: ' - 'Type and Code must be integers for ICMP protocol type', - 'icmp', " ", "81", '0.0.0.0/0') - # Invalid CIDR Address - _assert('Invalid CIDR', 'icmp', -1, -1, '0.0.0.0') - # Invalid CIDR Address - _assert('Invalid CIDR', 'icmp', -1, -1, '0.0.0.0/') - # Invalid Cidr ports - _assert('Invalid port range', 'icmp', 1, 256, '0.0.0.0/0') - - self.expect_http() - self.mox.ReplayAll() - - rv = self.ec2.get_all_security_groups() - - group = [grp for grp in rv if grp.name == security_group_name][0] - - self.assertEqual(len(group.rules), 8) - self.assertEqual(int(group.rules[0].from_port), 80) - self.assertEqual(int(group.rules[0].to_port), 81) - self.assertEqual(len(group.rules[0].grants), 1) - self.assertEqual(str(group.rules[0].grants[0]), '0.0.0.0/0') - - self.expect_http() - self.mox.ReplayAll() - group.connection = self.ec2 - - group.revoke('tcp', 80, 81, '0.0.0.0/0') - group.revoke('icmp', -1, -1, '0.0.0.0/0') - group.revoke('udp', 80, 81, '0.0.0.0/0') - group.revoke('tcp', 1, 65535, '0.0.0.0/0') - group.revoke('udp', 1, 65535, '0.0.0.0/0') - group.revoke('icmp', 1, 0, '0.0.0.0/0') - group.revoke('icmp', 0, 1, '0.0.0.0/0') - group.revoke('icmp', 0, 0, '0.0.0.0/0') - - self.expect_http() - self.mox.ReplayAll() - - self.ec2.delete_security_group(security_group_name) - - self.expect_http() - self.mox.ReplayAll() - group.connection = self.ec2 - - rv = self.ec2.get_all_security_groups() - - self.assertEqual(len(rv), 1) - self.assertEqual(rv[0].name, 'default') - - def test_authorize_revoke_security_group_cidr_v6(self): - """Test that we can add and remove CIDR based rules - to a security group for IPv6 - """ - self.expect_http() - self.mox.ReplayAll() - - security_group_name = "".join(random.choice("sdiuisudfsdcnpaqwertasd") - for x in range(random.randint(4, 8))) - - group = self.ec2.create_security_group(security_group_name, - 'test group') - - self.expect_http() - self.mox.ReplayAll() - group.connection = self.ec2 - - group.authorize('tcp', 80, 81, '::/0') - - self.expect_http() - self.mox.ReplayAll() - - rv = self.ec2.get_all_security_groups() - - group = [grp for grp in rv if grp.name == security_group_name][0] - self.assertEqual(len(group.rules), 1) - self.assertEqual(int(group.rules[0].from_port), 80) - self.assertEqual(int(group.rules[0].to_port), 81) - self.assertEqual(len(group.rules[0].grants), 1) - self.assertEqual(str(group.rules[0].grants[0]), '::/0') - - self.expect_http() - self.mox.ReplayAll() - group.connection = self.ec2 - - group.revoke('tcp', 80, 81, '::/0') - - self.expect_http() - self.mox.ReplayAll() - - self.ec2.delete_security_group(security_group_name) - - self.expect_http() - self.mox.ReplayAll() - group.connection = self.ec2 - - rv = self.ec2.get_all_security_groups() - - self.assertEqual(len(rv), 1) - self.assertEqual(rv[0].name, 'default') - - def test_authorize_revoke_security_group_foreign_group(self): - """Test that we can grant and revoke another security group access - to a security group - """ - self.expect_http() - self.mox.ReplayAll() - - rand_string = 'sdiuisudfsdcnpaqwertasd' - security_group_name = "".join(random.choice(rand_string) - for x in range(random.randint(4, 8))) - other_security_group_name = "".join(random.choice(rand_string) - for x in range(random.randint(4, 8))) - - group = self.ec2.create_security_group(security_group_name, - 'test group') - - self.expect_http() - self.mox.ReplayAll() - - other_group = self.ec2.create_security_group(other_security_group_name, - 'some other group') - - self.expect_http() - self.mox.ReplayAll() - group.connection = self.ec2 - - group.authorize(src_group=other_group) - - self.expect_http() - self.mox.ReplayAll() - - rv = self.ec2.get_all_security_groups() - - # I don't bother checkng that we actually find it here, - # because the create/delete unit test further up should - # be good enough for that. - for group in rv: - if group.name == security_group_name: - self.assertEqual(len(group.rules), 3) - self.assertEqual(len(group.rules[0].grants), 1) - self.assertEqual(str(group.rules[0].grants[0]), - '%s-%s' % (other_security_group_name, 'fake')) - - self.expect_http() - self.mox.ReplayAll() - - rv = self.ec2.get_all_security_groups() - - for group in rv: - if group.name == security_group_name: - self.expect_http() - self.mox.ReplayAll() - group.connection = self.ec2 - group.revoke(src_group=other_group) - - self.expect_http() - self.mox.ReplayAll() - - self.ec2.delete_security_group(security_group_name) - self.ec2.delete_security_group(other_security_group_name) diff --git a/nova/tests/unit/api/ec2/test_apirequest.py b/nova/tests/unit/api/ec2/test_apirequest.py deleted file mode 100644 index e4e9414f804d..000000000000 --- a/nova/tests/unit/api/ec2/test_apirequest.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2014 Hewlett-Packard Development Company, L.P. -# -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Unit tests for the API Request internals.""" - -import copy - -import six - -from oslo_utils import timeutils - -from nova.api.ec2 import apirequest -from nova import test - - -class APIRequestTestCase(test.NoDBTestCase): - - def setUp(self): - super(APIRequestTestCase, self).setUp() - self.req = apirequest.APIRequest("FakeController", "FakeAction", - "FakeVersion", {}) - self.resp = { - 'string': 'foo', - 'int': 1, - 'long': int(1), - 'bool': False, - 'dict': { - 'string': 'foo', - 'int': 1, - } - } - - # The previous will produce an output that looks like the - # following (excusing line wrap for 80 cols): - # - # - # uuid - # 1 - # - # 1 - # foo - # - # false - # foo - # - # - # We don't attempt to ever test for the full document because - # hash seed order might impact it's rendering order. The fact - # that running the function doesn't explode is a big part of - # the win. - - def test_render_response_ascii(self): - data = self.req._render_response(self.resp, 'uuid') - self.assertIn('= 500 or without code) should - be filtered as they might contain sensitive information. - """ - msg = "Test server failure." - err = ec2.ec2_error_ex(TestServerExceptionEC2(msg), self.req, - unexpected=True) - self._validate_ec2_error(err, TestServerExceptionEC2.code, - TestServerExceptionEC2.ec2_code, - unknown_msg=True) - - def test_unexpected_exception_builtin(self): - """Test response to builtin unexpected exception. - - Server exception messages (with code >= 500 or without code) should - be filtered as they might contain sensitive information. - """ - msg = "Test server failure." - err = ec2.ec2_error_ex(RuntimeError(msg), self.req, unexpected=True) - self._validate_ec2_error(err, 500, 'RuntimeError', unknown_msg=True) diff --git a/nova/tests/unit/api/ec2/test_faults.py b/nova/tests/unit/api/ec2/test_faults.py deleted file mode 100644 index 26941398bd77..000000000000 --- a/nova/tests/unit/api/ec2/test_faults.py +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from mox3 import mox -import webob - -from nova.api.ec2 import faults -from nova import test -from nova import wsgi - - -class TestFaults(test.NoDBTestCase): - """Tests covering ec2 Fault class.""" - - def test_fault_exception(self): - # Ensure the status_int is set correctly on faults. - fault = faults.Fault(webob.exc.HTTPBadRequest( - explanation='test')) - self.assertIsInstance(fault.wrapped_exc, webob.exc.HTTPBadRequest) - - def test_fault_exception_status_int(self): - # Ensure the status_int is set correctly on faults. - fault = faults.Fault(webob.exc.HTTPNotFound(explanation='test')) - self.assertEqual(fault.wrapped_exc.status_int, 404) - - def test_fault_call(self): - # Ensure proper EC2 response on faults. - message = 'test message' - ex = webob.exc.HTTPNotFound(explanation=message) - fault = faults.Fault(ex) - req = wsgi.Request.blank('/test') - req.GET['AWSAccessKeyId'] = "test_user_id:test_project_id" - self.mox.StubOutWithMock(faults, 'ec2_error_response') - faults.ec2_error_response(mox.IgnoreArg(), 'HTTPNotFound', - message=message, status=ex.status_int) - self.mox.ReplayAll() - fault(req) diff --git a/nova/tests/unit/api/ec2/test_middleware.py b/nova/tests/unit/api/ec2/test_middleware.py deleted file mode 100644 index c8d3ba4ae3f2..000000000000 --- a/nova/tests/unit/api/ec2/test_middleware.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from lxml import etree -import mock -from oslo_config import cfg -from oslo_utils import fixture as utils_fixture -import requests -from six.moves import range -import webob -import webob.dec -import webob.exc - -from nova.api import ec2 -from nova import context -from nova import exception -from nova import test -from nova import wsgi - -CONF = cfg.CONF - - -@webob.dec.wsgify -def conditional_forbid(req): - """Helper wsgi app returns 403 if param 'die' is 1.""" - if 'die' in req.params and req.params['die'] == '1': - raise webob.exc.HTTPForbidden() - return 'OK' - - -class LockoutTestCase(test.NoDBTestCase): - """Test case for the Lockout middleware.""" - def setUp(self): - super(LockoutTestCase, self).setUp() - self.time_fixture = self.useFixture(utils_fixture.TimeFixture()) - self.lockout = ec2.Lockout(conditional_forbid) - - def _send_bad_attempts(self, access_key, num_attempts=1): - """Fail x.""" - for i in range(num_attempts): - req = webob.Request.blank('/?AWSAccessKeyId=%s&die=1' % access_key) - self.assertEqual(req.get_response(self.lockout).status_int, 403) - - def _is_locked_out(self, access_key): - """Sends a test request to see if key is locked out.""" - req = webob.Request.blank('/?AWSAccessKeyId=%s' % access_key) - return (req.get_response(self.lockout).status_int == 403) - - def test_lockout(self): - self._send_bad_attempts('test', CONF.lockout_attempts) - self.assertTrue(self._is_locked_out('test')) - - def test_timeout(self): - self._send_bad_attempts('test', CONF.lockout_attempts) - self.assertTrue(self._is_locked_out('test')) - self.time_fixture.advance_time_seconds(CONF.lockout_minutes * 60) - self.assertFalse(self._is_locked_out('test')) - - def test_multiple_keys(self): - self._send_bad_attempts('test1', CONF.lockout_attempts) - self.assertTrue(self._is_locked_out('test1')) - self.assertFalse(self._is_locked_out('test2')) - self.time_fixture.advance_time_seconds(CONF.lockout_minutes * 60) - self.assertFalse(self._is_locked_out('test1')) - self.assertFalse(self._is_locked_out('test2')) - - def test_window_timeout(self): - self._send_bad_attempts('test', CONF.lockout_attempts - 1) - self.assertFalse(self._is_locked_out('test')) - self.time_fixture.advance_time_seconds(CONF.lockout_window * 60) - self._send_bad_attempts('test', CONF.lockout_attempts - 1) - self.assertFalse(self._is_locked_out('test')) - - -class ExecutorTestCase(test.NoDBTestCase): - def setUp(self): - super(ExecutorTestCase, self).setUp() - self.executor = ec2.Executor() - - def _execute(self, invoke): - class Fake(object): - pass - fake_ec2_request = Fake() - fake_ec2_request.invoke = invoke - - fake_wsgi_request = Fake() - - fake_wsgi_request.environ = { - 'nova.context': context.get_admin_context(), - 'ec2.request': fake_ec2_request, - } - return self.executor(fake_wsgi_request) - - def _extract_message(self, result): - tree = etree.fromstring(result.body) - return tree.findall('./Errors')[0].find('Error/Message').text - - def _extract_code(self, result): - tree = etree.fromstring(result.body) - return tree.findall('./Errors')[0].find('Error/Code').text - - def test_instance_not_found(self): - def not_found(context): - raise exception.InstanceNotFound(instance_id=5) - result = self._execute(not_found) - self.assertIn('i-00000005', self._extract_message(result)) - self.assertEqual('InvalidInstanceID.NotFound', - self._extract_code(result)) - - def test_instance_not_found_none(self): - def not_found(context): - raise exception.InstanceNotFound(instance_id=None) - - # NOTE(mikal): we want no exception to be raised here, which was what - # was happening in bug/1080406 - result = self._execute(not_found) - self.assertIn('None', self._extract_message(result)) - self.assertEqual('InvalidInstanceID.NotFound', - self._extract_code(result)) - - def test_snapshot_not_found(self): - def not_found(context): - raise exception.SnapshotNotFound(snapshot_id=5) - result = self._execute(not_found) - self.assertIn('snap-00000005', self._extract_message(result)) - self.assertEqual('InvalidSnapshot.NotFound', - self._extract_code(result)) - - def test_volume_not_found(self): - def not_found(context): - raise exception.VolumeNotFound(volume_id=5) - result = self._execute(not_found) - self.assertIn('vol-00000005', self._extract_message(result)) - self.assertEqual('InvalidVolume.NotFound', self._extract_code(result)) - - def test_floating_ip_bad_create_request(self): - def bad_request(context): - raise exception.FloatingIpBadRequest() - result = self._execute(bad_request) - self.assertIn('BadRequest', self._extract_message(result)) - self.assertEqual('UnsupportedOperation', self._extract_code(result)) - - -class FakeResponse(object): - reason = "Test Reason" - - def __init__(self, status_code=400): - self.status_code = status_code - - def json(self): - return {} - - -class KeystoneAuthTestCase(test.NoDBTestCase): - def setUp(self): - super(KeystoneAuthTestCase, self).setUp() - self.kauth = ec2.EC2KeystoneAuth(conditional_forbid) - - def _validate_ec2_error(self, response, http_status, ec2_code): - self.assertEqual(response.status_code, http_status, - 'Expected HTTP status %s' % http_status) - root_e = etree.XML(response.body) - self.assertEqual(root_e.tag, 'Response', - "Top element must be Response.") - errors_e = root_e.find('Errors') - error_e = errors_e[0] - code_e = error_e.find('Code') - self.assertIsNotNone(code_e, "Code element must be present.") - self.assertEqual(code_e.text, ec2_code) - - def test_no_signature(self): - req = wsgi.Request.blank('/test') - resp = self.kauth(req) - self._validate_ec2_error(resp, 400, 'AuthFailure') - - def test_no_key_id(self): - req = wsgi.Request.blank('/test') - req.GET['Signature'] = 'test-signature' - resp = self.kauth(req) - self._validate_ec2_error(resp, 400, 'AuthFailure') - - @mock.patch.object(requests, 'request', return_value=FakeResponse()) - def test_communication_failure(self, mock_request): - req = wsgi.Request.blank('/test') - req.GET['Signature'] = 'test-signature' - req.GET['AWSAccessKeyId'] = 'test-key-id' - resp = self.kauth(req) - self._validate_ec2_error(resp, 400, 'AuthFailure') - mock_request.assert_called_with('POST', CONF.keystone_ec2_url, - data=mock.ANY, headers=mock.ANY, - verify=mock.ANY, cert=mock.ANY) - - @mock.patch.object(requests, 'request', return_value=FakeResponse(200)) - def test_no_result_data(self, mock_request): - req = wsgi.Request.blank('/test') - req.GET['Signature'] = 'test-signature' - req.GET['AWSAccessKeyId'] = 'test-key-id' - resp = self.kauth(req) - self._validate_ec2_error(resp, 400, 'AuthFailure') - mock_request.assert_called_with('POST', CONF.keystone_ec2_url, - data=mock.ANY, headers=mock.ANY, - verify=mock.ANY, cert=mock.ANY) diff --git a/nova/tests/unit/api/test_validator.py b/nova/tests/unit/api/test_validator.py index e9e349194ade..55c922bef995 100644 --- a/nova/tests/unit/api/test_validator.py +++ b/nova/tests/unit/api/test_validator.py @@ -64,12 +64,6 @@ class ValidatorTestCase(test.NoDBTestCase): self.assertFalse(validator.validate_int(4)(5)) self.assertFalse(validator.validate_int()(None)) - def test_validate_ec2_id(self): - self.assertFalse(validator.validate_ec2_id('foobar')) - self.assertFalse(validator.validate_ec2_id('')) - self.assertFalse(validator.validate_ec2_id(1234)) - self.assertTrue(validator.validate_ec2_id('i-284f3a41')) - def test_validate_url_path(self): self.assertTrue(validator.validate_url_path('/path/to/file')) self.assertFalse(validator.validate_url_path('path/to/file')) @@ -89,15 +83,3 @@ class ValidatorTestCase(test.NoDBTestCase): self.assertTrue(validator.validate_user_data(fixture)) self.assertFalse(validator.validate_user_data(False)) self.assertFalse(validator.validate_user_data('hello, world!')) - - def test_default_validator(self): - expect_pass = { - 'attribute': 'foobar' - } - self.assertTrue(validator.validate(expect_pass, - validator.DEFAULT_VALIDATOR)) - expect_fail = { - 'attribute': 0 - } - self.assertFalse(validator.validate(expect_fail, - validator.DEFAULT_VALIDATOR)) diff --git a/nova/tests/unit/image/test_glance.py b/nova/tests/unit/image/test_glance.py index b2bfb3aa31fc..9fb2e0476135 100644 --- a/nova/tests/unit/image/test_glance.py +++ b/nova/tests/unit/image/test_glance.py @@ -20,6 +20,7 @@ from six.moves import StringIO import glanceclient.exc import mock from oslo_config import cfg +from oslo_service import sslutils from oslo_utils import netutils import six import testtools @@ -385,6 +386,7 @@ class TestGlanceClientWrapper(test.NoDBTestCase): @mock.patch('glanceclient.Client') def test_create_glance_client_with_ssl(self, client_mock, ssl_enable_mock): + sslutils.register_opts(CONF) self.flags(ca_file='foo.cert', cert_file='bar.cert', key_file='wut.key', group='ssl') ctxt = mock.sentinel.ctx diff --git a/nova/tests/unit/image/test_s3.py b/nova/tests/unit/image/test_s3.py deleted file mode 100644 index be05928f18cc..000000000000 --- a/nova/tests/unit/image/test_s3.py +++ /dev/null @@ -1,267 +0,0 @@ -# Copyright 2011 Isaku Yamahata -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import binascii -import os -import tempfile - -import eventlet -import fixtures -from mox3 import mox - -from nova.api.ec2 import ec2utils -from nova import context -from nova import db -from nova import exception -from nova.image import s3 -from nova import test -from nova.tests.unit.image import fake - - -ami_manifest_xml = """ - - 2011-06-17 - - test-s3 - 0 - 0 - - - x86_64 - - - ami - sda1 - - - root - /dev/sda1 - - - ephemeral0 - sda2 - - - swap - sda3 - - - aki-00000001 - ari-00000001 - - -""" - -file_manifest_xml = """ - - - foo - foo - foo - - - foo - - - - -""" - - -class TestS3ImageService(test.TestCase): - def setUp(self): - super(TestS3ImageService, self).setUp() - self.context = context.RequestContext(None, None) - self.useFixture(fixtures.FakeLogger('boto')) - - # set up 3 fixtures to test shows, should have id '1', '2', and '3' - db.s3_image_create(self.context, - '155d900f-4e14-4e4c-a73d-069cbf4541e6') - db.s3_image_create(self.context, - 'a2459075-d96c-40d5-893e-577ff92e721c') - db.s3_image_create(self.context, - '76fa36fc-c930-4bf3-8c8a-ea2a2420deb6') - - fake.stub_out_image_service(self) - self.image_service = s3.S3ImageService() - ec2utils.reset_cache() - - def tearDown(self): - super(TestS3ImageService, self).tearDown() - fake.FakeImageService_reset() - - def _assertEqualList(self, list0, list1, keys): - self.assertEqual(len(list0), len(list1)) - key = keys[0] - for x in list0: - self.assertEqual(len(x), len(keys)) - self.assertIn(key, x) - for y in list1: - self.assertIn(key, y) - if x[key] == y[key]: - for k in keys: - self.assertEqual(x[k], y[k]) - - def test_show_cannot_use_uuid(self): - self.assertRaises(exception.ImageNotFound, - self.image_service.show, self.context, - '155d900f-4e14-4e4c-a73d-069cbf4541e6') - - def test_show_translates_correctly(self): - self.image_service.show(self.context, '1') - - def test_show_translates_image_state_correctly(self): - def my_fake_show(self, context, image_id, **kwargs): - fake_state_map = { - '155d900f-4e14-4e4c-a73d-069cbf4541e6': 'downloading', - 'a2459075-d96c-40d5-893e-577ff92e721c': 'failed_decrypt', - '76fa36fc-c930-4bf3-8c8a-ea2a2420deb6': 'available'} - return {'id': image_id, - 'name': 'fakeimage123456', - 'deleted_at': None, - 'deleted': False, - 'status': 'active', - 'is_public': False, - 'container_format': 'raw', - 'disk_format': 'raw', - 'size': '25165824', - 'properties': {'image_state': fake_state_map[image_id]}} - - # Override part of the fake image service as well just for - # this test so we can set the image_state to various values - # and test that S3ImageService does the correct mapping for - # us. We can't put fake bad or pending states in the real fake - # image service as it causes other tests to fail - self.stubs.Set(fake._FakeImageService, 'show', my_fake_show) - ret_image = self.image_service.show(self.context, '1') - self.assertEqual(ret_image['properties']['image_state'], 'pending') - ret_image = self.image_service.show(self.context, '2') - self.assertEqual(ret_image['properties']['image_state'], 'failed') - ret_image = self.image_service.show(self.context, '3') - self.assertEqual(ret_image['properties']['image_state'], 'available') - - def test_detail(self): - self.image_service.detail(self.context) - - def test_s3_create(self): - metadata = {'properties': { - 'root_device_name': '/dev/sda1', - 'block_device_mapping': [ - {'device_name': '/dev/sda1', - 'snapshot_id': 'snap-12345678', - 'delete_on_termination': True}, - {'device_name': '/dev/sda2', - 'virtual_name': 'ephemeral0'}, - {'device_name': '/dev/sdb0', - 'no_device': True}]}} - _manifest, image, image_uuid = self.image_service._s3_parse_manifest( - self.context, metadata, ami_manifest_xml) - - ret_image = self.image_service.show(self.context, image['id']) - self.assertIn('properties', ret_image) - properties = ret_image['properties'] - - self.assertIn('mappings', properties) - mappings = properties['mappings'] - expected_mappings = [ - {"device": "sda1", "virtual": "ami"}, - {"device": "/dev/sda1", "virtual": "root"}, - {"device": "sda2", "virtual": "ephemeral0"}, - {"device": "sda3", "virtual": "swap"}] - self._assertEqualList(mappings, expected_mappings, - ['device', 'virtual']) - - self.assertIn('block_device_mapping', properties) - block_device_mapping = properties['block_device_mapping'] - expected_bdm = [ - {'device_name': '/dev/sda1', - 'snapshot_id': 'snap-12345678', - 'delete_on_termination': True}, - {'device_name': '/dev/sda2', - 'virtual_name': 'ephemeral0'}, - {'device_name': '/dev/sdb0', - 'no_device': True}] - self.assertEqual(block_device_mapping, expected_bdm) - - def _initialize_mocks(self): - handle, tempf = tempfile.mkstemp(dir='/tmp') - ignore = mox.IgnoreArg() - mockobj = self.mox.CreateMockAnything() - self.stubs.Set(self.image_service, '_conn', mockobj) - mockobj(ignore).AndReturn(mockobj) - self.stubs.Set(mockobj, 'get_bucket', mockobj) - mockobj(ignore).AndReturn(mockobj) - self.stubs.Set(mockobj, 'get_key', mockobj) - mockobj(ignore).AndReturn(mockobj) - self.stubs.Set(mockobj, 'get_contents_as_string', mockobj) - mockobj().AndReturn(file_manifest_xml) - self.stubs.Set(self.image_service, '_download_file', mockobj) - mockobj(ignore, ignore, ignore).AndReturn(tempf) - self.stubs.Set(binascii, 'a2b_hex', mockobj) - mockobj(ignore).AndReturn('foo') - mockobj(ignore).AndReturn('foo') - self.stubs.Set(self.image_service, '_decrypt_image', mockobj) - mockobj(ignore, ignore, ignore, ignore, ignore).AndReturn(mockobj) - self.stubs.Set(self.image_service, '_untarzip_image', mockobj) - mockobj(ignore, ignore).AndReturn(tempf) - self.mox.ReplayAll() - - def test_s3_create_image_locations(self): - image_location_1 = 'testbucket_1/test.img.manifest.xml' - # Use another location that starts with a '/' - image_location_2 = '/testbucket_2/test.img.manifest.xml' - - metadata = [{'properties': {'image_location': image_location_1}}, - {'properties': {'image_location': image_location_2}}] - - for mdata in metadata: - self._initialize_mocks() - image = self.image_service._s3_create(self.context, mdata) - eventlet.sleep() - translated = self.image_service._translate_id_to_uuid(self.context, - image) - uuid = translated['id'] - image_service = fake.FakeImageService() - updated_image = image_service.update(self.context, uuid, - {'properties': {'image_state': 'available'}}, - purge_props=False) - self.assertEqual(updated_image['properties']['image_state'], - 'available') - - def test_s3_create_is_public(self): - self._initialize_mocks() - metadata = {'properties': { - 'image_location': 'mybucket/my.img.manifest.xml'}, - 'name': 'mybucket/my.img'} - img = self.image_service._s3_create(self.context, metadata) - eventlet.sleep() - translated = self.image_service._translate_id_to_uuid(self.context, - img) - uuid = translated['id'] - image_service = fake.FakeImageService() - updated_image = image_service.update(self.context, uuid, - {'is_public': True}, purge_props=False) - self.assertTrue(updated_image['is_public']) - self.assertEqual(updated_image['status'], 'active') - self.assertEqual(updated_image['properties']['image_state'], - 'available') - - def test_s3_malicious_tarballs(self): - self.assertRaises(exception.NovaException, - self.image_service._test_for_malicious_tarball, - "/unused", os.path.join(os.path.dirname(__file__), 'abs.tar.gz')) - self.assertRaises(exception.NovaException, - self.image_service._test_for_malicious_tarball, - "/unused", os.path.join(os.path.dirname(__file__), 'rel.tar.gz')) diff --git a/nova/tests/unit/test_bdm.py b/nova/tests/unit/test_bdm.py deleted file mode 100644 index 52a0ca45ef6f..000000000000 --- a/nova/tests/unit/test_bdm.py +++ /dev/null @@ -1,248 +0,0 @@ -# Copyright 2011 Isaku Yamahata -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Tests for Block Device Mapping Code. -""" - -from nova.api.ec2 import cloud -from nova.api.ec2 import ec2utils -from nova import test -from nova.tests.unit import matchers - - -class BlockDeviceMappingEc2CloudTestCase(test.NoDBTestCase): - """Test Case for Block Device Mapping.""" - - def fake_ec2_vol_id_to_uuid(obj, ec2_id): - if ec2_id == 'vol-87654321': - return '22222222-3333-4444-5555-666666666666' - elif ec2_id == 'vol-98765432': - return '77777777-8888-9999-0000-aaaaaaaaaaaa' - else: - return 'OhNoooo' - - def fake_ec2_snap_id_to_uuid(obj, ec2_id): - if ec2_id == 'snap-12345678': - return '00000000-1111-2222-3333-444444444444' - elif ec2_id == 'snap-23456789': - return '11111111-2222-3333-4444-555555555555' - else: - return 'OhNoooo' - - def _assertApply(self, action, bdm_list): - for bdm, expected_result in bdm_list: - self.assertThat(action(bdm), matchers.DictMatches(expected_result)) - - def test_parse_block_device_mapping(self): - self.stubs.Set(ec2utils, - 'ec2_vol_id_to_uuid', - self.fake_ec2_vol_id_to_uuid) - self.stubs.Set(ec2utils, - 'ec2_snap_id_to_uuid', - self.fake_ec2_snap_id_to_uuid) - bdm_list = [ - ({'device_name': '/dev/fake0', - 'ebs': {'snapshot_id': 'snap-12345678', - 'volume_size': 1}}, - {'device_name': '/dev/fake0', - 'snapshot_id': '00000000-1111-2222-3333-444444444444', - 'volume_size': 1, - 'delete_on_termination': True}), - - ({'device_name': '/dev/fake1', - 'ebs': {'snapshot_id': 'snap-23456789', - 'delete_on_termination': False}}, - {'device_name': '/dev/fake1', - 'snapshot_id': '11111111-2222-3333-4444-555555555555', - 'delete_on_termination': False}), - - ({'device_name': '/dev/fake2', - 'ebs': {'snapshot_id': 'vol-87654321', - 'volume_size': 2}}, - {'device_name': '/dev/fake2', - 'volume_id': '22222222-3333-4444-5555-666666666666', - 'volume_size': 2, - 'delete_on_termination': True}), - - ({'device_name': '/dev/fake3', - 'ebs': {'snapshot_id': 'vol-98765432', - 'delete_on_termination': False}}, - {'device_name': '/dev/fake3', - 'volume_id': '77777777-8888-9999-0000-aaaaaaaaaaaa', - 'delete_on_termination': False}), - - ({'device_name': '/dev/fake4', - 'ebs': {'no_device': True}}, - {'device_name': '/dev/fake4', - 'no_device': True}), - - ({'device_name': '/dev/fake5', - 'virtual_name': 'ephemeral0'}, - {'device_name': '/dev/fake5', - 'virtual_name': 'ephemeral0'}), - - ({'device_name': '/dev/fake6', - 'virtual_name': 'swap'}, - {'device_name': '/dev/fake6', - 'virtual_name': 'swap'}), - ] - self._assertApply(cloud._parse_block_device_mapping, bdm_list) - - def test_format_block_device_mapping(self): - bdm_list = [ - ({'device_name': '/dev/fake0', - 'snapshot_id': 0x12345678, - 'volume_size': 1, - 'delete_on_termination': True}, - {'deviceName': '/dev/fake0', - 'ebs': {'snapshotId': 'snap-12345678', - 'volumeSize': 1, - 'deleteOnTermination': True}}), - - ({'device_name': '/dev/fake1', - 'snapshot_id': 0x23456789}, - {'deviceName': '/dev/fake1', - 'ebs': {'snapshotId': 'snap-23456789'}}), - - ({'device_name': '/dev/fake2', - 'snapshot_id': 0x23456789, - 'delete_on_termination': False}, - {'deviceName': '/dev/fake2', - 'ebs': {'snapshotId': 'snap-23456789', - 'deleteOnTermination': False}}), - - ({'device_name': '/dev/fake3', - 'volume_id': 0x12345678, - 'volume_size': 1, - 'delete_on_termination': True}, - {'deviceName': '/dev/fake3', - 'ebs': {'snapshotId': 'vol-12345678', - 'volumeSize': 1, - 'deleteOnTermination': True}}), - - ({'device_name': '/dev/fake4', - 'volume_id': 0x23456789}, - {'deviceName': '/dev/fake4', - 'ebs': {'snapshotId': 'vol-23456789'}}), - - ({'device_name': '/dev/fake5', - 'volume_id': 0x23456789, - 'delete_on_termination': False}, - {'deviceName': '/dev/fake5', - 'ebs': {'snapshotId': 'vol-23456789', - 'deleteOnTermination': False}}), - ] - self._assertApply(cloud._format_block_device_mapping, bdm_list) - - def test_format_mapping(self): - properties = { - 'mappings': [ - {'virtual': 'ami', - 'device': 'sda1'}, - {'virtual': 'root', - 'device': '/dev/sda1'}, - - {'virtual': 'swap', - 'device': 'sdb1'}, - {'virtual': 'swap', - 'device': 'sdb2'}, - {'virtual': 'swap', - 'device': 'sdb3'}, - {'virtual': 'swap', - 'device': 'sdb4'}, - - {'virtual': 'ephemeral0', - 'device': 'sdc1'}, - {'virtual': 'ephemeral1', - 'device': 'sdc2'}, - {'virtual': 'ephemeral2', - 'device': 'sdc3'}, - ], - - 'block_device_mapping': [ - # root - {'device_name': '/dev/sda1', - 'snapshot_id': 0x12345678, - 'delete_on_termination': False}, - - - # overwrite swap - {'device_name': '/dev/sdb2', - 'snapshot_id': 0x23456789, - 'delete_on_termination': False}, - {'device_name': '/dev/sdb3', - 'snapshot_id': 0x3456789A}, - {'device_name': '/dev/sdb4', - 'no_device': True}, - - # overwrite ephemeral - {'device_name': '/dev/sdc2', - 'snapshot_id': 0x3456789A, - 'delete_on_termination': False}, - {'device_name': '/dev/sdc3', - 'snapshot_id': 0x456789AB}, - {'device_name': '/dev/sdc4', - 'no_device': True}, - - # volume - {'device_name': '/dev/sdd1', - 'snapshot_id': 0x87654321, - 'delete_on_termination': False}, - {'device_name': '/dev/sdd2', - 'snapshot_id': 0x98765432}, - {'device_name': '/dev/sdd3', - 'snapshot_id': 0xA9875463}, - {'device_name': '/dev/sdd4', - 'no_device': True}]} - - expected_result = { - 'blockDeviceMapping': [ - # root - {'deviceName': '/dev/sda1', - 'ebs': {'snapshotId': 'snap-12345678', - 'deleteOnTermination': False}}, - - # swap - {'deviceName': '/dev/sdb1', - 'virtualName': 'swap'}, - {'deviceName': '/dev/sdb2', - 'ebs': {'snapshotId': 'snap-23456789', - 'deleteOnTermination': False}}, - {'deviceName': '/dev/sdb3', - 'ebs': {'snapshotId': 'snap-3456789a'}}, - - # ephemeral - {'deviceName': '/dev/sdc1', - 'virtualName': 'ephemeral0'}, - {'deviceName': '/dev/sdc2', - 'ebs': {'snapshotId': 'snap-3456789a', - 'deleteOnTermination': False}}, - {'deviceName': '/dev/sdc3', - 'ebs': {'snapshotId': 'snap-456789ab'}}, - - # volume - {'deviceName': '/dev/sdd1', - 'ebs': {'snapshotId': 'snap-87654321', - 'deleteOnTermination': False}}, - {'deviceName': '/dev/sdd2', - 'ebs': {'snapshotId': 'snap-98765432'}}, - {'deviceName': '/dev/sdd3', - 'ebs': {'snapshotId': 'snap-a9875463'}}]} - - result = {} - cloud._format_mappings(properties, result) - self.assertEqual(result['blockDeviceMapping'].sort(), - expected_result['blockDeviceMapping'].sort()) diff --git a/nova/tests/unit/test_objectstore.py b/nova/tests/unit/test_objectstore.py deleted file mode 100644 index e2f7942e8677..000000000000 --- a/nova/tests/unit/test_objectstore.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Unittets for S3 objectstore clone. -""" - -import os -import shutil -import tempfile - -import boto -from boto import exception as boto_exception -from boto.s3 import connection as s3 -from oslo_config import cfg - -from nova.objectstore import s3server -from nova import test -from nova import wsgi - -CONF = cfg.CONF -CONF.import_opt('s3_host', 'nova.image.s3') - -# Create a unique temporary directory. We don't delete after test to -# allow checking the contents after running tests. Users and/or tools -# running the tests need to remove the tests directories. -OSS_TEMPDIR = tempfile.mkdtemp(prefix='test_oss-') - -# Create bucket/images path -os.makedirs(os.path.join(OSS_TEMPDIR, 'images')) -os.makedirs(os.path.join(OSS_TEMPDIR, 'buckets')) - - -class S3APITestCase(test.NoDBTestCase): - """Test objectstore through S3 API.""" - - def setUp(self): - """Setup users, projects, and start a test server.""" - super(S3APITestCase, self).setUp() - self.flags(buckets_path=os.path.join(OSS_TEMPDIR, 'buckets'), - s3_host='127.0.0.1') - - shutil.rmtree(CONF.buckets_path) - os.mkdir(CONF.buckets_path) - - router = s3server.S3Application(CONF.buckets_path) - self.server = wsgi.Server("S3 Objectstore", - router, - host=CONF.s3_host, - port=0) - self.server.start() - - if not boto.config.has_section('Boto'): - boto.config.add_section('Boto') - - boto.config.set('Boto', 'num_retries', '0') - conn = s3.S3Connection(aws_access_key_id='fake', - aws_secret_access_key='fake', - host=CONF.s3_host, - port=self.server.port, - is_secure=False, - calling_format=s3.OrdinaryCallingFormat()) - self.conn = conn - - def get_http_connection(*args): - """Get a new S3 connection, don't attempt to reuse connections.""" - return self.conn.new_http_connection(*args) - - self.conn.get_http_connection = get_http_connection - - def _ensure_no_buckets(self, buckets): - self.assertEqual(len(buckets), 0, "Bucket list was not empty") - return True - - def _ensure_one_bucket(self, buckets, name): - self.assertEqual(len(buckets), 1, - "Bucket list didn't have exactly one element in it") - self.assertEqual(buckets[0].name, name, "Wrong name") - return True - - def test_list_buckets(self): - # Make sure we are starting with no buckets. - self._ensure_no_buckets(self.conn.get_all_buckets()) - - def test_create_and_delete_bucket(self): - # Test bucket creation and deletion. - bucket_name = 'testbucket' - - self.conn.create_bucket(bucket_name) - self._ensure_one_bucket(self.conn.get_all_buckets(), bucket_name) - self.conn.delete_bucket(bucket_name) - self._ensure_no_buckets(self.conn.get_all_buckets()) - - def test_create_bucket_and_key_and_delete_key_again(self): - # Test key operations on buckets. - bucket_name = 'testbucket' - key_name = 'somekey' - key_contents = 'somekey' - - b = self.conn.create_bucket(bucket_name) - k = b.new_key(key_name) - k.set_contents_from_string(key_contents) - - bucket = self.conn.get_bucket(bucket_name) - - # make sure the contents are correct - key = bucket.get_key(key_name) - self.assertEqual(key.get_contents_as_string(), key_contents, - "Bad contents") - - # delete the key - key.delete() - - self._ensure_no_buckets(bucket.get_all_keys()) - - def test_unknown_bucket(self): - # NOTE(unicell): Since Boto v2.25.0, the underlying implementation - # of get_bucket method changed from GET to HEAD. - # - # Prior to v2.25.0, default validate=True fetched a list of keys in the - # bucket and raises S3ResponseError. As a side effect of switching to - # HEAD request, get_bucket call now generates less error message. - # - # To keep original semantics, additional get_all_keys call is - # suggestted per Boto document. This case tests both validate=False and - # validate=True case for completeness. - # - # http://docs.pythonboto.org/en/latest/releasenotes/v2.25.0.html - # http://docs.pythonboto.org/en/latest/s3_tut.html#accessing-a-bucket - bucket_name = 'falalala' - self.assertRaises(boto_exception.S3ResponseError, - self.conn.get_bucket, - bucket_name) - bucket = self.conn.get_bucket(bucket_name, validate=False) - self.assertRaises(boto_exception.S3ResponseError, - bucket.get_all_keys, - maxkeys=0) - - def tearDown(self): - """Tear down test server.""" - self.server.stop() - super(S3APITestCase, self).tearDown() diff --git a/nova/utils.py b/nova/utils.py index e0b8fe2cc5df..0c013af5d7d0 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -67,7 +67,6 @@ monkey_patch_opts = [ help='Whether to apply monkey patching'), cfg.ListOpt('monkey_patch_modules', default=[ - 'nova.api.ec2.cloud:%s' % (notify_decorator), 'nova.compute.api:%s' % (notify_decorator) ], help='List of modules/decorators to monkey patch'), diff --git a/releasenotes/notes/remove_ec2_and_objectstore_api-4ccb539db1d171fa.yaml b/releasenotes/notes/remove_ec2_and_objectstore_api-4ccb539db1d171fa.yaml new file mode 100644 index 000000000000..67c5df643e72 --- /dev/null +++ b/releasenotes/notes/remove_ec2_and_objectstore_api-4ccb539db1d171fa.yaml @@ -0,0 +1,7 @@ +--- +upgrade: + - All code and tests for Nova's EC2 and ObjectStore API support which + was deprecated in Kilo + (https://wiki.openstack.org/wiki/ReleaseNotes/Kilo#Upgrade_Notes_2) has + been completely removed in Mitaka. This has been replaced by the new + ec2-api project (http://git.openstack.org/cgit/openstack/ec2-api/). \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 0535f2d478e9..f17b932094c0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -58,7 +58,6 @@ console_scripts = nova-manage = nova.cmd.manage:main nova-network = nova.cmd.network:main nova-novncproxy = nova.cmd.novncproxy:main - nova-objectstore = nova.cmd.objectstore:main nova-rootwrap = oslo_rootwrap.cmd:main nova-rootwrap-daemon = oslo_rootwrap.cmd:daemon nova-scheduler = nova.cmd.scheduler:main diff --git a/tests-py3.txt b/tests-py3.txt index 8e5cc5fed903..fa400630cc3e 100644 --- a/tests-py3.txt +++ b/tests-py3.txt @@ -1,12 +1,3 @@ -nova.tests.unit.api.ec2.test_api.ApiEc2TestCase -nova.tests.unit.api.ec2.test_apirequest.APIRequestTestCase -nova.tests.unit.api.ec2.test_cinder_cloud.CinderCloudTestCase -nova.tests.unit.api.ec2.test_cloud.CloudTestCase -nova.tests.unit.api.ec2.test_cloud.CloudTestCaseNeutronProxy -nova.tests.unit.api.ec2.test_ec2_validate.EC2ValidateTestCase -nova.tests.unit.api.ec2.test_error_response.Ec2ErrorResponseTestCase -nova.tests.unit.api.ec2.test_middleware.ExecutorTestCase -nova.tests.unit.api.ec2.test_middleware.KeystoneAuthTestCase nova.tests.unit.api.openstack.compute.legacy_v2.test_extensions.ActionExtensionTest nova.tests.unit.api.openstack.compute.legacy_v2.test_extensions.ControllerExtensionTest nova.tests.unit.api.openstack.compute.legacy_v2.test_extensions.ExtensionControllerIdFormatTest @@ -97,7 +88,6 @@ nova.tests.unit.db.test_migrations.TestNovaMigrationsMySQL nova.tests.unit.db.test_migrations.TestNovaMigrationsPostgreSQL nova.tests.unit.db.test_migrations.TestNovaMigrationsSQLite nova.tests.unit.image.test_fake.FakeImageServiceTestCase -nova.tests.unit.image.test_s3.TestS3ImageService nova.tests.unit.keymgr.test_barbican.BarbicanKeyManagerTestCase nova.tests.unit.keymgr.test_conf_key_mgr.ConfKeyManagerTestCase nova.tests.unit.keymgr.test_key.SymmetricKeyTestCase @@ -126,7 +116,6 @@ nova.tests.unit.test_metadata.MetadataPasswordTestCase nova.tests.unit.test_metadata.MetadataTestCase nova.tests.unit.test_metadata.OpenStackMetadataTestCase nova.tests.unit.test_nova_manage.CellCommandsTestCase -nova.tests.unit.test_objectstore.S3APITestCase nova.tests.unit.test_pipelib.PipelibTest nova.tests.unit.test_policy.AdminRolePolicyTestCase nova.tests.unit.test_quota.QuotaIntegrationTestCase