Merge "Add unittests for image upload functionality in v1"

This commit is contained in:
Jenkins 2013-05-23 10:36:49 +00:00 committed by Gerrit Code Review
commit 266bc4a4a6
3 changed files with 479 additions and 125 deletions

View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2010-2012 OpenStack LLC.
# Copyright 2013 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -38,6 +38,7 @@ from glance.api import policy
import glance.api.v1
from glance.api.v1 import controller
from glance.api.v1 import filters
from glance.api.v1 import upload_utils
from glance.common import exception
from glance.common import utils
from glance.common import wsgi
@ -411,7 +412,7 @@ class Controller(controller.BaseController):
image_data, image_size = self._get_from_store(req.context,
copy_from)
except Exception as e:
self._safe_kill(req, image_meta['id'])
upload_utils.safe_kill(req, image_meta['id'])
msg = _("Copy from external source failed: %s") % e
LOG.debug(msg)
return
@ -420,7 +421,7 @@ class Controller(controller.BaseController):
try:
req.get_content_type('application/octet-stream')
except exception.InvalidContentType:
self._safe_kill(req, image_meta['id'])
upload_utils.safe_kill(req, image_meta['id'])
msg = _("Content-Type must be application/octet-stream")
LOG.debug(msg)
raise HTTPBadRequest(explanation=msg)
@ -439,96 +440,17 @@ class Controller(controller.BaseController):
LOG.debug(_("Uploading image data for image %(image_id)s "
"to %(scheme)s store"), locals())
try:
self.notifier.info("image.prepare", redact_loc(image_meta))
location, size, checksum = store.add(
image_meta['id'],
utils.CooperativeReader(image_data),
image_meta['size'])
self.notifier.info("image.prepare", redact_loc(image_meta))
def _kill_mismatched(image_meta, attr, actual):
supplied = image_meta.get(attr)
if supplied and supplied != actual:
msg = _("Supplied %(attr)s (%(supplied)s) and "
"%(attr)s generated from uploaded image "
"(%(actual)s) did not match. Setting image "
"status to 'killed'.") % locals()
LOG.error(msg)
self._safe_kill(req, image_id)
self._initiate_deletion(req, location, image_id)
raise HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
image_meta, location = upload_utils.upload_data_to_store(req,
image_meta,
image_data,
store,
self.notifier)
# Verify any supplied size/checksum value matches size/checksum
# returned from store when adding image
_kill_mismatched(image_meta, 'size', size)
_kill_mismatched(image_meta, 'checksum', checksum)
self.notifier.info('image.upload', redact_loc(image_meta))
# Update the database with the checksum returned
# from the backend store
LOG.debug(_("Updating image %(image_id)s data. "
"Checksum set to %(checksum)s, size set "
"to %(size)d"), locals())
update_data = {'checksum': checksum,
'size': size}
image_meta = registry.update_image_metadata(req.context,
image_id,
update_data)
self.notifier.info('image.upload', redact_loc(image_meta))
return location
except exception.Duplicate as e:
msg = _("Attempt to upload duplicate image: %s") % e
LOG.debug(msg)
self._safe_kill(req, image_id)
raise HTTPConflict(explanation=msg, request=req)
except exception.Forbidden as e:
msg = _("Forbidden upload attempt: %s") % e
LOG.debug(msg)
self._safe_kill(req, image_id)
raise HTTPForbidden(explanation=msg,
request=req,
content_type="text/plain")
except exception.StorageFull as e:
msg = _("Image storage media is full: %s") % e
LOG.error(msg)
self._safe_kill(req, image_id)
self.notifier.error('image.upload', msg)
raise HTTPRequestEntityTooLarge(explanation=msg, request=req,
content_type='text/plain')
except exception.StorageWriteDenied as e:
msg = _("Insufficient permissions on image storage media: %s") % e
LOG.error(msg)
self._safe_kill(req, image_id)
self.notifier.error('image.upload', msg)
raise HTTPServiceUnavailable(explanation=msg, request=req,
content_type='text/plain')
except exception.ImageSizeLimitExceeded as e:
msg = _("Denying attempt to upload image larger than %d bytes."
% CONF.image_size_cap)
LOG.info(msg)
self._safe_kill(req, image_id)
raise HTTPRequestEntityTooLarge(explanation=msg, request=req,
content_type='text/plain')
except HTTPError as e:
self._safe_kill(req, image_id)
#NOTE(bcwaldon): Ideally, we would just call 'raise' here,
# but something in the above function calls is affecting the
# exception context and we must explicitly re-raise the
# caught exception.
raise e
except Exception as e:
LOG.exception(_("Failed to upload image"))
self._safe_kill(req, image_id)
raise HTTPInternalServerError(request=req)
return location
def _activate(self, req, image_id, location):
"""
@ -558,33 +480,6 @@ class Controller(controller.BaseController):
request=req,
content_type="text/plain")
def _kill(self, req, image_id):
"""
Marks the image status to `killed`.
:param req: The WSGI/Webob Request object
:param image_id: Opaque image identifier
"""
registry.update_image_metadata(req.context, image_id,
{'status': 'killed'})
def _safe_kill(self, req, image_id):
"""
Mark image killed without raising exceptions if it fails.
Since _kill is meant to be called from exceptions handlers, it should
not raise itself, rather it should just log its error.
:param req: The WSGI/Webob Request object
:param image_id: Opaque image identifier
"""
try:
self._kill(req, image_id)
except Exception as e:
LOG.error(_("Unable to kill image %(id)s: "
"%(exc)s") % ({'id': image_id,
'exc': repr(e)}))
def _upload_and_activate(self, req, image_meta):
"""
Safely uploads the image data in the request payload
@ -810,13 +705,6 @@ class Controller(controller.BaseController):
return {'image_meta': image_meta}
@staticmethod
def _initiate_deletion(req, location, id):
if CONF.delayed_delete:
schedule_delayed_delete_from_backend(location, id)
else:
safe_delete_from_backend(location, req.context, id)
@utils.mutating
def delete(self, req, id):
"""
@ -868,7 +756,8 @@ class Controller(controller.BaseController):
# to delete the image if the backend doesn't yet store it.
# See https://bugs.launchpad.net/glance/+bug/747799
if image['location']:
self._initiate_deletion(req, image['location'], id)
upload_utils.initiate_deletion(req, image['location'], id,
CONF.delayed_delete)
except exception.NotFound as e:
msg = (_("Failed to find image to delete: %(e)s") % locals())
for line in msg.split('\n'):

View File

@ -0,0 +1,172 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo.config import cfg
import webob.exc
from glance.common import exception
from glance.common import utils
import glance.openstack.common.log as logging
import glance.registry.client.v1.api as registry
import glance.store
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
def initiate_deletion(req, location, id, delayed_delete=False):
"""
Deletes image data from the backend store.
:param req: The WSGI/Webob Request object
:param location: URL to the image data in a data store
:param image_id: Opaque image identifier
:param delayed_delete: whether data deletion will be delayed
"""
if delayed_delete:
glance.store.schedule_delayed_delete_from_backend(location, id)
else:
glance.store.safe_delete_from_backend(location, req.context, id)
def _kill(req, image_id):
"""
Marks the image status to `killed`.
:param req: The WSGI/Webob Request object
:param image_id: Opaque image identifier
"""
registry.update_image_metadata(req.context, image_id,
{'status': 'killed'})
def safe_kill(req, image_id):
"""
Mark image killed without raising exceptions if it fails.
Since _kill is meant to be called from exceptions handlers, it should
not raise itself, rather it should just log its error.
:param req: The WSGI/Webob Request object
:param image_id: Opaque image identifier
"""
try:
_kill(req, image_id)
except Exception as e:
LOG.exception(_("Unable to kill image %(id)s: ") % {'id': image_id})
def upload_data_to_store(req, image_meta, image_data, store, notifier):
"""
Upload image data to specified store.
Upload image data to the store and cleans up on error.
"""
image_id = image_meta['id']
try:
location, size, checksum = store.add(
image_meta['id'],
utils.CooperativeReader(image_data),
image_meta['size'])
def _kill_mismatched(image_meta, attr, actual):
supplied = image_meta.get(attr)
if supplied and supplied != actual:
msg = _("Supplied %(attr)s (%(supplied)s) and "
"%(attr)s generated from uploaded image "
"(%(actual)s) did not match. Setting image "
"status to 'killed'.") % locals()
LOG.error(msg)
safe_kill(req, image_id)
initiate_deletion(req, location, image_id, CONF.delayed_delete)
raise webob.exc.HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
# Verify any supplied size/checksum value matches size/checksum
# returned from store when adding image
_kill_mismatched(image_meta, 'size', size)
_kill_mismatched(image_meta, 'checksum', checksum)
# Update the database with the checksum returned
# from the backend store
LOG.debug(_("Updating image %(image_id)s data. "
"Checksum set to %(checksum)s, size set "
"to %(size)d"), locals())
update_data = {'checksum': checksum,
'size': size}
image_meta = registry.update_image_metadata(req.context,
image_id,
update_data)
except exception.Duplicate as e:
msg = _("Attempt to upload duplicate image: %s") % e
LOG.debug(msg)
safe_kill(req, image_id)
raise webob.exc.HTTPConflict(explanation=msg, request=req)
except exception.Forbidden as e:
msg = _("Forbidden upload attempt: %s") % e
LOG.debug(msg)
safe_kill(req, image_id)
raise webob.exc.HTTPForbidden(explanation=msg,
request=req,
content_type="text/plain")
except exception.StorageFull as e:
msg = _("Image storage media is full: %s") % e
LOG.error(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except exception.StorageWriteDenied as e:
msg = _("Insufficient permissions on image storage media: %s") % e
LOG.error(msg)
safe_kill(req, image_id)
notifier.error('image.upload', msg)
raise webob.exc.HTTPServiceUnavailable(explanation=msg,
request=req,
content_type='text/plain')
except exception.ImageSizeLimitExceeded as e:
msg = _("Denying attempt to upload image larger than %d bytes."
% CONF.image_size_cap)
LOG.info(msg)
safe_kill(req, image_id)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except webob.exc.HTTPError as e:
LOG.exception(_("Received HTTP error while uploading image."))
safe_kill(req, image_id)
#NOTE(bcwaldon): Ideally, we would just call 'raise' here,
# but something in the above function calls is affecting the
# exception context and we must explicitly re-raise the
# caught exception.
raise e
except Exception as e:
LOG.exception(_("Failed to upload image"))
safe_kill(req, image_id)
raise webob.exc.HTTPInternalServerError(request=req)
return image_meta, location

View File

@ -0,0 +1,293 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mox
import webob.exc
from glance.api.v1 import upload_utils
from glance.common import exception
import glance.registry.client.v1.api as registry
import glance.store
from glance.tests.unit import base
import glance.tests.unit.utils as unit_test_utils
class TestUploadUtils(base.StoreClearingUnitTest):
def setUp(self):
super(TestUploadUtils, self).setUp()
self.config(verbose=True, debug=True)
self.mox = mox.Mox()
def tearDown(self):
super(TestUploadUtils, self).tearDown()
self.mox.UnsetStubs()
def test_initiate_delete(self):
req = unit_test_utils.get_fake_request()
location = "file://foo/bar"
id = unit_test_utils.UUID1
self.mox.StubOutWithMock(glance.store, "safe_delete_from_backend")
glance.store.safe_delete_from_backend(location, req.context, id)
self.mox.ReplayAll()
upload_utils.initiate_deletion(req, location, id)
self.mox.VerifyAll()
def test_initiate_delete_with_delayed_delete(self):
req = unit_test_utils.get_fake_request()
location = "file://foo/bar"
id = unit_test_utils.UUID1
self.mox.StubOutWithMock(glance.store,
"schedule_delayed_delete_from_backend")
glance.store.schedule_delayed_delete_from_backend(location,
id)
self.mox.ReplayAll()
upload_utils.initiate_deletion(req, location, id, True)
self.mox.VerifyAll()
def test_safe_kill(self):
req = unit_test_utils.get_fake_request()
id = unit_test_utils.UUID1
self.mox.StubOutWithMock(registry, "update_image_metadata")
registry.update_image_metadata(req.context, id, {'status': 'killed'})
self.mox.ReplayAll()
upload_utils.safe_kill(req, id)
self.mox.VerifyAll()
def test_safe_kill_with_error(self):
req = unit_test_utils.get_fake_request()
id = unit_test_utils.UUID1
self.mox.StubOutWithMock(registry, "update_image_metadata")
registry.update_image_metadata(req.context,
id,
{'status': 'killed'}
).AndRaise(Exception())
self.mox.ReplayAll()
upload_utils.safe_kill(req, id)
self.mox.VerifyAll()
def test_upload_data_to_store(self):
req = unit_test_utils.get_fake_request()
location = "file://foo/bar"
size = 10
checksum = "checksum"
image_meta = {'id': unit_test_utils.UUID1,
'size': size}
image_data = "blah"
notifier = self.mox.CreateMockAnything()
store = self.mox.CreateMockAnything()
store.add(
image_meta['id'],
mox.IgnoreArg(),
image_meta['size']).AndReturn((location, size, checksum))
self.mox.StubOutWithMock(registry, "update_image_metadata")
update_data = {'checksum': checksum,
'size': size}
registry.update_image_metadata(req.context,
image_meta['id'],
update_data
).AndReturn(
image_meta.update(update_data))
self.mox.ReplayAll()
actual_meta, actual_loc = upload_utils.upload_data_to_store(req,
image_meta,
image_data,
store,
notifier)
self.mox.VerifyAll()
self.assertEqual(actual_loc, location)
self.assertEqual(actual_meta, image_meta.update(update_data))
def test_upload_data_to_store_mismatch_size(self):
req = unit_test_utils.get_fake_request()
location = "file://foo/bar"
size = 10
checksum = "checksum"
image_meta = {'id': unit_test_utils.UUID1,
'size': size + 1} # Need incorrect size for test
image_data = "blah"
notifier = self.mox.CreateMockAnything()
store = self.mox.CreateMockAnything()
store.add(
image_meta['id'],
mox.IgnoreArg(),
image_meta['size']).AndReturn((location, size, checksum))
self.mox.StubOutWithMock(registry, "update_image_metadata")
update_data = {'checksum': checksum}
registry.update_image_metadata(req.context,
image_meta['id'],
update_data
).AndReturn(
image_meta.update(update_data))
self.mox.ReplayAll()
self.assertRaises(webob.exc.HTTPBadRequest,
upload_utils.upload_data_to_store,
req, image_meta, image_data, store, notifier)
self.mox.VerifyAll()
def test_upload_data_to_store_mismatch_checksum(self):
req = unit_test_utils.get_fake_request()
location = "file://foo/bar"
size = 10
checksum = "checksum"
image_meta = {'id': unit_test_utils.UUID1,
'size': size}
image_data = "blah"
notifier = self.mox.CreateMockAnything()
store = self.mox.CreateMockAnything()
store.add(
image_meta['id'],
mox.IgnoreArg(),
image_meta['size']).AndReturn((location, size, checksum + "NOT"))
self.mox.StubOutWithMock(registry, "update_image_metadata")
update_data = {'checksum': checksum}
registry.update_image_metadata(req.context,
image_meta['id'],
update_data).AndReturn(
image_meta.update(update_data))
self.mox.ReplayAll()
self.assertRaises(webob.exc.HTTPBadRequest,
upload_utils.upload_data_to_store,
req, image_meta, image_data, store, notifier)
self.mox.VerifyAll()
def _test_upload_data_to_store_exception(self, exc_class, expected_class):
req = unit_test_utils.get_fake_request()
location = "file://foo/bar"
size = 10
checksum = "checksum"
image_meta = {'id': unit_test_utils.UUID1,
'size': size}
image_data = "blah"
notifier = self.mox.CreateMockAnything()
store = self.mox.CreateMockAnything()
store.add(
image_meta['id'],
mox.IgnoreArg(),
image_meta['size']).AndRaise(exc_class)
self.mox.StubOutWithMock(upload_utils, "safe_kill")
upload_utils.safe_kill(req, image_meta['id'])
self.mox.ReplayAll()
self.assertRaises(expected_class,
upload_utils.upload_data_to_store,
req, image_meta, image_data, store, notifier)
self.mox.VerifyAll()
def _test_upload_data_to_store_exception_with_notify(self,
exc_class,
expected_class):
req = unit_test_utils.get_fake_request()
location = "file://foo/bar"
size = 10
checksum = "checksum"
image_meta = {'id': unit_test_utils.UUID1,
'size': size}
image_data = "blah"
store = self.mox.CreateMockAnything()
store.add(
image_meta['id'],
mox.IgnoreArg(),
image_meta['size']).AndRaise(exc_class)
self.mox.StubOutWithMock(upload_utils, "safe_kill")
upload_utils.safe_kill(req, image_meta['id'])
notifier = self.mox.CreateMockAnything()
notifier.error('image.upload', mox.IgnoreArg())
self.mox.ReplayAll()
self.assertRaises(expected_class,
upload_utils.upload_data_to_store,
req, image_meta, image_data, store, notifier)
self.mox.VerifyAll()
def test_upload_data_to_store_duplicate(self):
self._test_upload_data_to_store_exception(exception.Duplicate,
webob.exc.HTTPConflict)
def test_upload_data_to_store_forbidden(self):
self._test_upload_data_to_store_exception(exception.Forbidden,
webob.exc.HTTPForbidden)
def test_upload_data_to_store_storage_full(self):
self._test_upload_data_to_store_exception_with_notify(
exception.StorageFull,
webob.exc.HTTPRequestEntityTooLarge)
def test_upload_data_to_store_storage_write_denied(self):
self._test_upload_data_to_store_exception_with_notify(
exception.StorageWriteDenied,
webob.exc.HTTPServiceUnavailable)
def test_upload_data_to_store_size_limit_exceeded(self):
self._test_upload_data_to_store_exception(
exception.ImageSizeLimitExceeded,
webob.exc.HTTPRequestEntityTooLarge)
def test_upload_data_to_store_http_error(self):
self._test_upload_data_to_store_exception(
webob.exc.HTTPError,
webob.exc.HTTPError)
def test_upload_data_to_store_exception(self):
self._test_upload_data_to_store_exception(
Exception,
webob.exc.HTTPInternalServerError)