openstack
/
deb-glance
Code Issues Proposed changes

Merge "Accept Range requests and set appropriate response"

This commit is contained in:
Jenkins 2017-04-06 03:42:34 +00:00 committed by Gerrit Code Review
parent 11b16baa11 423f340174
commit 9463987cfc
6 changed files with 297 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
glance/api/v2/image_data.py
View File

@ -285,16 +285,43 @@ class RequestDeserializer(wsgi.JSONRequestDeserializer):
class ResponseSerializer(wsgi.JSONResponseSerializer):
def download(self, response, image):
offset, chunk_size = 0, None
# NOTE(dharinic): In case of a malformed content range,
# NOTE(dharinic): In case of a malformed range header,
# glance/common/wsgi.py will raise HTTPRequestRangeNotSatisfiable
# (setting status_code to 416)
range_val = response.request.get_content_range(image.size)
range_val = response.request.get_range_from_request(image.size)
if range_val:
# NOTE(flaper87): if not present, both, start
# and stop, will be None.
if range_val.start is not None and range_val.stop is not None:
if isinstance(range_val, webob.byterange.Range):
response_end = image.size - 1
# NOTE(dharinic): webob parsing is zero-indexed.
# i.e.,to download first 5 bytes of a 10 byte image,
# request should be "bytes=0-4" and the response would be
# "bytes 0-4/10".
# Range if validated, will never have 'start' object as None.
if range_val.start >= 0:
offset = range_val.start
else:
# NOTE(dharinic): Negative start values needs to be
# processed to allow suffix-length for Range request
# like "bytes=-2" as per rfc7233.
if abs(range_val.start) < image.size:
offset = image.size + range_val.start
if range_val.end is not None and range_val.end < image.size:
chunk_size = range_val.end - offset
response_end = range_val.end - 1
else:
chunk_size = image.size - offset
# NOTE(dharinic): For backward compatibility reasons, we maintain
# support for 'Content-Range' in requests even though it's not
# correct to use it in requests.
elif isinstance(range_val, webob.byterange.ContentRange):
response_end = range_val.stop - 1
# NOTE(flaper87): if not present, both, start
# and stop, will be None.
offset = range_val.start
chunk_size = range_val.stop - offset
@ -311,7 +338,12 @@ class ResponseSerializer(wsgi.JSONResponseSerializer):
# NOTE(dharinic): In case of a full image download, when
# chunk_size was none, reset it to image.size to set the
# response header's Content-Length.
if not chunk_size:
if chunk_size is not None:
response.headers['Content-Range'] = 'bytes %s-%s/%s'\
% (offset,
response_end,
image.size)
else:
chunk_size = image.size
except glance_store.NotFound as e:
raise webob.exc.HTTPNoContent(explanation=e.msg)

55
glance/common/wsgi.py
View File

@ -965,19 +965,56 @@ class Request(webob.Request):
langs = i18n.get_available_languages('glance')
return self.accept_language.best_match(langs)
def get_content_range(self, image_size):
def get_range_from_request(self, image_size):
"""Return the `Range` in a request."""
range_str = self.headers.get('Content-Range')
range_str = self.headers.get('Range')
if range_str is not None:
range_ = webob.byterange.ContentRange.parse(range_str)
# NOTE(dharinic): Ensure that a range like 1-4/* for an image
# size of 3 is invalidated.
if range_ is None or (range_.length is None and
range_.stop > image_size):
msg = _('Malformed Content-Range header: %s') % range_str
raise webob.exc.HTTPRequestRangeNotSatisfiable(explanation=msg)
# NOTE(dharinic): We do not support multi range requests.
if ',' in range_str:
msg = ("Requests with multiple ranges are not supported in "
"Glance. You may make multiple single-range requests "
"instead.")
raise webob.exc.HTTPBadRequest(explanation=msg)
range_ = webob.byterange.Range.parse(range_str)
if range_ is None:
msg = ("Invalid Range header.")
raise webob.exc.HTTPRequestRangeNotSatisfiable(msg)
# NOTE(dharinic): Ensure that a range like bytes=4- for an image
# size of 3 is invalidated as per rfc7233.
if range_.start >= image_size:
msg = ("Invalid start position in Range header. "
"Start position MUST be in the inclusive range [0, %s]."
% (image_size - 1))
raise webob.exc.HTTPRequestRangeNotSatisfiable(msg)
return range_
# NOTE(dharinic): For backward compatibility reasons, we maintain
# support for 'Content-Range' in requests even though it's not
# correct to use it in requests..
c_range_str = self.headers.get('Content-Range')
if c_range_str is not None:
content_range = webob.byterange.ContentRange.parse(c_range_str)
# NOTE(dharinic): Ensure that a content range like 1-4/* for an
# image size of 3 is invalidated.
if content_range is None:
msg = ("Invalid Content-Range header.")
raise webob.exc.HTTPRequestRangeNotSatisfiable(msg)
if (content_range.length is None and
content_range.stop > image_size):
msg = ("Invalid stop position in Content-Range header. "
"The stop position MUST be in the inclusive range "
"[0, %s]." % (image_size - 1))
raise webob.exc.HTTPRequestRangeNotSatisfiable(msg)
if content_range.start >= image_size:
msg = ("Invalid start position in Content-Range header. "
"Start position MUST be in the inclusive range [0, %s]."
% (image_size - 1))
raise webob.exc.HTTPRequestRangeNotSatisfiable(msg)
return content_range
class JSONRequestDeserializer(object):
valid_transfer_encoding = frozenset(['chunked', 'compress', 'deflate',

56
glance/tests/functional/v2/test_images.py
View File

@ -743,7 +743,54 @@ class TestImages(functional.FunctionalTest):
self.stop_servers()
def test_download_random_access(self):
def test_download_random_access_w_range_request(self):
"""
Test partial download 'Range' requests for images (random image access)
"""
self.start_servers(**self.__dict__.copy())
# Create an image (with two deployer-defined properties)
path = self._url('/v2/images')
headers = self._headers({'content-type': 'application/json'})
data = jsonutils.dumps({'name': 'image-2', 'type': 'kernel',
'bar': 'foo', 'disk_format': 'aki',
'container_format': 'aki', 'xyz': 'abc'})
response = requests.post(path, headers=headers, data=data)
self.assertEqual(http.CREATED, response.status_code)
image = jsonutils.loads(response.text)
image_id = image['id']
# Upload data to image
image_data = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
path = self._url('/v2/images/%s/file' % image_id)
headers = self._headers({'Content-Type': 'application/octet-stream'})
response = requests.put(path, headers=headers, data=image_data)
self.assertEqual(http.NO_CONTENT, response.status_code)
# test for success on satisfiable Range request.
range_ = 'bytes=3-10'
headers = self._headers({'Range': range_})
path = self._url('/v2/images/%s/file' % image_id)
response = requests.get(path, headers=headers)
self.assertEqual(http.PARTIAL_CONTENT, response.status_code)
self.assertEqual('DEFGHIJK', response.text)
# test for failure on unsatisfiable Range request.
range_ = 'bytes=10-5'
headers = self._headers({'Range': range_})
path = self._url('/v2/images/%s/file' % image_id)
response = requests.get(path, headers=headers)
self.assertEqual(http.REQUESTED_RANGE_NOT_SATISFIABLE,
response.status_code)
self.stop_servers()
def test_download_random_access_w_content_range(self):
"""
Even though Content-Range is incorrect on requests, we support it
for backward compatibility with clients written for pre-Pike Glance.
The following test is for 'Content-Range' requests, which we have
to ensure that we prevent regression.
"""
self.start_servers(**self.__dict__.copy())
# Create another image (with two deployer-defined properties)
path = self._url('/v2/images')
@ -772,17 +819,18 @@ class TestImages(functional.FunctionalTest):
headers = self._headers({'Content-Range': content_range})
path = self._url('/v2/images/%s/file' % image_id)
response = requests.get(path, headers=headers)
self.assertEqual(206, response.status_code)
self.assertEqual(http.PARTIAL_CONTENT, response.status_code)
result_body += response.text
self.assertEqual(result_body, image_data)
# test for failure on unsatisfiable request range.
# test for failure on unsatisfiable request for ContentRange.
content_range = 'bytes 3-16/15'
headers = self._headers({'Content-Range': content_range})
path = self._url('/v2/images/%s/file' % image_id)
response = requests.get(path, headers=headers)
self.assertEqual(416, response.status_code)
self.assertEqual(http.REQUESTED_RANGE_NOT_SATISFIABLE,
response.status_code)
self.stop_servers()

17
glance/tests/unit/common/test_wsgi.py
View File

@ -67,7 +67,7 @@ class RequestTest(test_utils.BaseTestCase):
def test_content_range(self):
request = wsgi.Request.blank('/tests/123')
request.headers["Content-Range"] = 'bytes 10-99/*'
range_ = request.get_content_range(120)
range_ = request.get_range_from_request(120)
self.assertEqual(10, range_.start)
self.assertEqual(100, range_.stop) # non-inclusive
self.assertIsNone(range_.length)
@ -76,7 +76,20 @@ class RequestTest(test_utils.BaseTestCase):
request = wsgi.Request.blank('/tests/123')
request.headers["Content-Range"] = 'bytes=0-99'
self.assertRaises(webob.exc.HTTPRequestRangeNotSatisfiable,
request.get_content_range, 120)
request.get_range_from_request, 120)
def test_range(self):
request = wsgi.Request.blank('/tests/123')
request.headers["Range"] = 'bytes=10-99'
range_ = request.get_range_from_request(120)
self.assertEqual(10, range_.start)
self.assertEqual(100, range_.end) # non-inclusive
def test_range_invalid(self):
request = wsgi.Request.blank('/tests/123')
request.headers["Range"] = 'bytes=150-'
self.assertRaises(webob.exc.HTTPRequestRangeNotSatisfiable,
request.get_range_from_request, 120)
def test_content_type_missing(self):
request = wsgi.Request.blank('/tests/123')

160
glance/tests/unit/v2/test_image_data_resource.py
View File

@ -63,8 +63,10 @@ class FakeImage(object):
else:
self._status = value
def get_data(self, *args, **kwargs):
return self.data
def get_data(self, offset=0, chunk_size=None):
if chunk_size:
return self.data[offset:offset + chunk_size]
return self.data[offset:]
def set_data(self, data, size=None):
self.data = ''.join(data)
@ -548,40 +550,144 @@ class TestImageDataSerializer(test_utils.BaseTestCase):
self.assertEqual('application/octet-stream',
response.headers['Content-Type'])
def _test_partial_download_successful(self, d_range):
def test_range_requests_for_image_downloads(self):
"""
Test partial download 'Range' requests for images (random image access)
"""
def download_successful_Range(d_range):
request = wsgi.Request.blank('/')
request.environ = {}
request.headers['Range'] = d_range
response = webob.Response()
response.request = request
image = FakeImage(size=3, data=[b'X', b'Y', b'Z'])
self.serializer.download(response, image)
self.assertEqual(206, response.status_code)
self.assertEqual('2', response.headers['Content-Length'])
self.assertEqual('bytes 1-2/3', response.headers['Content-Range'])
self.assertEqual(b'YZ', response.body)
download_successful_Range('bytes=1-2')
download_successful_Range('bytes=1-')
download_successful_Range('bytes=1-3')
download_successful_Range('bytes=-2')
download_successful_Range('bytes=1-100')
def full_image_download_w_range(d_range):
request = wsgi.Request.blank('/')
request.environ = {}
request.headers['Range'] = d_range
response = webob.Response()
response.request = request
image = FakeImage(size=3, data=[b'X', b'Y', b'Z'])
self.serializer.download(response, image)
self.assertEqual(206, response.status_code)
self.assertEqual('3', response.headers['Content-Length'])
self.assertEqual('bytes 0-2/3', response.headers['Content-Range'])
self.assertEqual(b'XYZ', response.body)
full_image_download_w_range('bytes=0-')
full_image_download_w_range('bytes=0-2')
full_image_download_w_range('bytes=0-3')
full_image_download_w_range('bytes=-3')
full_image_download_w_range('bytes=-4')
full_image_download_w_range('bytes=0-100')
full_image_download_w_range('bytes=-100')
def download_failures_Range(d_range):
request = wsgi.Request.blank('/')
request.environ = {}
request.headers['Range'] = d_range
response = webob.Response()
response.request = request
image = FakeImage(size=3, data=[b'Z', b'Z', b'Z'])
self.assertRaises(webob.exc.HTTPRequestRangeNotSatisfiable,
self.serializer.download,
response, image)
return
download_failures_Range('bytes=4-1')
download_failures_Range('bytes=4-')
download_failures_Range('bytes=3-')
download_failures_Range('bytes=1')
download_failures_Range('bytes=100')
download_failures_Range('bytes=100-')
download_failures_Range('bytes=')
def test_multi_range_requests_raises_bad_request_error(self):
request = wsgi.Request.blank('/')
request.environ = {}
request.headers['Content-Range'] = d_range
request.headers['Range'] = 'bytes=0-0,-1'
response = webob.Response()
response.request = request
image = FakeImage(size=3, data=[b'Z', b'Z', b'Z'])
self.serializer.download(response, image)
self.assertEqual(206, response.status_code)
self.assertEqual('2', response.headers['Content-Length'])
def test_partial_download_successful_with_range(self):
self._test_partial_download_successful('bytes 1-2/3')
self._test_partial_download_successful('bytes 1-2/*')
def _test_partial_download_failures(self, d_range):
request = wsgi.Request.blank('/')
request.environ = {}
request.headers['Content-Range'] = d_range
response = webob.Response()
response.request = request
image = FakeImage(size=3, data=[b'Z', b'Z', b'Z'])
self.assertRaises(webob.exc.HTTPRequestRangeNotSatisfiable,
self.assertRaises(webob.exc.HTTPBadRequest,
self.serializer.download,
response, image)
return
def test_partial_download_failure_with_range(self):
self._test_partial_download_failures('bytes 1-4/3')
self._test_partial_download_failures('bytes 1-4/*')
self._test_partial_download_failures('bytes 4-1/3')
self._test_partial_download_failures('bytes 4-1/*')
def test_download_failure_with_valid_range(self):
with mock.patch.object(glance.api.policy.ImageProxy,
'get_data') as mock_get_data:
mock_get_data.side_effect = glance_store.NotFound(image="image")
request = wsgi.Request.blank('/')
request.environ = {}
request.headers['Range'] = 'bytes=1-2'
response = webob.Response()
response.request = request
image = FakeImage(size=3, data=[b'Z', b'Z', b'Z'])
image.get_data = mock_get_data
self.assertRaises(webob.exc.HTTPNoContent,
self.serializer.download,
response, image)
def test_content_range_requests_for_image_downloads(self):
"""
Even though Content-Range is incorrect on requests, we support it
for backward compatibility with clients written for pre-Pike
Glance.
The following test is for 'Content-Range' requests, which we have
to ensure that we prevent regression.
"""
def download_successful_ContentRange(d_range):
request = wsgi.Request.blank('/')
request.environ = {}
request.headers['Content-Range'] = d_range
response = webob.Response()
response.request = request
image = FakeImage(size=3, data=[b'X', b'Y', b'Z'])
self.serializer.download(response, image)
self.assertEqual(206, response.status_code)
self.assertEqual('2', response.headers['Content-Length'])
self.assertEqual('bytes 1-2/3', response.headers['Content-Range'])
self.assertEqual(b'YZ', response.body)
download_successful_ContentRange('bytes 1-2/3')
download_successful_ContentRange('bytes 1-2/*')
def download_failures_ContentRange(d_range):
request = wsgi.Request.blank('/')
request.environ = {}
request.headers['Content-Range'] = d_range
response = webob.Response()
response.request = request
image = FakeImage(size=3, data=[b'Z', b'Z', b'Z'])
self.assertRaises(webob.exc.HTTPRequestRangeNotSatisfiable,
self.serializer.download,
response, image)
return
download_failures_ContentRange('bytes -3/3')
download_failures_ContentRange('bytes 1-/3')
download_failures_ContentRange('bytes 1-3/3')
download_failures_ContentRange('bytes 1-4/3')
download_failures_ContentRange('bytes 1-4/*')
download_failures_ContentRange('bytes 4-1/3')
download_failures_ContentRange('bytes 4-1/*')
download_failures_ContentRange('bytes 4-8/*')
download_failures_ContentRange('bytes 4-8/10')
download_failures_ContentRange('bytes 4-8/3')
def test_download_failure_with_valid_content_range(self):
with mock.patch.object(glance.api.policy.ImageProxy,
'get_data') as mock_get_data:
mock_get_data.side_effect = glance_store.NotFound(image="image")

13
releasenotes/notes/range-header-request-83cf11eebf865fb1.yaml Normal file
View File

@ -0,0 +1,13 @@
---
fixes:
- |
Glance had been accepting the Content-Range header for GET v2/images/{image_id}/file requests,
contrary to RFC 7233.
Following RFC 7233, Glance will now:
* Accept the Range header in requests to serve partial images.
* Include a ``Content-Range`` header upon successful delivery of the requested partial content.
Please note that not all Glance storage backends support partial downloads. A Range request to a
Glance server with such a backend will result in the entire image content being delivered
despite the 206 response code.