Use Chunked transfer encoding in the VMware store
Nova needs to upload streamOptimized disks to Glance. These streamOptimized disks are converted/compressed on the fly by vCenter. Consequently, it is not possible to know the size of the Glance image before upload. Without specifying the size or size zero, vCenter will reject the request (Broken Pipe). This patch adds the Chunked transfer encoding which allows to not specify a Content-Length header. Change-Id: I579084460e7f61ab4042632d17ec0f045fa6f5af Closes-Bug: #1313992
This commit is contained in:
parent
001dd0112e
commit
cb93eb60ab
|
@ -108,16 +108,40 @@ def http_response_iterator(conn, response, size):
|
|||
|
||||
class _Reader(object):
|
||||
|
||||
def __init__(self, data, checksum):
|
||||
def __init__(self, data, checksum, blocksize=8192):
|
||||
self.data = data
|
||||
self.checksum = checksum
|
||||
self._size = 0
|
||||
self.blocksize = blocksize
|
||||
self.current_chunk = ""
|
||||
self.closed = False
|
||||
|
||||
def read(self, length):
|
||||
result = self.data.read(length)
|
||||
self._size += len(result)
|
||||
self.checksum.update(result)
|
||||
return result
|
||||
def read(self, size=None):
|
||||
ret = ""
|
||||
while size is None or size >= len(self.current_chunk):
|
||||
ret += self.current_chunk
|
||||
if size is not None:
|
||||
size -= len(self.current_chunk)
|
||||
if self.closed:
|
||||
self.current_chunk = ""
|
||||
break
|
||||
self._get_chunk()
|
||||
else:
|
||||
ret += self.current_chunk[:size]
|
||||
self.current_chunk = self.current_chunk[size:]
|
||||
return ret
|
||||
|
||||
def _get_chunk(self):
|
||||
if not self.closed:
|
||||
chunk = self.data.read(self.blocksize)
|
||||
chunk_len = len(chunk)
|
||||
self._size += chunk_len
|
||||
self.checksum.update(chunk)
|
||||
if chunk:
|
||||
self.current_chunk = '%x\r\n%s\r\n' % (chunk_len, chunk)
|
||||
else:
|
||||
self.current_chunk = '0\r\n\r\n'
|
||||
self.closed = True
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
|
@ -270,7 +294,9 @@ class Store(glance.store.base.Store):
|
|||
'image_id': image_id})
|
||||
cookie = self._build_vim_cookie_header(
|
||||
self._session.vim.client.options.transport.cookiejar)
|
||||
headers = {'Cookie': cookie, 'Content-Length': image_size}
|
||||
headers = {'Connection': 'Keep-Alive',
|
||||
'Cookie': cookie,
|
||||
'Transfer-Encoding': 'chunked'}
|
||||
try:
|
||||
conn = self._get_http_conn('PUT', loc, headers,
|
||||
content=image_file)
|
||||
|
|
|
@ -250,3 +250,61 @@ class TestStore(base.StoreClearingUnitTest):
|
|||
with mock.patch('httplib.HTTPConnection') as HttpConn:
|
||||
HttpConn.return_value = FakeHTTPConnection(status=404)
|
||||
self.assertRaises(exception.NotFound, self.store.get_size, loc)
|
||||
|
||||
def test_reader_image_fits_in_blocksize(self):
|
||||
"""
|
||||
Test that the image file reader returns the expected chunk of data
|
||||
when the block size is larger than the image.
|
||||
"""
|
||||
content = 'XXX'
|
||||
image = six.StringIO(content)
|
||||
expected_checksum = hashlib.md5(content).hexdigest()
|
||||
checksum = hashlib.md5()
|
||||
reader = vm_store._Reader(image, checksum)
|
||||
ret = reader.read()
|
||||
expected_chunk = '%x\r\n%s\r\n' % (len(content), content)
|
||||
last_chunk = '0\r\n\r\n'
|
||||
self.assertEqual('%s%s' % (expected_chunk, last_chunk), ret)
|
||||
self.assertEqual(image.len, reader.size)
|
||||
self.assertEqual(expected_checksum, reader.checksum.hexdigest())
|
||||
self.assertTrue(reader.closed)
|
||||
ret = reader.read()
|
||||
self.assertEqual(image.len, reader.size)
|
||||
self.assertEqual(expected_checksum, reader.checksum.hexdigest())
|
||||
self.assertTrue(reader.closed)
|
||||
self.assertEqual('', ret)
|
||||
|
||||
def test_reader_image_larger_blocksize(self):
|
||||
"""
|
||||
Test that the image file reader returns the expected chunks when
|
||||
the block size specified is smaller than the image.
|
||||
"""
|
||||
content = 'XXX'
|
||||
image = six.StringIO(content)
|
||||
expected_checksum = hashlib.md5(content).hexdigest()
|
||||
checksum = hashlib.md5()
|
||||
last_chunk = '0\r\n\r\n'
|
||||
reader = vm_store._Reader(image, checksum, blocksize=1)
|
||||
ret = reader.read()
|
||||
expected_chunk = '1\r\nX\r\n'
|
||||
self.assertEqual('%s%s%s%s' % (expected_chunk, expected_chunk,
|
||||
expected_chunk, last_chunk), ret)
|
||||
self.assertEqual(expected_checksum, reader.checksum.hexdigest())
|
||||
self.assertEqual(image.len, reader.size)
|
||||
self.assertTrue(reader.closed)
|
||||
|
||||
def test_reader_size(self):
|
||||
"""Test that the image reader takes into account the specified size."""
|
||||
content = 'XXX'
|
||||
image = six.StringIO(content)
|
||||
expected_checksum = hashlib.md5(content).hexdigest()
|
||||
checksum = hashlib.md5()
|
||||
reader = vm_store._Reader(image, checksum, blocksize=1)
|
||||
ret = reader.read(size=3)
|
||||
self.assertEqual('1\r\n', ret)
|
||||
ret = reader.read(size=1)
|
||||
self.assertEqual('X', ret)
|
||||
ret = reader.read()
|
||||
self.assertEqual(expected_checksum, reader.checksum.hexdigest())
|
||||
self.assertEqual(image.len, reader.size)
|
||||
self.assertTrue(reader.closed)
|
||||
|
|
Loading…
Reference in New Issue