From e03525bf1adc0a9966018ab33288bdda8c610267 Mon Sep 17 00:00:00 2001 From: Christopher Lefelhocz Date: Wed, 8 Oct 2014 13:18:19 -0500 Subject: [PATCH] Fix xen plugin to retry on upload failure The xen plugin does not catch all cases where upload can throw an exception due to connection failures and timeouts. This change extends the try/execpt window to catch these cases. Also added original connect which in part keeps us from lazy-loading the connection and logging of exception information so we can figure out later what happened. Change-Id: If643a7da9f602ab7bb5d255031a841c674fa6c6f Fixes-bug: 1378903 --- .../xenapi/etc/xapi.d/plugins/glance | 65 ++++++++++--------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index f3658c21547f..9e21d1c9b403 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -146,7 +146,9 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, conn = httplib.HTTPSConnection(glance_host, glance_port) else: conn = httplib.HTTPConnection(glance_host, glance_port) + conn.connect() except Exception, error: + logging.exception('Failed to connect %(url)s' % locals()) raise RetryableError(error) try: @@ -154,39 +156,41 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, # NOTE(sirp): httplib under python2.4 won't accept # a file-like object to request conn.putrequest('PUT', '/v1/images/%s' % image_id) + + # NOTE(sirp): There is some confusion around OVF. Here's a summary + # of where we currently stand: + # 1. OVF as a container format is misnamed. We really should be + # using OVA since that is the name for the container format; + # OVF is the standard applied to the manifest file contained + # within. + # 2. We're currently uploading a vanilla tarball. In order to be + # OVF/OVA compliant, we'll need to embed a minimal OVF + # manifest as the first file. + + # NOTE(dprince): In order to preserve existing Glance properties + # we set X-Glance-Registry-Purge-Props on this request. + headers = { + 'content-type': 'application/octet-stream', + 'transfer-encoding': 'chunked', + 'x-image-meta-is-public': 'False', + 'x-image-meta-status': 'queued', + 'x-image-meta-disk-format': 'vhd', + 'x-image-meta-container-format': 'ovf', + 'x-glance-registry-purge-props': 'False'} + + headers.update(**extra_headers) + + for key, value in properties.iteritems(): + header_key = "x-image-meta-property-%s" % key.replace('_', '-') + headers[header_key] = str(value) + + for header, value in headers.iteritems(): + conn.putheader(header, value) + conn.endheaders() except Exception, error: + logging.exception('Failed to upload %(url)s' % locals()) raise RetryableError(error) - # NOTE(sirp): There is some confusion around OVF. Here's a summary of - # where we currently stand: - # 1. OVF as a container format is misnamed. We really should be using - # OVA since that is the name for the container format; OVF is the - # standard applied to the manifest file contained within. - # 2. We're currently uploading a vanilla tarball. In order to be - # OVF/OVA compliant, we'll need to embed a minimal OVF manifest - # as the first file. - - # NOTE(dprince): In order to preserve existing Glance properties - # we set X-Glance-Registry-Purge-Props on this request. - headers = { - 'content-type': 'application/octet-stream', - 'transfer-encoding': 'chunked', - 'x-image-meta-is-public': 'False', - 'x-image-meta-status': 'queued', - 'x-image-meta-disk-format': 'vhd', - 'x-image-meta-container-format': 'ovf', - 'x-glance-registry-purge-props': 'False'} - - headers.update(**extra_headers) - - for key, value in properties.iteritems(): - header_key = "x-image-meta-property-%s" % key.replace('_', '-') - headers[header_key] = str(value) - - for header, value in headers.iteritems(): - conn.putheader(header, value) - conn.endheaders() - callback_data = {'bytes_written': 0} def send_chunked_transfer_encoded(chunk): @@ -195,6 +199,7 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, try: conn.send("%x\r\n%s\r\n" % (chunk_len, chunk)) except Exception, error: + logging.exception('Failed to upload when sending chunks') raise RetryableError(error) compression_level = properties.get('xenapi_image_compression_level')