Add method to cleanup autocreated image objects

This shouldn't really be needed, as the objects should get cleaned up
automatically. BUT - if things leak, this method can be used to delete
any objects shade has uploaded on behalf of the user for deleting
images.

While in there, clean up test_image to use a few more good practices.

Change-Id: Ifb697944856e1922517074d84a7c00a4af75b1e6
This commit is contained in:
Monty Taylor 2017-11-16 10:18:54 -06:00
parent 371e0b667e
commit 5996a0313b
No known key found for this signature in database
GPG Key ID: 7BAE94BC7141A594
4 changed files with 140 additions and 22 deletions

View File

@ -0,0 +1,5 @@
---
features:
- Added new method, delete_autocreated_image_objects
that can be used to delete any leaked objects shade
may have created on behalf of the user.

View File

@ -50,6 +50,8 @@ from shade import _utils
OBJECT_MD5_KEY = 'x-object-meta-x-shade-md5'
OBJECT_SHA256_KEY = 'x-object-meta-x-shade-sha256'
OBJECT_AUTOCREATE_KEY = 'x-object-meta-x-shade-autocreated'
OBJECT_AUTOCREATE_CONTAINER = 'images'
IMAGE_MD5_KEY = 'owner_specified.shade.md5'
IMAGE_SHA256_KEY = 'owner_specified.shade.sha256'
IMAGE_OBJECT_KEY = 'owner_specified.shade.object'
@ -4526,7 +4528,7 @@ class OpenStackCloud(
return up_to_date
def create_image(
self, name, filename=None, container='images',
self, name, filename=None, container=OBJECT_AUTOCREATE_CONTAINER,
md5=None, sha256=None,
disk_format=None, container_format=None,
disable_vendor_agent=True,
@ -4823,6 +4825,7 @@ class OpenStackCloud(
self.create_object(
container, name, filename,
md5=md5, sha256=sha256,
metadata={OBJECT_AUTOCREATE_KEY: 'true'},
**{'content-type': 'application/octet-stream'})
if not current_image:
current_image = self.get_image(name)
@ -7561,11 +7564,13 @@ class OpenStackCloud(
return self._object_store_client.get(
container, params=dict(format='json'))
def delete_object(self, container, name):
def delete_object(self, container, name, meta=None):
"""Delete an object from a container.
:param string container: Name of the container holding the object.
:param string name: Name of the object to delete.
:param dict meta: Metadata for the object in question. (optional, will
be fetched if not provided)
:returns: True if delete succeeded, False if the object was not found.
@ -7580,7 +7585,8 @@ class OpenStackCloud(
# Errors:
# We should ultimately do something with that
try:
meta = self.get_object_metadata(container, name)
if not meta:
meta = self.get_object_metadata(container, name)
if not meta:
return False
params = {}
@ -7594,6 +7600,28 @@ class OpenStackCloud(
except OpenStackCloudHTTPError:
return False
def delete_autocreated_image_objects(
self, container=OBJECT_AUTOCREATE_CONTAINER):
"""Delete all objects autocreated for image uploads.
This method should generally not be needed, as shade should clean up
the objects it uses for object-based image creation. If something
goes wrong and it is found that there are leaked objects, this method
can be used to delete any objects that shade has created on the user's
behalf in service of image uploads.
"""
# This method only makes sense on clouds that use tasks
if not self.image_api_use_tasks:
return False
deleted = False
for obj in self.list_objects(container):
meta = self.get_object_metadata(container, obj['name'])
if meta.get(OBJECT_AUTOCREATE_KEY) == 'true':
if self.delete_object(container, obj['name'], meta):
deleted = True
return deleted
def get_object_metadata(self, container, name):
try:
return self._object_store_client.head(

View File

@ -408,6 +408,10 @@ class RequestsMockTestCase(BaseTestCase):
return _RoleData(role_id, role_name, {'role': response},
{'role': request})
def use_nothing(self):
self.calls = []
self._uri_registry.clear()
def use_keystone_v3(self, catalog='catalog-v3.json'):
self.adapter = self.useFixture(rm_fixture.Fixture())
self.calls = []

View File

@ -24,6 +24,7 @@ import munch
import six
import shade
import shade.openstackcloud
from shade import exc
from shade import meta
from shade.tests import fakes
@ -44,6 +45,8 @@ class BaseTestImage(base.RequestsMockTestCase):
self.fake_image_dict = fakes.make_fake_image(image_id=self.image_id)
self.fake_search_return = {'images': [self.fake_image_dict]}
self.output = uuid.uuid4().bytes
self.image_name = self.getUniqueString('image')
self.container_name = self.getUniqueString('container')
class TestImage(BaseTestImage):
@ -258,8 +261,6 @@ class TestImage(BaseTestImage):
def test_create_image_task(self):
self.cloud.image_api_use_tasks = True
image_name = 'name-99'
container_name = 'image_upload_v2_test_container'
endpoint = self.cloud._object_store_client.get_endpoint()
task_id = str(uuid.uuid4())
@ -286,18 +287,18 @@ class TestImage(BaseTestImage):
slo={'min_segment_size': 500})),
dict(method='HEAD',
uri='{endpoint}/{container}'.format(
endpoint=endpoint, container=container_name),
endpoint=endpoint, container=self.container_name),
status_code=404),
dict(method='PUT',
uri='{endpoint}/{container}'.format(
endpoint=endpoint, container=container_name),
endpoint=endpoint, container=self.container_name),
status_code=201,
headers={'Date': 'Fri, 16 Dec 2016 18:21:20 GMT',
'Content-Length': '0',
'Content-Type': 'text/html; charset=UTF-8'}),
dict(method='HEAD',
uri='{endpoint}/{container}'.format(
endpoint=endpoint, container=container_name),
endpoint=endpoint, container=self.container_name),
headers={'Content-Length': '0',
'X-Container-Object-Count': '0',
'Accept-Ranges': 'bytes',
@ -309,13 +310,13 @@ class TestImage(BaseTestImage):
'Content-Type': 'text/plain; charset=utf-8'}),
dict(method='HEAD',
uri='{endpoint}/{container}/{object}'.format(
endpoint=endpoint, container=container_name,
object=image_name),
endpoint=endpoint, container=self.container_name,
object=self.image_name),
status_code=404),
dict(method='PUT',
uri='{endpoint}/{container}/{object}'.format(
endpoint=endpoint, container=container_name,
object=image_name),
endpoint=endpoint, container=self.container_name,
object=self.image_name),
status_code=201,
validate=dict(
headers={'x-object-meta-x-shade-md5': fakes.NO_MD5,
@ -329,8 +330,9 @@ class TestImage(BaseTestImage):
json=dict(
type='import', input={
'import_from': '{container}/{object}'.format(
container=container_name, object=image_name),
'image_properties': {'name': image_name}}))
container=self.container_name,
object=self.image_name),
'image_properties': {'name': self.image_name}}))
),
dict(method='GET',
uri='https://image.example.com/v2/tasks/{id}'.format(
@ -348,8 +350,8 @@ class TestImage(BaseTestImage):
validate=dict(
json=sorted([{u'op': u'add',
u'value': '{container}/{object}'.format(
container=container_name,
object=image_name),
container=self.container_name,
object=self.image_name),
u'path': u'/owner_specified.shade.object'},
{u'op': u'add', u'value': fakes.NO_MD5,
u'path': u'/owner_specified.shade.md5'},
@ -362,8 +364,8 @@ class TestImage(BaseTestImage):
),
dict(method='HEAD',
uri='{endpoint}/{container}/{object}'.format(
endpoint=endpoint, container=container_name,
object=image_name),
endpoint=endpoint, container=self.container_name,
object=self.image_name),
headers={
'X-Timestamp': '1429036140.50253',
'X-Trans-Id': 'txbbb825960a3243b49a36f-005a0dadaedfw1',
@ -377,15 +379,94 @@ class TestImage(BaseTestImage):
'Etag': fakes.NO_MD5}),
dict(method='DELETE',
uri='{endpoint}/{container}/{object}'.format(
endpoint=endpoint, container=container_name,
object=image_name)),
endpoint=endpoint, container=self.container_name,
object=self.image_name)),
dict(method='GET', uri='https://image.example.com/v2/images',
json=self.fake_search_return)
])
self.cloud.create_image(
image_name, self.imagefile.name, wait=True, timeout=1,
is_public=False, container=container_name)
self.image_name, self.imagefile.name, wait=True, timeout=1,
is_public=False, container=self.container_name)
self.assert_calls()
def test_delete_autocreated_no_tasks(self):
self.use_nothing()
self.cloud.image_api_use_tasks = False
deleted = self.cloud.delete_autocreated_image_objects(
container=self.container_name)
self.assertFalse(deleted)
self.assert_calls()
def test_delete_autocreated_image_objects(self):
self.use_keystone_v3()
self.cloud.image_api_use_tasks = True
endpoint = self.cloud._object_store_client.get_endpoint()
other_image = self.getUniqueString('no-delete')
self.register_uris([
dict(method='GET',
uri=self.get_mock_url(
service_type='object-store',
resource=self.container_name,
qs_elements=['format=json']),
json=[{
'content_type': 'application/octet-stream',
'bytes': 1437258240,
'hash': '249219347276c331b87bf1ac2152d9af',
'last_modified': '2015-02-16T17:50:05.289600',
'name': other_image,
}, {
'content_type': 'application/octet-stream',
'bytes': 1290170880,
'hash': fakes.NO_MD5,
'last_modified': '2015-04-14T18:29:00.502530',
'name': self.image_name,
}]),
dict(method='HEAD',
uri=self.get_mock_url(
service_type='object-store',
resource=self.container_name,
append=[other_image]),
headers={
'X-Timestamp': '1429036140.50253',
'X-Trans-Id': 'txbbb825960a3243b49a36f-005a0dadaedfw1',
'Content-Length': '1290170880',
'Last-Modified': 'Tue, 14 Apr 2015 18:29:01 GMT',
'X-Object-Meta-X-Shade-Sha256': 'does not matter',
'X-Object-Meta-X-Shade-Md5': 'does not matter',
'Date': 'Thu, 16 Nov 2017 15:24:30 GMT',
'Accept-Ranges': 'bytes',
'Content-Type': 'application/octet-stream',
'Etag': '249219347276c331b87bf1ac2152d9af',
}),
dict(method='HEAD',
uri=self.get_mock_url(
service_type='object-store',
resource=self.container_name,
append=[self.image_name]),
headers={
'X-Timestamp': '1429036140.50253',
'X-Trans-Id': 'txbbb825960a3243b49a36f-005a0dadaedfw1',
'Content-Length': '1290170880',
'Last-Modified': 'Tue, 14 Apr 2015 18:29:01 GMT',
'X-Object-Meta-X-Shade-Sha256': fakes.NO_SHA256,
'X-Object-Meta-X-Shade-Md5': fakes.NO_MD5,
'Date': 'Thu, 16 Nov 2017 15:24:30 GMT',
'Accept-Ranges': 'bytes',
'Content-Type': 'application/octet-stream',
shade.openstackcloud.OBJECT_AUTOCREATE_KEY: 'true',
'Etag': fakes.NO_MD5}),
dict(method='DELETE',
uri='{endpoint}/{container}/{object}'.format(
endpoint=endpoint, container=self.container_name,
object=self.image_name)),
])
deleted = self.cloud.delete_autocreated_image_objects(
container=self.container_name)
self.assertTrue(deleted)
self.assert_calls()