From 5996a0313bbee5f09fee3eb8be3a78bc206e9725 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 16 Nov 2017 10:18:54 -0600 Subject: [PATCH] Add method to cleanup autocreated image objects This shouldn't really be needed, as the objects should get cleaned up automatically. BUT - if things leak, this method can be used to delete any objects shade has uploaded on behalf of the user for deleting images. While in there, clean up test_image to use a few more good practices. Change-Id: Ifb697944856e1922517074d84a7c00a4af75b1e6 --- .../delete-autocreated-1839187b0aa35022.yaml | 5 + shade/openstackcloud.py | 34 ++++- shade/tests/unit/base.py | 4 + shade/tests/unit/test_image.py | 119 +++++++++++++++--- 4 files changed, 140 insertions(+), 22 deletions(-) create mode 100644 releasenotes/notes/delete-autocreated-1839187b0aa35022.yaml diff --git a/releasenotes/notes/delete-autocreated-1839187b0aa35022.yaml b/releasenotes/notes/delete-autocreated-1839187b0aa35022.yaml new file mode 100644 index 000000000..a0c2f8d76 --- /dev/null +++ b/releasenotes/notes/delete-autocreated-1839187b0aa35022.yaml @@ -0,0 +1,5 @@ +--- +features: + - Added new method, delete_autocreated_image_objects + that can be used to delete any leaked objects shade + may have created on behalf of the user. diff --git a/shade/openstackcloud.py b/shade/openstackcloud.py index db7b18827..651dc0d1f 100644 --- a/shade/openstackcloud.py +++ b/shade/openstackcloud.py @@ -50,6 +50,8 @@ from shade import _utils OBJECT_MD5_KEY = 'x-object-meta-x-shade-md5' OBJECT_SHA256_KEY = 'x-object-meta-x-shade-sha256' +OBJECT_AUTOCREATE_KEY = 'x-object-meta-x-shade-autocreated' +OBJECT_AUTOCREATE_CONTAINER = 'images' IMAGE_MD5_KEY = 'owner_specified.shade.md5' IMAGE_SHA256_KEY = 'owner_specified.shade.sha256' IMAGE_OBJECT_KEY = 'owner_specified.shade.object' @@ -4526,7 +4528,7 @@ class OpenStackCloud( return up_to_date def create_image( - self, name, filename=None, container='images', + self, name, filename=None, container=OBJECT_AUTOCREATE_CONTAINER, md5=None, sha256=None, disk_format=None, container_format=None, disable_vendor_agent=True, @@ -4823,6 +4825,7 @@ class OpenStackCloud( self.create_object( container, name, filename, md5=md5, sha256=sha256, + metadata={OBJECT_AUTOCREATE_KEY: 'true'}, **{'content-type': 'application/octet-stream'}) if not current_image: current_image = self.get_image(name) @@ -7561,11 +7564,13 @@ class OpenStackCloud( return self._object_store_client.get( container, params=dict(format='json')) - def delete_object(self, container, name): + def delete_object(self, container, name, meta=None): """Delete an object from a container. :param string container: Name of the container holding the object. :param string name: Name of the object to delete. + :param dict meta: Metadata for the object in question. (optional, will + be fetched if not provided) :returns: True if delete succeeded, False if the object was not found. @@ -7580,7 +7585,8 @@ class OpenStackCloud( # Errors: # We should ultimately do something with that try: - meta = self.get_object_metadata(container, name) + if not meta: + meta = self.get_object_metadata(container, name) if not meta: return False params = {} @@ -7594,6 +7600,28 @@ class OpenStackCloud( except OpenStackCloudHTTPError: return False + def delete_autocreated_image_objects( + self, container=OBJECT_AUTOCREATE_CONTAINER): + """Delete all objects autocreated for image uploads. + + This method should generally not be needed, as shade should clean up + the objects it uses for object-based image creation. If something + goes wrong and it is found that there are leaked objects, this method + can be used to delete any objects that shade has created on the user's + behalf in service of image uploads. + """ + # This method only makes sense on clouds that use tasks + if not self.image_api_use_tasks: + return False + + deleted = False + for obj in self.list_objects(container): + meta = self.get_object_metadata(container, obj['name']) + if meta.get(OBJECT_AUTOCREATE_KEY) == 'true': + if self.delete_object(container, obj['name'], meta): + deleted = True + return deleted + def get_object_metadata(self, container, name): try: return self._object_store_client.head( diff --git a/shade/tests/unit/base.py b/shade/tests/unit/base.py index 381bbca70..b2a32e8ba 100644 --- a/shade/tests/unit/base.py +++ b/shade/tests/unit/base.py @@ -408,6 +408,10 @@ class RequestsMockTestCase(BaseTestCase): return _RoleData(role_id, role_name, {'role': response}, {'role': request}) + def use_nothing(self): + self.calls = [] + self._uri_registry.clear() + def use_keystone_v3(self, catalog='catalog-v3.json'): self.adapter = self.useFixture(rm_fixture.Fixture()) self.calls = [] diff --git a/shade/tests/unit/test_image.py b/shade/tests/unit/test_image.py index 71d723437..b9ec47937 100644 --- a/shade/tests/unit/test_image.py +++ b/shade/tests/unit/test_image.py @@ -24,6 +24,7 @@ import munch import six import shade +import shade.openstackcloud from shade import exc from shade import meta from shade.tests import fakes @@ -44,6 +45,8 @@ class BaseTestImage(base.RequestsMockTestCase): self.fake_image_dict = fakes.make_fake_image(image_id=self.image_id) self.fake_search_return = {'images': [self.fake_image_dict]} self.output = uuid.uuid4().bytes + self.image_name = self.getUniqueString('image') + self.container_name = self.getUniqueString('container') class TestImage(BaseTestImage): @@ -258,8 +261,6 @@ class TestImage(BaseTestImage): def test_create_image_task(self): self.cloud.image_api_use_tasks = True - image_name = 'name-99' - container_name = 'image_upload_v2_test_container' endpoint = self.cloud._object_store_client.get_endpoint() task_id = str(uuid.uuid4()) @@ -286,18 +287,18 @@ class TestImage(BaseTestImage): slo={'min_segment_size': 500})), dict(method='HEAD', uri='{endpoint}/{container}'.format( - endpoint=endpoint, container=container_name), + endpoint=endpoint, container=self.container_name), status_code=404), dict(method='PUT', uri='{endpoint}/{container}'.format( - endpoint=endpoint, container=container_name), + endpoint=endpoint, container=self.container_name), status_code=201, headers={'Date': 'Fri, 16 Dec 2016 18:21:20 GMT', 'Content-Length': '0', 'Content-Type': 'text/html; charset=UTF-8'}), dict(method='HEAD', uri='{endpoint}/{container}'.format( - endpoint=endpoint, container=container_name), + endpoint=endpoint, container=self.container_name), headers={'Content-Length': '0', 'X-Container-Object-Count': '0', 'Accept-Ranges': 'bytes', @@ -309,13 +310,13 @@ class TestImage(BaseTestImage): 'Content-Type': 'text/plain; charset=utf-8'}), dict(method='HEAD', uri='{endpoint}/{container}/{object}'.format( - endpoint=endpoint, container=container_name, - object=image_name), + endpoint=endpoint, container=self.container_name, + object=self.image_name), status_code=404), dict(method='PUT', uri='{endpoint}/{container}/{object}'.format( - endpoint=endpoint, container=container_name, - object=image_name), + endpoint=endpoint, container=self.container_name, + object=self.image_name), status_code=201, validate=dict( headers={'x-object-meta-x-shade-md5': fakes.NO_MD5, @@ -329,8 +330,9 @@ class TestImage(BaseTestImage): json=dict( type='import', input={ 'import_from': '{container}/{object}'.format( - container=container_name, object=image_name), - 'image_properties': {'name': image_name}})) + container=self.container_name, + object=self.image_name), + 'image_properties': {'name': self.image_name}})) ), dict(method='GET', uri='https://image.example.com/v2/tasks/{id}'.format( @@ -348,8 +350,8 @@ class TestImage(BaseTestImage): validate=dict( json=sorted([{u'op': u'add', u'value': '{container}/{object}'.format( - container=container_name, - object=image_name), + container=self.container_name, + object=self.image_name), u'path': u'/owner_specified.shade.object'}, {u'op': u'add', u'value': fakes.NO_MD5, u'path': u'/owner_specified.shade.md5'}, @@ -362,8 +364,8 @@ class TestImage(BaseTestImage): ), dict(method='HEAD', uri='{endpoint}/{container}/{object}'.format( - endpoint=endpoint, container=container_name, - object=image_name), + endpoint=endpoint, container=self.container_name, + object=self.image_name), headers={ 'X-Timestamp': '1429036140.50253', 'X-Trans-Id': 'txbbb825960a3243b49a36f-005a0dadaedfw1', @@ -377,15 +379,94 @@ class TestImage(BaseTestImage): 'Etag': fakes.NO_MD5}), dict(method='DELETE', uri='{endpoint}/{container}/{object}'.format( - endpoint=endpoint, container=container_name, - object=image_name)), + endpoint=endpoint, container=self.container_name, + object=self.image_name)), dict(method='GET', uri='https://image.example.com/v2/images', json=self.fake_search_return) ]) self.cloud.create_image( - image_name, self.imagefile.name, wait=True, timeout=1, - is_public=False, container=container_name) + self.image_name, self.imagefile.name, wait=True, timeout=1, + is_public=False, container=self.container_name) + + self.assert_calls() + + def test_delete_autocreated_no_tasks(self): + self.use_nothing() + self.cloud.image_api_use_tasks = False + deleted = self.cloud.delete_autocreated_image_objects( + container=self.container_name) + self.assertFalse(deleted) + self.assert_calls() + + def test_delete_autocreated_image_objects(self): + self.use_keystone_v3() + self.cloud.image_api_use_tasks = True + endpoint = self.cloud._object_store_client.get_endpoint() + other_image = self.getUniqueString('no-delete') + + self.register_uris([ + dict(method='GET', + uri=self.get_mock_url( + service_type='object-store', + resource=self.container_name, + qs_elements=['format=json']), + json=[{ + 'content_type': 'application/octet-stream', + 'bytes': 1437258240, + 'hash': '249219347276c331b87bf1ac2152d9af', + 'last_modified': '2015-02-16T17:50:05.289600', + 'name': other_image, + }, { + 'content_type': 'application/octet-stream', + 'bytes': 1290170880, + 'hash': fakes.NO_MD5, + 'last_modified': '2015-04-14T18:29:00.502530', + 'name': self.image_name, + }]), + dict(method='HEAD', + uri=self.get_mock_url( + service_type='object-store', + resource=self.container_name, + append=[other_image]), + headers={ + 'X-Timestamp': '1429036140.50253', + 'X-Trans-Id': 'txbbb825960a3243b49a36f-005a0dadaedfw1', + 'Content-Length': '1290170880', + 'Last-Modified': 'Tue, 14 Apr 2015 18:29:01 GMT', + 'X-Object-Meta-X-Shade-Sha256': 'does not matter', + 'X-Object-Meta-X-Shade-Md5': 'does not matter', + 'Date': 'Thu, 16 Nov 2017 15:24:30 GMT', + 'Accept-Ranges': 'bytes', + 'Content-Type': 'application/octet-stream', + 'Etag': '249219347276c331b87bf1ac2152d9af', + }), + dict(method='HEAD', + uri=self.get_mock_url( + service_type='object-store', + resource=self.container_name, + append=[self.image_name]), + headers={ + 'X-Timestamp': '1429036140.50253', + 'X-Trans-Id': 'txbbb825960a3243b49a36f-005a0dadaedfw1', + 'Content-Length': '1290170880', + 'Last-Modified': 'Tue, 14 Apr 2015 18:29:01 GMT', + 'X-Object-Meta-X-Shade-Sha256': fakes.NO_SHA256, + 'X-Object-Meta-X-Shade-Md5': fakes.NO_MD5, + 'Date': 'Thu, 16 Nov 2017 15:24:30 GMT', + 'Accept-Ranges': 'bytes', + 'Content-Type': 'application/octet-stream', + shade.openstackcloud.OBJECT_AUTOCREATE_KEY: 'true', + 'Etag': fakes.NO_MD5}), + dict(method='DELETE', + uri='{endpoint}/{container}/{object}'.format( + endpoint=endpoint, container=self.container_name, + object=self.image_name)), + ]) + + deleted = self.cloud.delete_autocreated_image_objects( + container=self.container_name) + self.assertTrue(deleted) self.assert_calls()