From 3766836d9308d7cfc959a1ed8bfb417fb720eec2 Mon Sep 17 00:00:00 2001 From: Michael Ionkin Date: Mon, 27 Jun 2016 20:00:28 +0300 Subject: [PATCH] Fix glanceclient.v2.images This patch overwrites migration from novaclient.images to glanceclient.v2.images. Glanceclient uses Warlock library and we want to be independent of this library because its 1.3.0 release had broken all our CI. The idea is we speak to glanceclient which returns image dict, then we transform this dict by adding our sahara related key-values. Change-Id: I07e9875622ace6b0aa3cd098d36b2eeed59c6d54 co-authored-by: Vitaly Gridnev --- sahara/conductor/objects.py | 10 ++ sahara/conductor/resource.py | 20 +++ sahara/exceptions.py | 13 +- sahara/service/api/v10.py | 36 +++--- sahara/service/api/v2/images.py | 36 +++--- sahara/service/engine.py | 4 +- sahara/service/validations/base.py | 4 +- sahara/tests/unit/service/test_engine.py | 10 +- .../test_cluster_create_validation.py | 1 - sahara/tests/unit/service/validation/utils.py | 9 +- .../tests/unit/utils/openstack/test_images.py | 33 ++--- sahara/utils/openstack/glance.py | 2 - sahara/utils/openstack/images.py | 121 +++++++++++------- 13 files changed, 184 insertions(+), 115 deletions(-) diff --git a/sahara/conductor/objects.py b/sahara/conductor/objects.py index 11b9373d2d..37cc7bb468 100644 --- a/sahara/conductor/objects.py +++ b/sahara/conductor/objects.py @@ -223,6 +223,16 @@ class NodeGroupTemplate(object): """ +class Image(object): + """An object representing Image. + + id + tags + username + description + """ + + # EDP Objects class DataSource(object): diff --git a/sahara/conductor/resource.py b/sahara/conductor/resource.py index f8e4b31d74..352feb686f 100644 --- a/sahara/conductor/resource.py +++ b/sahara/conductor/resource.py @@ -238,6 +238,26 @@ class ClusterResource(Resource, objects.Cluster): _sanitize_fields = {'cluster_configs': sanitize_cluster_configs} +class ImageResource(Resource, objects.Image): + + _resource_name = 'image' + + @property + def dict(self): + return self.to_dict() + + @property + def wrapped_dict(self): + return {'image': self.dict} + + def _sanitize_image_properties(self, image_props): + if 'links' in image_props: + del image_props['links'] + return image_props + + _sanitize_fields = {'links': _sanitize_image_properties} + + # EDP Resources class DataSource(Resource, objects.DataSource): diff --git a/sahara/exceptions.py b/sahara/exceptions.py index f4fcadc807..0792b95f81 100644 --- a/sahara/exceptions.py +++ b/sahara/exceptions.py @@ -53,8 +53,8 @@ class SaharaException(Exception): class NotFoundException(SaharaException): code = "NOT_FOUND" message_template = _("Object '%s' is not found") - # It could be a various property of object which was not found + # It could be a various property of object which was not found def __init__(self, value, message_template=None): self.value = value if message_template: @@ -65,6 +65,17 @@ class NotFoundException(SaharaException): super(NotFoundException, self).__init__(formatted_message) +class NoUniqueMatchException(SaharaException): + code = "NO_UNIQUE_MATCH" + message_template = _( + "Response {response} is not unique for this query {query}.") + + def __init__(self, response, query, message_template=None): + template = message_template or self.message_template + formatted_message = template.format(response=response, query=query) + super(NoUniqueMatchException, self).__init__(formatted_message) + + class NameAlreadyExistsException(SaharaException): code = "NAME_ALREADY_EXISTS" message = _("Name already exists") diff --git a/sahara/service/api/v10.py b/sahara/service/api/v10.py index fe9b442d87..e8aae54245 100644 --- a/sahara/service/api/v10.py +++ b/sahara/service/api/v10.py @@ -28,7 +28,7 @@ from sahara.utils import cluster as c_u from sahara.utils import general as g from sahara.utils.notification import sender from sahara.utils.openstack import base as b -from sahara.utils.openstack import glance +from sahara.utils.openstack import images as sahara_images conductor = c.API @@ -253,41 +253,43 @@ def construct_ngs_for_scaling(cluster, additional_node_groups): def get_images(name, tags): return b.execute_with_retries( - glance.client().images.list_registered, name, tags) + sahara_images.image_manager().list_registered, name, tags) def get_image(**kwargs): if len(kwargs) == 1 and 'id' in kwargs: - return b.execute_with_retries(glance.client().images.get, kwargs['id']) + return b.execute_with_retries( + sahara_images.image_manager().get, kwargs['id']) else: - return b.execute_with_retries(glance.client().images.find, **kwargs) + return b.execute_with_retries( + sahara_images.image_manager().find, **kwargs) def get_registered_image(image_id): return b.execute_with_retries( - glance.client().images.get_registered_image, image_id) + sahara_images.image_manager().get_registered_image, image_id) def register_image(image_id, username, description=None): - client = glance.client() + manager = sahara_images.image_manager() b.execute_with_retries( - client.images.set_description, image_id, username, description) - return b.execute_with_retries(client.images.get, image_id) + manager.set_image_info, image_id, username, description) + return b.execute_with_retries(manager.get, image_id) def unregister_image(image_id): - client = glance.client() - b.execute_with_retries(client.images.unset_description, image_id) - return b.execute_with_retries(client.images.get, image_id) + manager = sahara_images.image_manager() + b.execute_with_retries(manager.unset_image_info, image_id) + return b.execute_with_retries(manager.get, image_id) def add_image_tags(image_id, tags): - client = glance.client() - b.execute_with_retries(client.images.tag, image_id, tags) - return b.execute_with_retries(client.images.get, image_id) + manager = sahara_images.image_manager() + b.execute_with_retries(manager.tag, image_id, tags) + return b.execute_with_retries(manager.get, image_id) def remove_image_tags(image_id, tags): - client = glance.client() - b.execute_with_retries(client.images.untag, image_id, tags) - return b.execute_with_retries(client.images.get, image_id) + manager = sahara_images.image_manager() + b.execute_with_retries(manager.untag, image_id, tags) + return b.execute_with_retries(manager.get, image_id) diff --git a/sahara/service/api/v2/images.py b/sahara/service/api/v2/images.py index 52edd4a79a..934be67a89 100644 --- a/sahara/service/api/v2/images.py +++ b/sahara/service/api/v2/images.py @@ -15,7 +15,7 @@ from sahara import conductor as c from sahara.utils.openstack import base as b -from sahara.utils.openstack import glance +from sahara.utils.openstack import images as sahara_images conductor = c.API @@ -26,41 +26,43 @@ conductor = c.API def get_images(name, tags): return b.execute_with_retries( - glance.client().images.list_registered, name, tags) + sahara_images.image_manager().list_registered, name, tags) def get_image(**kwargs): if len(kwargs) == 1 and 'id' in kwargs: - return b.execute_with_retries(glance.client().images.get, kwargs['id']) + return b.execute_with_retries( + sahara_images.image_manager().get, kwargs['id']) else: - return b.execute_with_retries(glance.client().images.find, **kwargs) + return b.execute_with_retries( + sahara_images.image_manager().find, **kwargs) def get_registered_image(id): return b.execute_with_retries( - glance.client().images.get_registered_image, id) + sahara_images.image_manager().get_registered_image, id) def register_image(image_id, username, description=None): - client = glance.client() + manager = sahara_images.image_manager() b.execute_with_retries( - client.images.set_description, image_id, username, description) - return b.execute_with_retries(client.images.get, image_id) + manager.set_image_info, image_id, username, description) + return b.execute_with_retries(manager.get, image_id) def unregister_image(image_id): - client = glance.client() - b.execute_with_retries(client.images.unset_description, image_id) - return b.execute_with_retries(client.images.get, image_id) + manager = sahara_images.image_manager() + b.execute_with_retries(manager.unset_image_info, image_id) + return b.execute_with_retries(manager.get, image_id) def add_image_tags(image_id, tags): - client = glance.client() - b.execute_with_retries(client.images.tag, image_id, tags) - return b.execute_with_retries(client.images.get, image_id) + manager = sahara_images.image_manager() + b.execute_with_retries(manager.tag, image_id, tags) + return b.execute_with_retries(manager.get, image_id) def remove_image_tags(image_id, tags): - client = glance.client() - b.execute_with_retries(client.images.untag, image_id, tags) - return b.execute_with_retries(client.images.get, image_id) + manager = sahara_images.image_manager() + b.execute_with_retries(manager.untag, image_id, tags) + return b.execute_with_retries(manager.get, image_id) diff --git a/sahara/service/engine.py b/sahara/service/engine.py index bf5c64809b..eb17c4372f 100644 --- a/sahara/service/engine.py +++ b/sahara/service/engine.py @@ -34,7 +34,7 @@ from sahara.utils import cluster_progress_ops as cpo from sahara.utils import edp from sahara.utils import general as g from sahara.utils.openstack import base as b -from sahara.utils.openstack import glance +from sahara.utils.openstack import images as sahara_images from sahara.utils.openstack import nova from sahara.utils import poll_utils from sahara.utils import remote @@ -71,7 +71,7 @@ class Engine(object): def get_node_group_image_username(self, node_group): image_id = node_group.get_image_id() return b.execute_with_retries( - glance.client().images.get, image_id).username + sahara_images.image_manager().get, image_id).username @poll_utils.poll_status('ips_assign_timeout', _("Assign IPs"), sleep=1) def _ips_assign(self, ips_assigned, cluster, instances): diff --git a/sahara/service/validations/base.py b/sahara/service/validations/base.py index fc07ff1bb6..81aa69f723 100644 --- a/sahara/service/validations/base.py +++ b/sahara/service/validations/base.py @@ -27,7 +27,7 @@ import sahara.plugins.base as plugin_base from sahara.service.api import v10 as api from sahara.utils import general as g import sahara.utils.openstack.cinder as cinder -from sahara.utils.openstack import glance +from sahara.utils.openstack import images as sahara_images import sahara.utils.openstack.nova as nova @@ -76,7 +76,7 @@ def check_plugin_supports_version(p_name, version): def check_image_registered(image_id): if image_id not in ( - [i.id for i in glance.client().images.list_registered()]): + [i.id for i in sahara_images.image_manager().list_registered()]): raise ex.InvalidReferenceException( _("Requested image '%s' is not registered") % image_id) diff --git a/sahara/tests/unit/service/test_engine.py b/sahara/tests/unit/service/test_engine.py index 60c2b2d3da..0d6954e981 100644 --- a/sahara/tests/unit/service/test_engine.py +++ b/sahara/tests/unit/service/test_engine.py @@ -50,12 +50,12 @@ class TestEngine(base.SaharaWithDbTestCase): super(TestEngine, self).setUp() self.eng = EngineTest() - @mock.patch('sahara.utils.openstack.glance.client') - def test_get_node_group_image_username(self, glance_client): + @mock.patch('sahara.utils.openstack.images.SaharaImageManager') + def test_get_node_group_image_username(self, mock_manager): ng = mock.Mock() - client = mock.Mock() - client.images.get.return_value = mock.Mock(username='username') - glance_client.return_value = client + manager = mock.Mock() + manager.get.return_value = mock.Mock(username='username') + mock_manager.return_value = manager self.assertEqual( 'username', self.eng.get_node_group_image_username(ng)) diff --git a/sahara/tests/unit/service/validation/test_cluster_create_validation.py b/sahara/tests/unit/service/validation/test_cluster_create_validation.py index 9f55b53b4c..714cf0ec62 100644 --- a/sahara/tests/unit/service/validation/test_cluster_create_validation.py +++ b/sahara/tests/unit/service/validation/test_cluster_create_validation.py @@ -486,7 +486,6 @@ class TestClusterCreateFlavorValidation(base.SaharaWithDbTestCase): "sahara.service.validations.base.check_plugin_supports_version", "sahara.service.validations.base._get_plugin_configs", "sahara.service.validations.base.check_node_processes", - "sahara.utils.openstack.nova.client", ] self.patchers = [] for module in modules: diff --git a/sahara/tests/unit/service/validation/utils.py b/sahara/tests/unit/service/validation/utils.py index e739e64fb6..2e425c0f23 100644 --- a/sahara/tests/unit/service/validation/utils.py +++ b/sahara/tests/unit/service/validation/utils.py @@ -135,7 +135,8 @@ def start_patch(patch_templates=True): "sahara.service.api.v10.get_cluster_template") nova_p = mock.patch("sahara.utils.openstack.nova.client") heat_p = mock.patch("sahara.utils.openstack.heat.client") - glance_p = mock.patch("sahara.utils.openstack.glance.client") + image_manager_p = mock.patch( + "sahara.utils.openstack.images.SaharaImageManager") cinder_p = mock.patch("sahara.utils.openstack.cinder.client") cinder_exists_p = mock.patch( "sahara.utils.openstack.cinder.check_cinder_exists") @@ -167,7 +168,7 @@ def start_patch(patch_templates=True): heat = heat_p.start() heat().stacks.list.side_effect = _get_heat_stack_list - glance = glance_p.start() + image_manager = image_manager_p.start() cinder = cinder_p.start() cinder().availability_zones.list.side_effect = _get_availability_zone_list @@ -200,7 +201,7 @@ def start_patch(patch_templates=True): return Image('wrong_test') get_image.side_effect = _get_image - glance().images.list_registered.return_value = [Image(), + image_manager().list_registered.return_value = [Image(), Image(name='wrong_name')] ng_dict = tu.make_ng_dict('ng', '42', ['namenode'], 1) cluster = tu.create_cluster('test', 't', 'fake', '0.1', [ng_dict], @@ -233,7 +234,7 @@ def start_patch(patch_templates=True): get_ng_template.side_effect = _get_ng_template # request data to validate patchers = [get_clusters_p, get_cluster_p, - nova_p, get_image_p, heat_p, glance_p, cinder_p, + nova_p, get_image_p, heat_p, image_manager_p, cinder_p, cinder_exists_p] if patch_templates: patchers.extend([get_ng_template_p, get_ng_templates_p, diff --git a/sahara/tests/unit/utils/openstack/test_images.py b/sahara/tests/unit/utils/openstack/test_images.py index 006b78040e..6860162302 100644 --- a/sahara/tests/unit/utils/openstack/test_images.py +++ b/sahara/tests/unit/utils/openstack/test_images.py @@ -16,7 +16,7 @@ import mock from sahara.tests.unit import base -from sahara.utils.openstack import glance as glance_client +from sahara.utils.openstack import images as sahara_images class FakeImage(object): @@ -39,37 +39,38 @@ class TestImages(base.SaharaTestCase): FakeImage('baz', [], 'test'), FakeImage('spam', [], "")] - with mock.patch('glanceclient.v2.images.Controller.list', - return_value=some_images): - glance = glance_client.client() + with mock.patch( + 'sahara.utils.openstack.images.SaharaImageManager.list', + return_value=some_images): + manager = sahara_images.image_manager() - images = glance.images.list_registered() + images = manager.list_registered() self.assertEqual(2, len(images)) - images = glance.images.list_registered(name='foo') + images = manager.list_registered(name='foo') self.assertEqual(1, len(images)) self.assertEqual('foo', images[0].name) self.assertEqual('test', images[0].username) - images = glance.images.list_registered(name='eggs') + images = manager.list_registered(name='eggs') self.assertEqual(0, len(images)) - images = glance.images.list_registered(tags=['bar']) + images = manager.list_registered(tags=['bar']) self.assertEqual(1, len(images)) self.assertEqual('foo', images[0].name) - images = glance.images.list_registered(tags=['bar', 'eggs']) + images = manager.list_registered(tags=['bar', 'eggs']) self.assertEqual(0, len(images)) @mock.patch('sahara.utils.openstack.images.SaharaImageManager.set_meta') - def test_set_description(self, set_meta): + def test_set_image_info(self, set_meta): with mock.patch('sahara.utils.openstack.base.url_for'): - glance = glance_client.client() - glance.images.set_description('id', 'ubuntu') + manager = sahara_images.image_manager() + manager.set_image_info('id', 'ubuntu') self.assertEqual( ('id', {'_sahara_username': 'ubuntu'}), set_meta.call_args[0]) - glance.images.set_description('id', 'ubuntu', 'descr') + manager.set_image_info('id', 'ubuntu', 'descr') self.assertEqual( ('id', {'_sahara_description': 'descr', '_sahara_username': 'ubuntu'}), @@ -77,14 +78,14 @@ class TestImages(base.SaharaTestCase): @mock.patch('sahara.utils.openstack.images.SaharaImageManager.get') @mock.patch('sahara.utils.openstack.images.SaharaImageManager.delete_meta') - def test_unset_description(self, delete_meta, get_image): - glance = glance_client.client() + def test_unset_image_info(self, delete_meta, get_image): + manager = sahara_images.image_manager() image = mock.MagicMock() image.tags = ['fake', 'fake_2.0'] image.username = 'ubuntu' image.description = 'some description' get_image.return_value = image - glance.images.unset_description('id') + manager.unset_image_info('id') self.assertEqual( ('id', ['_sahara_tag_fake', '_sahara_tag_fake_2.0', '_sahara_description', '_sahara_username']), diff --git a/sahara/utils/openstack/glance.py b/sahara/utils/openstack/glance.py index 32a64ee8b0..811e99bafa 100644 --- a/sahara/utils/openstack/glance.py +++ b/sahara/utils/openstack/glance.py @@ -18,7 +18,6 @@ from glanceclient import client as glance_client from oslo_config import cfg from sahara.service import sessions -from sahara.utils.openstack import images from sahara.utils.openstack import keystone @@ -45,5 +44,4 @@ CONF.register_opts(opts, group=glance_group) def client(): session = sessions.cache().get_session(sessions.SESSION_TYPE_GLANCE) glance = glance_client.Client('2', session=session, auth=keystone.auth()) - glance.images = images.SaharaImageManager(glance) return glance diff --git a/sahara/utils/openstack/images.py b/sahara/utils/openstack/images.py index 9fe4ca9648..8506d9ab05 100644 --- a/sahara/utils/openstack/images.py +++ b/sahara/utils/openstack/images.py @@ -13,19 +13,39 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy +import functools -from glanceclient.v2 import images -from glanceclient.v2 import schemas import six +from sahara.conductor import resource from sahara import exceptions as exc +from sahara.utils.openstack import glance PROP_DESCR = '_sahara_description' PROP_USERNAME = '_sahara_username' PROP_TAG = '_sahara_tag_' -PROP_TAGS = '_all_tags' +PROP_ALL_TAGS = '_all_tags' + + +def image_manager(): + return SaharaImageManager() + + +def wrap_entity(func): + @functools.wraps(func) + def handle(*args, **kwargs): + res = func(*args, **kwargs) + if isinstance(res, list): + images = [] + for image in res: + _transform_image_props(image) + images.append(resource.ImageResource(image)) + return images + else: + _transform_image_props(res) + return resource.ImageResource(res) + return handle def _get_all_tags(image_props): @@ -36,62 +56,67 @@ def _get_all_tags(image_props): return tags +def _get_meta_prop(image_props, prop, default=None): + if PROP_ALL_TAGS == prop: + return _get_all_tags(image_props) + return image_props.get(prop, default) + + +def _parse_tags(image_props): + tags = _get_meta_prop(image_props, PROP_ALL_TAGS) + return [t.replace(PROP_TAG, "") for t in tags] + + +def _transform_image_props(image): + image['username'] = _get_meta_prop(image, PROP_USERNAME, "") + image['description'] = _get_meta_prop(image, PROP_DESCR, "") + image['tags'] = _parse_tags(image) + return image + + def _ensure_tags(tags): if not tags: return [] return [tags] if isinstance(tags, six.string_types) else tags -class SaharaImageModel(schemas.SchemaBasedModel): +class SaharaImageManager(object): + """SaharaImageManager - def __init__(self, *args, **kwargs): - super(SaharaImageModel, self).__init__(*args, **kwargs) - self.username = self._get_meta_prop(PROP_USERNAME, "") - self.description = self._get_meta_prop(PROP_DESCR, "") - self.tags = self._parse_tags() - - def _get_meta_prop(self, prop, default=None): - if PROP_TAGS == prop: - return _get_all_tags(self) - return self.get(prop, default) - - def _parse_tags(self): - tags = self._get_meta_prop(PROP_TAGS) - return [t.replace(PROP_TAG, "") for t in tags] - - @property - def dict(self): - return self.to_dict() - - @property - def wrapped_dict(self): - return {'image': self.dict} - - def to_dict(self): - result = copy.deepcopy(dict(self)) - if 'links' in result: - del result['links'] - return result - - -class SaharaImageManager(images.Controller): - """Manage :class:`SaharaImageModel` resources. - - This is an extended version of glance client's Controller with support of - additional description and image tags stored as image properties. + This class is intermediate layer between sahara and glanceclient.v2.images. + It provides additional sahara properties for image such as description, + image tags and image username. """ - def __init__(self, glance_client): - schemas.SchemaBasedModel = SaharaImageModel - super(SaharaImageManager, self).__init__(glance_client.http_client, - glance_client.schemas) + def __init__(self): + self.client = glance.client().images + + @wrap_entity + def get(self, image_id): + image = self.client.get(image_id) + return image + + @wrap_entity + def find(self, **kwargs): + images = self.client.list(**kwargs) + num_matches = len(images) + if num_matches == 0: + raise exc.NotFoundException(kwargs, "No images matching %s.") + elif num_matches > 1: + raise exc.NoUniqueMatchException(response=images, query=kwargs) + else: + return images[0] + + @wrap_entity + def list(self): + return list(self.client.list()) def set_meta(self, image_id, meta): - self.update(image_id, remove_props=None, **meta) + self.client.update(image_id, remove_props=None, **meta) def delete_meta(self, image_id, meta_list): - self.update(image_id, remove_props=meta_list) + self.client.update(image_id, remove_props=meta_list) - def set_description(self, image_id, username, description=None): + def set_image_info(self, image_id, username, description=None): """Sets human-readable information for image. For example: @@ -102,7 +127,7 @@ class SaharaImageManager(images.Controller): meta[PROP_DESCR] = description self.set_meta(image_id, meta) - def unset_description(self, image_id): + def unset_image_info(self, image_id): """Unsets all Sahara-related information. It removes username, description and tags from the specified image.