novaclient.v2.images to glanceclient migration

The novaclient.v2.images module is deprecated and will be removed
after Nova 15.0.0 is released so we need to migrate to glanceclient.

note: glanceclient doesn't have any Image class. It have image json
schema and use it to generate warlock models when creates image
object.

closes-bug: 1573595
Depends-on: I2e13867bed72fd2ae6749491dd5b032c2c08f9f6
Change-Id: Ie0f04c15e6e49e8e7c77f7b7d870a368cdd5c7c7

devstack/plugin.sh

@@ -123,6 +123,10 @@ function configure_sahara {
         iniset $SAHARA_CONF_FILE keystone ca_file $SSL_BUNDLE_FILE
     fi
 
+    if is_ssl_enabled_service "glance" || is_service_enabled tls-proxy; then
+        iniset $SAHARA_CONF_FILE glance ca_file $SSL_BUNDLE_FILE
+    fi
+
     # Register SSL certificates if provided
     if is_ssl_enabled_service sahara; then
         ensure_certificates SAHARA

releasenotes/notes/novaclient_images_to_glanceclient-0266a2bd92b4be05.yaml

@@ -0,0 +1,3 @@
+---
+upgrade:
+  - Migration from novaclient.v2.images to glanceclient

requirements.txt

@@ -36,6 +36,7 @@ python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
 python-swiftclient>=2.2.0 # Apache-2.0
 python-neutronclient>=4.2.0 # Apache-2.0
 python-heatclient>=1.1.0 # Apache-2.0
+python-glanceclient>=2.0.0  # Apache-2.0
 six>=1.9.0 # MIT
 stevedore>=1.10.0 # Apache-2.0
 SQLAlchemy<1.1.0,>=1.0.10 # MIT

sahara/api/v10.py

@@ -231,7 +231,7 @@ def images_list():
 @acl.enforce("data-processing:images:get")
 @v.check_exists(api.get_image, id='image_id')
 def images_get(image_id):
-    return u.render(api.get_registered_image(id=image_id).wrapped_dict)
+    return u.render(api.get_registered_image(image_id=image_id).wrapped_dict)
 
 
 @rest.post('/images/<image_id>')

sahara/service/api/v10.py

@@ -28,7 +28,7 @@ from sahara.utils import cluster as c_u
 from sahara.utils import general as g
 from sahara.utils.notification import sender
 from sahara.utils.openstack import base as b
-from sahara.utils.openstack import nova
+from sahara.utils.openstack import glance
 
 
 conductor = c.API
@@ -253,41 +253,41 @@ def construct_ngs_for_scaling(cluster, additional_node_groups):
 
 def get_images(name, tags):
     return b.execute_with_retries(
-        nova.client().images.list_registered, name, tags)
+        glance.client().images.list_registered, name, tags)
 
 
 def get_image(**kwargs):
     if len(kwargs) == 1 and 'id' in kwargs:
-        return b.execute_with_retries(nova.client().images.get, kwargs['id'])
+        return b.execute_with_retries(glance.client().images.get, kwargs['id'])
     else:
-        return b.execute_with_retries(nova.client().images.find, **kwargs)
+        return b.execute_with_retries(glance.client().images.find, **kwargs)
 
 
-def get_registered_image(id):
+def get_registered_image(image_id):
     return b.execute_with_retries(
-        nova.client().images.get_registered_image, id)
+        glance.client().images.get_registered_image, image_id)
 
 
 def register_image(image_id, username, description=None):
-    client = nova.client()
+    client = glance.client()
     b.execute_with_retries(
         client.images.set_description, image_id, username, description)
     return b.execute_with_retries(client.images.get, image_id)
 
 
 def unregister_image(image_id):
-    client = nova.client()
+    client = glance.client()
     b.execute_with_retries(client.images.unset_description, image_id)
     return b.execute_with_retries(client.images.get, image_id)
 
 
 def add_image_tags(image_id, tags):
-    client = nova.client()
+    client = glance.client()
     b.execute_with_retries(client.images.tag, image_id, tags)
     return b.execute_with_retries(client.images.get, image_id)
 
 
 def remove_image_tags(image_id, tags):
-    client = nova.client()
+    client = glance.client()
     b.execute_with_retries(client.images.untag, image_id, tags)
     return b.execute_with_retries(client.images.get, image_id)

sahara/service/api/v2/images.py

@@ -15,7 +15,7 @@
 
 from sahara import conductor as c
 from sahara.utils.openstack import base as b
-from sahara.utils.openstack import nova
+from sahara.utils.openstack import glance
 
 
 conductor = c.API
@@ -26,41 +26,41 @@ conductor = c.API
 
 def get_images(name, tags):
     return b.execute_with_retries(
-        nova.client().images.list_registered, name, tags)
+        glance.client().images.list_registered, name, tags)
 
 
 def get_image(**kwargs):
     if len(kwargs) == 1 and 'id' in kwargs:
-        return b.execute_with_retries(nova.client().images.get, kwargs['id'])
+        return b.execute_with_retries(glance.client().images.get, kwargs['id'])
     else:
-        return b.execute_with_retries(nova.client().images.find, **kwargs)
+        return b.execute_with_retries(glance.client().images.find, **kwargs)
 
 
 def get_registered_image(id):
     return b.execute_with_retries(
-        nova.client().images.get_registered_image, id)
+        glance.client().images.get_registered_image, id)
 
 
 def register_image(image_id, username, description=None):
-    client = nova.client()
+    client = glance.client()
     b.execute_with_retries(
         client.images.set_description, image_id, username, description)
     return b.execute_with_retries(client.images.get, image_id)
 
 
 def unregister_image(image_id):
-    client = nova.client()
+    client = glance.client()
     b.execute_with_retries(client.images.unset_description, image_id)
     return b.execute_with_retries(client.images.get, image_id)
 
 
 def add_image_tags(image_id, tags):
-    client = nova.client()
+    client = glance.client()
     b.execute_with_retries(client.images.tag, image_id, tags)
     return b.execute_with_retries(client.images.get, image_id)
 
 
 def remove_image_tags(image_id, tags):
-    client = nova.client()
+    client = glance.client()
     b.execute_with_retries(client.images.untag, image_id, tags)
     return b.execute_with_retries(client.images.get, image_id)

sahara/service/engine.py

@@ -34,6 +34,7 @@ from sahara.utils import cluster_progress_ops as cpo
 from sahara.utils import edp
 from sahara.utils import general as g
 from sahara.utils.openstack import base as b
+from sahara.utils.openstack import glance
 from sahara.utils.openstack import nova
 from sahara.utils import poll_utils
 from sahara.utils import remote
@@ -70,7 +71,7 @@ class Engine(object):
     def get_node_group_image_username(self, node_group):
         image_id = node_group.get_image_id()
         return b.execute_with_retries(
-            nova.client().images.get, image_id).username
+            glance.client().images.get, image_id).username
 
     @poll_utils.poll_status('ips_assign_timeout', _("Assign IPs"), sleep=1)
     def _ips_assign(self, ips_assigned, cluster, instances):

sahara/service/sessions.py

@@ -31,6 +31,7 @@ SESSION_TYPE_CINDER = 'cinder'
 SESSION_TYPE_KEYSTONE = 'keystone'
 SESSION_TYPE_NEUTRON = 'neutron'
 SESSION_TYPE_NOVA = 'nova'
+SESSION_TYPE_GLANCE = 'glance'
 SESSION_TYPE_INSECURE = 'insecure'
 
 
@@ -61,6 +62,7 @@ class SessionCache(object):
             SESSION_TYPE_KEYSTONE: self.get_keystone_session,
             SESSION_TYPE_NEUTRON: self.get_neutron_session,
             SESSION_TYPE_NOVA: self.get_nova_session,
+            SESSION_TYPE_GLANCE: self.get_glance_session,
             SESSION_TYPE_INSECURE: self.get_insecure_session,
         }
 
@@ -144,6 +146,16 @@ class SessionCache(object):
             self._set_session(SESSION_TYPE_NOVA, session)
         return session
 
+    def get_glance_session(self):
+        session = self._sessions.get(SESSION_TYPE_GLANCE)
+        if not session:
+            if not CONF.glance.api_insecure:
+                session = keystone.Session(verify=CONF.glance.ca_file or True)
+            else:
+                session = self.get_insecure_session()
+            self._set_session(SESSION_TYPE_GLANCE, session)
+        return session
+
     def token_for_auth(self, auth):
         return self.get_keystone_session().get_auth_headers(auth).get(
             'X-Auth-Token')

sahara/service/validations/base.py

@@ -27,6 +27,7 @@ import sahara.plugins.base as plugin_base
 from sahara.service.api import v10 as api
 from sahara.utils import general as g
 import sahara.utils.openstack.cinder as cinder
+from sahara.utils.openstack import glance
 import sahara.utils.openstack.nova as nova
 
 
@@ -74,7 +75,8 @@ def check_plugin_supports_version(p_name, version):
 
 
 def check_image_registered(image_id):
-    if image_id not in [i.id for i in nova.client().images.list_registered()]:
+    if image_id not in (
+            [i.id for i in glance.client().images.list_registered()]):
         raise ex.InvalidReferenceException(
             _("Requested image '%s' is not registered") % image_id)
 

sahara/tests/unit/service/test_engine.py

@@ -50,12 +50,12 @@ class TestEngine(base.SaharaWithDbTestCase):
         super(TestEngine, self).setUp()
         self.eng = EngineTest()
 
-    @mock.patch('sahara.utils.openstack.nova.client')
+    @mock.patch('sahara.utils.openstack.glance.client')
-    def test_get_node_group_image_username(self, nova_client):
+    def test_get_node_group_image_username(self, glance_client):
         ng = mock.Mock()
         client = mock.Mock()
         client.images.get.return_value = mock.Mock(username='username')
-        nova_client.return_value = client
+        glance_client.return_value = client
 
         self.assertEqual(
             'username', self.eng.get_node_group_image_username(ng))

sahara/tests/unit/service/test_sessions.py

@@ -112,6 +112,25 @@ class TestSessionCache(base.SaharaTestCase):
         sc.get_session(sessions.SESSION_TYPE_NEUTRON)
         self.assertFalse(keystone_session.called)
 
+    @mock.patch('keystoneauth1.session.Session')
+    def test_get_glance_session(self, keystone_session):
+        sc = sessions.SessionCache()
+        self.override_config('ca_file', '/some/cacert', group='glance')
+        self.override_config('api_insecure', False, group='glance')
+        sc.get_session(sessions.SESSION_TYPE_GLANCE)
+        keystone_session.assert_called_once_with(verify='/some/cacert')
+
+        sc = sessions.SessionCache()
+        keystone_session.reset_mock()
+        self.override_config('ca_file', None, group='glance')
+        self.override_config('api_insecure', True, group='glance')
+        sc.get_session(sessions.SESSION_TYPE_GLANCE)
+        keystone_session.assert_called_once_with(verify=False)
+
+        keystone_session.reset_mock()
+        sc.get_session(sessions.SESSION_TYPE_GLANCE)
+        self.assertFalse(keystone_session.called)
+
     @mock.patch('keystoneauth1.session.Session')
     def test_insecure_session(self, session):
         sc = sessions.SessionCache()

sahara/tests/unit/service/validation/utils.py

@@ -135,6 +135,7 @@ def start_patch(patch_templates=True):
             "sahara.service.api.v10.get_cluster_template")
     nova_p = mock.patch("sahara.utils.openstack.nova.client")
     heat_p = mock.patch("sahara.utils.openstack.heat.client")
+    glance_p = mock.patch("sahara.utils.openstack.glance.client")
     cinder_p = mock.patch("sahara.utils.openstack.cinder.client")
     cinder_exists_p = mock.patch(
         "sahara.utils.openstack.cinder.check_cinder_exists")
@@ -166,6 +167,8 @@ def start_patch(patch_templates=True):
     heat = heat_p.start()
     heat().stacks.list.side_effect = _get_heat_stack_list
 
+    glance = glance_p.start()
+
     cinder = cinder_p.start()
     cinder().availability_zones.list.side_effect = _get_availability_zone_list
 
@@ -197,8 +200,8 @@ def start_patch(patch_templates=True):
             return Image('wrong_test')
 
     get_image.side_effect = _get_image
-    nova().images.list_registered.return_value = [Image(),
+    glance().images.list_registered.return_value = [Image(),
-                                                  Image(name='wrong_name')]
+                                                    Image(name='wrong_name')]
     ng_dict = tu.make_ng_dict('ng', '42', ['namenode'], 1)
     cluster = tu.create_cluster('test', 't', 'fake', '0.1', [ng_dict],
                                 id=1, status=c_u.CLUSTER_STATUS_ACTIVE)
@@ -230,7 +233,7 @@ def start_patch(patch_templates=True):
         get_ng_template.side_effect = _get_ng_template
     # request data to validate
     patchers = [get_clusters_p, get_cluster_p,
-                nova_p, get_image_p, heat_p, cinder_p,
+                nova_p, get_image_p, heat_p, glance_p, cinder_p,
                 cinder_exists_p]
     if patch_templates:
         patchers.extend([get_ng_template_p, get_ng_templates_p,

sahara/tests/unit/utils/openstack/test_images.py

@@ -16,7 +16,7 @@
 import mock
 
 from sahara.tests.unit import base
-from sahara.utils.openstack import nova as nova_client
+from sahara.utils.openstack import glance as glance_client
 
 
 class FakeImage(object):
@@ -39,38 +39,53 @@ class TestImages(base.SaharaTestCase):
             FakeImage('baz', [], 'test'),
             FakeImage('spam', [], "")]
 
-        with mock.patch('novaclient.v2.images.ImageManager.list',
+        with mock.patch('glanceclient.v2.images.Controller.list',
                         return_value=some_images):
-            nova = nova_client.client()
+            glance = glance_client.client()
 
-            images = nova.images.list_registered()
+            images = glance.images.list_registered()
             self.assertEqual(2, len(images))
 
-            images = nova.images.list_registered(name='foo')
+            images = glance.images.list_registered(name='foo')
             self.assertEqual(1, len(images))
             self.assertEqual('foo', images[0].name)
             self.assertEqual('test', images[0].username)
 
-            images = nova.images.list_registered(name='eggs')
+            images = glance.images.list_registered(name='eggs')
             self.assertEqual(0, len(images))
 
-            images = nova.images.list_registered(tags=['bar'])
+            images = glance.images.list_registered(tags=['bar'])
             self.assertEqual(1, len(images))
             self.assertEqual('foo', images[0].name)
 
-            images = nova.images.list_registered(tags=['bar', 'eggs'])
+            images = glance.images.list_registered(tags=['bar', 'eggs'])
             self.assertEqual(0, len(images))
 
-    @mock.patch('novaclient.v2.images.ImageManager.set_meta')
+    @mock.patch('sahara.utils.openstack.images.SaharaImageManager.set_meta')
     def test_set_description(self, set_meta):
         with mock.patch('sahara.utils.openstack.base.url_for'):
-            nova = nova_client.client()
+            glance = glance_client.client()
-            nova.images.set_description('id', 'ubuntu')
+            glance.images.set_description('id', 'ubuntu')
             self.assertEqual(
                 ('id', {'_sahara_username': 'ubuntu'}), set_meta.call_args[0])
 
-            nova.images.set_description('id', 'ubuntu', 'descr')
+            glance.images.set_description('id', 'ubuntu', 'descr')
             self.assertEqual(
                 ('id', {'_sahara_description': 'descr',
                         '_sahara_username': 'ubuntu'}),
                 set_meta.call_args[0])
+
+    @mock.patch('sahara.utils.openstack.images.SaharaImageManager.get')
+    @mock.patch('sahara.utils.openstack.images.SaharaImageManager.delete_meta')
+    def test_unset_description(self, delete_meta, get_image):
+            glance = glance_client.client()
+            image = mock.MagicMock()
+            image.tags = ['fake', 'fake_2.0']
+            image.username = 'ubuntu'
+            image.description = 'some description'
+            get_image.return_value = image
+            glance.images.unset_description('id')
+            self.assertEqual(
+                ('id', ['_sahara_tag_fake', '_sahara_tag_fake_2.0',
+                        '_sahara_description', '_sahara_username']),
+                delete_meta.call_args[0])

sahara/utils/openstack/glance.py

@@ -0,0 +1,49 @@
+# Copyright (c) 2016 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from glanceclient import client as glance_client
+from oslo_config import cfg
+
+from sahara.service import sessions
+from sahara.utils.openstack import images
+from sahara.utils.openstack import keystone
+
+
+opts = [
+    cfg.BoolOpt('api_insecure',
+                default=False,
+                help='Allow to perform insecure SSL requests to glance.'),
+    cfg.StrOpt('ca_file',
+               help='Location of ca certificates file to use for glance '
+                    'client requests.'),
+    cfg.StrOpt("endpoint_type",
+               default="internalURL",
+               help="Endpoint type for glance client requests"),
+]
+
+glance_group = cfg.OptGroup(name='glance',
+                            title='Glance client options')
+
+CONF = cfg.CONF
+CONF.register_group(glance_group)
+CONF.register_opts(opts, group=glance_group)
+
+
+def client():
+    session = sessions.cache().get_session(sessions.SESSION_TYPE_GLANCE)
+    glance = glance_client.Client('2', session=session, auth=keystone.auth())
+    glance.images = images.SaharaImageManager(glance)
+    return glance

sahara/utils/openstack/images.py

@@ -13,7 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from novaclient.v2 import images
+import copy
+
+from glanceclient.v2 import images
+from glanceclient.v2 import schemas
 import six
 
 from sahara import exceptions as exc
@@ -22,12 +25,15 @@ from sahara import exceptions as exc
 PROP_DESCR = '_sahara_description'
 PROP_USERNAME = '_sahara_username'
 PROP_TAG = '_sahara_tag_'
+PROP_TAGS = '_all_tags'
 
 
-def _iter_tags(meta):
+def _get_all_tags(image_props):
-    for key in meta:
+    tags = []
-        if key.startswith(PROP_TAG) and meta[key]:
+    for key, value in image_props.iteritems():
-            yield key[len(PROP_TAG):]
+        if key.startswith(PROP_TAG) and value:
+            tags.append(key)
+    return tags
 
 
 def _ensure_tags(tags):
@@ -36,24 +42,22 @@ def _ensure_tags(tags):
     return [tags] if isinstance(tags, six.string_types) else tags
 
 
-class SaharaImage(images.Image):
+class SaharaImageModel(schemas.SchemaBasedModel):
-    def __init__(self, manager, info, **kwargs):
-        info['description'] = info.get('metadata', {}).get(PROP_DESCR)
-        info['username'] = info.get('metadata', {}).get(PROP_USERNAME)
-        info['tags'] = [tag for tag in _iter_tags(info.get('metadata', {}))]
-        super(SaharaImage, self).__init__(manager, info, **kwargs)
 
-    def tag(self, tags):
+    def __init__(self, *args, **kwargs):
-        self.manager.tag(self, tags)
+        super(SaharaImageModel, self).__init__(*args, **kwargs)
+        self.username = self._get_meta_prop(PROP_USERNAME, "")
+        self.description = self._get_meta_prop(PROP_DESCR, "")
+        self.tags = self._parse_tags()
 
-    def untag(self, tags):
+    def _get_meta_prop(self, prop, default=None):
-        self.manager.untag(self, tags)
+        if PROP_TAGS == prop:
+            return _get_all_tags(self)
+        return self.get(prop, default)
 
-    def set_description(self, username, description=None):
+    def _parse_tags(self):
-        self.manager.set_description(self, username, description)
+        tags = self._get_meta_prop(PROP_TAGS)
-
+        return [t.replace(PROP_TAG, "") for t in tags]
-    def unset_description(self):
-        self.manager.unset_description(self)
 
     @property
     def dict(self):
@@ -64,56 +68,62 @@ class SaharaImage(images.Image):
         return {'image': self.dict}
 
     def to_dict(self):
-        result = self._info.copy()
+        result = copy.deepcopy(dict(self))
         if 'links' in result:
             del result['links']
         return result
 
 
-class SaharaImageManager(images.ImageManager):
+class SaharaImageManager(images.Controller):
-    """Manage :class:`SaharaImage` resources.
+    """Manage :class:`SaharaImageModel` resources.
 
-    This is an extended version of nova client's ImageManager with support of
+    This is an extended version of glance client's Controller with support of
-    additional description and image tags stored in images' meta.
+    additional description and image tags stored as image properties.
     """
-    resource_class = SaharaImage
+    def __init__(self, glance_client):
+        schemas.SchemaBasedModel = SaharaImageModel
+        super(SaharaImageManager, self).__init__(glance_client.http_client,
+                                                 glance_client.schemas)
 
-    def set_description(self, image, username, description=None):
+    def set_meta(self, image_id, meta):
+        self.update(image_id, remove_props=None, **meta)
+
+    def delete_meta(self, image_id, meta_list):
+        self.update(image_id, remove_props=meta_list)
+
+    def set_description(self, image_id, username, description=None):
         """Sets human-readable information for image.
 
         For example:
-
+            Ubuntu 15 x64 with Java 1.7 and Apache Hadoop 2.1, ubuntu
-            Ubuntu 13.04 x64 with Java 1.7u21 and Apache Hadoop 1.1.1, ubuntu
         """
         meta = {PROP_USERNAME: username}
         if description:
             meta[PROP_DESCR] = description
-        self.set_meta(image, meta)
+        self.set_meta(image_id, meta)
 
-    def unset_description(self, image):
+    def unset_description(self, image_id):
         """Unsets all Sahara-related information.
 
         It removes username, description and tags from the specified image.
         """
-        image = self.get(image)
+        image = self.get(image_id)
         meta = [PROP_TAG + tag for tag in image.tags]
         if image.description is not None:
             meta += [PROP_DESCR]
         if image.username is not None:
             meta += [PROP_USERNAME]
-        self.delete_meta(image, meta)
+        self.delete_meta(image_id, meta)
 
-    def tag(self, image, tags):
+    def tag(self, image_id, tags):
         """Adds tags to the specified image."""
         tags = _ensure_tags(tags)
+        self.set_meta(image_id, {PROP_TAG + tag: 'True' for tag in tags})
 
-        self.set_meta(image, {PROP_TAG + tag: 'True' for tag in tags})
+    def untag(self, image_id, tags):
-
-    def untag(self, image, tags):
         """Removes tags from the specified image."""
         tags = _ensure_tags(tags)
-
+        self.delete_meta(image_id, [PROP_TAG + tag for tag in tags])
-        self.delete_meta(image, [PROP_TAG + tag for tag in tags])
 
     def list_by_tags(self, tags):
         """Returns images having all of the specified tags."""
@@ -122,16 +132,16 @@ class SaharaImageManager(images.ImageManager):
 
     def list_registered(self, name=None, tags=None):
         tags = _ensure_tags(tags)
-        images = [i for i in self.list()
+        images_list = [i for i in self.list()
-                  if i.username and set(tags).issubset(i.tags)]
+                       if i.username and set(tags).issubset(i.tags)]
         if name:
-            return [i for i in images if i.name == name]
+            return [i for i in images_list if i.name == name]
         else:
-            return images
+            return images_list
 
-    def get_registered_image(self, image):
+    def get_registered_image(self, image_id):
-        img = self.get(image)
+        img = self.get(image_id)
         if img.username:
             return img
         else:
-            raise exc.ImageNotRegistered(image)
+            raise exc.ImageNotRegistered(image_id)

sahara/utils/openstack/nova.py

@@ -19,7 +19,6 @@ from oslo_config import cfg
 
 from sahara.service import sessions
 import sahara.utils.openstack.base as base
-from sahara.utils.openstack import images
 from sahara.utils.openstack import keystone
 
 
@@ -48,7 +47,6 @@ def client():
     nova = nova_client.Client('2', session=session, auth=keystone.auth(),
                               endpoint_type=CONF.nova.endpoint_type,
                               region_name=CONF.os_region_name)
-    nova.images = images.SaharaImageManager(nova)
     return nova