openstack
/
sahara
Code Issues Proposed changes

novaclient.v2.images to glanceclient migration 

The novaclient.v2.images module is deprecated and will be removed
after Nova 15.0.0 is released so we need to migrate to glanceclient.

note: glanceclient doesn't have any Image class. It have image json
schema and use it to generate warlock models when creates image
object.

closes-bug: 1573595
Depends-on: I2e13867bed72fd2ae6749491dd5b032c2c08f9f6
Change-Id: Ie0f04c15e6e49e8e7c77f7b7d870a368cdd5c7c7
This commit is contained in:
Michael Ionkin 2016-05-25 20:05:19 +03:00
parent 5fb5358fa6
commit fba2d67ed9
16 changed files with 205 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
devstack/plugin.sh
View File

@ -123,6 +123,10 @@ function configure_sahara {
iniset $SAHARA_CONF_FILE keystone ca_file $SSL_BUNDLE_FILE
fi
if is_ssl_enabled_service "glance" || is_service_enabled tls-proxy; then
iniset $SAHARA_CONF_FILE glance ca_file $SSL_BUNDLE_FILE
fi
# Register SSL certificates if provided
if is_ssl_enabled_service sahara; then
ensure_certificates SAHARA

3
releasenotes/notes/novaclient_images_to_glanceclient-0266a2bd92b4be05.yaml Normal file
View File

@ -0,0 +1,3 @@
---
upgrade:
- Migration from novaclient.v2.images to glanceclient

1
requirements.txt
View File

@ -36,6 +36,7 @@ python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
python-swiftclient>=2.2.0 # Apache-2.0
python-neutronclient>=4.2.0 # Apache-2.0
python-heatclient>=1.1.0 # Apache-2.0
python-glanceclient>=2.0.0 # Apache-2.0
six>=1.9.0 # MIT
stevedore>=1.10.0 # Apache-2.0
SQLAlchemy<1.1.0,>=1.0.10 # MIT

2
sahara/api/v10.py
View File

@ -231,7 +231,7 @@ def images_list():
@acl.enforce("data-processing:images:get")
@v.check_exists(api.get_image, id='image_id')
def images_get(image_id):
return u.render(api.get_registered_image(id=image_id).wrapped_dict)
return u.render(api.get_registered_image(image_id=image_id).wrapped_dict)
@rest.post('/images/<image_id>')

20
sahara/service/api/v10.py
View File

@ -28,7 +28,7 @@ from sahara.utils import cluster as c_u
from sahara.utils import general as g
from sahara.utils.notification import sender
from sahara.utils.openstack import base as b
from sahara.utils.openstack import nova
from sahara.utils.openstack import glance
conductor = c.API
@ -253,41 +253,41 @@ def construct_ngs_for_scaling(cluster, additional_node_groups):
def get_images(name, tags):
return b.execute_with_retries(
nova.client().images.list_registered, name, tags)
glance.client().images.list_registered, name, tags)
def get_image(**kwargs):
if len(kwargs) == 1 and 'id' in kwargs:
return b.execute_with_retries(nova.client().images.get, kwargs['id'])
return b.execute_with_retries(glance.client().images.get, kwargs['id'])
else:
return b.execute_with_retries(nova.client().images.find, **kwargs)
return b.execute_with_retries(glance.client().images.find, **kwargs)
def get_registered_image(id):
def get_registered_image(image_id):
return b.execute_with_retries(
nova.client().images.get_registered_image, id)
glance.client().images.get_registered_image, image_id)
def register_image(image_id, username, description=None):
client = nova.client()
client = glance.client()
b.execute_with_retries(
client.images.set_description, image_id, username, description)
return b.execute_with_retries(client.images.get, image_id)
def unregister_image(image_id):
client = nova.client()
client = glance.client()
b.execute_with_retries(client.images.unset_description, image_id)
return b.execute_with_retries(client.images.get, image_id)
def add_image_tags(image_id, tags):
client = nova.client()
client = glance.client()
b.execute_with_retries(client.images.tag, image_id, tags)
return b.execute_with_retries(client.images.get, image_id)
def remove_image_tags(image_id, tags):
client = nova.client()
client = glance.client()
b.execute_with_retries(client.images.untag, image_id, tags)
return b.execute_with_retries(client.images.get, image_id)

18
sahara/service/api/v2/images.py
View File

@ -15,7 +15,7 @@
from sahara import conductor as c
from sahara.utils.openstack import base as b
from sahara.utils.openstack import nova
from sahara.utils.openstack import glance
conductor = c.API
@ -26,41 +26,41 @@ conductor = c.API
def get_images(name, tags):
return b.execute_with_retries(
nova.client().images.list_registered, name, tags)
glance.client().images.list_registered, name, tags)
def get_image(**kwargs):
if len(kwargs) == 1 and 'id' in kwargs:
return b.execute_with_retries(nova.client().images.get, kwargs['id'])
return b.execute_with_retries(glance.client().images.get, kwargs['id'])
else:
return b.execute_with_retries(nova.client().images.find, **kwargs)
return b.execute_with_retries(glance.client().images.find, **kwargs)
def get_registered_image(id):
return b.execute_with_retries(
nova.client().images.get_registered_image, id)
glance.client().images.get_registered_image, id)
def register_image(image_id, username, description=None):
client = nova.client()
client = glance.client()
b.execute_with_retries(
client.images.set_description, image_id, username, description)
return b.execute_with_retries(client.images.get, image_id)
def unregister_image(image_id):
client = nova.client()
client = glance.client()
b.execute_with_retries(client.images.unset_description, image_id)
return b.execute_with_retries(client.images.get, image_id)
def add_image_tags(image_id, tags):
client = nova.client()
client = glance.client()
b.execute_with_retries(client.images.tag, image_id, tags)
return b.execute_with_retries(client.images.get, image_id)
def remove_image_tags(image_id, tags):
client = nova.client()
client = glance.client()
b.execute_with_retries(client.images.untag, image_id, tags)
return b.execute_with_retries(client.images.get, image_id)

3
sahara/service/engine.py
View File

@ -34,6 +34,7 @@ from sahara.utils import cluster_progress_ops as cpo
from sahara.utils import edp
from sahara.utils import general as g
from sahara.utils.openstack import base as b
from sahara.utils.openstack import glance
from sahara.utils.openstack import nova
from sahara.utils import poll_utils
from sahara.utils import remote
@ -70,7 +71,7 @@ class Engine(object):
def get_node_group_image_username(self, node_group):
image_id = node_group.get_image_id()
return b.execute_with_retries(
nova.client().images.get, image_id).username
glance.client().images.get, image_id).username
@poll_utils.poll_status('ips_assign_timeout', _("Assign IPs"), sleep=1)
def _ips_assign(self, ips_assigned, cluster, instances):

12
sahara/service/sessions.py
View File

@ -31,6 +31,7 @@ SESSION_TYPE_CINDER = 'cinder'
SESSION_TYPE_KEYSTONE = 'keystone'
SESSION_TYPE_NEUTRON = 'neutron'
SESSION_TYPE_NOVA = 'nova'
SESSION_TYPE_GLANCE = 'glance'
SESSION_TYPE_INSECURE = 'insecure'
@ -61,6 +62,7 @@ class SessionCache(object):
SESSION_TYPE_KEYSTONE: self.get_keystone_session,
SESSION_TYPE_NEUTRON: self.get_neutron_session,
SESSION_TYPE_NOVA: self.get_nova_session,
SESSION_TYPE_GLANCE: self.get_glance_session,
SESSION_TYPE_INSECURE: self.get_insecure_session,
}
@ -144,6 +146,16 @@ class SessionCache(object):
self._set_session(SESSION_TYPE_NOVA, session)
return session
def get_glance_session(self):
session = self._sessions.get(SESSION_TYPE_GLANCE)
if not session:
if not CONF.glance.api_insecure:
session = keystone.Session(verify=CONF.glance.ca_file or True)
else:
session = self.get_insecure_session()
self._set_session(SESSION_TYPE_GLANCE, session)
return session
def token_for_auth(self, auth):
return self.get_keystone_session().get_auth_headers(auth).get(
'X-Auth-Token')

4
sahara/service/validations/base.py
View File

@ -27,6 +27,7 @@ import sahara.plugins.base as plugin_base
from sahara.service.api import v10 as api
from sahara.utils import general as g
import sahara.utils.openstack.cinder as cinder
from sahara.utils.openstack import glance
import sahara.utils.openstack.nova as nova
@ -74,7 +75,8 @@ def check_plugin_supports_version(p_name, version):
def check_image_registered(image_id):
if image_id not in [i.id for i in nova.client().images.list_registered()]:
if image_id not in (
[i.id for i in glance.client().images.list_registered()]):
raise ex.InvalidReferenceException(
_("Requested image '%s' is not registered") % image_id)

6
sahara/tests/unit/service/test_engine.py
View File

@ -50,12 +50,12 @@ class TestEngine(base.SaharaWithDbTestCase):
super(TestEngine, self).setUp()
self.eng = EngineTest()
@mock.patch('sahara.utils.openstack.nova.client')
def test_get_node_group_image_username(self, nova_client):
@mock.patch('sahara.utils.openstack.glance.client')
def test_get_node_group_image_username(self, glance_client):
ng = mock.Mock()
client = mock.Mock()
client.images.get.return_value = mock.Mock(username='username')
nova_client.return_value = client
glance_client.return_value = client
self.assertEqual(
'username', self.eng.get_node_group_image_username(ng))

19
sahara/tests/unit/service/test_sessions.py
View File

@ -112,6 +112,25 @@ class TestSessionCache(base.SaharaTestCase):
sc.get_session(sessions.SESSION_TYPE_NEUTRON)
self.assertFalse(keystone_session.called)
@mock.patch('keystoneauth1.session.Session')
def test_get_glance_session(self, keystone_session):
sc = sessions.SessionCache()
self.override_config('ca_file', '/some/cacert', group='glance')
self.override_config('api_insecure', False, group='glance')
sc.get_session(sessions.SESSION_TYPE_GLANCE)
keystone_session.assert_called_once_with(verify='/some/cacert')
sc = sessions.SessionCache()
keystone_session.reset_mock()
self.override_config('ca_file', None, group='glance')
self.override_config('api_insecure', True, group='glance')
sc.get_session(sessions.SESSION_TYPE_GLANCE)
keystone_session.assert_called_once_with(verify=False)
keystone_session.reset_mock()
sc.get_session(sessions.SESSION_TYPE_GLANCE)
self.assertFalse(keystone_session.called)
@mock.patch('keystoneauth1.session.Session')
def test_insecure_session(self, session):
sc = sessions.SessionCache()

9
sahara/tests/unit/service/validation/utils.py
View File

@ -135,6 +135,7 @@ def start_patch(patch_templates=True):
"sahara.service.api.v10.get_cluster_template")
nova_p = mock.patch("sahara.utils.openstack.nova.client")
heat_p = mock.patch("sahara.utils.openstack.heat.client")
glance_p = mock.patch("sahara.utils.openstack.glance.client")
cinder_p = mock.patch("sahara.utils.openstack.cinder.client")
cinder_exists_p = mock.patch(
"sahara.utils.openstack.cinder.check_cinder_exists")
@ -166,6 +167,8 @@ def start_patch(patch_templates=True):
heat = heat_p.start()
heat().stacks.list.side_effect = _get_heat_stack_list
glance = glance_p.start()
cinder = cinder_p.start()
cinder().availability_zones.list.side_effect = _get_availability_zone_list
@ -197,8 +200,8 @@ def start_patch(patch_templates=True):
return Image('wrong_test')
get_image.side_effect = _get_image
nova().images.list_registered.return_value = [Image(),
Image(name='wrong_name')]
glance().images.list_registered.return_value = [Image(),
Image(name='wrong_name')]
ng_dict = tu.make_ng_dict('ng', '42', ['namenode'], 1)
cluster = tu.create_cluster('test', 't', 'fake', '0.1', [ng_dict],
id=1, status=c_u.CLUSTER_STATUS_ACTIVE)
@ -230,7 +233,7 @@ def start_patch(patch_templates=True):
get_ng_template.side_effect = _get_ng_template
# request data to validate
patchers = [get_clusters_p, get_cluster_p,
nova_p, get_image_p, heat_p, cinder_p,
nova_p, get_image_p, heat_p, glance_p, cinder_p,
cinder_exists_p]
if patch_templates:
patchers.extend([get_ng_template_p, get_ng_templates_p,

39
sahara/tests/unit/utils/openstack/test_images.py
View File

@ -16,7 +16,7 @@
import mock
from sahara.tests.unit import base
from sahara.utils.openstack import nova as nova_client
from sahara.utils.openstack import glance as glance_client
class FakeImage(object):
@ -39,38 +39,53 @@ class TestImages(base.SaharaTestCase):
FakeImage('baz', [], 'test'),
FakeImage('spam', [], "")]
with mock.patch('novaclient.v2.images.ImageManager.list',
with mock.patch('glanceclient.v2.images.Controller.list',
return_value=some_images):
nova = nova_client.client()
glance = glance_client.client()
images = nova.images.list_registered()
images = glance.images.list_registered()
self.assertEqual(2, len(images))
images = nova.images.list_registered(name='foo')
images = glance.images.list_registered(name='foo')
self.assertEqual(1, len(images))
self.assertEqual('foo', images[0].name)
self.assertEqual('test', images[0].username)
images = nova.images.list_registered(name='eggs')
images = glance.images.list_registered(name='eggs')
self.assertEqual(0, len(images))
images = nova.images.list_registered(tags=['bar'])
images = glance.images.list_registered(tags=['bar'])
self.assertEqual(1, len(images))
self.assertEqual('foo', images[0].name)
images = nova.images.list_registered(tags=['bar', 'eggs'])
images = glance.images.list_registered(tags=['bar', 'eggs'])
self.assertEqual(0, len(images))
@mock.patch('novaclient.v2.images.ImageManager.set_meta')
@mock.patch('sahara.utils.openstack.images.SaharaImageManager.set_meta')
def test_set_description(self, set_meta):
with mock.patch('sahara.utils.openstack.base.url_for'):
nova = nova_client.client()
nova.images.set_description('id', 'ubuntu')
glance = glance_client.client()
glance.images.set_description('id', 'ubuntu')
self.assertEqual(
('id', {'_sahara_username': 'ubuntu'}), set_meta.call_args[0])
nova.images.set_description('id', 'ubuntu', 'descr')
glance.images.set_description('id', 'ubuntu', 'descr')
self.assertEqual(
('id', {'_sahara_description': 'descr',
'_sahara_username': 'ubuntu'}),
set_meta.call_args[0])
@mock.patch('sahara.utils.openstack.images.SaharaImageManager.get')
@mock.patch('sahara.utils.openstack.images.SaharaImageManager.delete_meta')
def test_unset_description(self, delete_meta, get_image):
glance = glance_client.client()
image = mock.MagicMock()
image.tags = ['fake', 'fake_2.0']
image.username = 'ubuntu'
image.description = 'some description'
get_image.return_value = image
glance.images.unset_description('id')
self.assertEqual(
('id', ['_sahara_tag_fake', '_sahara_tag_fake_2.0',
'_sahara_description', '_sahara_username']),
delete_meta.call_args[0])

49
sahara/utils/openstack/glance.py Normal file
View File

@ -0,0 +1,49 @@
# Copyright (c) 2016 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from glanceclient import client as glance_client
from oslo_config import cfg
from sahara.service import sessions
from sahara.utils.openstack import images
from sahara.utils.openstack import keystone
opts = [
cfg.BoolOpt('api_insecure',
default=False,
help='Allow to perform insecure SSL requests to glance.'),
cfg.StrOpt('ca_file',
help='Location of ca certificates file to use for glance '
'client requests.'),
cfg.StrOpt("endpoint_type",
default="internalURL",
help="Endpoint type for glance client requests"),
]
glance_group = cfg.OptGroup(name='glance',
title='Glance client options')
CONF = cfg.CONF
CONF.register_group(glance_group)
CONF.register_opts(opts, group=glance_group)
def client():
session = sessions.cache().get_session(sessions.SESSION_TYPE_GLANCE)
glance = glance_client.Client('2', session=session, auth=keystone.auth())
glance.images = images.SaharaImageManager(glance)
return glance

102
sahara/utils/openstack/images.py
View File

@ -13,7 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from novaclient.v2 import images
import copy
from glanceclient.v2 import images
from glanceclient.v2 import schemas
import six
from sahara import exceptions as exc
@ -22,12 +25,15 @@ from sahara import exceptions as exc
PROP_DESCR = '_sahara_description'
PROP_USERNAME = '_sahara_username'
PROP_TAG = '_sahara_tag_'
PROP_TAGS = '_all_tags'
def _iter_tags(meta):
for key in meta:
if key.startswith(PROP_TAG) and meta[key]:
yield key[len(PROP_TAG):]
def _get_all_tags(image_props):
tags = []
for key, value in image_props.iteritems():
if key.startswith(PROP_TAG) and value:
tags.append(key)
return tags
def _ensure_tags(tags):
@ -36,24 +42,22 @@ def _ensure_tags(tags):
return [tags] if isinstance(tags, six.string_types) else tags
class SaharaImage(images.Image):
def __init__(self, manager, info, **kwargs):
info['description'] = info.get('metadata', {}).get(PROP_DESCR)
info['username'] = info.get('metadata', {}).get(PROP_USERNAME)
info['tags'] = [tag for tag in _iter_tags(info.get('metadata', {}))]
super(SaharaImage, self).__init__(manager, info, **kwargs)
class SaharaImageModel(schemas.SchemaBasedModel):
def tag(self, tags):
self.manager.tag(self, tags)
def __init__(self, *args, **kwargs):
super(SaharaImageModel, self).__init__(*args, **kwargs)
self.username = self._get_meta_prop(PROP_USERNAME, "")
self.description = self._get_meta_prop(PROP_DESCR, "")
self.tags = self._parse_tags()
def untag(self, tags):
self.manager.untag(self, tags)
def _get_meta_prop(self, prop, default=None):
if PROP_TAGS == prop:
return _get_all_tags(self)
return self.get(prop, default)
def set_description(self, username, description=None):
self.manager.set_description(self, username, description)
def unset_description(self):
self.manager.unset_description(self)
def _parse_tags(self):
tags = self._get_meta_prop(PROP_TAGS)
return [t.replace(PROP_TAG, "") for t in tags]
@property
def dict(self):
@ -64,56 +68,62 @@ class SaharaImage(images.Image):
return {'image': self.dict}
def to_dict(self):
result = self._info.copy()
result = copy.deepcopy(dict(self))
if 'links' in result:
del result['links']
return result
class SaharaImageManager(images.ImageManager):
"""Manage :class:`SaharaImage` resources.
class SaharaImageManager(images.Controller):
"""Manage :class:`SaharaImageModel` resources.
This is an extended version of nova client's ImageManager with support of
additional description and image tags stored in images' meta.
This is an extended version of glance client's Controller with support of
additional description and image tags stored as image properties.
"""
resource_class = SaharaImage
def __init__(self, glance_client):
schemas.SchemaBasedModel = SaharaImageModel
super(SaharaImageManager, self).__init__(glance_client.http_client,
glance_client.schemas)
def set_description(self, image, username, description=None):
def set_meta(self, image_id, meta):
self.update(image_id, remove_props=None, **meta)
def delete_meta(self, image_id, meta_list):
self.update(image_id, remove_props=meta_list)
def set_description(self, image_id, username, description=None):
"""Sets human-readable information for image.
For example:
Ubuntu 13.04 x64 with Java 1.7u21 and Apache Hadoop 1.1.1, ubuntu
Ubuntu 15 x64 with Java 1.7 and Apache Hadoop 2.1, ubuntu
"""
meta = {PROP_USERNAME: username}
if description:
meta[PROP_DESCR] = description
self.set_meta(image, meta)
self.set_meta(image_id, meta)
def unset_description(self, image):
def unset_description(self, image_id):
"""Unsets all Sahara-related information.
It removes username, description and tags from the specified image.
"""
image = self.get(image)
image = self.get(image_id)
meta = [PROP_TAG + tag for tag in image.tags]
if image.description is not None:
meta += [PROP_DESCR]
if image.username is not None:
meta += [PROP_USERNAME]
self.delete_meta(image, meta)
self.delete_meta(image_id, meta)
def tag(self, image, tags):
def tag(self, image_id, tags):
"""Adds tags to the specified image."""
tags = _ensure_tags(tags)
self.set_meta(image_id, {PROP_TAG + tag: 'True' for tag in tags})
self.set_meta(image, {PROP_TAG + tag: 'True' for tag in tags})
def untag(self, image, tags):
def untag(self, image_id, tags):
"""Removes tags from the specified image."""
tags = _ensure_tags(tags)
self.delete_meta(image, [PROP_TAG + tag for tag in tags])
self.delete_meta(image_id, [PROP_TAG + tag for tag in tags])
def list_by_tags(self, tags):
"""Returns images having all of the specified tags."""
@ -122,16 +132,16 @@ class SaharaImageManager(images.ImageManager):
def list_registered(self, name=None, tags=None):
tags = _ensure_tags(tags)
images = [i for i in self.list()
if i.username and set(tags).issubset(i.tags)]
images_list = [i for i in self.list()
if i.username and set(tags).issubset(i.tags)]
if name:
return [i for i in images if i.name == name]
return [i for i in images_list if i.name == name]
else:
return images
return images_list
def get_registered_image(self, image):
img = self.get(image)
def get_registered_image(self, image_id):
img = self.get(image_id)
if img.username:
return img
else:
raise exc.ImageNotRegistered(image)
raise exc.ImageNotRegistered(image_id)

2
sahara/utils/openstack/nova.py
View File

@ -19,7 +19,6 @@ from oslo_config import cfg
from sahara.service import sessions
import sahara.utils.openstack.base as base
from sahara.utils.openstack import images
from sahara.utils.openstack import keystone
@ -48,7 +47,6 @@ def client():
nova = nova_client.Client('2', session=session, auth=keystone.auth(),
endpoint_type=CONF.nova.endpoint_type,
region_name=CONF.os_region_name)
nova.images = images.SaharaImageManager(nova)
return nova