openstack
/
deb-ceilometer
Code Issues Proposed changes

Switch to use Glance v2 in image pollsters 

Since the Glance v1 APIs won't be maintained any more, it is better to
switch use Glance v2 for in image pollsters.

Change-Id: Ib2df3bb4fdd12649bddffd624714707e1369f6af
This commit is contained in:
liusheng 2016-08-01 15:35:14 +08:00
parent d191741cdd
commit f8933f4abd
7 changed files with 158 additions and 267 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
ceilometer/image/discovery.py Normal file
View File

@ -0,0 +1,43 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import glanceclient
from oslo_config import cfg
from ceilometer.agent import plugin_base
from ceilometer import keystone_client
SERVICE_OPTS = [
cfg.StrOpt('glance',
default='image',
help='Glance service type.'),
]
cfg.CONF.register_opts(SERVICE_OPTS, group='service_types')
cfg.CONF.import_group('service_credentials', 'ceilometer.keystone_client')
class ImagesDiscovery(plugin_base.DiscoveryBase):
def __init__(self):
super(ImagesDiscovery, self).__init__()
conf = cfg.CONF.service_credentials
self.glance_client = glanceclient.Client(
version='2',
session=keystone_client.get_session(),
region_name=conf.region_name,
interface=conf.interface,
service_type=cfg.CONF.service_types.glance)
def discover(self, manager, param=None):
"""Discover resources to monitor."""
return self.glance_client.images.list()

105
ceilometer/image/glance.py
View File

@ -17,63 +17,14 @@
from __future__ import absolute_import
import glanceclient
from oslo_config import cfg
from ceilometer.agent import plugin_base
from ceilometer import keystone_client
from ceilometer import sample
OPTS = [
cfg.IntOpt('glance_page_size',
default=0,
help="Number of items to request in "
"each paginated Glance API request "
"(parameter used by glanceclient). "
"If this is less than or equal to 0, "
"page size is not specified "
"(default value in glanceclient is used)."),
]
SERVICE_OPTS = [
cfg.StrOpt('glance',
default='image',
help='Glance service type.'),
]
cfg.CONF.register_opts(OPTS)
cfg.CONF.register_opts(SERVICE_OPTS, group='service_types')
class _Base(plugin_base.PollsterBase):
@property
def default_discovery(self):
return 'endpoint:%s' % cfg.CONF.service_types.glance
@staticmethod
def get_glance_client(ksclient, endpoint):
# hard-code v1 glance API version selection while v2 API matures
return glanceclient.Client('1',
session=keystone_client.get_session(),
endpoint=endpoint,
auth=ksclient.session.auth)
def _get_images(self, ksclient, endpoint):
client = self.get_glance_client(ksclient, endpoint)
page_size = cfg.CONF.glance_page_size
kwargs = {}
if page_size > 0:
kwargs['page_size'] = page_size
return client.images.list(filters={"is_public": None}, **kwargs)
def _iter_images(self, ksclient, cache, endpoint):
"""Iterate over all images."""
key = '%s-images' % endpoint
if key not in cache:
cache[key] = list(self._get_images(ksclient, endpoint))
return iter(cache[key])
return 'images'
@staticmethod
def extract_image_metadata(image):
@ -81,49 +32,45 @@ class _Base(plugin_base.PollsterBase):
for k in
[
"status",
"is_public",
"visibility",
"name",
"deleted",
"container_format",
"created_at",
"disk_format",
"updated_at",
"properties",
"min_disk",
"protected",
"checksum",
"deleted_at",
"min_ram",
"size", ])
"tags",
"virtual_size"])
class ImagePollster(_Base):
def get_samples(self, manager, cache, resources):
for endpoint in resources:
for image in self._iter_images(manager.keystone, cache, endpoint):
yield sample.Sample(
name='image',
type=sample.TYPE_GAUGE,
unit='image',
volume=1,
user_id=None,
project_id=image.owner,
resource_id=image.id,
resource_metadata=self.extract_image_metadata(image),
)
for image in resources:
yield sample.Sample(
name='image',
type=sample.TYPE_GAUGE,
unit='image',
volume=1,
user_id=None,
project_id=image.owner,
resource_id=image.id,
resource_metadata=self.extract_image_metadata(image),
)
class ImageSizePollster(_Base):
def get_samples(self, manager, cache, resources):
for endpoint in resources:
for image in self._iter_images(manager.keystone, cache, endpoint):
yield sample.Sample(
name='image.size',
type=sample.TYPE_GAUGE,
unit='B',
volume=image.size,
user_id=None,
project_id=image.owner,
resource_id=image.id,
resource_metadata=self.extract_image_metadata(image),
)
for image in resources:
yield sample.Sample(
name='image.size',
type=sample.TYPE_GAUGE,
unit='B',
volume=image.size,
user_id=None,
project_id=image.owner,
resource_id=image.id,
resource_metadata=self.extract_image_metadata(image),
)

2
ceilometer/nova_client.py
View File

@ -40,7 +40,7 @@ SERVICE_OPTS = [
cfg.CONF.register_opts(OPTS)
cfg.CONF.register_opts(SERVICE_OPTS, group='service_types')
cfg.CONF.import_opt('http_timeout', 'ceilometer.service')
cfg.CONF.import_opt('glance', 'ceilometer.image.glance', 'service_types')
cfg.CONF.import_opt('glance', 'ceilometer.image.discovery', 'service_types')
cfg.CONF.import_group('service_credentials', 'ceilometer.keystone_client')
LOG = log.getLogger(__name__)

5
ceilometer/opts.py
View File

@ -34,7 +34,7 @@ import ceilometer.dispatcher.gnocchi
import ceilometer.energy.kwapi
import ceilometer.event.converter
import ceilometer.hardware.discovery
import ceilometer.image.glance
import ceilometer.image.discovery
import ceilometer.ipmi.notifications.ironic
import ceilometer.ipmi.platform.intel_node_manager
import ceilometer.ipmi.pollsters
@ -67,7 +67,6 @@ def list_opts():
ceilometer.compute.virt.inspector.OPTS,
ceilometer.compute.virt.libvirt.inspector.OPTS,
ceilometer.dispatcher.OPTS,
ceilometer.image.glance.OPTS,
ceilometer.ipmi.notifications.ironic.OPTS,
ceilometer.middleware.OPTS,
ceilometer.network.notifications.OPTS,
@ -113,7 +112,7 @@ def list_opts():
loading.get_auth_plugin_conf_options('password'))),
('service_types',
itertools.chain(ceilometer.energy.kwapi.SERVICE_OPTS,
ceilometer.image.glance.SERVICE_OPTS,
ceilometer.image.discovery.SERVICE_OPTS,
ceilometer.neutron_client.SERVICE_OPTS,
ceilometer.nova_client.SERVICE_OPTS,
ceilometer.objectstore.rgw.SERVICE_OPTS,

261
ceilometer/tests/unit/image/test_glance.py
View File

@ -14,214 +14,107 @@
# under the License.
import mock
from oslo_config import fixture as fixture_config
from oslotest import base
from oslotest import mockpatch
from ceilometer.agent import manager
from ceilometer.image import glance
import ceilometer.tests.base as base
IMAGE_LIST = [
type('Image', (object,),
{u'status': u'queued',
u'name': "some name",
u'deleted': False,
u'container_format': None,
u'created_at': u'2012-09-18T16:29:46',
u'disk_format': None,
u'updated_at': u'2012-09-18T16:29:46',
u'properties': {},
u'min_disk': 0,
u'protected': False,
u'id': u'1d21a8d0-25f4-4e0a-b4ec-85f40237676b',
u'location': None,
u'checksum': None,
u'owner': u'4c8364fc20184ed7971b76602aa96184',
u'is_public': True,
u'deleted_at': None,
{u'status': u'active',
u'tags': [],
u'kernel_id': u'fd24d91a-dfd5-4a3c-b990-d4563eb27396',
u'container_format': u'ami',
u'min_ram': 0,
u'size': 2048}),
u'ramdisk_id': u'd629522b-ebaa-4c92-9514-9e31fe760d18',
u'updated_at': u'2016-06-20T13: 34: 41Z',
u'visibility': u'public',
u'owner': u'6824974c08974d4db864bbaa6bc08303',
u'file': u'/v2/images/fda54a44-3f96-40bf-ab07-0a4ce9e1761d/file',
u'min_disk': 0,
u'virtual_size': None,
u'id': u'fda54a44-3f96-40bf-ab07-0a4ce9e1761d',
u'size': 25165824,
u'name': u'cirros-0.3.4-x86_64-uec',
u'checksum': u'eb9139e4942121f22bbc2afc0400b2a4',
u'created_at': u'2016-06-20T13: 34: 40Z',
u'disk_format': u'ami',
u'protected': False,
u'schema': u'/v2/schemas/image'}),
type('Image', (object,),
{u'status': u'active',
u'name': "hello world",
u'deleted': False,
u'container_format': None,
u'created_at': u'2012-09-18T16:27:41',
u'disk_format': None,
u'updated_at': u'2012-09-18T16:27:41',
u'properties': {},
u'min_disk': 0,
u'protected': False,
u'id': u'22be9f90-864d-494c-aa74-8035fd535989',
u'location': None,
u'checksum': None,
u'owner': u'9e4f98287a0246daa42eaf4025db99d4',
u'is_public': True,
u'deleted_at': None,
u'tags': [],
u'container_format': u'ari',
u'min_ram': 0,
u'size': 0}),
u'updated_at': u'2016-06-20T13: 34: 38Z',
u'visibility': u'public',
u'owner': u'6824974c08974d4db864bbaa6bc08303',
u'file': u'/v2/images/d629522b-ebaa-4c92-9514-9e31fe760d18/file',
u'min_disk': 0,
u'virtual_size': None,
u'id': u'd629522b-ebaa-4c92-9514-9e31fe760d18',
u'size': 3740163,
u'name': u'cirros-0.3.4-x86_64-uec-ramdisk',
u'checksum': u'be575a2b939972276ef675752936977f',
u'created_at': u'2016-06-20T13: 34: 37Z',
u'disk_format': u'ari',
u'protected': False,
u'schema': u'/v2/schemas/image'}),
type('Image', (object,),
{u'status': u'queued',
u'name': None,
u'deleted': False,
u'container_format': None,
u'created_at': u'2012-09-18T16:23:27',
u'disk_format': "raw",
u'updated_at': u'2012-09-18T16:23:27',
u'properties': {},
u'min_disk': 0,
u'protected': False,
u'id': u'8d133f6c-38a8-403c-b02c-7071b69b432d',
u'location': None,
u'checksum': None,
u'owner': u'5f8806a76aa34ee8b8fc8397bd154319',
u'is_public': True,
u'deleted_at': None,
{u'status': u'active',
u'tags': [],
u'container_format': u'aki',
u'min_ram': 0,
u'size': 1024}),
type('Image', (object,),
{u'status': u'queued',
u'name': "some name",
u'deleted': False,
u'container_format': None,
u'created_at': u'2012-09-18T16:29:46',
u'disk_format': None,
u'updated_at': u'2012-09-18T16:29:46',
u'properties': {},
u'updated_at': u'2016-06-20T13: 34: 35Z',
u'visibility': u'public',
u'owner': u'6824974c08974d4db864bbaa6bc08303',
u'file': u'/v2/images/fd24d91a-dfd5-4a3c-b990-d4563eb27396/file',
u'min_disk': 0,
u'virtual_size': None,
u'id': u'fd24d91a-dfd5-4a3c-b990-d4563eb27396',
u'size': 4979632,
u'name': u'cirros-0.3.4-x86_64-uec-kernel',
u'checksum': u'8a40c862b5735975d82605c1dd395796',
u'created_at': u'2016-06-20T13: 34: 35Z',
u'disk_format': u'aki',
u'protected': False,
u'id': u'e753b196-49b4-48e8-8ca5-09ebd9805f40',
u'location': None,
u'checksum': None,
u'owner': u'4c8364fc20184ed7971b76602aa96184',
u'is_public': True,
u'deleted_at': None,
u'min_ram': 0,
u'size': 2048}),
u'schema': u'/v2/schemas/image'}),
]
ENDPOINT = 'end://point'
class _BaseObject(object):
pass
class FakeGlanceClient(object):
class images(object):
pass
class TestManager(manager.AgentManager):
def __init__(self):
super(TestManager, self).__init__()
self._keystone = mock.Mock()
access = self._keystone.session.auth.get_access.return_value
access.service_catalog.get_endpoints = mock.Mock(
return_value={'image': mock.ANY})
class TestImagePollsterPageSize(base.BaseTestCase):
@staticmethod
def fake_get_glance_client(ksclient, endpoint):
glanceclient = FakeGlanceClient()
glanceclient.images.list = mock.MagicMock(return_value=IMAGE_LIST)
return glanceclient
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def setUp(self):
super(TestImagePollsterPageSize, self).setUp()
self.manager = TestManager()
self.useFixture(mockpatch.PatchObject(
glance._Base, 'get_glance_client',
side_effect=self.fake_get_glance_client))
self.CONF = self.useFixture(fixture_config.Config()).conf
self.manager = manager.AgentManager()
self.pollster = glance.ImageSizePollster()
def _do_test_iter_images(self, page_size=0, length=0):
self.CONF.set_override("glance_page_size", page_size)
images = list(glance.ImagePollster().
_iter_images(self.manager.keystone, {}, ENDPOINT))
kwargs = {}
if page_size > 0:
kwargs['page_size'] = page_size
FakeGlanceClient.images.list.assert_called_with(
filters={'is_public': None}, **kwargs)
self.assertEqual(length, len(images))
def test_page_size(self):
self._do_test_iter_images(100, 4)
def test_page_size_default(self):
self._do_test_iter_images(length=4)
def test_page_size_negative_number(self):
self._do_test_iter_images(-1, 4)
def test_image_pollster(self):
image_samples = list(
self.pollster.get_samples(self.manager, {}, resources=IMAGE_LIST))
self.assertEqual(3, len(image_samples))
self.assertEqual('image.size', image_samples[0].name)
self.assertEqual(25165824, image_samples[0].volume)
self.assertEqual('6824974c08974d4db864bbaa6bc08303',
image_samples[0].project_id)
self.assertEqual('fda54a44-3f96-40bf-ab07-0a4ce9e1761d',
image_samples[0].resource_id)
class TestImagePollster(base.BaseTestCase):
@staticmethod
def fake_get_glance_client(ksclient, endpoint):
glanceclient = _BaseObject()
setattr(glanceclient, "images", _BaseObject())
setattr(glanceclient.images,
"list", lambda *args, **kwargs: iter(IMAGE_LIST))
return glanceclient
class TestImagePageSize(base.BaseTestCase):
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
def setUp(self):
super(TestImagePollster, self).setUp()
self.manager = TestManager()
self.useFixture(mockpatch.PatchObject(
glance._Base, 'get_glance_client',
side_effect=self.fake_get_glance_client))
super(TestImagePageSize, self).setUp()
self.manager = manager.AgentManager()
self.pollster = glance.ImagePollster()
def test_default_discovery(self):
pollster = glance.ImagePollster()
self.assertEqual('endpoint:image', pollster.default_discovery)
def test_iter_images(self):
# Tests whether the iter_images method returns a unique image
# list when there is nothing in the cache
images = list(glance.ImagePollster().
_iter_images(self.manager.keystone, {}, ENDPOINT))
self.assertEqual(len(set(image.id for image in images)), len(images))
def test_iter_images_cached(self):
# Tests whether the iter_images method returns the values from
# the cache
cache = {'%s-images' % ENDPOINT: []}
images = list(glance.ImagePollster().
_iter_images(self.manager.keystone, cache,
ENDPOINT))
self.assertEqual([], images)
def test_image(self):
samples = list(glance.ImagePollster().get_samples(self.manager, {},
[ENDPOINT]))
self.assertEqual(4, len(samples))
for sample in samples:
self.assertEqual(1, sample.volume)
def test_image_size(self):
samples = list(glance.ImageSizePollster().get_samples(self.manager,
{},
[ENDPOINT]))
self.assertEqual(4, len(samples))
for image in IMAGE_LIST:
self.assertTrue(
any(map(lambda sample: sample.volume == image.size,
samples)))
def test_image_get_sample_names(self):
samples = list(glance.ImagePollster().get_samples(self.manager, {},
[ENDPOINT]))
self.assertEqual(set(['image']), set([s.name for s in samples]))
def test_image_size_get_sample_names(self):
samples = list(glance.ImageSizePollster().get_samples(self.manager,
{},
[ENDPOINT]))
self.assertEqual(set(['image.size']), set([s.name for s in samples]))
def test_image_pollster(self):
image_samples = list(
self.pollster.get_samples(self.manager, {}, resources=IMAGE_LIST))
self.assertEqual(3, len(image_samples))
self.assertEqual('image', image_samples[0].name)
self.assertEqual(1, image_samples[0].volume)
self.assertEqual('6824974c08974d4db864bbaa6bc08303',
image_samples[0].project_id)
self.assertEqual('fda54a44-3f96-40bf-ab07-0a4ce9e1761d',
image_samples[0].resource_id)

8
releasenotes/notes/use-glance-v2-in-image-pollsters-137a315577d5dc4c.yaml Normal file
View File

@ -0,0 +1,8 @@
---
features:
- Since the Glance v1 APIs won't be maintained any more, this change
add the support of glance v2 in images pollsters.
upgrade:
- >
The option `glance_page_size' has been removed because it's not actually needed.

1
setup.cfg
View File

@ -75,6 +75,7 @@ ceilometer.discover.central =
fw_policy = ceilometer.network.services.discovery:FirewallPolicyDiscovery
tripleo_overcloud_nodes = ceilometer.hardware.discovery:NodesDiscoveryTripleO
fip_services = ceilometer.network.services.discovery:FloatingIPDiscovery
images = ceilometer.image.discovery:ImagesDiscovery
ceilometer.discover.ipmi =
local_node = ceilometer.agent.discovery.localnode:LocalNodeDiscovery