From 1f9f4e1072a5e5037b93734bafcc65e4211eb19f Mon Sep 17 00:00:00 2001 From: Pradeep Kilambi Date: Mon, 18 Jan 2016 18:38:00 -0500 Subject: [PATCH] Fix ceilometer floatingip pollster The existing floatingip pollster talks to nova api to get the floatingip data. There are limitations in nova api wrt returning this info for all the tenants as stated in the bug#1402514. This patch changes the pollster to use the neutron api to get this data instead. Considering this is a network related pollster and in most cases network networking manages the floating ips now, it makes sense to get this data from neutron. Closes-Bug: #1536338 Change-Id: I372e3a85b34f90ff9aba842d9598b468f90fe94f --- ceilometer/network/floatingip.py | 85 ++++++++------- ceilometer/neutron_client.py | 5 + .../tests/unit/network/test_floating_ip.py | 103 ++++++++++++++++++ .../tests/unit/network/test_floatingip.py | 98 ----------------- ...-floatingip-pollster-f5172060c626b19e.yaml | 9 ++ 5 files changed, 163 insertions(+), 137 deletions(-) create mode 100644 ceilometer/tests/unit/network/test_floating_ip.py delete mode 100644 ceilometer/tests/unit/network/test_floatingip.py create mode 100644 releasenotes/notes/fix-floatingip-pollster-f5172060c626b19e.yaml diff --git a/ceilometer/network/floatingip.py b/ceilometer/network/floatingip.py index 7258cf5970..ebbc13a933 100644 --- a/ceilometer/network/floatingip.py +++ b/ceilometer/network/floatingip.py @@ -1,6 +1,6 @@ -# +# Copyright 2016 Sungard Availability Services +# Copyright 2016 Red Hat # Copyright 2012 eNovance -# # Copyright 2013 IBM Corp # All Rights Reserved. # @@ -21,53 +21,60 @@ from oslo_log import log from oslo_utils import timeutils from ceilometer.agent import plugin_base -from ceilometer.i18n import _LI -from ceilometer import nova_client +from ceilometer.i18n import _LW +from ceilometer import neutron_client from ceilometer import sample - LOG = log.getLogger(__name__) +cfg.CONF.import_group('service_types', 'ceilometer.neutron_client') + class FloatingIPPollster(plugin_base.PollsterBase): - @staticmethod - def _get_floating_ips(ksclient, endpoint): - nv = nova_client.Client( - auth=ksclient.session.auth, - endpoint_override=endpoint) - return nv.floating_ip_get_all() + STATUS = { + 'inactive': 0, + 'active': 1, + 'pending_create': 2, + } - def _iter_floating_ips(self, ksclient, cache, endpoint): - key = '%s-floating_ips' % endpoint - if key not in cache: - cache[key] = list(self._get_floating_ips(ksclient, endpoint)) - return iter(cache[key]) + def __init__(self): + self.neutron_cli = neutron_client.Client() @property def default_discovery(self): - return 'endpoint:%s' % cfg.CONF.service_types.nova + return 'endpoint:%s' % cfg.CONF.service_types.neutron + + @staticmethod + def _form_metadata_for_fip(fip): + """Return a metadata dictionary for the fip usage data.""" + metadata = { + 'router_id': fip.get("router_id"), + 'status': fip.get("status"), + 'floating_network_id': fip.get("floating_network_id"), + 'fixed_ip_address': fip.get("fixed_ip_address"), + 'port_id': fip.get("port_id"), + 'floating_ip_address': fip.get("floating_ip_address") + } + return metadata def get_samples(self, manager, cache, resources): - for endpoint in resources: - for ip in self._iter_floating_ips(manager.keystone, cache, - endpoint): - LOG.info(_LI("FLOATING IP USAGE: %s") % ip.ip) - # FIXME (flwang) Now Nova API /os-floating-ips can't provide - # those attributes were used by Ceilometer, such as project - # id, host. In this fix, those attributes usage will be - # removed temporarily. And they will be back after fix the - # Nova bug 1174802. - yield sample.Sample( - name='ip.floating', - type=sample.TYPE_GAUGE, - unit='ip', - volume=1, - user_id=None, - project_id=None, - resource_id=ip.id, - timestamp=timeutils.utcnow().isoformat(), - resource_metadata={ - 'address': ip.ip, - 'pool': ip.pool - }) + + for fip in self.neutron_cli.fip_get_all(): + status = self.STATUS.get(fip['status'].lower()) + if status is None: + LOG.warn(_LW("Invalid status, skipping IP address %s") % + fip['floating_ip_address']) + continue + res_metadata = self._form_metadata_for_fip(fip) + yield sample.Sample( + name='ip.floating', + type=sample.TYPE_GAUGE, + unit='ip', + volume=status, + user_id=fip.get('user_id'), + project_id=fip['tenant_id'], + resource_id=fip['id'], + timestamp=timeutils.utcnow().isoformat(), + resource_metadata=res_metadata + ) diff --git a/ceilometer/neutron_client.py b/ceilometer/neutron_client.py index 150221eacb..b0c9df3c12 100644 --- a/ceilometer/neutron_client.py +++ b/ceilometer/neutron_client.py @@ -113,3 +113,8 @@ class Client(object): def fw_policy_get_all(self): resp = self.client.list_firewall_policies() return resp.get('firewall_policies') + + @logged + def fip_get_all(self): + fips = self.client.list_floatingips()['floatingips'] + return fips diff --git a/ceilometer/tests/unit/network/test_floating_ip.py b/ceilometer/tests/unit/network/test_floating_ip.py new file mode 100644 index 0000000000..a68a4a71cf --- /dev/null +++ b/ceilometer/tests/unit/network/test_floating_ip.py @@ -0,0 +1,103 @@ +# #!/usr/bin/env python +# +# Copyright 2016 Sungard Availability Services +# Copyright 2016 Red Hat +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from oslotest import base +from oslotest import mockpatch + +from ceilometer.agent import manager +from ceilometer.agent import plugin_base +from ceilometer.network import floatingip + + +class _BaseTestFloatingIPPollster(base.BaseTestCase): + + @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock()) + def setUp(self): + super(_BaseTestFloatingIPPollster, self).setUp() + self.manager = manager.AgentManager() + plugin_base._get_keystone = mock.Mock() + + +class TestFloatingIPPollster(_BaseTestFloatingIPPollster): + + def setUp(self): + super(TestFloatingIPPollster, self).setUp() + self.pollster = floatingip.FloatingIPPollster() + fake_fip = self.fake_get_fip_service() + self.useFixture(mockpatch.Patch('ceilometer.neutron_client.Client.' + 'fip_get_all', + return_value=fake_fip)) + + @staticmethod + def fake_get_fip_service(): + return [{'router_id': 'e24f8a37-1bb7-49e4-833c-049bb21986d2', + 'status': 'ACTIVE', + 'tenant_id': '54a00c50ee4c4396b2f8dc220a2bed57', + 'floating_network_id': + 'f41f399e-d63e-47c6-9a19-21c4e4fbbba0', + 'fixed_ip_address': '10.0.0.6', + 'floating_ip_address': '65.79.162.11', + 'port_id': '93a0d2c7-a397-444c-9d75-d2ac89b6f209', + 'id': '18ca27bf-72bc-40c8-9c13-414d564ea367'}, + {'router_id': 'astf8a37-1bb7-49e4-833c-049bb21986d2', + 'status': 'DOWN', + 'tenant_id': '34a00c50ee4c4396b2f8dc220a2bed57', + 'floating_network_id': + 'gh1f399e-d63e-47c6-9a19-21c4e4fbbba0', + 'fixed_ip_address': '10.0.0.7', + 'floating_ip_address': '65.79.162.12', + 'port_id': '453a0d2c7-a397-444c-9d75-d2ac89b6f209', + 'id': 'jkca27bf-72bc-40c8-9c13-414d564ea367'}, + {'router_id': 'e2478937-1bb7-49e4-833c-049bb21986d2', + 'status': 'error', + 'tenant_id': '54a0gggg50ee4c4396b2f8dc220a2bed57', + 'floating_network_id': + 'po1f399e-d63e-47c6-9a19-21c4e4fbbba0', + 'fixed_ip_address': '10.0.0.8', + 'floating_ip_address': '65.79.162.13', + 'port_id': '67a0d2c7-a397-444c-9d75-d2ac89b6f209', + 'id': '90ca27bf-72bc-40c8-9c13-414d564ea367'}] + + def test_default_discovery(self): + self.assertEqual('endpoint:network', self.pollster.default_discovery) + + def test_fip_get_samples(self): + samples = list(self.pollster.get_samples( + self.manager, {}, + resources=['http://localhost:9696/'])) + self.assertEqual(1, len(samples)) + self.assertEqual('18ca27bf-72bc-40c8-9c13-414d564ea367', + samples[0].resource_id) + self.assertEqual("65.79.162.11", samples[0].resource_metadata[ + "floating_ip_address"]) + self.assertEqual("10.0.0.6", samples[0].resource_metadata[ + "fixed_ip_address"]) + + def test_fip_volume(self): + samples = list(self.pollster.get_samples( + self.manager, {}, + resources=['http://localhost:9696/'])) + self.assertEqual(1, samples[0].volume) + + def test_get_fip_meter_names(self): + samples = list(self.pollster.get_samples( + self.manager, {}, + resources=['http://localhost:9696/'])) + self.assertEqual(set(['ip.floating']), + set([s.name for s in samples])) diff --git a/ceilometer/tests/unit/network/test_floatingip.py b/ceilometer/tests/unit/network/test_floatingip.py deleted file mode 100644 index 9395d84f53..0000000000 --- a/ceilometer/tests/unit/network/test_floatingip.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2012 eNovance -# -# Copyright 2013 IBM Corp -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import mock -from oslo_context import context -from oslotest import base - -from ceilometer.agent import manager -from ceilometer.network import floatingip - - -class TestFloatingIPPollster(base.BaseTestCase): - - @mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock()) - def setUp(self): - super(TestFloatingIPPollster, self).setUp() - self.addCleanup(mock.patch.stopall) - self.context = context.get_admin_context() - self.manager = manager.AgentManager() - self.manager._keystone = mock.Mock() - catalog = (self.manager._keystone.session.auth. - get_access.return_value.service_catalog) - catalog.get_endpoints = mock.Mock(return_value={'network': mock.ANY}) - self.pollster = floatingip.FloatingIPPollster() - fake_ips = self.fake_get_ips() - patch_virt = mock.patch('ceilometer.nova_client.Client.' - 'floating_ip_get_all', - return_value=fake_ips) - patch_virt.start() - - @staticmethod - def fake_get_ips(): - ips = [] - for i in range(1, 4): - ip = mock.MagicMock() - ip.id = i - ip.ip = '1.1.1.%d' % i - ip.pool = 'public' - ips.append(ip) - return ips - - def test_default_discovery(self): - self.assertEqual('endpoint:compute', self.pollster.default_discovery) - - # FIXME(dhellmann): Is there a useful way to define this - # test without a database? - # - # def test_get_samples_none_defined(self): - # try: - # list(self.pollster.get_samples(self.manager, - # self.context) - # ) - # except exception.NoFloatingIpsDefined: - # pass - # else: - # assert False, 'Should have seen an error' - - def test_get_samples_not_empty(self): - samples = list(self.pollster.get_samples(self.manager, {}, ['e'])) - self.assertEqual(3, len(samples)) - # It's necessary to verify all the attributes extracted by Nova - # API /os-floating-ips to make sure they're available and correct. - self.assertEqual(1, samples[0].resource_id) - self.assertEqual("1.1.1.1", samples[0].resource_metadata["address"]) - self.assertEqual("public", samples[0].resource_metadata["pool"]) - - self.assertEqual(2, samples[1].resource_id) - self.assertEqual("1.1.1.2", samples[1].resource_metadata["address"]) - self.assertEqual("public", samples[1].resource_metadata["pool"]) - - self.assertEqual(3, samples[2].resource_id) - self.assertEqual("1.1.1.3", samples[2].resource_metadata["address"]) - self.assertEqual("public", samples[2].resource_metadata["pool"]) - - def test_get_meter_names(self): - samples = list(self.pollster.get_samples(self.manager, {}, ['e'])) - self.assertEqual(set(['ip.floating']), set([s.name for s in samples])) - - def test_get_samples_cached(self): - cache = {'e-floating_ips': self.fake_get_ips()[:2]} - samples = list(self.pollster.get_samples(self.manager, cache, ['e'])) - self.assertEqual(2, len(samples)) diff --git a/releasenotes/notes/fix-floatingip-pollster-f5172060c626b19e.yaml b/releasenotes/notes/fix-floatingip-pollster-f5172060c626b19e.yaml new file mode 100644 index 0000000000..1bd295abfe --- /dev/null +++ b/releasenotes/notes/fix-floatingip-pollster-f5172060c626b19e.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - > + [`bug 1536338 `_] + Patch was added to fix the broken floatingip pollster + that polled data from nova api, but since the nova api + filtered the data by tenant, ceilometer was not getting + any data back. The fix changes the pollster to use the + neutron api instead to get the floating ip info.