Drop timestamping in pollsters

Many of the pollsters define the timestamp individually for each
generated sample, which should really be timestamping based on when
the data was polled and not when each sample is generated. We need to
set the timestamp of the polled data to the timestamp when the polling
starts for unity.

Previous commit be3f7fb8300e8853ebd35986cdf159422aa247a already unified
the timestamp in samples, and now drop the timestamping in pollsters.

Change-Id: I9c3047e70fcd6c02278dee91603e2aeb2ca8b8c8
Closes-Bug: #1491509
Implements: blueprint unify-timestamp-of-polled-data
This commit is contained in:
Wenzhi Yu 2016-04-29 11:14:39 +08:00
parent 615b1209f0
commit 8dd821a03d
20 changed files with 46 additions and 121 deletions

View File

@ -13,7 +13,6 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_utils import timeutils
from ceilometer.compute import util as compute_util
from ceilometer import sample
@ -88,7 +87,6 @@ def make_sample_from_instance(instance, name, type, unit, volume,
user_id=instance.user_id,
project_id=instance.tenant_id,
resource_id=resource_id or instance.id,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=resource_metadata,
)

View File

@ -12,8 +12,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from keystoneauth1 import exceptions
from oslo_config import cfg
from oslo_log import log
@ -104,8 +102,6 @@ class EnergyPollster(_Base):
user_id=None,
project_id=None,
resource_id=probe['id'],
timestamp=datetime.datetime.fromtimestamp(
probe['timestamp']).isoformat(),
resource_metadata={}
)
@ -124,7 +120,5 @@ class PowerPollster(_Base):
user_id=None,
project_id=None,
resource_id=probe['id'],
timestamp=datetime.datetime.fromtimestamp(
probe['timestamp']).isoformat(),
resource_metadata={}
)

View File

@ -20,7 +20,6 @@
import copy
from oslo_utils import timeutils
from six.moves.urllib import parse as urlparse
from ceilometer import sample
@ -59,7 +58,6 @@ def make_sample_from_host(host_url, name, sample_type, unit, volume,
user_id=user_id or extra.get('user_id'),
project_id=project_id or extra.get('project_id'),
resource_id=res_id,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=resource_metadata,
source='hardware',
)

View File

@ -19,7 +19,6 @@ from __future__ import absolute_import
import glanceclient
from oslo_config import cfg
from oslo_utils import timeutils
from ceilometer.agent import plugin_base
from ceilometer import keystone_client
@ -110,7 +109,6 @@ class ImagePollster(_Base):
user_id=None,
project_id=image.owner,
resource_id=image.id,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_image_metadata(image),
)
@ -127,6 +125,5 @@ class ImageSizePollster(_Base):
user_id=None,
project_id=image.owner,
resource_id=image.id,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_image_metadata(image),
)

View File

@ -16,7 +16,6 @@ import abc
from oslo_config import cfg
from oslo_log import log
from oslo_utils import timeutils
import six
from ceilometer.agent import plugin_base
@ -89,7 +88,6 @@ class _Base(plugin_base.PollsterBase):
user_id=None,
project_id=None,
resource_id=CONF.host,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=metadata)

View File

@ -14,7 +14,6 @@
from oslo_config import cfg
from oslo_log import log
from oslo_utils import timeutils
from ceilometer.agent import plugin_base
from ceilometer.i18n import _
@ -112,7 +111,6 @@ class SensorPollster(plugin_base.PollsterBase):
user_id=None,
project_id=None,
resource_id=resource_id,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=metadata)

View File

@ -18,7 +18,6 @@
from oslo_config import cfg
from oslo_log import log
from oslo_utils import timeutils
from ceilometer.i18n import _LW
from ceilometer.network.services import base
@ -59,6 +58,5 @@ class FloatingIPPollster(base.BaseServicesPollster):
user_id=fip.get('user_id'),
project_id=fip['tenant_id'],
resource_id=fip['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(fip)
)

View File

@ -14,7 +14,6 @@
# under the License.
from oslo_log import log
from oslo_utils import timeutils
from ceilometer.i18n import _
from ceilometer.network.services import base
@ -58,7 +57,6 @@ class FirewallPollster(base.BaseServicesPollster):
user_id=None,
project_id=fw['tenant_id'],
resource_id=fw['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(fw)
)
@ -92,6 +90,5 @@ class FirewallPolicyPollster(base.BaseServicesPollster):
user_id=None,
project_id=fw['tenant_id'],
resource_id=fw['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(fw)
)

View File

@ -18,7 +18,6 @@ import collections
from oslo_config import cfg
from oslo_log import log
from oslo_utils import timeutils
import six
from ceilometer.i18n import _
@ -98,7 +97,6 @@ class LBPoolPollster(BaseLBPollster):
user_id=None,
project_id=pool['tenant_id'],
resource_id=pool['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(pool)
)
@ -146,7 +144,6 @@ class LBVipPollster(base.BaseServicesPollster):
user_id=None,
project_id=vip['tenant_id'],
resource_id=vip['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(vip)
)
@ -186,7 +183,6 @@ class LBMemberPollster(BaseLBPollster):
user_id=None,
project_id=member['tenant_id'],
resource_id=member['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(member)
)
@ -217,7 +213,6 @@ class LBHealthMonitorPollster(base.BaseServicesPollster):
user_id=None,
project_id=probe['tenant_id'],
resource_id=probe['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(probe)
)
@ -248,7 +243,6 @@ class _LBStatsPollster(base.BaseServicesPollster):
user_id=None,
project_id=pool['tenant_id'],
resource_id=pool['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=resource_metadata,
)
@ -377,7 +371,6 @@ def make_sample_from_pool(pool, name, type, unit, volume,
user_id=None,
project_id=pool['tenant_id'],
resource_id=pool['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=resource_metadata,
)
@ -423,7 +416,6 @@ class LBListenerPollster(BaseLBPollster):
user_id=None,
project_id=listener['tenant_id'],
resource_id=listener['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(listener)
)
@ -468,6 +460,5 @@ class LBLoadBalancerPollster(BaseLBPollster):
user_id=None,
project_id=loadbalancer['tenant_id'],
resource_id=loadbalancer['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(loadbalancer)
)

View File

@ -14,7 +14,6 @@
# under the License.
from oslo_log import log
from oslo_utils import timeutils
from ceilometer.i18n import _
from ceilometer.network.services import base
@ -59,7 +58,6 @@ class VPNServicesPollster(base.BaseServicesPollster):
user_id=None,
project_id=vpn['tenant_id'],
resource_id=vpn['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(vpn)
)
@ -102,6 +100,5 @@ class IPSecConnectionsPollster(base.BaseServicesPollster):
user_id=None,
project_id=conn['tenant_id'],
resource_id=conn['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(conn)
)

View File

@ -86,7 +86,7 @@ class _Base(plugin_base.PollsterBase):
if not isinstance(data, list):
data = [data]
for (volume, resource_id,
resource_metadata, timestamp) in data:
resource_metadata) in data:
yield sample.Sample(
name=self.meter_name,
@ -96,6 +96,5 @@ class _Base(plugin_base.PollsterBase):
user_id=None,
project_id=None,
resource_id=resource_id,
timestamp=timestamp,
resource_metadata=resource_metadata
)

View File

@ -14,7 +14,6 @@
import re
from oslo_utils import timeutils
from six.moves.urllib import parse as urlparse
from ceilometer.network.statistics import driver
@ -104,7 +103,6 @@ class OpencontrailDriver(driver.Driver):
fqdn_uuid = params.get('fqdn_uuid', ['*'])[0]
virtual_network = params.get('virtual_network', [None])[0]
timestamp = timeutils.utcnow().isoformat()
statistics = data['o_client'].networks.get_vm_statistics(fqdn_uuid)
if not statistics:
return
@ -113,7 +111,7 @@ class OpencontrailDriver(driver.Driver):
for sample in iter(extractor, value, ports_map,
resource, virtual_network):
if sample is not None:
yield sample + (timestamp, )
yield sample
def _get_iter(self, meter_name):
if meter_name.startswith('switch.port'):

View File

@ -14,7 +14,6 @@
# under the License.
from oslo_log import log
from oslo_utils import timeutils
import six
from six import moves
from six.moves.urllib import parse as urlparse
@ -161,9 +160,6 @@ class OpenDayLightDriver(driver.Driver):
cs.host_tracker.get_active_hosts(container_name))
container_data['inactive_hosts'] = (
cs.host_tracker.get_inactive_hosts(container_name))
container_data['timestamp'] = timeutils.utcnow().isoformat()
data[container_name] = container_data
except Exception:
LOG.exception(_('Request failed to connect to OpenDaylight'
@ -199,7 +195,6 @@ class OpenDayLightDriver(driver.Driver):
samples = []
for name, value in six.iteritems(data):
timestamp = value['timestamp']
for sample in iter(extractor, value):
if sample is not None:
# set controller name and container name
@ -207,7 +202,7 @@ class OpenDayLightDriver(driver.Driver):
sample[2]['controller'] = 'OpenDaylight'
sample[2]['container'] = name
samples.append(sample + (timestamp, ))
samples.append(sample)
return samples

View File

@ -18,7 +18,6 @@
from keystoneauth1 import exceptions
from oslo_config import cfg
from oslo_log import log
from oslo_utils import timeutils
import six.moves.urllib.parse as urlparse
from ceilometer.agent import plugin_base
@ -117,7 +116,6 @@ class ContainersObjectsPollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant + '/' + it.name,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -137,7 +135,6 @@ class ContainersSizePollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant + '/' + it.name,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -156,7 +153,6 @@ class ObjectsSizePollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -175,7 +171,6 @@ class ObjectsPollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -192,7 +187,6 @@ class ObjectsContainersPollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -212,6 +206,5 @@ class UsagePollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)

View File

@ -20,7 +20,6 @@ from __future__ import absolute_import
from keystoneauth1 import exceptions
from oslo_config import cfg
from oslo_log import log
from oslo_utils import timeutils
import six.moves.urllib.parse as urlparse
from swiftclient import client as swift
@ -115,7 +114,6 @@ class ObjectsPollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -134,7 +132,6 @@ class ObjectsSizePollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -153,7 +150,6 @@ class ObjectsContainersPollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -177,7 +173,6 @@ class ContainersObjectsPollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant + '/' + container['name'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -201,6 +196,5 @@ class ContainersSizePollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant + '/' + container['name'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)

View File

@ -12,8 +12,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from keystoneauth1 import exceptions
import mock
from oslotest import base
@ -107,10 +105,6 @@ class TestEnergyPollster(_BaseTestCase):
samples_by_name = dict((s.resource_id, s) for s in samples)
for name, probe in PROBE_DICT['probes'].items():
sample = samples_by_name[name]
expected = datetime.datetime.fromtimestamp(
probe['timestamp']
).isoformat()
self.assertEqual(expected, sample.timestamp)
self.assertEqual(probe[self.unit], sample.volume)

View File

@ -135,8 +135,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY),
'resource': 'if_stats_list'},),
(13,
'96d49cc3-4e01-40ce-9cac-c0e32642a442',
{'device_owner_id':
@ -145,8 +144,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY)]
'resource': 'if_stats_list'},)]
self._test_meter('switch.port.receive.packets', expected,
self.fake_port_stats_with_node())
@ -159,8 +157,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY)]
'resource': 'if_stats_list'},)]
self._test_meter('switch.port.receive.packets', expected)
def test_switch_port_transmit_packets(self):
@ -172,8 +169,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY)]
'resource': 'if_stats_list'},)]
self._test_meter('switch.port.transmit.packets', expected)
def test_switch_port_receive_bytes(self):
@ -185,8 +181,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY)]
'resource': 'if_stats_list'},)]
self._test_meter('switch.port.receive.bytes', expected)
def test_switch_port_transmit_bytes(self):
@ -198,8 +193,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY)]
'resource': 'if_stats_list'},)]
self._test_meter('switch.port.transmit.bytes', expected)
def test_switch_port_receive_packets_fip(self):
@ -212,8 +206,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'fip_stats_list'},
mock.ANY)]
'resource': 'fip_stats_list'},)]
self._test_meter('switch.port.receive.packets', expected)
def test_switch_port_transmit_packets_fip(self):
@ -226,8 +219,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'fip_stats_list'},
mock.ANY)]
'resource': 'fip_stats_list'},)]
self._test_meter('switch.port.transmit.packets', expected)
def test_switch_port_receive_bytes_fip(self):
@ -240,8 +232,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'fip_stats_list'},
mock.ANY)]
'resource': 'fip_stats_list'},)]
self._test_meter('switch.port.receive.bytes', expected)
def test_switch_port_transmit_bytes_fip(self):
@ -254,8 +245,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'fip_stats_list'},
mock.ANY)]
'resource': 'fip_stats_list'},)]
self._test_meter('switch.port.transmit.bytes', expected)
def test_switch_port_transmit_bytes_non_existing_network(self):

View File

@ -122,7 +122,6 @@ class _Base(base.BaseTestCase):
self.assertEqual(expected[0], sample[0]) # check volume
self.assertEqual(expected[1], sample[1]) # check resource id
self.assertEqual(expected[2], sample[2]) # check resource metadata
self.assertIsNotNone(sample[3]) # timestamp
class TestOpenDayLightDriverSpecial(_Base):

View File

@ -105,75 +105,65 @@ class TestBaseGetSamples(base.BaseTestCase):
return [v for v in self.pollster.get_samples(self, {}, resources)]
def _assert_sample(self, s, volume, resource_id, resource_metadata,
timestamp):
self.assertEqual('foo', s.name)
self.assertEqual(sample.TYPE_CUMULATIVE, s.type)
self.assertEqual('bar', s.unit)
self.assertEqual(volume, s.volume)
self.assertIsNone(s.user_id)
self.assertIsNone(s.project_id)
self.assertEqual(resource_id, s.resource_id)
self.assertEqual(timestamp, s.timestamp)
self.assertEqual(resource_metadata, s.resource_metadata)
def _assert_sample(self, s, volume, resource_id, resource_metadata):
self.assertEqual('foo', s.name)
self.assertEqual(sample.TYPE_CUMULATIVE, s.type)
self.assertEqual('bar', s.unit)
self.assertEqual(volume, s.volume)
self.assertIsNone(s.user_id)
self.assertIsNone(s.project_id)
self.assertEqual(resource_id, s.resource_id)
self.assertEqual(resource_metadata, s.resource_metadata)
def test_get_samples_one_driver_one_resource(self):
times = self._make_timestamps(2)
fake_driver = self._make_fake_driver((1, 'a', {'spam': 'egg'},
times[0]),
(2, 'b', None, times[1]))
fake_driver = self._make_fake_driver((1, 'a', {'spam': 'egg'},),
(2, 'b', None))
self._setup_ext_mgr(http=fake_driver())
samples = self._get_samples('http://foo')
self.assertEqual(1, len(samples))
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'}, times[0])
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'})
def test_get_samples_one_driver_two_resource(self):
times = self._make_timestamps(3)
fake_driver = self._make_fake_driver((1, 'a', {'spam': 'egg'},
times[0]),
(2, 'b', None, times[1]),
(3, 'c', None, times[2]))
fake_driver = self._make_fake_driver((1, 'a', {'spam': 'egg'},),
(2, 'b', None),
(3, 'c', None))
self._setup_ext_mgr(http=fake_driver())
samples = self._get_samples('http://foo', 'http://bar')
self.assertEqual(2, len(samples))
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'}, times[0])
self._assert_sample(samples[1], 2, 'b', {}, times[1])
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'})
self._assert_sample(samples[1], 2, 'b', {})
def test_get_samples_two_driver_one_resource(self):
times = self._make_timestamps(4)
fake_driver1 = self._make_fake_driver((1, 'a', {'spam': 'egg'},
times[0]),
(2, 'b', None), times[1])
fake_driver1 = self._make_fake_driver((1, 'a', {'spam': 'egg'},),
(2, 'b', None))
fake_driver2 = self._make_fake_driver((11, 'A', None, times[2]),
(12, 'B', None, times[3]))
fake_driver2 = self._make_fake_driver((11, 'A', None),
(12, 'B', None))
self._setup_ext_mgr(http=fake_driver1(), https=fake_driver2())
samples = self._get_samples('http://foo')
self.assertEqual(1, len(samples))
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'}, times[0])
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'})
def test_get_samples_multi_samples(self):
times = self._make_timestamps(2)
fake_driver = self._make_fake_driver([(1, 'a', {'spam': 'egg'},
times[0]),
(2, 'b', None, times[1])])
fake_driver = self._make_fake_driver([(1, 'a', {'spam': 'egg'},),
(2, 'b', None)])
self._setup_ext_mgr(http=fake_driver())
samples = self._get_samples('http://foo')
self.assertEqual(2, len(samples))
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'}, times[0])
self._assert_sample(samples[1], 2, 'b', {}, times[1])
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'})
self._assert_sample(samples[1], 2, 'b', {})
def test_get_samples_return_none(self):
fake_driver = self._make_fake_driver(None)

View File

@ -0,0 +1,7 @@
---
fixes:
- >
[`bug 1491509 <https://bugs.launchpad.net/ceilometer/+bug/1491509>`_]
Patch to unify timestamp in samples polled by pollsters. Set the time
point polling starts as timestamp of samples, and drop timetamping in
pollsters.