Set the time point polling starts as timestamp of samples

Many of the pollsters define the timestamp individually for each
generated sample, which should really be timestamping based on when
the data was polled and not when each sample is generated. We need to
set the timestamp of the polled data to the timestamp when the polling
starts for unity.

This commit set the time point when the polling starts as the timestamp
of samples, and make 'timestamp' argument of Sample.__init__ method
optional.

The timestamping in polling agents will be dropped in following patches.

Change-Id: I3be3f7fb8300e8853ebd35986cdf159422aa247a
Partial-Bug: #1491509
Partially-Implements: blueprint unify-timestamp-of-polled-data

ceilometer/agent/manager.py

@@ -28,6 +28,7 @@ from oslo_config import cfg
 from oslo_log import log
 import oslo_messaging
 from oslo_utils import fnmatch
+from oslo_utils import timeutils
 from six import moves
 from six.moves.urllib import parse as urlparse
 from stevedore import extension
@@ -190,6 +191,7 @@ class PollingTask(object):
                              "%(src)s"),
                          dict(poll=pollster.name, src=source_name))
                 try:
+                    polling_timestamp = timeutils.utcnow().isoformat()
                     samples = pollster.obj.get_samples(
                         manager=self.manager,
                         cache=cache,
@@ -198,6 +200,8 @@ class PollingTask(object):
                     sample_batch = []
 
                     for sample in samples:
+                        # Note(yuywz): Unify the timestamp of polled samples
+                        sample.set_timestamp(polling_timestamp)
                         sample_dict = (
                             publisher_utils.meter_message_from_counter(
                                 sample, self._telemetry_secret

ceilometer/sample.py

@@ -57,8 +57,8 @@ cfg.CONF.register_opts(OPTS)
 class Sample(object):
 
     def __init__(self, name, type, unit, volume, user_id, project_id,
-                 resource_id, timestamp, resource_metadata, source=None,
-                 id=None):
+                 resource_id, timestamp=None, resource_metadata=None,
+                 source=None, id=None):
         self.name = name
         self.type = type
         self.unit = unit
@@ -67,7 +67,7 @@ class Sample(object):
         self.project_id = project_id
         self.resource_id = resource_id
         self.timestamp = timestamp
-        self.resource_metadata = resource_metadata
+        self.resource_metadata = resource_metadata or {}
         self.source = source or cfg.CONF.sample_source
         self.id = id or str(uuid.uuid1())
 
@@ -99,6 +99,9 @@ class Sample(object):
                    resource_metadata=metadata,
                    source=source)
 
+    def set_timestamp(self, timestamp):
+        self.timestamp = timestamp
+
 TYPE_GAUGE = 'gauge'
 TYPE_DELTA = 'delta'
 TYPE_CUMULATIVE = 'cumulative'

ceilometer/tests/unit/agent/agentbase.py

@@ -42,7 +42,8 @@ from ceilometer import utils
 
 class TestSample(sample.Sample):
     def __init__(self, name, type, unit, volume, user_id, project_id,
-                 resource_id, timestamp, resource_metadata, source=None):
+                 resource_id, timestamp=None, resource_metadata=None,
+                 source=None):
         super(TestSample, self).__init__(name, type, unit, volume, user_id,
                                          project_id, resource_id, timestamp,
                                          resource_metadata, source)
@@ -717,3 +718,21 @@ class BaseAgentManagerTestCase(base.BaseTestCase):
         LOG.info.assert_called_with(
             'Skip pollster %(name)s, no %(p_context)sresources found this '
             'cycle', {'name': 'test', 'p_context': 'new '})
+
+    @mock.patch('oslo_utils.timeutils.utcnow')
+    def test_polling_samples_timestamp(self, mock_utc):
+        polled_samples = []
+        timestamp = '2222-11-22T00:11:22.333333'
+
+        def fake_send_notification(samples):
+            polled_samples.extend(samples)
+
+        mock_utc.return_value = datetime.datetime.strptime(
+            timestamp, "%Y-%m-%dT%H:%M:%S.%f")
+
+        self.setup_polling()
+        polling_task = list(self.mgr.setup_polling_tasks().values())[0]
+        polling_task._send_notification = mock.Mock(
+            side_effect=fake_send_notification)
+        polling_task.poll_and_notify()
+        self.assertEqual(timestamp, polled_samples[0]['timestamp'])

ceilometer/tests/unit/network/statistics/test_statistics.py

@@ -143,7 +143,7 @@ class TestBaseGetSamples(base.BaseTestCase):
 
         self.assertEqual(2, len(samples))
         self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'}, times[0])
-        self._assert_sample(samples[1], 2, 'b', None, times[1])
+        self._assert_sample(samples[1], 2, 'b', {}, times[1])
 
     def test_get_samples_two_driver_one_resource(self):
         times = self._make_timestamps(4)
@@ -173,7 +173,7 @@ class TestBaseGetSamples(base.BaseTestCase):
 
         self.assertEqual(2, len(samples))
         self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'}, times[0])
-        self._assert_sample(samples[1], 2, 'b', None, times[1])
+        self._assert_sample(samples[1], 2, 'b', {}, times[1])
 
     def test_get_samples_return_none(self):
         fake_driver = self._make_fake_driver(None)