Re-amp recorder and only get stuff that's a day back from current time as default

billistix/recorder/base.py

@@ -15,12 +15,25 @@
 # under the License.
 import abc
 
+from billistix.openstack.common import cfg
+
+
+cfg.CONF.register_opt(
+    cfg.BoolOpt('record_audit_logging', default=False,
+                help='Logs individual records pr get_records()')
+)
+
+
+cfg.CONF.register_opt(
+    cfg.IntOpt('poll_age', default=86400,
+                help='How far back to pull data from the source service')
+)
+
 
 class RecorderEngine(object):
     """
     Base Record engine for getting Records from external systems
     """
-
     __metaclass__ = abc.ABCMeta
 
     @abc.abstractmethod

billistix/recorder/impl_ceilometer.py

@@ -14,8 +14,11 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 # NOTE(zykes): Copied verbatim from ceilometerclient
+from datetime import datetime, timedelta
+
 import ceilometerclient
 
+from billistix.openstack.common import cfg
 from billistix.openstack.common import log
 from billistix.recorder.openstack import OpenstackEngine
 
@@ -31,6 +34,9 @@ class RecordEngine(OpenstackEngine):
         return ceilometerclient.Client(keystone_client=keystone_client)
 
     def get_records(self):
+        """
+        Get the records between a period of time
+        """
         records = []
 
         try:
@@ -40,41 +46,81 @@ class RecordEngine(OpenstackEngine):
             LOG.exception(e)
             return
 
+        poll_start = datetime.now() - timedelta(seconds=cfg.CONF.poll_age)
+
         for project_id in projects:
             if project_id is None:
                 continue
-            for record in self.get_project_records(project_id):
+            project_records = self.get_project_records_between(project_id,
-                import pprint
+                start_timestamp=poll_start)
-                print pprint.pformat(record)
+            for record in project_records:
                 records.append(record)
         return records
 
-    def get_project_records(self, project_id):
+    def get_project_records_between(self, project_id, start_timestamp=None,
+                                    end_timestamp=None):
+        """
+        Get the given project id's records between given timestamps
+
+        :param project_id: Project ID to get Records for.
+        :param start_timestamp: Start timestamp
+        :param end_timestamp: End timestamp
+        """
         records = []
         for resource in self.client.get_resources(project_id=project_id):
             meters = [item.get('counter_name') for item in resource['meter']]
             for meter in meters:
-                type_, volume, metadata = self.get_meter_data(resource, meter)
+                record = self.get_record_between(
-
+                    resource, meter,
-                if type_ is not None:
+                    start_timestamp=start_timestamp,
-                    duration_info = self.client.get_resource_duration_info(
+                    end_timestamp=end_timestamp)
-                        resource_id=resource['resource_id'],
+                if record is not None:
-                        meter=meter,
-                    )
-                    record = dict(
-                        resource_id=resource['resource_id'],
-                        type=type_,
-                        volume=volume,
-                        extra=metadata,
-                        start_timestamp=duration_info.get('start_timestamp'),
-                        end_timestamp=duration_info.get('end_timestamp'),
-                        duration=duration_info.get('duration')
-                    )
                     records.append(record)
-        LOG.debug("Returning %d records", len(records))
+        LOG.debug("Returning %d records for project %s", len(records),
+                  project_id)
         return records
 
-    def get_meter_data(self, resource, meter):
+    def get_record_between(self, resource, meter,
+                           start_timestamp=None, end_timestamp=None):
+        """
+        :param resource: A resource in Dict form
+        :param meter: Meter name
+        :param start_timestamp: Start timestamp
+        :param end_timestamp: End timestamp
+        """
+        # NOTE: No type, skip it. Needs re-amp
+        type_, volume, metadata = self._get_meter_data(resource, meter)
+        if type_ is None:
+            return
+
+        duration_info = self.client.get_resource_duration_info(
+            resource_id=resource['resource_id'], meter=meter,
+            start_timestamp=start_timestamp, end_timestamp=end_timestamp
+        )
+
+        # NOTE: Not sure on this but I think we can skip returning events that
+        # don't have volume or duration
+        if not volume and not duration_info.get('duration'):
+            return
+
+        record = dict(
+            resource_id=resource['resource_id'],
+            type=type_,
+            volume=volume,
+            extra=metadata,
+            start_timestamp=duration_info.get('start_timestamp'),
+            end_timestamp=duration_info.get('end_timestamp'),
+            duration=duration_info.get('duration')
+        )
+        if cfg.CONF.record_audit_logging:
+            LOG.debug("Record: %s", record)
+        return record
+
+    def _get_meter_data(self, resource, meter):
+        """
+        :param resource: A resource in Dict form
+        :param meter: Meter name
+        """
         type_ = None
         volume = resource['metadata'].get('size')
         metadata = {}