Support grouping by timestamp in GET /v2/summary
Work items: * Updated Elasticsearch and InfluxDB v2 storage backends in order to support grouping on time. * Added a "dt_from_ts()" function to "cloudkitty.tzutils", along with unit tests. This function converts an epoch timestamp to a timezone-aware datetime object. * Added unit tests for GET /v2/summary to validate grouping by time works as expected. * Updated the API documentation. Change-Id: Ia01fced0bdb3a9b389a89d56f02029b9456781c2 Story: 2006730 Task: 37153
This commit is contained in:
parent
33edfb483d
commit
0d8a636755
|
@ -184,6 +184,9 @@ class ElasticsearchStorage(v2_storage.BaseStorage):
|
||||||
# Means we had a composite aggregation
|
# Means we had a composite aggregation
|
||||||
if 'key' in doc.keys():
|
if 'key' in doc.keys():
|
||||||
for key, value in doc['key'].items():
|
for key, value in doc['key'].items():
|
||||||
|
if key == 'begin' or key == 'end':
|
||||||
|
# Elasticsearch returns ts in milliseconds
|
||||||
|
value = tzutils.dt_from_ts(value // 1000)
|
||||||
output[key] = value
|
output[key] = value
|
||||||
return output
|
return output
|
||||||
|
|
||||||
|
|
|
@ -92,14 +92,18 @@ class ElasticsearchClient(object):
|
||||||
{'term': {'metadata.' + k: v}}]
|
{'term': {'metadata.' + k: v}}]
|
||||||
return should
|
return should
|
||||||
|
|
||||||
@staticmethod
|
def _build_composite(self, groupby):
|
||||||
def _build_composite(groupby):
|
|
||||||
if not groupby:
|
if not groupby:
|
||||||
return []
|
return []
|
||||||
sources = []
|
sources = []
|
||||||
for elem in groupby:
|
for elem in groupby:
|
||||||
if elem == 'type':
|
if elem == 'type':
|
||||||
sources.append({'type': {'terms': {'field': 'type'}}})
|
sources.append({'type': {'terms': {'field': 'type'}}})
|
||||||
|
elif elem == 'time':
|
||||||
|
# Not doing a date_histogram aggregation because we don't know
|
||||||
|
# the period
|
||||||
|
sources.append({'begin': {'terms': {'field': 'start'}}})
|
||||||
|
sources.append({'end': {'terms': {'field': 'end'}}})
|
||||||
else:
|
else:
|
||||||
sources.append({elem: {'terms': {'field': 'groupby.' + elem}}})
|
sources.append({elem: {'terms': {'field': 'groupby.' + elem}}})
|
||||||
|
|
||||||
|
|
|
@ -65,19 +65,27 @@ CONF.register_opts(influx_storage_opts, INFLUX_STORAGE_GROUP)
|
||||||
PERIOD_FIELD_NAME = '__ck_collect_period'
|
PERIOD_FIELD_NAME = '__ck_collect_period'
|
||||||
|
|
||||||
|
|
||||||
|
def _sanitized_groupby(groupby):
|
||||||
|
forbidden = ('time',)
|
||||||
|
return [g for g in groupby if g not in forbidden] if groupby else []
|
||||||
|
|
||||||
|
|
||||||
class InfluxClient(object):
|
class InfluxClient(object):
|
||||||
"""Classe used to ease interaction with InfluxDB"""
|
"""Classe used to ease interaction with InfluxDB"""
|
||||||
|
|
||||||
def __init__(self, chunk_size=500, autocommit=True):
|
def __init__(self, chunk_size=500, autocommit=True, default_period=3600):
|
||||||
"""Creates an InfluxClient object.
|
"""Creates an InfluxClient object.
|
||||||
|
|
||||||
:param chunk_size: Size after which points should be pushed.
|
:param chunk_size: Size after which points should be pushed.
|
||||||
:param autocommit: Set to false to disable autocommit
|
:param autocommit: Set to false to disable autocommit
|
||||||
|
:param default_period: Placeholder for the period in cae it can't
|
||||||
|
be determined.
|
||||||
"""
|
"""
|
||||||
self._conn = self._get_influx_client()
|
self._conn = self._get_influx_client()
|
||||||
self._chunk_size = chunk_size
|
self._chunk_size = chunk_size
|
||||||
self._autocommit = autocommit
|
self._autocommit = autocommit
|
||||||
self._retention_policy = CONF.storage_influxdb.retention_policy
|
self._retention_policy = CONF.storage_influxdb.retention_policy
|
||||||
|
self._default_period = default_period
|
||||||
self._points = []
|
self._points = []
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
@ -185,7 +193,13 @@ class InfluxClient(object):
|
||||||
query += self._get_type_query(types)
|
query += self._get_type_query(types)
|
||||||
|
|
||||||
if groupby:
|
if groupby:
|
||||||
groupby_query = '"' + '","'.join(groupby) + '"'
|
groupby_query = ''
|
||||||
|
if 'time' in groupby:
|
||||||
|
groupby_query += 'time(' + str(self._default_period) + 's)'
|
||||||
|
groupby_query += ',' if groupby else ''
|
||||||
|
if groupby:
|
||||||
|
groupby_query += '"' + '","'.join(
|
||||||
|
_sanitized_groupby(groupby)) + '"'
|
||||||
query += ' GROUP BY ' + groupby_query
|
query += ' GROUP BY ' + groupby_query
|
||||||
|
|
||||||
query += ';'
|
query += ';'
|
||||||
|
@ -243,7 +257,7 @@ class InfluxStorage(v2_storage.BaseStorage):
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
super(InfluxStorage, self).__init__(*args, **kwargs)
|
super(InfluxStorage, self).__init__(*args, **kwargs)
|
||||||
self._default_period = kwargs.get('period') or CONF.collect.period
|
self._default_period = kwargs.get('period') or CONF.collect.period
|
||||||
self._conn = InfluxClient()
|
self._conn = InfluxClient(default_period=self._default_period)
|
||||||
|
|
||||||
def init(self):
|
def init(self):
|
||||||
policy = CONF.storage_influxdb.retention_policy
|
policy = CONF.storage_influxdb.retention_policy
|
||||||
|
@ -326,8 +340,11 @@ class InfluxStorage(v2_storage.BaseStorage):
|
||||||
def delete(self, begin=None, end=None, filters=None):
|
def delete(self, begin=None, end=None, filters=None):
|
||||||
self._conn.delete(begin, end, filters)
|
self._conn.delete(begin, end, filters)
|
||||||
|
|
||||||
@staticmethod
|
def _get_total_elem(self, begin, end, groupby, series_groupby, point):
|
||||||
def _get_total_elem(begin, end, groupby, series_groupby, point):
|
if groupby and 'time' in groupby:
|
||||||
|
begin = tzutils.dt_from_iso(point['time'])
|
||||||
|
period = point.get(PERIOD_FIELD_NAME) or self._default_period
|
||||||
|
end = tzutils.add_delta(begin, datetime.timedelta(seconds=period))
|
||||||
output = {
|
output = {
|
||||||
'begin': begin,
|
'begin': begin,
|
||||||
'end': end,
|
'end': end,
|
||||||
|
@ -335,7 +352,7 @@ class InfluxStorage(v2_storage.BaseStorage):
|
||||||
'rate': point['price'],
|
'rate': point['price'],
|
||||||
}
|
}
|
||||||
if groupby:
|
if groupby:
|
||||||
for group in groupby:
|
for group in _sanitized_groupby(groupby):
|
||||||
output[group] = series_groupby.get(group, '')
|
output[group] = series_groupby.get(group, '')
|
||||||
return output
|
return output
|
||||||
|
|
||||||
|
@ -353,12 +370,18 @@ class InfluxStorage(v2_storage.BaseStorage):
|
||||||
output = []
|
output = []
|
||||||
for (series_name, series_groupby), points in total.items():
|
for (series_name, series_groupby), points in total.items():
|
||||||
for point in points:
|
for point in points:
|
||||||
output.append(self._get_total_elem(
|
# NOTE(peschk_l): InfluxDB returns all timestamps for a given
|
||||||
begin, end,
|
# period and interval, even those with no data. This filters
|
||||||
groupby,
|
# out periods with no data
|
||||||
series_groupby,
|
if point['qty'] is not None and point['price'] is not None:
|
||||||
point))
|
output.append(self._get_total_elem(
|
||||||
|
tzutils.utc_to_local(begin),
|
||||||
|
tzutils.utc_to_local(end),
|
||||||
|
groupby,
|
||||||
|
series_groupby,
|
||||||
|
point))
|
||||||
|
|
||||||
|
groupby = _sanitized_groupby(groupby)
|
||||||
if groupby:
|
if groupby:
|
||||||
output.sort(key=lambda x: [x[group] for group in groupby])
|
output.sort(key=lambda x: [x[group] for group in groupby])
|
||||||
return {
|
return {
|
||||||
|
|
|
@ -68,3 +68,21 @@ tests:
|
||||||
response_json_paths:
|
response_json_paths:
|
||||||
$.results.`len`: 0
|
$.results.`len`: 0
|
||||||
$.total: 0
|
$.total: 0
|
||||||
|
|
||||||
|
- name: Get a summary grouped by time
|
||||||
|
url: /v2/summary
|
||||||
|
status: 200
|
||||||
|
query_parameters:
|
||||||
|
groupby: [time]
|
||||||
|
response_json_paths:
|
||||||
|
$.results.`len`: 1
|
||||||
|
$.total: 1
|
||||||
|
|
||||||
|
- name: Get a summary grouped by time and project_id
|
||||||
|
url: /v2/summary
|
||||||
|
status: 200
|
||||||
|
query_parameters:
|
||||||
|
groupby: [time, project_id]
|
||||||
|
response_json_paths:
|
||||||
|
$.results.`len`: 2
|
||||||
|
$.total: 2
|
||||||
|
|
|
@ -17,6 +17,7 @@ import functools
|
||||||
|
|
||||||
from influxdb import resultset
|
from influxdb import resultset
|
||||||
|
|
||||||
|
from cloudkitty.storage.v2.influx import _sanitized_groupby
|
||||||
from cloudkitty.storage.v2.influx import InfluxClient
|
from cloudkitty.storage.v2.influx import InfluxClient
|
||||||
|
|
||||||
|
|
||||||
|
@ -63,6 +64,12 @@ class FakeInfluxClient(InfluxClient):
|
||||||
break
|
break
|
||||||
valid = True
|
valid = True
|
||||||
for tag in serie['tags'].keys():
|
for tag in serie['tags'].keys():
|
||||||
|
if tag == 'time':
|
||||||
|
if point['time'].isoformat() != serie['values'][0][0]:
|
||||||
|
valid = False
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
continue
|
||||||
if tag not in point['tags'].keys() or \
|
if tag not in point['tags'].keys() or \
|
||||||
point['tags'][tag] != serie['tags'][tag]:
|
point['tags'][tag] != serie['tags'][tag]:
|
||||||
valid = False
|
valid = False
|
||||||
|
@ -74,10 +81,11 @@ class FakeInfluxClient(InfluxClient):
|
||||||
if target_serie is None:
|
if target_serie is None:
|
||||||
target_serie = copy.deepcopy(self.total_series_sample)
|
target_serie = copy.deepcopy(self.total_series_sample)
|
||||||
if groupby:
|
if groupby:
|
||||||
target_serie['tags'] = {k: point['tags'][k] for k in groupby}
|
target_serie['tags'] = {k: point['tags'][k] for k in
|
||||||
|
_sanitized_groupby(groupby)}
|
||||||
else:
|
else:
|
||||||
target_serie['tags'] = {}
|
target_serie['tags'] = {}
|
||||||
target_serie['values'] = [['1970-01-01T00:00:00Z', 0, 0]]
|
target_serie['values'] = [[point['time'].isoformat(), 0, 0]]
|
||||||
series.append(target_serie)
|
series.append(target_serie)
|
||||||
return target_serie
|
return target_serie
|
||||||
|
|
||||||
|
|
|
@ -16,6 +16,7 @@ import datetime
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from dateutil import tz
|
from dateutil import tz
|
||||||
|
import mock
|
||||||
from oslo_utils import timeutils
|
from oslo_utils import timeutils
|
||||||
|
|
||||||
from cloudkitty import tzutils
|
from cloudkitty import tzutils
|
||||||
|
@ -132,3 +133,15 @@ class TestTZUtils(unittest.TestCase):
|
||||||
two = datetime.datetime(2019, 3, 31, 3,
|
two = datetime.datetime(2019, 3, 31, 3,
|
||||||
tzinfo=tz.gettz('Europe/Paris'))
|
tzinfo=tz.gettz('Europe/Paris'))
|
||||||
self.assertEqual(tzutils.diff_seconds(two, one), 3600)
|
self.assertEqual(tzutils.diff_seconds(two, one), 3600)
|
||||||
|
|
||||||
|
def test_cloudkitty_dt_from_ts_as_utc(self):
|
||||||
|
ts = 1569902400
|
||||||
|
dt = datetime.datetime(2019, 10, 1, 4, tzinfo=tz.UTC)
|
||||||
|
self.assertEqual(dt, tzutils.dt_from_ts(ts, as_utc=True))
|
||||||
|
|
||||||
|
def test_cloudkitty_dt_from_ts_local_tz(self):
|
||||||
|
ts = 1569902400
|
||||||
|
timezone = tz.gettz('Europe/Paris')
|
||||||
|
dt = datetime.datetime(2019, 10, 1, 6, tzinfo=timezone)
|
||||||
|
with mock.patch.object(tzutils, '_LOCAL_TZ', new=timezone):
|
||||||
|
self.assertEqual(dt, tzutils.dt_from_ts(ts))
|
||||||
|
|
|
@ -84,6 +84,14 @@ def dt_from_iso(time_str, as_utc=False):
|
||||||
tz.UTC if as_utc else _LOCAL_TZ).replace(microsecond=0)
|
tz.UTC if as_utc else _LOCAL_TZ).replace(microsecond=0)
|
||||||
|
|
||||||
|
|
||||||
|
def dt_from_ts(ts, as_utc=False):
|
||||||
|
"""Parses a timezone-aware datetime object from an epoch timestamp.
|
||||||
|
|
||||||
|
Returns the object as being from the local timezone.
|
||||||
|
"""
|
||||||
|
return datetime.datetime.fromtimestamp(ts, tz.UTC if as_utc else _LOCAL_TZ)
|
||||||
|
|
||||||
|
|
||||||
def add_delta(dt, delta):
|
def add_delta(dt, delta):
|
||||||
"""Adds a timedelta to a datetime object.
|
"""Adds a timedelta to a datetime object.
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,33 @@
|
||||||
|
{
|
||||||
|
"total": 232,
|
||||||
|
"columns": [
|
||||||
|
"begin",
|
||||||
|
"end",
|
||||||
|
"qty",
|
||||||
|
"rate",
|
||||||
|
"project_id"
|
||||||
|
],
|
||||||
|
"results": [
|
||||||
|
[
|
||||||
|
"2019-10-01T06:00:00+02:00",
|
||||||
|
"2019-10-01T07:00:00+02:00",
|
||||||
|
3.5533905029296875,
|
||||||
|
1.7766952514648438,
|
||||||
|
"84631866b2d84db49b29828052bdc287"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"2019-10-01T07:00:00+02:00",
|
||||||
|
"2019-10-01T08:00:00+02:00",
|
||||||
|
3.5533905029296875,
|
||||||
|
1.7766952514648438,
|
||||||
|
"84631866b2d84db49b29828052bdc287"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"2019-10-01T08:00:00+02:00",
|
||||||
|
"2019-10-01T09:00:00+02:00",
|
||||||
|
3.5533905029296875,
|
||||||
|
1.7766952514648438,
|
||||||
|
"84631866b2d84db49b29828052bdc287"
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
|
@ -65,6 +65,11 @@ the columns for each element of ``results``. The columns are the four mandatory
|
||||||
(``begin``, ``end``, ``qty``, ``rate``) along with each attribute the result is
|
(``begin``, ``end``, ``qty``, ``rate``) along with each attribute the result is
|
||||||
grouped by.
|
grouped by.
|
||||||
|
|
||||||
|
.. note:: It is also possible to group data by time, in order to obtain timeseries.
|
||||||
|
In order to do this, group by ``time``. No extra column will be added,
|
||||||
|
but you'll get one entry per collect period in the queried timeframe.
|
||||||
|
See examples below.
|
||||||
|
|
||||||
.. rest_parameters:: summary/summary_parameters.yml
|
.. rest_parameters:: summary/summary_parameters.yml
|
||||||
|
|
||||||
- begin: begin_resp
|
- begin: begin_resp
|
||||||
|
@ -75,6 +80,15 @@ grouped by.
|
||||||
Response Example
|
Response Example
|
||||||
----------------
|
----------------
|
||||||
|
|
||||||
|
Grouping by time and project_id:
|
||||||
|
|
||||||
|
.. code-block:: shell
|
||||||
|
|
||||||
|
curl "http://cloudkitty-api:8889/v2/summary?groupby=time&groupby=project_id&limit=3"
|
||||||
|
|
||||||
|
.. literalinclude:: ./api_samples/summary/summary_get_groupby_time.json
|
||||||
|
:language: javascript
|
||||||
|
|
||||||
.. code-block:: shell
|
.. code-block:: shell
|
||||||
|
|
||||||
curl "http://cloudkitty-api:8889/v2/summary?filters=project_id%3Afe9c35372db6420089883805b37a34af&groupby=type&groupby=project_id"
|
curl "http://cloudkitty-api:8889/v2/summary?filters=project_id%3Afe9c35372db6420089883805b37a34af&groupby=type&groupby=project_id"
|
||||||
|
|
|
@ -0,0 +1,6 @@
|
||||||
|
---
|
||||||
|
upgrade:
|
||||||
|
- |
|
||||||
|
It is now possible to group v2 summaries by timestamp. In order to do this,
|
||||||
|
the ``time`` parameter must be specified in the ``groupby`` list:
|
||||||
|
``cloudkitty summary get -g time,type``.
|
Loading…
Reference in New Issue