openstack
/
monasca-api
Code Issues Proposed changes

Support time range to query dimension names/values 

At present, dimensions are not scoped by time window, which makes
dimension related queries to large databases timeout because it searches
all of time instead of a time window specified on the grafana app.

This commit implements the server side changes required to scope the
search query by the time window specified on the app.

Change-Id: Ia760c6789ac0063b8a25e52c9e0c3cc3b790ad2d
Story: 2005204
Task: 35790
This commit is contained in:
Bharat Kunwar 2019-07-25 12:28:14 +01:00
parent fb23e914b1
commit 233ea9c51b
7 changed files with 128 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.zuul.yaml
View File

@ -101,6 +101,7 @@
MONASCA_PERSISTER_IMPLEMENTATION_LANG: python
MONASCA_METRICS_DB: cassandra
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api)
- job:
name: monasca-tempest-python3-cassandra
@ -113,6 +114,7 @@
MONASCA_PERSISTER_IMPLEMENTATION_LANG: python
MONASCA_METRICS_DB: cassandra
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api)
- job:
name: monasca-tempest-python2-java-cassandra
@ -123,6 +125,7 @@
MONASCA_PERSISTER_IMPLEMENTATION_LANG: java
MONASCA_METRICS_DB: cassandra
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api)
- job:
name: monasca-tempest-python3-java-cassandra
@ -135,6 +138,7 @@
MONASCA_PERSISTER_IMPLEMENTATION_LANG: java
MONASCA_METRICS_DB: cassandra
TEMPEST_PLUGINS: /opt/stack/monasca-tempest-plugin
tempest_test_regex: (?!.*\[.*\btimerange\b.*\])(^monasca_tempest_tests.tests.api)
- project:

4
docs/monasca-api-spec.md
View File

@ -1229,6 +1229,8 @@ None.
* tenant_id (string, optional, restricted) - Tenant ID to from which to get dimension values. This parameter can be used to get dimension values from a tenant other than the tenant the request auth token is scoped to. Usage of this query parameter is restricted to users with the monasca admin role, as defined in the monasca api configuration file, which defaults to `monasca-admin`.
* metric_name (string(255), optional) - A metric name to filter dimension values by.
* dimension_name (string(255), required) - A dimension name to filter dimension values by.
* start_time (string, optional) - The start time in ISO 8601 combined date and time format in UTC.
* end_time (string, optional) - The end time in ISO 8601 combined date and time format in UTC.
* offset (string(255), optional) - The dimension values are returned in alphabetic order, and the offset is the dimension name after which to return in the next pagination request.
* limit (integer, optional)
@ -1291,6 +1293,8 @@ None.
#### Query Parameters
* tenant_id (string, optional, restricted) - Tenant ID from which to get dimension names. This parameter can be used to get dimension names from a tenant other than the tenant the request auth token is scoped to. Usage of this query parameter is restricted to users with the monasca admin role, as defined in the monasca api configuration file, which defaults to `monasca-admin`.
* metric_name (string(255), optional) - A metric name to filter dimension names by.
* start_time (string, optional) - The start time in ISO 8601 combined date and time format in UTC.
* end_time (string, optional) - The end time in ISO 8601 combined date and time format in UTC.
* offset (string(255), optional) - The dimension names are returned in alphabetic order, and the offset is the dimension name after which will return in the next pagination request.
* limit (integer, optional)

16
monasca_api/common/repositories/cassandra/metrics_repository.py
View File

@ -157,7 +157,13 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
self.epoch = datetime.utcfromtimestamp(0)
def list_dimension_values(self, tenant_id, region, metric_name,
dimension_name):
dimension_name, start_timestamp=None,
end_timestamp=None):
if start_timestamp or end_timestamp:
# NOTE(brtknr): For more details, see story
# https://storyboard.openstack.org/#!/story/2006204
LOG.info("Scoping by timestamp not implemented for cassandra.")
try:
if metric_name:
@ -185,7 +191,13 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
return json_dim_value_list
def list_dimension_names(self, tenant_id, region, metric_name):
def list_dimension_names(self, tenant_id, region, metric_name,
start_timestamp=None, end_timestamp=None):
if start_timestamp or end_timestamp:
# NOTE(brtknr): For more details, see story
# https://storyboard.openstack.org/#!/story/2006204
LOG.info("Scoping by timestamp not implemented for cassandra.")
try:
if metric_name:

26
monasca_api/common/repositories/influxdb/metrics_repository.py
View File

@ -159,7 +159,8 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
return query
def _build_show_tag_values_query(self, metric_name, dimension_name,
tenant_id, region):
tenant_id, region, start_timestamp,
end_timestamp):
from_with_clause = ''
if metric_name:
from_with_clause += ' from "{}"'.format(metric_name)
@ -167,18 +168,21 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
if dimension_name:
from_with_clause += ' with key = "{}"'.format(dimension_name)
where_clause = self._build_where_clause(None, None, tenant_id, region)
where_clause = self._build_where_clause(None, None, tenant_id, region,
start_timestamp, end_timestamp)
query = 'show tag values' + from_with_clause + where_clause
return query
def _build_show_tag_keys_query(self, metric_name, tenant_id, region):
def _build_show_tag_keys_query(self, metric_name, tenant_id, region,
start_timestamp, end_timestamp):
from_with_clause = ''
if metric_name:
from_with_clause += ' from "{}"'.format(metric_name)
where_clause = self._build_where_clause(None, None, tenant_id, region)
where_clause = self._build_where_clause(None, None, tenant_id, region,
start_timestamp, end_timestamp)
query = 'show tag keys' + from_with_clause + where_clause
@ -919,11 +923,14 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
return int((dt - datetime(1970, 1, 1)).total_seconds() * 1000)
def list_dimension_values(self, tenant_id, region, metric_name,
dimension_name):
dimension_name, start_timestamp=None,
end_timestamp=None):
try:
query = self._build_show_tag_values_query(metric_name,
dimension_name,
tenant_id, region)
tenant_id, region,
start_timestamp,
end_timestamp)
result = self.influxdb_client.query(query)
json_dim_name_list = self._build_serie_dimension_values(
result, dimension_name)
@ -932,10 +939,13 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
LOG.exception(ex)
raise exceptions.RepositoryException(ex)
def list_dimension_names(self, tenant_id, region, metric_name):
def list_dimension_names(self, tenant_id, region, metric_name,
start_timestamp=None, end_timestamp=None):
try:
query = self._build_show_tag_keys_query(metric_name,
tenant_id, region)
tenant_id, region,
start_timestamp,
end_timestamp)
result = self.influxdb_client.query(query)
json_dim_name_list = self._build_serie_dimension_names(result)
return json_dim_name_list

88
monasca_api/tests/test_repositories.py
View File

@ -192,53 +192,76 @@ class TestRepoMetricsInfluxDB(base.BaseTestCase):
@patch("monasca_api.common.repositories.influxdb."
"metrics_repository.client.InfluxDBClient")
def test_list_dimension_values(self, influxdb_client_mock):
def test_list_dimension_values(self, influxdb_client_mock, timestamp=True):
mock_client = influxdb_client_mock.return_value
tenant = u'38dc2a2549f94d2e9a4fa1cc45a4970c'
region = u'useast'
metric = u'custom_metric'
column = u'hostname'
hostname = u'custom_host'
start_timestamp = 1571917171275
end_timestamp = 1572917171275
mock_client.query.return_value.raw = {
u'series': [
{
u'values': [[u'custom_host']],
u'name': u'custom_metric',
u'columns': [u'hostname']
}]
u'series': [{
u'values': [[hostname]],
u'name': metric,
u'columns': [column]
}]
}
repo = influxdb_repo.MetricsRepository()
mock_client.query.reset_mock()
if timestamp:
result = repo.list_dimension_values(tenant, region, metric, column,
start_timestamp, end_timestamp)
else:
result = repo.list_dimension_values(tenant, region, metric, column)
result = repo.list_dimension_values(
"38dc2a2549f94d2e9a4fa1cc45a4970c",
"useast",
"custom_metric",
"hostname")
self.assertEqual(result, [{u'dimension_value': hostname}])
self.assertEqual(result, [{u'dimension_value': u'custom_host'}])
expected_query = ('show tag values from "{metric}"'
' with key = "{column}"'
' where _tenant_id = \'{tenant}\''
' and _region = \'{region}\' '
.format(tenant=tenant, region=region,
metric=metric, column=column))
expected_query += (' and time >= {start_timestamp}000000u'
' and time < {end_timestamp}000000u'
.format(start_timestamp=start_timestamp,
end_timestamp=end_timestamp)
if timestamp else '')
mock_client.query.assert_called_once_with(expected_query)
mock_client.query.assert_called_once_with(
'show tag values from "custom_metric" with key = "hostname"'
' where _tenant_id = \'{tenant}\''
' and _region = \'{region}\' '.format(tenant='38dc2a2549f94d2e9a4fa1cc45a4970c',
region='useast'))
def test_list_dimension_values_with_timestamp(self):
self.test_list_dimension_values(timestamp=True)
@patch("monasca_api.common.repositories.influxdb."
"metrics_repository.client.InfluxDBClient")
def test_list_dimension_names(self, influxdb_client_mock):
def test_list_dimension_names(self, influxdb_client_mock, timestamp=False):
mock_client = influxdb_client_mock.return_value
tenant = u'38dc2a2549f94d2e9a4fa1cc45a4970c'
region = u'useast'
metric = u'custom_metric'
start_timestamp = 1571917171275
end_timestamp = 1572917171275
mock_client.query.return_value.raw = {
u'series': [{
u'values': [[u'_region'], [u'_tenant_id'], [u'hostname'],
[u'service']],
u'name': u'custom_metric',
u'name': metric,
u'columns': [u'tagKey']
}]
}
repo = influxdb_repo.MetricsRepository()
result = repo.list_dimension_names(
"38dc2a2549f94d2e9a4fa1cc45a4970c",
"useast",
"custom_metric")
mock_client.query.reset_mock()
if timestamp:
result = repo.list_dimension_names(tenant, region, metric,
start_timestamp, end_timestamp)
else:
result = repo.list_dimension_names(tenant, region, metric)
self.assertEqual(result,
[
@ -246,6 +269,21 @@ class TestRepoMetricsInfluxDB(base.BaseTestCase):
{u'dimension_name': u'service'}
])
expected_query = ('show tag keys from "{metric}"'
' where _tenant_id = \'{tenant}\''
' and _region = \'{region}\' '
.format(tenant=tenant, region=region, metric=metric))
expected_query += (' and time >= {start_timestamp}000000u'
' and time < {end_timestamp}000000u'
.format(start_timestamp=start_timestamp,
end_timestamp=end_timestamp)
if timestamp else '')
mock_client.query.assert_called_once_with(expected_query)
def test_list_dimension_names_with_timestamp(self):
self.test_list_dimension_names(timestamp=True)
@patch("monasca_api.common.repositories.influxdb."
"metrics_repository.requests.head")
def test_check_status(self, head_mock):

24
monasca_api/v2/reference/metrics.py
View File

@ -289,18 +289,24 @@ class DimensionValues(metrics_api_v2.DimensionValuesV2API):
dimension_name = helpers.get_query_param(req, 'dimension_name',
required=True)
offset = helpers.get_query_param(req, 'offset')
start_timestamp = helpers.get_query_starttime_timestamp(req, False)
end_timestamp = helpers.get_query_endtime_timestamp(req, False)
result = self._dimension_values(tenant_id, req.uri, metric_name,
dimension_name, offset, req.limit)
dimension_name, offset, req.limit,
start_timestamp, end_timestamp)
res.body = helpers.to_json(result)
res.status = falcon.HTTP_200
def _dimension_values(self, tenant_id, req_uri, metric_name,
dimension_name, offset, limit):
dimension_name, offset, limit, start_timestamp,
end_timestamp):
result = self._metrics_repo.list_dimension_values(tenant_id,
self._region,
metric_name,
dimension_name)
dimension_name,
start_timestamp,
end_timestamp)
return helpers.paginate_with_no_id(result, req_uri, offset, limit)
@ -324,15 +330,21 @@ class DimensionNames(metrics_api_v2.DimensionNamesV2API):
tenant_id = helpers.get_x_tenant_or_tenant_id(req, ['api:delegate'])
metric_name = helpers.get_query_param(req, 'metric_name')
offset = helpers.get_query_param(req, 'offset')
start_timestamp = helpers.get_query_starttime_timestamp(req, False)
end_timestamp = helpers.get_query_endtime_timestamp(req, False)
result = self._dimension_names(tenant_id, req.uri, metric_name,
offset, req.limit)
offset, req.limit,
start_timestamp, end_timestamp)
res.body = helpers.to_json(result)
res.status = falcon.HTTP_200
def _dimension_names(self, tenant_id, req_uri, metric_name, offset, limit):
def _dimension_names(self, tenant_id, req_uri, metric_name, offset, limit,
start_timestamp, end_timestamp):
result = self._metrics_repo.list_dimension_names(tenant_id,
self._region,
metric_name)
metric_name,
start_timestamp,
end_timestamp)
return helpers.paginate_with_no_id(result, req_uri, offset, limit)

7
releasenotes/notes/support-timerange-for-dimension-names-and-values-e5a2ba64700dcd0b.yaml Normal file
View File

@ -0,0 +1,7 @@
---
features:
- |
Dimensions names and values can be scoped by a timerange which can make
dimension related queries to large databases much faster because only the
relevant shards are searched. Users that upgrade their Monasca Grafana
Datasource plugin to version 1.3.0 will benefit from this feature.