Remove id from monasca metric name list

Since returned metric names are distinct, we can remove the id from
each element. Also change the returned metric name into alphabetical
order.

Change-Id: Id981dafd00778a6d4a376b9ceab011231e94c0c6
This commit is contained in:
Kaiyan Sheng 2016-09-12 14:17:54 -06:00
parent 1e56736bba
commit 55549ff54c
9 changed files with 271 additions and 149 deletions

View File

@ -1,3 +1,4 @@
# Monasca API
Date: November 5, 2014
@ -1352,7 +1353,7 @@ None.
#### Query Parameters
* tenant_id (string, optional, restricted) - Tenant ID from which to get metric names. This parameter can be used to get metric names from a tenant other than the tenant the request auth token is scoped to. Usage of this query parameter is restricted to users with the monasca admin role, as defined in the monasca api configuration file, which defaults to `monasca-admin`.
* dimensions (string, optional) - A dictionary to filter metrics by specified as a comma separated array of (key, value) pairs as `key1:value1,key2:value2, ...`
* offset (integer, optional)
* offset (string, optional)
* limit (integer, optional)
#### Request Body
@ -1372,31 +1373,37 @@ Cache-Control: no-cache
* 200 - OK
#### Response Body
Returns a JSON object with a 'links' array of links and an 'elements' array of metric name objects for each unique metric name (not including dimensions) with the following fields:
Returns a JSON object with a 'links' array of links and an 'elements' array of metric name objects for each unique metric name (not including dimensions) in alphabetical order with the following fields:
* name (string(255)) - A name of a metric.
#### Response Examples
```
{
"elements": [
{
"name":"name1"
},
{
"name":"name2"
}
],
"links": [
{
"rel": "self",
"href": "http://192.168.10.4:8070/v2.0/metrics/names?offset=tenantId%3region%26name1%26dimensionKey1%3DdimensionValue1%26dimensionKey2%3DdimensionValue2"
},
{
"rel": "next"
"href": http://192.168.10.4:8070/v2.0/metrics/names?offset=tenantId%3region%26name3%26dimensionKey1%3DdimensionValue1%26dimensionKey2%3DdimensionValue2
}
]
"elements": [
{
"name": "cpu.idle_perc"
},
{
"name": "cpu.idle_time"
},
{
"name": "cpu.percent"
},
{
"name": "cpu.stolen_perc"
}
],
"links": [
{
"href": "http://192.168.10.6:8070/v2.0/metrics/names?offset=cpu.frequency_mhz&limit=4",
"rel": "self"
},
{
"href": "http://192.168.10.6:8070/v2.0/metrics/names?offset=cpu.stolen_perc&limit=4",
"rel": "next"
}
]
}
```
___

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
* (C) Copyright 2014, 2016 Hewlett Packard Enterprise Development LP
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@ -13,15 +13,18 @@
*/
package monasca.api.domain.model.metric;
import com.fasterxml.jackson.annotation.JsonIgnore;
import monasca.common.model.domain.common.AbstractEntity;
public class MetricName extends AbstractEntity implements Comparable<MetricName> {
private String id;
private String name;
public MetricName(String id, String name) {
this.id = id;
public MetricName(String name) {
this.id = name;
this.name = name;
}
@ -54,6 +57,7 @@ public class MetricName extends AbstractEntity implements Comparable<MetricName>
return true;
}
@JsonIgnore
public String getId() {return id;}
public String getName() {return name;}
@ -61,14 +65,12 @@ public class MetricName extends AbstractEntity implements Comparable<MetricName>
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
int result = 17;
result = prime * result + ((id == null) ? 0 : id.hashCode());
result = prime * result + ((name == null) ? 0 : name.hashCode());
return result;
}
public void setId(String id) {this.id = id;}
public void setName(String name) {this.name = name;}
@Override

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
* (C) Copyright 2014, 2016 Hewlett Packard Enterprise Development LP
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@ -13,6 +13,7 @@
*/
package monasca.api.infrastructure.persistence.influxdb;
import com.google.common.base.Strings;
import com.google.inject.Inject;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -22,12 +23,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
import java.util.Set;
import monasca.api.ApiConfig;
import monasca.api.domain.model.measurement.Measurements;
import monasca.api.domain.model.metric.MetricDefinitionRepo;
import monasca.api.domain.model.metric.MetricName;
import monasca.common.model.metric.MetricDefinition;
@ -124,30 +125,60 @@ public class InfluxV9MetricDefinitionRepo implements MetricDefinitionRepo {
@Override
public List<MetricName> findNames(String tenantId, Map<String, String> dimensions,
String offset, int limit) throws Exception {
//
// Use treeset to keep list in alphabetic/predictable order
// for string based offset.
//
List<MetricName> metricNameList = new ArrayList<>();
Set<String> matchingNames = new TreeSet<>();
int startIndex = this.influxV9Utils.startIndex(offset);
String q = String.format("show measurements "
+ "where %1$s %2$s %3$s %4$s %5$s",
String q = String.format("show series "
+ "where %1$s %2$s %3$s",
this.influxV9Utils.privateTenantIdPart(tenantId),
this.influxV9Utils.privateRegionPart(this.region),
this.influxV9Utils.dimPart(dimensions),
this.influxV9Utils.limitPart(limit),
this.influxV9Utils.offsetPart(startIndex));
this.influxV9Utils.dimPart(dimensions));
logger.debug("Metric name query: {}", q);
String r = this.influxV9RepoReader.read(q);
Series series = this.objectMapper.readValue(r, Series.class);
if (!series.isEmpty()) {
for (Serie serie : series.getSeries()) {
matchingNames.add(serie.getName());
}
}
List<MetricName> metricNameList = metricNameList(series, startIndex);
List<String> filteredNames = filterMetricNames(matchingNames, limit, offset);
for (String filteredName : filteredNames) {
MetricName dimName = new MetricName(filteredName);
metricNameList.add(dimName);
}
logger.debug("Found {} metric definitions matching query", metricNameList.size());
return metricNameList;
}
private List<String> filterMetricNames(Set<String> matchingNames,
int limit,
String offset) {
Boolean haveOffset = !Strings.isNullOrEmpty(offset);
List<String> filteredNames = new ArrayList<>();
int remaining_limit = limit + 1;
for (String dimName : matchingNames) {
if (remaining_limit <= 0) {
break;
}
if (haveOffset && dimName.compareTo(offset) <= 0) {
continue;
}
filteredNames.add(dimName);
remaining_limit--;
}
return filteredNames;
}
private List<MetricDefinition> metricDefinitionList(Series series,
String tenantId,
String name,
@ -183,18 +214,15 @@ public class InfluxV9MetricDefinitionRepo implements MetricDefinitionRepo {
return metricDefinitionList;
}
private List<MetricName> metricNameList(Series series, int startIndex) {
private List<MetricName> metricNameList(Series series) {
List<MetricName> metricNameList = new ArrayList<>();
if (!series.isEmpty()) {
int index = startIndex;
Serie serie = series.getSeries()[0];
for (String[] values : serie.getValues()) {
MetricName m =
new MetricName(String.valueOf(index++), values[0]);
MetricName m = new MetricName(values[0]);
metricNameList.add(m);
}
@ -236,7 +264,7 @@ public class InfluxV9MetricDefinitionRepo implements MetricDefinitionRepo {
// checking if there are current measurements, default to
// existing behavior and return the definition.
//
logger.error("Failed to query for measuremnts for: {}", m.name, e);
logger.error("Failed to query for measurements for: {}", m.name, e);
hasMeasurements = true;
}

View File

@ -1,5 +1,5 @@
/*
* (C) Copyright 2014,2016 Hewlett Packard Enterprise Development Company LP
* (C) Copyright 2014, 2016 Hewlett Packard Enterprise Development LP
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
@ -56,21 +56,12 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo {
+ "%s "; // limit goes here
private static final String FIND_METRIC_NAMES_SQL =
"SELECT %s distinct def.id, def.name "
"SELECT distinct def.name "
+ "FROM MonMetrics.Definitions def "
+ "WHERE def.id IN (%s) " // Subselect goes here
+ "ORDER BY def.id ASC ";
private static final String METRIC_NAMES_SUB_SELECT =
"SELECT distinct MAX(defSub.id) as max_id " // The aggregation function gives us one id per name
+ "FROM MonMetrics.Definitions defSub "
+ "JOIN MonMetrics.DefinitionDimensions defDimsSub ON defDimsSub.definition_id = defSub.id "
+ "WHERE defSub.tenant_id = :tenantId "
+ "%s " // Offset goes here.
+ "WHERE def.tenant_id = :tenantId " // tenantId
+ "%s " // optional offset goes here
+ "%s " // Dimensions and clause goes here
+ "GROUP BY defSub.name " // This is to reduce the (id, name) sets to only include unique names
+ "ORDER BY max_id ASC %s"; // Limit goes here.
+ "ORDER BY def.name ASC %s "; // Limit goes here.
private static final String TABLE_TO_JOIN_ON = "defDimsSub";
@ -98,11 +89,9 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo {
for (Map<String, Object> row : rows) {
byte[] defId = (byte[]) row.get("id");
String name = (String) row.get("name");
MetricName metricName = new MetricName(Hex.encodeHexString(defId), name);
MetricName metricName = new MetricName(name);
metricNameList.add(metricName);
@ -122,41 +111,19 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo {
if (offset != null && !offset.isEmpty()) {
offsetPart = " and defSub.id > :offset ";
offsetPart = " and def.name > '" + offset + "' ";
}
// Can't bind limit in a nested sub query. So, just tack on as String.
String limitPart = " limit " + Integer.toString(limit + 1);
String defSubSelect =
String.format(METRIC_NAMES_SUB_SELECT,
offsetPart,
MetricQueries.buildDimensionAndClause(dimensions,
TABLE_TO_JOIN_ON),
limitPart);
String sql = String.format(FIND_METRIC_NAMES_SQL, this.dbHint, defSubSelect);
String sql = String.format(FIND_METRIC_NAMES_SQL, this.dbHint, offsetPart, limitPart);
try (Handle h = db.open()) {
Query<Map<String, Object>> query = h.createQuery(sql).bind("tenantId", tenantId);
if (offset != null && !offset.isEmpty()) {
logger.debug("binding offset: {}", offset);
try {
query.bind("offset", Hex.decodeHex(offset.toCharArray()));
} catch (DecoderException e) {
throw Exceptions.badRequest("failed to decode offset " + offset, e);
}
}
MetricQueries.bindDimensionsToQuery(query, dimensions);
return query.list();

View File

@ -296,17 +296,11 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
return json_metric_list
if 'series' in series_names.raw:
id = 0
for series in series_names.raw['series']:
id += 1
name = {u'id': str(id),
u'name': series[u'name']}
name = {u'name': series[u'name']}
json_metric_list.append(name)
json_metric_list = sorted(json_metric_list)
return json_metric_list
def _get_dimensions(self, tenant_id, region, name, dimensions):
@ -399,22 +393,14 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
raise exceptions.RepositoryException(ex)
def list_metric_names(self, tenant_id, region, dimensions, offset, limit):
def list_metric_names(self, tenant_id, region, dimensions):
try:
query = self._build_show_series_query(dimensions, None, tenant_id,
region)
query += " limit {}".format(limit + 1)
if offset:
query += ' offset {}'.format(int(offset) + 1)
result = self.influxdb_client.query(query)
json_name_list = self._build_serie_name_list(result)
return json_name_list
except Exception as ex:

View File

@ -35,7 +35,7 @@ class AbstractMetricsRepository(object):
pass
@abc.abstractmethod
def list_metric_names(self, tenant_id, region, dimensions, offset, limit):
def list_metric_names(self, tenant_id, region, dimensions):
pass
@abc.abstractmethod

View File

@ -1,6 +1,5 @@
# Copyright 2014 Hewlett-Packard
# Copyright 2015 Cray Inc. All Rights Reserved.
# Copyright 2016 Hewlett Packard Enterprise Development Company LP
# Copyright 2014, 2016 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -355,6 +354,72 @@ def paginate(resource, uri, limit):
return resource
def paginate_with_no_id(dictionary_list, uri, offset, limit):
"""This method is to paginate a list of dictionaries with no id in it.
For example, metric name list, directory name list and directory
value list.
"""
parsed_uri = urlparse.urlparse(uri)
self_link = build_base_uri(parsed_uri)
old_query_params = _get_old_query_params(parsed_uri)
if old_query_params:
self_link += '?' + '&'.join(old_query_params)
value_list = []
for item in dictionary_list:
value_list.extend(item.values())
if value_list:
# Truncate dictionary list with offset first
truncated_list_offset = _truncate_with_offset(
dictionary_list, value_list, offset)
# Then truncate it with limit
truncated_list_offset_limit = truncated_list_offset[:limit]
links = [{u'rel': u'self', u'href': self_link.decode('utf8')}]
if len(truncated_list_offset) > limit:
new_offset = truncated_list_offset_limit[limit - 1].values()[0]
next_link = build_base_uri(parsed_uri)
new_query_params = [u'offset' + '=' + new_offset]
_get_old_query_params_except_offset(new_query_params, parsed_uri)
if new_query_params:
next_link += '?' + '&'.join(new_query_params)
links.append({u'rel': u'next', u'href': next_link.decode('utf8')})
resource = {u'links': links,
u'elements': truncated_list_offset_limit}
else:
resource = {u'links': ([{u'rel': u'self',
u'href': self_link.decode('utf8')}]),
u'elements': dictionary_list}
return resource
def _truncate_with_offset(resource, value_list, offset):
"""Truncate a list of dictionaries with a given offset.
"""
if not offset:
return resource
offset = offset.lower()
for i, j in enumerate(value_list):
# if offset matches one of the values in value_list,
# the truncated list should start with the one after current offset
if j == offset:
return resource[i + 1:]
# if offset does not exist in value_list, find the nearest
# location and truncate from that location.
if j > offset:
return resource[i:]
return []
def paginate_alarming(resource, uri, limit):
parsed_uri = urlparse.urlparse(uri)

View File

@ -302,10 +302,9 @@ class MetricsNames(metrics_api_v2.MetricsNamesV2API):
result = self._metrics_repo.list_metric_names(tenant_id,
self._region,
dimensions,
offset, limit)
dimensions)
return helpers.paginate(result, req_uri, limit)
return helpers.paginate_with_no_id(result, req_uri, offset, limit)
class DimensionValues(metrics_api_v2.DimensionValuesV2API):

View File

@ -1,4 +1,4 @@
# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP
# (C) Copyright 2015-2016 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -14,13 +14,12 @@
import time
from oslo_utils import timeutils
from monasca_tempest_tests.tests.api import base
from monasca_tempest_tests.tests.api import constants
from monasca_tempest_tests.tests.api import helpers
from tempest.common.utils import data_utils
from tempest import test
from urllib import urlencode
class TestMetricsNames(base.BaseMonascaTest):
@ -28,27 +27,41 @@ class TestMetricsNames(base.BaseMonascaTest):
@classmethod
def resource_setup(cls):
super(TestMetricsNames, cls).resource_setup()
name = data_utils.rand_name()
name1 = data_utils.rand_name('name1')
name2 = data_utils.rand_name('name2')
name3 = data_utils.rand_name('name3')
key = data_utils.rand_name()
key1 = data_utils.rand_name()
value = data_utils.rand_name()
cls._param = key + ':' + value
metric = helpers.create_metric(name=name,
dimensions={key: value})
cls._test_metric = metric
cls.monasca_client.create_metrics(metric)
value1 = data_utils.rand_name()
start_time = str(timeutils.iso8601_from_timestamp(
metric['timestamp'] / 1000.0))
query_params = '?name=' + str(cls._test_metric['name']) +\
'&start_time=' + start_time
timestamp = int(round(time.time() * 1000))
time_iso = helpers.timestamp_to_iso(timestamp)
metric1 = helpers.create_metric(name=name1,
dimensions={key: value})
metric2 = helpers.create_metric(name=name2,
dimensions={key1: value1})
metric3 = helpers.create_metric(name=name3,
dimensions={key: value})
cls._test_metric_names = {name1, name2, name3}
cls._expected_names_list = list(cls._test_metric_names)
cls._expected_names_list.sort()
cls._test_metric_names_with_same_dim = [name1, name3]
cls._test_metrics = [metric1, metric2, metric3]
cls._dimensions_param = key + ':' + value
cls.monasca_client.create_metrics(cls._test_metrics)
query_param = '?start_time=' + time_iso
returned_name_set = set()
for i in xrange(constants.MAX_RETRIES):
resp, response_body = cls.monasca_client.list_metrics(
query_params)
resp, response_body = cls.monasca_client.list_metrics(query_param)
elements = response_body['elements']
for element in elements:
if str(element['name']) == cls._test_metric['name']:
return
returned_name_set.add(str(element['name']))
if cls._test_metric_names.issubset(returned_name_set):
return
time.sleep(constants.RETRY_WAIT_SECS)
assert False, 'Unable to initialize metrics'
@ -60,35 +73,90 @@ class TestMetricsNames(base.BaseMonascaTest):
@test.attr(type='gate')
def test_list_metrics_names(self):
resp, response_body = self.monasca_client.list_metrics_names()
self.assertEqual(200, resp.status)
self.assertTrue(set(['links', 'elements']) == set(response_body))
if self._is_name_in_list(response_body):
return
self.fail('Metric name not found')
metric_names = self._verify_response(resp, response_body)
self.assertEqual(metric_names, self._expected_names_list)
@test.attr(type='gate')
def test_list_metrics_names_with_dimensions(self):
query_params = '?dimensions=' + self._param
query_params = '?dimensions=' + self._dimensions_param
resp, response_body = self.monasca_client.list_metrics_names(
query_params)
self.assertEqual(200, resp.status)
self.assertTrue(set(['links', 'elements']) == set(response_body))
if self._is_name_in_list(response_body):
return
self.fail('Metric name not found')
metric_names = self._verify_response(resp, response_body)
self.assertEqual(metric_names,
self._test_metric_names_with_same_dim)
@test.attr(type='gate')
def test_list_metrics_names_with_limit_offset(self):
# Can not test list_metrics_names_with_limit_offset for now because
# list_metrics_names returns a list of metric names with no
# duplicates. But the limit and offset are using the original list
# with duplicates as reference.
self.skipException('list_metrics_names_with_limit_offset')
def _is_name_in_list(self, response_body):
resp, response_body = self.monasca_client.list_metrics_names()
self.assertEqual(200, resp.status)
elements = response_body['elements']
for element in elements:
self.assertTrue(set(['id', 'name']) == set(element))
if str(element['name']) == self._test_metric['name']:
return True
return False
num_names = len(elements)
for limit in xrange(1, num_names):
start_index = 0
params = [('limit', limit)]
offset = None
while True:
num_expected_elements = limit
if (num_expected_elements + start_index) > num_names:
num_expected_elements = num_names - start_index
these_params = list(params)
# If not the first call, use the offset returned by the last
# call
if offset:
these_params.extend([('offset', str(offset))])
query_params = '?' + urlencode(these_params)
resp, response_body = \
self.monasca_client.list_metrics_names(query_params)
new_elements = self._verify_response(resp, response_body)
self.assertEqual(num_expected_elements, len(new_elements))
expected_elements = elements[start_index:start_index+limit]
expected_names = \
[expected_elements[i]['name'] for i in xrange(
len(expected_elements))]
self.assertEqual(expected_names, new_elements)
start_index += num_expected_elements
if start_index >= num_names:
break
# Get the next set
offset = self._get_offset(response_body)
@test.attr(type='gate')
def test_list_metrics_names_with_offset_not_in_metrics_names_list(self):
offset1 = 'tempest-abc'
offset2 = 'tempest-name111'
offset3 = 'tempest-name4-random'
query_param1 = '?' + urlencode([('offset', offset1)])
query_param2 = '?' + urlencode([('offset', offset2)])
query_param3 = '?' + urlencode([('offset', offset3)])
resp, response_body = self.monasca_client.list_metrics_names(
query_param1)
metric_names = self._verify_response(resp, response_body)
self.assertEqual(metric_names, self._expected_names_list[:])
resp, response_body = self.monasca_client.list_metrics_names(
query_param2)
metric_names = self._verify_response(resp, response_body)
self.assertEqual(metric_names, self._expected_names_list[1:])
resp, response_body = self.monasca_client.list_metrics_names(
query_param3)
self.assertEqual(response_body['elements'], [])
def _verify_response(self, resp, response_body):
self.assertEqual(200, resp.status)
self.assertTrue(set(['links', 'elements']) == set(response_body))
response_names_length = len(response_body['elements'])
if response_names_length == 0:
self.fail("No metric names returned")
metric_names = [str(response_body['elements'][i]['name']) for i in
xrange(response_names_length)]
return metric_names