openstack
/
gnocchi
Code Issues Proposed changes

add support to process measures on GET 

depending on the backlog, the returned measures from get-measures
may not include what sits in carbonara backlog. to allow for more
flexibility on metricd deployments, this patch offers the ability
to allow users to get all measures even if they don't have enough
metricd workers, at the expense of responsiveness.

refresh blocks until it can process measures

Change-Id: I588ae6879474d780e8ec9e893d4ecc2b367b832e
Closes-Bug: #1603495
This commit is contained in:
gord chung 2016-08-05 09:47:09 -04:00
parent 2df8f95f6e
commit 72a2091727
5 changed files with 85 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
doc/source/rest.j2
View File

@ -73,6 +73,17 @@ endpoint:
{{ scenarios['get-measures']['doc'] }}
Depending on the driver, there may be some lag after POSTing measures before
they are processed and queryable. To ensure your query returns all measures
that have been POSTed, you can force any unprocessed measures to be handled:
{{ scenarios['get-measures-refresh']['doc'] }}
.. note::
Depending on the amount of data that is unprocessed, `refresh` may add
some overhead to your query.
The list of points returned is composed of tuples with (timestamp, granularity,
value) sorted by timestamp. The granularity is the timespan covered by
aggregation for this point.
@ -474,6 +485,10 @@ requested resource type, and the compute the aggregation:
{{ scenarios['get-across-metrics-measures-by-attributes-lookup-groupby']['doc'] }}
Similar to retrieving measures for a single metric, the `refresh` parameter
can be provided to force all POSTed measures to be processed across all
metrics before computing the result.
Also aggregation across metrics have different behavior depending
on if boundary are set ('start' and 'stop') and if 'needed_overlap' is set.

3
doc/source/rest.yaml
View File

@ -214,6 +214,9 @@
- name: get-measures-granularity
request: GET /v1/metric/{{ scenarios['create-metric']['response'].json['id'] }}/measures?granularity=1 HTTP/1.1
- name: get-measures-refresh
request: GET /v1/metric/{{ scenarios['create-metric']['response'].json['id'] }}/measures?refresh=true HTTP/1.1
- name: create-resource-generic
request: |
POST /v1/resource/generic HTTP/1.1

45
gnocchi/rest/__init__.py
View File

@ -449,7 +449,7 @@ class MetricController(rest.RestController):
@pecan.expose('json')
def get_measures(self, start=None, stop=None, aggregation='mean',
granularity=None, **param):
granularity=None, refresh=False, **param):
self.enforce_metric("get measures")
if not (aggregation
in archive_policy.ArchivePolicy.VALID_AGGREGATION_METHODS
@ -473,6 +473,10 @@ class MetricController(rest.RestController):
except Exception:
abort(400, "Invalid value for stop")
if strutils.bool_from_string(refresh):
pecan.request.storage.process_new_measures(
pecan.request.indexer, [six.text_type(self.metric.id)], True)
try:
if aggregation in self.custom_agg:
measures = self.custom_agg[aggregation].compute(
@ -1244,9 +1248,8 @@ class AggregationResourceController(rest.RestController):
@pecan.expose('json')
def post(self, start=None, stop=None, aggregation='mean',
reaggregation=None,
granularity=None, needed_overlap=100.0,
groupby=None):
reaggregation=None, granularity=None, needed_overlap=100.0,
groupby=None, refresh=False):
# First, set groupby in the right format: a sorted list of unique
# strings.
groupby = sorted(set(arg_to_list(groupby)))
@ -1270,7 +1273,7 @@ class AggregationResourceController(rest.RestController):
for r in resources)))
return AggregationController.get_cross_metric_measures_from_objs(
metrics, start, stop, aggregation, reaggregation,
granularity, needed_overlap)
granularity, needed_overlap, refresh)
def groupper(r):
return tuple((attr, r[attr]) for attr in groupby)
@ -1284,7 +1287,7 @@ class AggregationResourceController(rest.RestController):
"group": dict(key),
"measures": AggregationController.get_cross_metric_measures_from_objs( # noqa
metrics, start, stop, aggregation, reaggregation,
granularity, needed_overlap)
granularity, needed_overlap, refresh)
})
return results
@ -1314,7 +1317,8 @@ class AggregationController(rest.RestController):
aggregation='mean',
reaggregation=None,
granularity=None,
needed_overlap=100.0):
needed_overlap=100.0,
refresh=False):
try:
needed_overlap = float(needed_overlap)
except ValueError:
@ -1344,14 +1348,18 @@ class AggregationController(rest.RestController):
enforce("get metric", metric)
number_of_metrics = len(metrics)
if number_of_metrics == 0:
return []
if granularity is not None:
try:
granularity = float(granularity)
except ValueError as e:
abort(400, "granularity must be a float: %s" % e)
try:
if number_of_metrics == 0:
return []
if granularity is not None:
try:
granularity = float(granularity)
except ValueError as e:
abort(400, "granularity must be a float: %s" % e)
if strutils.bool_from_string(refresh):
pecan.request.storage.process_new_measures(
pecan.request.indexer,
[six.text_type(m.id) for m in metrics], True)
if number_of_metrics == 1:
# NOTE(sileht): don't do the aggregation if we only have one
# metric
@ -1376,10 +1384,9 @@ class AggregationController(rest.RestController):
abort(404, e)
@pecan.expose('json')
def get_metric(self, metric=None, start=None,
stop=None, aggregation='mean',
reaggregation=None,
granularity=None, needed_overlap=100.0):
def get_metric(self, metric=None, start=None, stop=None,
aggregation='mean', reaggregation=None, granularity=None,
needed_overlap=100.0, refresh=False):
# Check RBAC policy
metric_ids = arg_to_list(metric)
metrics = pecan.request.indexer.list_metrics(ids=metric_ids)
@ -1391,7 +1398,7 @@ class AggregationController(rest.RestController):
missing_metric_ids.pop()))
return self.get_cross_metric_measures_from_objs(
metrics, start, stop, aggregation, reaggregation,
granularity, needed_overlap)
granularity, needed_overlap, refresh)
class CapabilityController(rest.RestController):

20
gnocchi/tests/gabbi/gabbits-live/live.yaml
View File

@ -617,6 +617,26 @@ tests:
$[0][2]: 2
$[1][2]: 2
- name: post some more measures to the metric on myresource
POST: /v1/resource/myresource/2ae35573-7f9f-4bb1-aae8-dad8dff5706e/metric/vcpus/measures
request_headers:
content-type: application/json
data:
- timestamp: "2015-03-06T14:34:15"
value: 5
- timestamp: "2015-03-06T14:34:20"
value: 5
status: 202
- name: get myresource measures with refresh
GET: /v1/resource/myresource/2ae35573-7f9f-4bb1-aae8-dad8dff5706e/metric/vcpus/measures?refresh=true
response_json_paths:
$[0][2]: 2
$[1][2]: 4
$[2][2]: 2
$[3][2]: 2
$[4][2]: 5
$[5][2]: 5
#
# Search for resources

21
gnocchi/tests/gabbi/gabbits/aggregation.yaml
View File

@ -68,6 +68,16 @@ tests:
GET: /v1/aggregation/metric?metric=$RESPONSE['$[0].id']&metric=$RESPONSE['$[1].id']&granularity=foobar
status: 400
- name: get metric list to get aggregates for get with refresh
GET: /v1/metric
- name: get measure aggregates by granularity with refresh
GET: /v1/aggregation/metric?metric=$RESPONSE['$[0].id']&metric=$RESPONSE['$[1].id']&granularity=1&refresh=true
response_json_paths:
$:
- ['2015-03-06T14:33:57+00:00', 1.0, 23.1]
- ['2015-03-06T14:34:12+00:00', 1.0, 7.0]
- name: get metric list to get aggregates 2
GET: /v1/metric
@ -162,6 +172,17 @@ tests:
value: 2
status: 202
- name: get measure aggregates by granularity from resources with refresh
POST: /v1/aggregation/resource/generic/metric/agg_meter?granularity=1&refresh=true
request_headers:
x-user-id: 0fbb231484614b1a80131fc22f6afc9c
x-project-id: f3d41b770cc14f0bb94a1d5be9c0e3ea
content-type: application/json
response_json_paths:
$:
- ['2015-03-06T14:33:57+00:00', 1.0, 23.1]
- ['2015-03-06T14:34:12+00:00', 1.0, 7.0]
- name: get measure aggregates by granularity from resources
POST: /v1/aggregation/resource/generic/metric/agg_meter?granularity=1
request_headers: