Add response handlers to support different response types
Problem description =================== The dynamic pollsters only support APIs that produce JSON responses. Therefore the dynamic pollsters do not support APIs where the response is an XML or not Restful compliant APIs with HTTP 200 within a plain text message on errors. Proposal ======== To allow the dynamic pollsters to support other APIs response formats, we propose to add a response handling that supports multiple response types. It must be configurable in the dynamic pollsters YAML. The default continues to be JSON. Change-Id: I4886cefe06eccac2dc24adbc2fad2166bcbfdd2c
This commit is contained in:
parent
ce52d50c84
commit
225f1cd776
|
@ -49,6 +49,10 @@ class DynamicPollsterDefinitionException(DynamicPollsterException):
|
|||
pass
|
||||
|
||||
|
||||
class InvalidResponseTypeException(DynamicPollsterException):
|
||||
pass
|
||||
|
||||
|
||||
class NonOpenStackApisDynamicPollsterException\
|
||||
(DynamicPollsterDefinitionException):
|
||||
pass
|
||||
|
|
|
@ -18,8 +18,10 @@
|
|||
similar to the idea used for handling notifications.
|
||||
"""
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import xmltodict
|
||||
|
||||
from oslo_log import log
|
||||
|
||||
|
@ -46,6 +48,80 @@ def validate_sample_type(sample_type):
|
|||
% (sample_type, ceilometer_sample.TYPES))
|
||||
|
||||
|
||||
class XMLResponseHandler(object):
|
||||
"""This response handler converts an XML in string format to a dict"""
|
||||
|
||||
@staticmethod
|
||||
def handle(response):
|
||||
return xmltodict.parse(response)
|
||||
|
||||
|
||||
class JsonResponseHandler(object):
|
||||
"""This response handler converts a JSON in string format to a dict"""
|
||||
|
||||
@staticmethod
|
||||
def handle(response):
|
||||
return json.loads(response)
|
||||
|
||||
|
||||
class PlainTextResponseHandler(object):
|
||||
"""This response handler converts a string to a dict {'out'=<string>}"""
|
||||
|
||||
@staticmethod
|
||||
def handle(response):
|
||||
return {'out': str(response)}
|
||||
|
||||
|
||||
VALID_HANDLERS = {
|
||||
'json': JsonResponseHandler,
|
||||
'xml': XMLResponseHandler,
|
||||
'text': PlainTextResponseHandler
|
||||
}
|
||||
|
||||
|
||||
def validate_response_handler(val):
|
||||
if not isinstance(val, list):
|
||||
raise declarative.DynamicPollsterDefinitionException(
|
||||
"Invalid response_handlers configuration. It must be a list. "
|
||||
"Provided value type: %s" % type(val).__name__)
|
||||
|
||||
for value in val:
|
||||
if value not in VALID_HANDLERS:
|
||||
raise declarative.DynamicPollsterDefinitionException(
|
||||
"Invalid response_handler value [%s]. Accepted values "
|
||||
"are [%s]" % (value, ', '.join(list(VALID_HANDLERS))))
|
||||
|
||||
|
||||
class ResponseHandlerChain(object):
|
||||
"""Tries to convert a string to a dict using the response handlers"""
|
||||
|
||||
def __init__(self, response_handlers, **meta):
|
||||
if not isinstance(response_handlers, list):
|
||||
response_handlers = list(response_handlers)
|
||||
|
||||
self.response_handlers = response_handlers
|
||||
self.meta = meta
|
||||
|
||||
def handle(self, response):
|
||||
failed_handlers = []
|
||||
for handler in self.response_handlers:
|
||||
try:
|
||||
return handler.handle(response)
|
||||
except Exception as e:
|
||||
handler_name = handler.__name__
|
||||
failed_handlers.append(handler_name)
|
||||
LOG.debug(
|
||||
"Error handling response [%s] with handler [%s]: %s. "
|
||||
"We will try the next one, if multiple handlers were "
|
||||
"configured.",
|
||||
response, handler_name, e)
|
||||
|
||||
handlers_str = ', '.join(failed_handlers)
|
||||
raise declarative.InvalidResponseTypeException(
|
||||
"No remaining handlers to handle the response [%s], "
|
||||
"used handlers [%s]. [%s]." % (response, handlers_str, self.meta))
|
||||
|
||||
|
||||
class PollsterDefinitionBuilder(object):
|
||||
|
||||
def __init__(self, definitions):
|
||||
|
@ -440,7 +516,9 @@ class PollsterDefinitions(object):
|
|||
PollsterDefinition(name='timeout', default=30),
|
||||
PollsterDefinition(name='extra_metadata_fields_cache_seconds',
|
||||
default=3600),
|
||||
PollsterDefinition(name='extra_metadata_fields')
|
||||
PollsterDefinition(name='extra_metadata_fields'),
|
||||
PollsterDefinition(name='response_handlers', default=['json'],
|
||||
validator=validate_response_handler)
|
||||
]
|
||||
|
||||
extra_definitions = []
|
||||
|
@ -655,6 +733,11 @@ class PollsterSampleGatherer(object):
|
|||
|
||||
def __init__(self, definitions):
|
||||
self.definitions = definitions
|
||||
self.response_handler_chain = ResponseHandlerChain(
|
||||
map(VALID_HANDLERS.get,
|
||||
self.definitions.configurations['response_handlers']),
|
||||
url_path=definitions.configurations['url_path']
|
||||
)
|
||||
|
||||
@property
|
||||
def default_discovery(self):
|
||||
|
@ -668,17 +751,17 @@ class PollsterSampleGatherer(object):
|
|||
resp, url = self._internal_execute_request_get_samples(
|
||||
definitions=definitions, **kwargs)
|
||||
|
||||
response_json = resp.json()
|
||||
entry_size = len(response_json)
|
||||
LOG.debug("Entries [%s] in the JSON for request [%s] "
|
||||
response_dict = self.response_handler_chain.handle(resp.text)
|
||||
entry_size = len(response_dict)
|
||||
LOG.debug("Entries [%s] in the DICT for request [%s] "
|
||||
"for dynamic pollster [%s].",
|
||||
response_json, url, definitions['name'])
|
||||
response_dict, url, definitions['name'])
|
||||
|
||||
if entry_size > 0:
|
||||
samples = self.retrieve_entries_from_response(
|
||||
response_json, definitions)
|
||||
response_dict, definitions)
|
||||
url_to_next_sample = self.get_url_to_next_sample(
|
||||
response_json, definitions)
|
||||
response_dict, definitions)
|
||||
|
||||
self.prepare_samples(definitions, samples, **kwargs)
|
||||
|
||||
|
|
|
@ -14,13 +14,14 @@
|
|||
"""Tests for OpenStack dynamic pollster
|
||||
"""
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
from unittest import mock
|
||||
|
||||
import requests
|
||||
from urllib import parse as urlparse
|
||||
|
||||
from ceilometer.declarative import DynamicPollsterDefinitionException
|
||||
from ceilometer import declarative
|
||||
from ceilometer.polling import dynamic_pollster
|
||||
from ceilometer import sample
|
||||
from oslotest import base
|
||||
|
@ -107,6 +108,11 @@ class TestDynamicPollster(base.BaseTestCase):
|
|||
class FakeResponse(object):
|
||||
status_code = None
|
||||
json_object = None
|
||||
_text = None
|
||||
|
||||
@property
|
||||
def text(self):
|
||||
return self._text or json.dumps(self.json_object)
|
||||
|
||||
def json(self):
|
||||
return self.json_object
|
||||
|
@ -242,9 +248,10 @@ class TestDynamicPollster(base.BaseTestCase):
|
|||
pollster_definition = copy.deepcopy(
|
||||
self.pollster_definition_only_required_fields)
|
||||
pollster_definition.pop(key)
|
||||
exception = self.assertRaises(DynamicPollsterDefinitionException,
|
||||
dynamic_pollster.DynamicPollster,
|
||||
pollster_definition)
|
||||
exception = self.assertRaises(
|
||||
declarative.DynamicPollsterDefinitionException,
|
||||
dynamic_pollster.DynamicPollster,
|
||||
pollster_definition)
|
||||
self.assertEqual("Required fields ['%s'] not specified."
|
||||
% key, exception.brief_message)
|
||||
|
||||
|
@ -252,7 +259,7 @@ class TestDynamicPollster(base.BaseTestCase):
|
|||
self.pollster_definition_only_required_fields[
|
||||
'sample_type'] = "invalid_sample_type"
|
||||
exception = self.assertRaises(
|
||||
DynamicPollsterDefinitionException,
|
||||
declarative.DynamicPollsterDefinitionException,
|
||||
dynamic_pollster.DynamicPollster,
|
||||
self.pollster_definition_only_required_fields)
|
||||
self.assertEqual("Invalid sample type [invalid_sample_type]. "
|
||||
|
@ -313,6 +320,147 @@ class TestDynamicPollster(base.BaseTestCase):
|
|||
|
||||
self.assertEqual(3, len(samples))
|
||||
|
||||
@mock.patch('keystoneclient.v2_0.client.Client')
|
||||
def test_execute_request_json_response_handler(
|
||||
self, client_mock):
|
||||
pollster = dynamic_pollster.DynamicPollster(
|
||||
self.pollster_definition_only_required_fields)
|
||||
|
||||
return_value = self.FakeResponse()
|
||||
return_value.status_code = requests.codes.ok
|
||||
return_value._text = '{"test": [1,2,3]}'
|
||||
|
||||
client_mock.session.get.return_value = return_value
|
||||
|
||||
samples = pollster.definitions.sample_gatherer. \
|
||||
execute_request_get_samples(
|
||||
keystone_client=client_mock,
|
||||
resource="https://endpoint.server.name/")
|
||||
|
||||
self.assertEqual(3, len(samples))
|
||||
|
||||
@mock.patch('keystoneclient.v2_0.client.Client')
|
||||
def test_execute_request_xml_response_handler(
|
||||
self, client_mock):
|
||||
definitions = copy.deepcopy(
|
||||
self.pollster_definition_only_required_fields)
|
||||
definitions['response_handlers'] = ['xml']
|
||||
pollster = dynamic_pollster.DynamicPollster(definitions)
|
||||
|
||||
return_value = self.FakeResponse()
|
||||
return_value.status_code = requests.codes.ok
|
||||
return_value._text = '<test>123</test>'
|
||||
client_mock.session.get.return_value = return_value
|
||||
|
||||
samples = pollster.definitions.sample_gatherer. \
|
||||
execute_request_get_samples(
|
||||
keystone_client=client_mock,
|
||||
resource="https://endpoint.server.name/")
|
||||
|
||||
self.assertEqual(3, len(samples))
|
||||
|
||||
@mock.patch('keystoneclient.v2_0.client.Client')
|
||||
def test_execute_request_xml_json_response_handler(
|
||||
self, client_mock):
|
||||
definitions = copy.deepcopy(
|
||||
self.pollster_definition_only_required_fields)
|
||||
definitions['response_handlers'] = ['xml', 'json']
|
||||
pollster = dynamic_pollster.DynamicPollster(definitions)
|
||||
|
||||
return_value = self.FakeResponse()
|
||||
return_value.status_code = requests.codes.ok
|
||||
return_value._text = '<test>123</test>'
|
||||
client_mock.session.get.return_value = return_value
|
||||
|
||||
samples = pollster.definitions.sample_gatherer. \
|
||||
execute_request_get_samples(
|
||||
keystone_client=client_mock,
|
||||
resource="https://endpoint.server.name/")
|
||||
|
||||
self.assertEqual(3, len(samples))
|
||||
|
||||
return_value._text = '{"test": [1,2,3,4]}'
|
||||
|
||||
samples = pollster.definitions.sample_gatherer. \
|
||||
execute_request_get_samples(
|
||||
keystone_client=client_mock,
|
||||
resource="https://endpoint.server.name/")
|
||||
|
||||
self.assertEqual(4, len(samples))
|
||||
|
||||
@mock.patch('keystoneclient.v2_0.client.Client')
|
||||
def test_execute_request_xml_json_response_handler_invalid_response(
|
||||
self, client_mock):
|
||||
definitions = copy.deepcopy(
|
||||
self.pollster_definition_only_required_fields)
|
||||
definitions['response_handlers'] = ['xml', 'json']
|
||||
pollster = dynamic_pollster.DynamicPollster(definitions)
|
||||
|
||||
return_value = self.FakeResponse()
|
||||
return_value.status_code = requests.codes.ok
|
||||
return_value._text = 'Invalid response'
|
||||
client_mock.session.get.return_value = return_value
|
||||
|
||||
with self.assertLogs('ceilometer.polling.dynamic_pollster',
|
||||
level='DEBUG') as logs:
|
||||
gatherer = pollster.definitions.sample_gatherer
|
||||
exception = self.assertRaises(
|
||||
declarative.InvalidResponseTypeException,
|
||||
gatherer.execute_request_get_samples,
|
||||
keystone_client=client_mock,
|
||||
resource="https://endpoint.server.name/")
|
||||
|
||||
xml_handling_error = logs.output[2]
|
||||
json_handling_error = logs.output[3]
|
||||
|
||||
self.assertIn(
|
||||
'DEBUG:ceilometer.polling.dynamic_pollster:'
|
||||
'Error handling response [Invalid response] '
|
||||
'with handler [XMLResponseHandler]',
|
||||
xml_handling_error)
|
||||
|
||||
self.assertIn(
|
||||
'DEBUG:ceilometer.polling.dynamic_pollster:'
|
||||
'Error handling response [Invalid response] '
|
||||
'with handler [JsonResponseHandler]',
|
||||
json_handling_error)
|
||||
|
||||
self.assertEqual(
|
||||
"InvalidResponseTypeException None: "
|
||||
"No remaining handlers to handle the response "
|
||||
"[Invalid response], used handlers "
|
||||
"[XMLResponseHandler, JsonResponseHandler]. "
|
||||
"[{'url_path': 'v1/test/endpoint/fake'}].",
|
||||
str(exception))
|
||||
|
||||
def test_configure_response_handler_definition_invalid_value(self):
|
||||
definitions = copy.deepcopy(
|
||||
self.pollster_definition_only_required_fields)
|
||||
definitions['response_handlers'] = ['jason']
|
||||
|
||||
exception = self.assertRaises(
|
||||
declarative.DynamicPollsterDefinitionException,
|
||||
dynamic_pollster.DynamicPollster,
|
||||
pollster_definitions=definitions)
|
||||
self.assertEqual("DynamicPollsterDefinitionException None: "
|
||||
"Invalid response_handler value [jason]. "
|
||||
"Accepted values are [json, xml, text]",
|
||||
str(exception))
|
||||
|
||||
def test_configure_response_handler_definition_invalid_type(self):
|
||||
definitions = copy.deepcopy(
|
||||
self.pollster_definition_only_required_fields)
|
||||
definitions['response_handlers'] = 'json'
|
||||
|
||||
exception = self.assertRaises(
|
||||
declarative.DynamicPollsterDefinitionException,
|
||||
dynamic_pollster.DynamicPollster,
|
||||
pollster_definitions=definitions)
|
||||
self.assertEqual("DynamicPollsterDefinitionException None: "
|
||||
"Invalid response_handlers configuration. "
|
||||
"It must be a list. Provided value type: str",
|
||||
str(exception))
|
||||
|
||||
@mock.patch('keystoneclient.v2_0.client.Client')
|
||||
def test_execute_request_get_samples_exception_on_request(
|
||||
self, client_mock):
|
||||
|
@ -728,6 +876,10 @@ class TestDynamicPollster(base.BaseTestCase):
|
|||
|
||||
def internal_execute_request_get_samples_mock(self, **kwargs):
|
||||
class Response:
|
||||
@property
|
||||
def text(self):
|
||||
return json.dumps([sample])
|
||||
|
||||
def json(self):
|
||||
return [sample]
|
||||
return Response(), "url"
|
||||
|
|
|
@ -15,6 +15,7 @@
|
|||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
from unittest import mock
|
||||
|
||||
|
@ -312,6 +313,11 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
|
|||
def internal_execute_request_get_samples_mock(
|
||||
self, definitions, **kwargs):
|
||||
class Response:
|
||||
|
||||
@property
|
||||
def text(self):
|
||||
return json.dumps([sample])
|
||||
|
||||
def json(self):
|
||||
return [sample]
|
||||
return Response(), "url"
|
||||
|
|
|
@ -45,7 +45,7 @@ attributes to define a dynamic pollster:
|
|||
the unit or some other meaningful String value;
|
||||
|
||||
* ``value_attribute``: mandatory attribute; defines the attribute in the
|
||||
JSON response from the URL of the component being polled. We also accept
|
||||
response from the URL of the component being polled. We also accept
|
||||
nested values dictionaries. To use a nested value one can simply use
|
||||
``attribute1.attribute2.<asMuchAsNeeded>.lastattribute``. It is also
|
||||
possible to reference the sample itself using ``"." (dot)``; the self
|
||||
|
@ -281,6 +281,117 @@ desires):
|
|||
name: "display_name"
|
||||
default_value: 0
|
||||
|
||||
* ``response_handlers``: optional parameter. Defines the response
|
||||
handlers used to handle the response. For now, the supported values
|
||||
are:
|
||||
|
||||
``json``: This handler will interpret the response as a `JSON` and will
|
||||
convert it to a `dictionary` which can be manipulated using the
|
||||
operations options when mapping the attributes:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
---
|
||||
|
||||
- name: "dynamic.json.response"
|
||||
sample_type: "gauge"
|
||||
[...]
|
||||
response_handlers:
|
||||
- json
|
||||
|
||||
Response to handle:
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
{
|
||||
"test": {
|
||||
"list": [1, 2, 3]
|
||||
}
|
||||
}
|
||||
|
||||
Response handled:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
{
|
||||
'test': {
|
||||
'list': [1, 2, 3]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
``xml``: This handler will interpret the response as an `XML` and will
|
||||
convert it to a `dictionary` which can be manipulated using the
|
||||
operations options when mapping the attributes:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
---
|
||||
|
||||
- name: "dynamic.json.response"
|
||||
sample_type: "gauge"
|
||||
[...]
|
||||
response_handlers:
|
||||
- xml
|
||||
|
||||
Response to handle:
|
||||
|
||||
.. code-block:: xml
|
||||
|
||||
<test>
|
||||
<list>1</list>
|
||||
<list>2</list>
|
||||
<list>3</list>
|
||||
</test>
|
||||
|
||||
Response handled:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
{
|
||||
'test': {
|
||||
'list': [1, 2, 3]
|
||||
}
|
||||
}
|
||||
|
||||
``text``: This handler will interpret the response as a `PlainText` and
|
||||
will convert it to a `dictionary` which can be manipulated using the
|
||||
operations options when mapping the attributes:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
---
|
||||
|
||||
- name: "dynamic.json.response"
|
||||
sample_type: "gauge"
|
||||
[...]
|
||||
response_handlers:
|
||||
- text
|
||||
|
||||
Response to handle:
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
Plain text response
|
||||
|
||||
Response handled:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
{
|
||||
'out': "Plain text response"
|
||||
}
|
||||
|
||||
They can be used together or individually. If not defined, the
|
||||
`default` value will be `json`. If you set 2 or more response
|
||||
handlers, the first configured handler will be used to try to
|
||||
handle the response, if it is not possible, a `DEBUG` log
|
||||
message will be displayed, then the next will be used
|
||||
and so on. If no configured handler was able to handle
|
||||
the response, an empty dict will be returned and a `WARNING`
|
||||
log will be displayed to warn operators that the response was
|
||||
not able to be handled by any configured handler.
|
||||
|
||||
The dynamic pollsters system configuration (for non-OpenStack APIs)
|
||||
-------------------------------------------------------------------
|
||||
|
||||
|
|
|
@ -6,6 +6,7 @@
|
|||
# of appearance. Changing the order has an impact on the overall integration
|
||||
# process, which may cause wedges in the gate later.
|
||||
|
||||
xmltodict>=0.13.0 # MIT License
|
||||
cachetools>=2.1.0 # MIT License
|
||||
cotyledon>=1.3.0 #Apache-2.0
|
||||
futurist>=1.8.0 # Apache-2.0
|
||||
|
|
Loading…
Reference in New Issue