diff --git a/docs/api/index.rst b/docs/api/index.rst index a389b91..3e6e72d 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -9,6 +9,7 @@ Classes and Functions cookies status errors + media redirects middleware hooks diff --git a/docs/api/media.rst b/docs/api/media.rst new file mode 100644 index 0000000..412726e --- /dev/null +++ b/docs/api/media.rst @@ -0,0 +1,138 @@ +.. _media: + +Media +===== + +Falcon allows for easy and customizable internet media type handling. By default +Falcon only enables a single JSON handler. However, additional handlers +can be configured through the :any:`falcon.RequestOptions` and +:any:`falcon.ResponseOptions` objects specified on your :any:`falcon.API`. + +.. note:: + + To avoid unnecessary overhead, Falcon will only process request media + the first time the media property is referenced. Once it has been + referenced, it'll use the cached result for subsequent interactions. + +Usage +----- + +Zero configuration is needed if you're creating a JSON API. Just access +or set the ``media`` attribute as appropriate and let Falcon do the heavy +lifting for you. + +.. code:: python + + import falcon + + + class EchoResource(object): + def on_post(self, req, resp): + message = req.media.get('message') + + resp.media = {'message': message} + resp.status = falcon.HTTP_200 + +.. warning:: + + Once `media` is called on a request, it'll consume the request's stream. + +Validating Media +---------------- + +Falcon currently only provides a JSON Schema media validator; however, +JSON Schema is very versatile and can be used to validate any deserialized +media type that JSON also supports (i.e. dicts, lists, etc). + +.. autofunction:: falcon.media.validators.jsonschema.validate + + +Content-Type Negotiation +------------------------ + +Falcon currently only supports partial negotiation out of the box. By default, +when the ``media`` attribute is used it attempts to de/serialize based on the +``Content-Type`` header value. The missing link that Falcon doesn't provide +is the connection between the :any:`falcon.Request` ``Accept`` header provided +by a user and the :any:`falcon.Response` ``Content-Type`` header. + +If you do need full negotiation, it is very easy to bridge the gap using +middleware. Here is an example of how this can be done: + +.. code-block:: python + + class NegotiationMiddleware(object): + def process_request(self, req, resp): + resp.content_type = req.accept + + +Replacing the Default Handlers +------------------------------ + +When creating your API object you can either add or completely +replace all of the handlers. For example, lets say you want to write an API +that sends and receives MessagePack. We can easily do this by telling our +Falcon API that we want a default media-type of ``application/msgpack`` and +then create a new :any:`Handlers` object specifying the desired media type and +a handler that can process that data. + +.. code:: python + + import falcon + from falcon import media + + + handlers = media.Handlers({ + 'application/msgpack': media.MessagePackHandler(), + }) + + api = falcon.API(media_type='application/msgpack') + + api.req_options.media_handlers = handlers + api.resp_options.media_handlers = handlers + +Alternatively, if you would like to add an additional handler such as +MessagePack, this can be easily done in the following manner: + +.. code-block:: python + + import falcon + from falcon import media + + + extra_handlers = { + 'application/msgpack': media.MessagePackHandler(), + } + + api = falcon.API() + + api.req_options.media_handlers.update(extra_handlers) + api.resp_options.media_handlers.update(extra_handlers) + + +Supported Handler Types +----------------------- + +.. autoclass:: falcon.media.JSONHandler + :members: + +.. autoclass:: falcon.media.MessagePackHandler + :members: + +Custom Handler Type +------------------- + +If Falcon doesn't have an internet media type handler that supports your +use case, you can easily implement your own using the abstract base class +provided by Falcon: + +.. autoclass:: falcon.media.BaseHandler + :members: + :member-order: bysource + + +Handlers +-------- + +.. autoclass:: falcon.media.Handlers + :members: diff --git a/falcon/api.py b/falcon/api.py index 96d06b8..837e0c7 100644 --- a/falcon/api.py +++ b/falcon/api.py @@ -155,6 +155,9 @@ class API(object): self.req_options = RequestOptions() self.resp_options = ResponseOptions() + self.req_options.default_media_type = media_type + self.resp_options.default_media_type = media_type + # NOTE(kgriffs): Add default error handlers self.add_error_handler(falcon.HTTPError, self._http_error_handler) self.add_error_handler(falcon.HTTPStatus, self._http_status_handler) diff --git a/falcon/media/__init__.py b/falcon/media/__init__.py new file mode 100644 index 0000000..44cb8d8 --- /dev/null +++ b/falcon/media/__init__.py @@ -0,0 +1,5 @@ +from .base import BaseHandler # NOQA +from .json import JSONHandler # NOQA +from .msgpack import MessagePackHandler # NOQA + +from .handlers import Handlers # NOQA diff --git a/falcon/media/base.py b/falcon/media/base.py new file mode 100644 index 0000000..7693729 --- /dev/null +++ b/falcon/media/base.py @@ -0,0 +1,30 @@ +import abc + +import six + + +@six.add_metaclass(abc.ABCMeta) +class BaseHandler(object): + """Abstract Base Class for an internet media type handler""" + + @abc.abstractmethod + def serialize(self, obj): + """Serialize the media object on a :any:`falcon.Response` + + Args: + obj (object): A serializable object. + + Returns: + bytes: The resulting serialized bytes from the input object. + """ + + @abc.abstractmethod + def deserialize(self, raw): + """Deserialize the :any:`falcon.Request` body. + + Args: + raw (bytes): Input bytes to deserialize + + Returns: + object: A deserialized object. + """ diff --git a/falcon/media/handlers.py b/falcon/media/handlers.py new file mode 100644 index 0000000..0ec8cf6 --- /dev/null +++ b/falcon/media/handlers.py @@ -0,0 +1,54 @@ +import mimeparse + +from six.moves import UserDict + +from falcon import errors +from falcon.media import JSONHandler + + +class Handlers(UserDict): + """A dictionary like object that manages internet media type handlers.""" + def __init__(self, initial=None): + handlers = initial or { + 'application/json': JSONHandler(), + 'application/json; charset=UTF-8': JSONHandler(), + } + + # NOTE(jmvrbanac): Directly calling UserDict as it's not inheritable. + # Also, this results in self.update(...) being called. + UserDict.__init__(self, handlers) + + def _resolve_media_type(self, media_type, all_media_types): + resolved = None + + try: + # NOTE(jmvrbanac): Mimeparse will return an empty string if it can + # parse the media type, but cannot find a suitable type. + resolved = mimeparse.best_match( + all_media_types, + media_type + ) + except ValueError: + pass + + return resolved + + def find_by_media_type(self, media_type, default): + # PERF(jmvrbanac): Check via a quick methods first for performance + if media_type == '*/*' or not media_type: + return self.data[default] + + try: + return self.data[media_type] + except KeyError: + pass + + # PERF(jmvrbanac): Fallback to the slower method + resolved = self._resolve_media_type(media_type, self.data.keys()) + + if not resolved: + raise errors.HTTPUnsupportedMediaType( + '{0} is an unsupported media type.'.format(media_type) + ) + + return self.data[resolved] diff --git a/falcon/media/json.py b/falcon/media/json.py new file mode 100644 index 0000000..c7d67c8 --- /dev/null +++ b/falcon/media/json.py @@ -0,0 +1,22 @@ +from __future__ import absolute_import + +import json + +from falcon import errors +from falcon.media import BaseHandler + + +class JSONHandler(BaseHandler): + """Handler built using Python's :py:mod:`json` module.""" + + def deserialize(self, raw): + try: + return json.loads(raw.decode('utf-8')) + except ValueError as err: + raise errors.HTTPBadRequest( + 'Invalid JSON', + 'Could not parse JSON body - {0}'.format(err) + ) + + def serialize(self, media): + return json.dumps(media, ensure_ascii=False).encode('utf-8') diff --git a/falcon/media/msgpack.py b/falcon/media/msgpack.py new file mode 100644 index 0000000..e193fb0 --- /dev/null +++ b/falcon/media/msgpack.py @@ -0,0 +1,40 @@ +from __future__ import absolute_import + +from falcon import errors +from falcon.media import BaseHandler + + +class MessagePackHandler(BaseHandler): + """Handler built using the :py:mod:`msgpack` module from python-msgpack + + Note: + This handler uses the `bin` type option which expects bytes instead + of strings. + + Note: + This handler requires the ``python-msgpack`` package to be installed. + """ + + def __init__(self): + import msgpack + + self.msgpack = msgpack + self.packer = msgpack.Packer( + encoding='utf-8', + autoreset=True, + use_bin_type=True, + ) + + def deserialize(self, raw): + try: + # NOTE(jmvrbanac): Using unpackb since we would need to manage + # a buffer for Unpacker() which wouldn't gain us much. + return self.msgpack.unpackb(raw, encoding='utf-8') + except ValueError as err: + raise errors.HTTPBadRequest( + 'Invalid MessagePack', + 'Could not parse MessagePack body - {0}'.format(err) + ) + + def serialize(self, media): + return self.packer.pack(media) diff --git a/falcon/media/validators/__init__.py b/falcon/media/validators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/falcon/media/validators/jsonschema.py b/falcon/media/validators/jsonschema.py new file mode 100644 index 0000000..54386cd --- /dev/null +++ b/falcon/media/validators/jsonschema.py @@ -0,0 +1,47 @@ +from __future__ import absolute_import + +import falcon + +try: + import jsonschema +except ImportError: + pass + + +def validate(schema): + """Decorator that validates ``req.media`` using JSON Schema + + Args: + schema (dict): A dictionary that follows the JSON Schema specification. + See `json-schema.org `_ for more + information on defining a compatible dictionary. + + Example: + .. code:: python + + from falcon.media.validators import jsonschema + + # -- snip -- + + @jsonschema.validate(my_post_schema) + def on_post(self, req, resp): + + # -- snip -- + + Note: + This validator requires the ``jsonschema`` library available via + PyPI. The library also requires Python 2.7+. + """ + def decorator(func): + def wrapper(self, req, resp, *args, **kwargs): + try: + jsonschema.validate(req.media, schema) + except jsonschema.ValidationError as e: + raise falcon.HTTPBadRequest( + 'Failed data validation', + description=e.message + ) + + return func(self, req, resp, *args, **kwargs) + return wrapper + return decorator diff --git a/falcon/request.py b/falcon/request.py index dd06bcc..8b8f972 100644 --- a/falcon/request.py +++ b/falcon/request.py @@ -36,9 +36,11 @@ import mimeparse import six from six.moves import http_cookies +from falcon import DEFAULT_MEDIA_TYPE from falcon import errors from falcon import request_helpers as helpers from falcon import util +from falcon.media import Handlers from falcon.util.uri import parse_host, parse_query_string, unquote_string # NOTE(tbug): In some cases, http_cookies is not a module @@ -251,6 +253,18 @@ class Request(object): doc = json.load(req.bounded_stream) + media (object): Returns a deserialized form of the request stream. + When called, it will attempt to deserialize the request stream + using the Content-Type header as well as the media-type handlers + configured via :class:`falcon.RequestOptions`. + + See :ref:`media` for more information regarding media handling. + + Warning: + This operation will consume the request stream the first time + it's called and cache the results. Follow-up calls will just + retrieve a cached version of the object. + date (datetime): Value of the Date header, converted to a ``datetime`` instance. The header value is assumed to conform to RFC 1123. @@ -320,6 +334,7 @@ class Request(object): '_cached_access_route', '__dict__', 'uri_template', + '_media', ) # Child classes may override this @@ -336,6 +351,7 @@ class Request(object): self.method = env['REQUEST_METHOD'] self.uri_template = None + self._media = None # Normalize path path = env['PATH_INFO'] @@ -738,6 +754,23 @@ class Request(object): return netloc_value + @property + def media(self): + if self._media: + return self._media + + handler = self.options.media_handlers.find_by_media_type( + self.content_type, + self.options.default_media_type + ) + + # Consume the stream + raw = self.bounded_stream.read() + + # Deserialize and Return + self._media = handler.deserialize(raw) + return self._media + # ------------------------------------------------------------------------ # Methods # ------------------------------------------------------------------------ @@ -1394,12 +1427,25 @@ class RequestOptions(object): forward slash. However, this behavior can be problematic in certain cases, such as when working with authentication schemes that employ URL-based signatures. + + default_media_type (str): The default media-type to use when + deserializing a response. This value is normally set to the media + type provided when a :class:`falcon.API` is initialized; however, + if created independently, this will default to the + ``DEFAULT_MEDIA_TYPE`` specified by Falcon. + + media_handlers (Handlers): A dict-like object that allows you to + configure the media-types that you would like to handle. + By default, a handler is provided for the ``application/json`` + media type. """ __slots__ = ( 'keep_blank_qs_values', 'auto_parse_form_urlencoded', 'auto_parse_qs_csv', 'strip_url_path_trailing_slash', + 'default_media_type', + 'media_handlers', ) def __init__(self): @@ -1407,3 +1453,5 @@ class RequestOptions(object): self.auto_parse_form_urlencoded = False self.auto_parse_qs_csv = True self.strip_url_path_trailing_slash = True + self.default_media_type = DEFAULT_MEDIA_TYPE + self.media_handlers = Handlers() diff --git a/falcon/response.py b/falcon/response.py index 36fc919..998c52d 100644 --- a/falcon/response.py +++ b/falcon/response.py @@ -22,6 +22,8 @@ from six import string_types as STRING_TYPES # See issue https://github.com/falconry/falcon/issues/556 from six.moves import http_cookies +from falcon import DEFAULT_MEDIA_TYPE +from falcon.media import Handlers from falcon.response_helpers import ( format_header_value_list, format_range, @@ -86,6 +88,11 @@ class Response(object): ensure Unicode characters are properly encoded in the HTTP response. + media (object): A serializable object supported by the media handlers + configured via :class:`falcon.RequestOptions`. + + See :ref:`media` for more information regarding media handling. + stream: Either a file-like object with a `read()` method that takes an optional size argument and returns a block of bytes, or an iterable object, representing response content, and yielding @@ -141,6 +148,7 @@ class Response(object): # NOTE(tbug): will be set to a SimpleCookie object # when cookie is set via set_cookie self._cookies = None + self._media = None self.body = None self.data = None @@ -153,6 +161,23 @@ class Response(object): else: self.context = self.context_type() + @property + def media(self): + return self._media + + @media.setter + def media(self, obj): + self._media = obj + + if not self.content_type: + self.content_type = self.options.default_media_type + + handler = self.options.media_handlers.find_by_media_type( + self.content_type, + self.options.default_media_type + ) + self.data = handler.serialize(self._media) + def __repr__(self): return '<%s: %s>' % (self.__class__.__name__, self.status) @@ -782,10 +807,25 @@ class ResponseOptions(object): default to ``False``. This can make testing easier by not requiring HTTPS. Note, however, that this setting can be overridden via `set_cookie()`'s `secure` kwarg. + + default_media_type (str): The default media-type to use when + deserializing a response. This value is normally set to the media + type provided when a :class:`falcon.API` is initialized; however, + if created independently, this will default to the + ``DEFAULT_MEDIA_TYPE`` specified by Falcon. + + media_handlers (Handlers): A dict-like object that allows you to + configure the media-types that you would like to handle. + By default, a handler is provided for the ``application/json`` + media type. """ __slots__ = ( 'secure_cookies_by_default', + 'default_media_type', + 'media_handlers', ) def __init__(self): self.secure_cookies_by_default = True + self.default_media_type = DEFAULT_MEDIA_TYPE + self.media_handlers = Handlers() diff --git a/requirements/tests b/requirements/tests index 5b8554c..29b4f7f 100644 --- a/requirements/tests +++ b/requirements/tests @@ -6,3 +6,9 @@ pyyaml requests six testtools + +# Handler Specific +msgpack-python + +# Validator Specific +jsonschema diff --git a/tests/test_media_handlers.py b/tests/test_media_handlers.py new file mode 100644 index 0000000..1a8341f --- /dev/null +++ b/tests/test_media_handlers.py @@ -0,0 +1,13 @@ +import pytest + +from falcon import media + + +def test_base_handler_contract(): + class TestHandler(media.BaseHandler): + pass + + with pytest.raises(TypeError) as err: + TestHandler() + + assert 'abstract methods deserialize, serialize' in str(err.value) diff --git a/tests/test_request_media.py b/tests/test_request_media.py new file mode 100644 index 0000000..175aef4 --- /dev/null +++ b/tests/test_request_media.py @@ -0,0 +1,127 @@ +import pytest + +import falcon +from falcon import errors, media, testing + + +def create_client(handlers=None): + res = testing.SimpleTestResource() + + app = falcon.API() + app.add_route('/', res) + + if handlers: + app.req_options.media_handlers.update(handlers) + + client = testing.TestClient(app) + client.resource = res + + return client + + +@pytest.mark.parametrize('media_type', [ + (None), + ('*/*'), + ('application/json'), + ('application/json; charset=utf-8'), +]) +def test_json(media_type): + client = create_client() + expected_body = b'{"something": true}' + headers = {'Content-Type': media_type} + client.simulate_post('/', body=expected_body, headers=headers) + + media = client.resource.captured_req.media + assert media is not None + assert media.get('something') is True + + +@pytest.mark.parametrize('media_type', [ + ('application/msgpack'), + ('application/msgpack; charset=utf-8'), + ('application/x-msgpack'), +]) +def test_msgpack(media_type): + client = create_client({ + 'application/msgpack': media.MessagePackHandler(), + 'application/x-msgpack': media.MessagePackHandler(), + }) + headers = {'Content-Type': media_type} + + # Bytes + expected_body = b'\x81\xc4\tsomething\xc3' + client.simulate_post('/', body=expected_body, headers=headers) + + req_media = client.resource.captured_req.media + assert req_media.get(b'something') is True + + # Unicode + expected_body = b'\x81\xa9something\xc3' + client.simulate_post('/', body=expected_body, headers=headers) + + req_media = client.resource.captured_req.media + assert req_media.get(u'something') is True + + +@pytest.mark.parametrize('media_type', [ + ('nope/json'), +]) +def test_unknown_media_type(media_type): + client = create_client() + headers = {'Content-Type': media_type} + client.simulate_post('/', body='', headers=headers) + + with pytest.raises(errors.HTTPUnsupportedMediaType) as err: + client.resource.captured_req.media + + msg = '{0} is an unsupported media type.'.format(media_type) + assert err.value.description == msg + + +def test_invalid_json(): + client = create_client() + expected_body = b'{' + headers = {'Content-Type': 'application/json'} + client.simulate_post('/', body=expected_body, headers=headers) + + with pytest.raises(errors.HTTPBadRequest) as err: + client.resource.captured_req.media + + assert 'Could not parse JSON body' in err.value.description + + +def test_invalid_msgpack(): + client = create_client({'application/msgpack': media.MessagePackHandler()}) + expected_body = '/////////////////////' + headers = {'Content-Type': 'application/msgpack'} + client.simulate_post('/', body=expected_body, headers=headers) + + with pytest.raises(errors.HTTPBadRequest) as err: + client.resource.captured_req.media + + desc = 'Could not parse MessagePack body - unpack(b) received extra data.' + assert err.value.description == desc + + +def test_invalid_stream_fails_gracefully(): + client = create_client() + client.simulate_post('/') + + req = client.resource.captured_req + req.headers['Content-Type'] = 'application/json' + req._bounded_stream = None + + with pytest.raises(errors.HTTPBadRequest) as err: + req.media + + assert 'Could not parse JSON body' in err.value.description + + +def test_use_cached_media(): + client = create_client() + client.simulate_post('/') + + req = client.resource.captured_req + req._media = {'something': True} + + assert req.media == {'something': True} diff --git a/tests/test_response_media.py b/tests/test_response_media.py new file mode 100644 index 0000000..00a572f --- /dev/null +++ b/tests/test_response_media.py @@ -0,0 +1,111 @@ +import pytest +import six + +import falcon +from falcon import errors, media, testing + + +def create_client(handlers=None): + res = testing.SimpleTestResource() + + app = falcon.API() + app.add_route('/', res) + + if handlers: + app.resp_options.media_handlers.update(handlers) + + client = testing.TestClient(app) + client.resource = res + + return client + + +@pytest.mark.parametrize('media_type', [ + ('*/*'), + ('application/json'), + ('application/json; charset=utf-8'), +]) +def test_json(media_type): + client = create_client() + client.simulate_get('/') + + resp = client.resource.captured_resp + resp.content_type = media_type + resp.media = {'something': True} + + assert resp.data == b'{"something": true}' + + +@pytest.mark.parametrize('media_type', [ + ('application/msgpack'), + ('application/msgpack; charset=utf-8'), + ('application/x-msgpack'), +]) +def test_msgpack(media_type): + client = create_client({ + 'application/msgpack': media.MessagePackHandler(), + 'application/x-msgpack': media.MessagePackHandler(), + }) + client.simulate_get('/') + + resp = client.resource.captured_resp + resp.content_type = media_type + + # Bytes + resp.media = {b'something': True} + assert resp.data == b'\x81\xc4\tsomething\xc3' + + # Unicode + resp.media = {u'something': True} + assert resp.data == b'\x81\xa9something\xc3' + + +def test_unknown_media_type(): + client = create_client() + client.simulate_get('/') + + resp = client.resource.captured_resp + with pytest.raises(errors.HTTPUnsupportedMediaType) as err: + resp.content_type = 'nope/json' + resp.media = {'something': True} + + assert err.value.description == 'nope/json is an unsupported media type.' + + +def test_use_cached_media(): + expected = {'something': True} + + client = create_client() + client.simulate_get('/') + + resp = client.resource.captured_resp + resp._media = expected + + assert resp.media == expected + + +@pytest.mark.parametrize('media_type', [ + (''), + pytest.mark.skipif(six.PY2, reason='PY3 only')(None), +]) +def test_default_media_type(media_type): + client = create_client() + client.simulate_get('/') + + resp = client.resource.captured_resp + resp.content_type = media_type + resp.media = {'something': True} + + assert resp.data == b'{"something": true}' + assert resp.content_type == 'application/json; charset=UTF-8' + + +@pytest.mark.skipif(six.PY3, reason='Python 2 edge-case only') +def test_mimeparse_edgecases(): + client = create_client() + client.simulate_get('/') + + resp = client.resource.captured_resp + with pytest.raises(errors.HTTPUnsupportedMediaType): + resp.content_type = None + resp.media = {'something': True} diff --git a/tests/test_validators.py b/tests/test_validators.py new file mode 100644 index 0000000..1a907cd --- /dev/null +++ b/tests/test_validators.py @@ -0,0 +1,51 @@ +import sys + +import pytest + +import falcon +from falcon.media.validators import jsonschema + +skip_py26 = pytest.mark.skipif( + sys.version_info[:2] == (2, 6), + reason='Minimum Python version for this feature is 2.7.x' +) + +basic_schema = { + 'type': 'object', + 'properies': { + 'message': { + 'type': 'string', + }, + }, + 'required': ['message'], +} + + +class SampleResource(object): + @jsonschema.validate(basic_schema) + def on_get(self, req, resp): + assert req.media is not None + + +class RequestStub(object): + media = {'message': 'something'} + + +@skip_py26 +def test_jsonschema_validation_success(): + req = RequestStub() + + res = SampleResource() + assert res.on_get(req, None) is None + + +@skip_py26 +def test_jsonschema_validation_failure(): + req = RequestStub() + req.media = {} + + res = SampleResource() + with pytest.raises(falcon.HTTPBadRequest) as err: + res.on_get(req, None) + + assert err.value.description == '\'message\' is a required property'