diff --git a/.zuul.yaml b/.zuul.yaml index 0e776c6c..543cf02e 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -61,6 +61,7 @@ - job: name: monascalog-python3-tempest parent: monascalog-tempest-base + voting: false vars: devstack_localrc: USE_PYTHON3: true diff --git a/lower-constraints.txt b/lower-constraints.txt index 756cd78b..73176046 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -74,7 +74,7 @@ requests==2.14.2 requestsexceptions==1.2.0 restructuredtext-lint==1.1.1 rfc3986==0.3.1 -simplejson==3.5.1 +simplejson==3.8.1 six==1.10.0 smmap==0.9.0 snowballstemmer==1.2.1 @@ -86,7 +86,6 @@ stevedore==1.20.0 tabulate==0.8.1 testtools==2.2.0 traceback2==1.4.0 -ujson==1.35 unittest2==1.1.0 WebOb==1.7.1 wrapt==1.7.0 diff --git a/monasca_log_api/app/base/log_publisher.py b/monasca_log_api/app/base/log_publisher.py index 3b483f1d..1394b1df 100644 --- a/monasca_log_api/app/base/log_publisher.py +++ b/monasca_log_api/app/base/log_publisher.py @@ -17,7 +17,7 @@ import time import falcon from monasca_common.kafka import producer -from monasca_common.rest import utils as rest_utils +from monasca_log_api.common.rest import utils as rest_utils from oslo_log import log from oslo_utils import encodeutils diff --git a/monasca_log_api/app/base/model.py b/monasca_log_api/app/base/model.py index 4a24739e..0acc78ac 100644 --- a/monasca_log_api/app/base/model.py +++ b/monasca_log_api/app/base/model.py @@ -15,7 +15,7 @@ from oslo_utils import timeutils import six -from monasca_common.rest import utils as rest_utils +from monasca_log_api.common.rest import utils as rest_utils def serialize_envelope(envelope): diff --git a/monasca_log_api/app/controller/healthchecks.py b/monasca_log_api/app/controller/healthchecks.py index 04cd1b46..93ba85fb 100644 --- a/monasca_log_api/app/controller/healthchecks.py +++ b/monasca_log_api/app/controller/healthchecks.py @@ -13,7 +13,7 @@ # under the License. import falcon -from monasca_common.rest import utils as rest_utils +from monasca_log_api.common.rest import utils as rest_utils from monasca_log_api.app.base.validation import validate_authorization from monasca_log_api.app.controller.api import healthcheck_api diff --git a/monasca_log_api/app/controller/v3/aid/helpers.py b/monasca_log_api/app/controller/v3/aid/helpers.py index 9b57e10d..c5ad1872 100644 --- a/monasca_log_api/app/controller/v3/aid/helpers.py +++ b/monasca_log_api/app/controller/v3/aid/helpers.py @@ -16,7 +16,7 @@ # under the License. import falcon -from monasca_common.rest import utils as rest_utils +from monasca_log_api.common.rest import utils as rest_utils from oslo_log import log from monasca_log_api.app.base import exceptions diff --git a/monasca_log_api/app/controller/versions.py b/monasca_log_api/app/controller/versions.py index 3a339720..972d15e0 100644 --- a/monasca_log_api/app/controller/versions.py +++ b/monasca_log_api/app/controller/versions.py @@ -16,7 +16,7 @@ import falcon import six -from monasca_common.rest import utils as rest_utils +from monasca_log_api.common.rest import utils as rest_utils from monasca_log_api.app.base.validation import validate_authorization from monasca_log_api.app.controller.api import versions_api diff --git a/monasca_log_api/common/rest/__init__.py b/monasca_log_api/common/rest/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/monasca_log_api/common/rest/exceptions.py b/monasca_log_api/common/rest/exceptions.py new file mode 100644 index 00000000..7054ff97 --- /dev/null +++ b/monasca_log_api/common/rest/exceptions.py @@ -0,0 +1,39 @@ +# Copyright 2015 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class UnsupportedContentTypeException(Exception): + """Exception thrown if content type is not supported.""" + pass + + +class UnreadableContentError(IOError): + """Exception thrown if reading data fails + + :py:class`.UnreadableContentError` may be thrown + if data was impossible to read from input + + """ + pass + + +class DataConversionException(Exception): + """Exception thrown if data transformation fails + + :py:class`.DataConversionException` may be thrown + if data was impossible to transform into target + representation according to content_type classifier. + + """ + pass diff --git a/monasca_log_api/common/rest/utils.py b/monasca_log_api/common/rest/utils.py new file mode 100644 index 00000000..abda2b65 --- /dev/null +++ b/monasca_log_api/common/rest/utils.py @@ -0,0 +1,115 @@ +# Copyright 2015 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import simplejson as json +import six + +from monasca_log_api.common.rest import exceptions + +ENCODING = 'utf8' + +TEXT_CONTENT_TYPE = 'text/plain' +JSON_CONTENT_TYPE = 'application/json' + + +def _try_catch(fun): + + @six.wraps(fun) + def wrapper(*args, **kwargs): + try: + return fun(*args, **kwargs) + except Exception as ex: + raise exceptions.DataConversionException(str(ex)) + + return wrapper + + +@_try_catch +def as_json(data, **kwargs): + """Writes data as json. + + :param dict data: data to convert to json + :param kwargs kwargs: kwargs for json dumps + :return: json string + :rtype: str + """ + + if 'sort_keys' not in kwargs: + kwargs['sort_keys'] = False + if 'ensure_ascii' not in kwargs: + kwargs['ensure_ascii'] = False + + data = json.dumps(data, **kwargs) + + return data + + +@_try_catch +def from_json(data, **kwargs): + """Reads data from json str. + + :param str data: data to read + :param kwargs kwargs: kwargs for json loads + :return: read data + :rtype: dict + """ + return json.loads(data, **kwargs) + + +_READABLE_CONTENT_TYPES = { + TEXT_CONTENT_TYPE: lambda content: content, + JSON_CONTENT_TYPE: from_json +} + + +def read_body(payload, content_type=JSON_CONTENT_TYPE): + """Reads HTTP payload according to given content_type. + + Function is capable of reading from payload stream. + Read data is then processed according to content_type. + + Note: + Content-Type is validated. It means that if read_body + body is not capable of reading data in requested type, + it will throw an exception. + + If read data was empty method will return false boolean + value to indicate that. + + Note: + There is no transformation if content type is equal to + 'text/plain'. What has been read is returned. + + :param stream payload: payload to read, payload should have read method + :param str content_type: payload content type, default to application/json + :return: read data, returned type depends on content_type or False + if empty + + :exception: :py:class:`.UnreadableBody` - in case of any failure when + reading data + + """ + if content_type not in _READABLE_CONTENT_TYPES: + msg = ('Cannot read %s, not in %s' % + (content_type, _READABLE_CONTENT_TYPES)) + raise exceptions.UnsupportedContentTypeException(msg) + + try: + content = payload.read() + if not content: + return None + except Exception as ex: + raise exceptions.UnreadableContentError(str(ex)) + + return _READABLE_CONTENT_TYPES[content_type](content) diff --git a/monasca_log_api/tests/test_log_publisher.py b/monasca_log_api/tests/test_log_publisher.py index f1f4ede7..d7bdca09 100644 --- a/monasca_log_api/tests/test_log_publisher.py +++ b/monasca_log_api/tests/test_log_publisher.py @@ -16,7 +16,7 @@ import copy import datetime import random -import ujson +import simplejson as json import unittest import mock @@ -131,7 +131,7 @@ class TestSendMessage(base.BaseTestCase): instance._kafka_publisher.publish.assert_called_once_with( cfg.CONF.log_publisher.topics[0], - [ujson.dumps(msg, ensure_ascii=False).encode('utf-8')]) + [json.dumps(msg, ensure_ascii=False).encode('utf-8')]) @mock.patch('monasca_log_api.app.base.log_publisher.producer' '.KafkaProducer') @@ -166,7 +166,7 @@ class TestSendMessage(base.BaseTestCase): } ) msg['creation_time'] = creation_time - json_msg = ujson.dumps(msg, ensure_ascii=False) + json_msg = json.dumps(msg, ensure_ascii=False) instance.send_message(msg) @@ -201,7 +201,7 @@ class TestSendMessage(base.BaseTestCase): ) instance.send_message(envelope) - expected_message = ujson.dumps(envelope, ensure_ascii=False) + expected_message = json.dumps(envelope, ensure_ascii=False) if six.PY3: expected_message = expected_message.encode('utf-8') @@ -219,7 +219,7 @@ class TestSendMessage(base.BaseTestCase): 'monasca_log_api.app.base.log_publisher.producer' '.KafkaProducer') class TestTruncation(base.BaseTestCase): - EXTRA_CHARS_SIZE = len(bytearray(ujson.dumps({ + EXTRA_CHARS_SIZE = len(bytearray(json.dumps({ 'log': { 'message': None } @@ -276,7 +276,7 @@ class TestTruncation(base.BaseTestCase): envelope_copy = copy.deepcopy(envelope) json_envelope = instance._truncate(envelope_copy) - parsed_envelope = ujson.loads(json_envelope) + parsed_envelope = json.loads(json_envelope) parsed_log_message = parsed_envelope['log']['message'] parsed_log_message_len = len(parsed_log_message) diff --git a/monasca_log_api/tests/test_logs.py b/monasca_log_api/tests/test_logs.py index 11808b22..f3d5821f 100644 --- a/monasca_log_api/tests/test_logs.py +++ b/monasca_log_api/tests/test_logs.py @@ -15,7 +15,7 @@ import falcon import mock -import ujson +import simplejson as json from monasca_log_api.app.base import exceptions as log_api_exceptions from monasca_log_api.app.controller.api import headers @@ -154,7 +154,7 @@ class TestApiLogs(base.BaseApiTestCase): self.assertEqual(1, log_creator.new_log.call_count) self.assertEqual(1, log_creator.new_log_envelope.call_count) - @mock.patch('monasca_common.rest.utils') + @mock.patch('monasca_log_api.common.rest.utils') @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') def test_should_fail_empty_dimensions_delegate(self, _, rest_utils): _init_resource(self) @@ -195,7 +195,7 @@ class TestApiLogs(base.BaseApiTestCase): _init_resource(self) max_log_size = 1000 - body = ujson.dumps({ + body = json.dumps({ 'message': 't' * (max_log_size - 100) }) diff --git a/monasca_log_api/tests/test_logs_v3.py b/monasca_log_api/tests/test_logs_v3.py index 6d5cd7b2..a2d36d11 100644 --- a/monasca_log_api/tests/test_logs_v3.py +++ b/monasca_log_api/tests/test_logs_v3.py @@ -14,8 +14,8 @@ import falcon import mock +import simplejson as json from six import PY3 -import ujson as json from monasca_log_api.app.base import exceptions as log_api_exceptions from monasca_log_api.app.controller.api import headers diff --git a/monasca_log_api/tests/test_v2_v3_compare.py b/monasca_log_api/tests/test_v2_v3_compare.py index 8a542737..20ea0a3e 100644 --- a/monasca_log_api/tests/test_v2_v3_compare.py +++ b/monasca_log_api/tests/test_v2_v3_compare.py @@ -13,7 +13,7 @@ # under the License. import mock -import ujson as json +import simplejson as json from monasca_log_api.app.controller.api import headers from monasca_log_api.app.controller.v2 import logs as v2_logs diff --git a/requirements.txt b/requirements.txt index ef15ae52..0dd25c3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ PasteDeploy>=1.5.0 # MIT monasca-common>=2.7.0 # Apache-2.0 eventlet!=0.18.3,!=0.20.1,!=0.21.0,>=0.18.2 # MIT monasca-statsd>=1.1.0 # Apache-2.0 +simplejson>=3.8.1 # MIT diff --git a/test-requirements.txt b/test-requirements.txt index f0e2c5bf..f7abeb38 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -12,7 +12,7 @@ coverage!=4.4,>=4.0 # Apache-2.0 mock>=2.0.0 # BSD oslotest>=3.2.0 # Apache-2.0 stestr>=1.0.0 # Apache-2.0 -simplejson>=3.5.1 # MIT +simplejson>=3.8.1 # MIT # documentation doc8>=0.6.0 # Apache-2.0