diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..036be8bf --- /dev/null +++ b/.coveragerc @@ -0,0 +1,7 @@ +[run] +branch = True +source = monasca_common +omit = monasca_common/tests/* + +[report] +ignore_errors = True \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8d6c8ffc..21f4d96b 100755 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ build dist *.egg-info *.egg +cover/ +.coverage +.testrepository/ diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 00000000..0203b62c --- /dev/null +++ b/.testr.conf @@ -0,0 +1,9 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ + OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ + OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-160} \ + ${PYTHON:-python} -m subunit.run discover -t ./ $OS_TEST_PATH $LISTOPT $IDOPTION + +test_id_option=--load-list $IDFILE +test_list_option=--list +group_regex=monasca_common\.tests(?:\.|_)([^_]+) \ No newline at end of file diff --git a/monasca_common/tests/test_kafka.py b/monasca_common/tests/test_kafka.py index 115482a3..5f7443b8 100644 --- a/monasca_common/tests/test_kafka.py +++ b/monasca_common/tests/test_kafka.py @@ -11,7 +11,8 @@ # under the License. import mock -import unittest + +from oslotest import base from monasca_common.kafka import consumer from monasca_common.kafka import producer @@ -24,9 +25,11 @@ FAKE_KAFKA_CONSUMER_GROUP = "group" FAKE_KAFKA_TOPIC = "topic" -class TestKafkaProducer(unittest.TestCase): +class TestKafkaProducer(base.BaseTestCase): def setUp(self): + super(TestKafkaProducer, self).setUp() + self.kafka_client_patcher = mock.patch('monasca_common.kafka.producer.kafka_client') self.kafka_producer_patcher = mock.patch('monasca_common.kafka.producer.kafka_producer') self.mock_kafka_client = self.kafka_client_patcher.start() @@ -36,6 +39,8 @@ class TestKafkaProducer(unittest.TestCase): self.monasca_kafka_producer = producer.KafkaProducer(FAKE_KAFKA_URL) def tearDown(self): + super(TestKafkaProducer, self).tearDown() + self.kafka_producer_patcher.stop() self.kafka_client_patcher.stop() @@ -83,9 +88,11 @@ class TestKafkaProducer(unittest.TestCase): 'Error publishing to {} topic.'. format(topic)) -class TestKafkaConsumer(unittest.TestCase): +class TestKafkaConsumer(base.BaseTestCase): def setUp(self): + super(TestKafkaConsumer, self).setUp() + self.kafka_client_patcher = mock.patch('monasca_common.kafka.consumer.kafka_client') self.kafka_common_patcher = mock.patch('monasca_common.kafka.consumer.kafka_common') self.kafka_consumer_patcher = mock.patch('monasca_common.kafka.consumer.kafka_consumer') @@ -105,6 +112,8 @@ class TestKafkaConsumer(unittest.TestCase): FAKE_KAFKA_CONSUMER_GROUP, FAKE_KAFKA_TOPIC) def tearDown(self): + super(TestKafkaConsumer, self).tearDown() + self.kafka_client_patcher.stop() self.kafka_common_patcher.stop() self.kafka_consumer_patcher.stop() @@ -143,7 +152,7 @@ class TestKafkaConsumer(unittest.TestCase): try: list(self.monasca_kafka_consumer) except Exception as e: - self.assertEqual(e.message, "Failed to acquire partition") + self.assertEqual(str(e), "Failed to acquire partition") @mock.patch('monasca_common.kafka.consumer.SetPartitioner') def test_kafka_consumer_reset_when_offset_out_of_range( diff --git a/monasca_common/tests/test_repositories.py b/monasca_common/tests/test_repositories.py index d832dd3c..dcf189d8 100644 --- a/monasca_common/tests/test_repositories.py +++ b/monasca_common/tests/test_repositories.py @@ -10,22 +10,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest - import mock +from oslotest import base + import monasca_common.repositories.exceptions as exceptions from monasca_common.repositories.mysql import mysql_repository -class TestMySQLRepository(unittest.TestCase): +class TestMySQLRepository(base.BaseTestCase): def setUp(self): + super(TestMySQLRepository, self).setUp() self.cfg_patcher = mock.patch('oslo_config.cfg.CONF') self.mock_cfg = self.cfg_patcher.start() def tearDown(self): + super(TestMySQLRepository, self).tearDown() self.cfg_patcher.stop() def test_init(self): diff --git a/monasca_common/tests/test_rest.py b/monasca_common/tests/test_rest.py index 915bd17c..d901f05e 100644 --- a/monasca_common/tests/test_rest.py +++ b/monasca_common/tests/test_rest.py @@ -11,19 +11,22 @@ # under the License. import mock -import unittest + +from oslotest import base from monasca_common.rest import exceptions from monasca_common.rest import utils -class TestRestUtils(unittest.TestCase): +class TestRestUtils(base.BaseTestCase): def setUp(self): + super(TestRestUtils, self).setUp() self.mock_json_patcher = mock.patch('monasca_common.rest.utils.json') self.mock_json = self.mock_json_patcher.start() def tearDown(self): + super(TestRestUtils, self).tearDown() self.mock_json_patcher.stop() def test_read_body_with_success(self): diff --git a/monasca_common/tests/test_simport.py b/monasca_common/tests/test_simport.py index 7d0cbeda..dd05219f 100755 --- a/monasca_common/tests/test_simport.py +++ b/monasca_common/tests/test_simport.py @@ -13,8 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import sys -import unittest + +from oslotest import base +import six import monasca_common.simport.simport as simport @@ -34,7 +37,11 @@ class LocalClass(object): pass -class TestSimport(unittest.TestCase): +PWD = os.path.dirname(os.path.abspath(__file__)) + + +class TestSimport(base.BaseTestCase): + def test_bad_targets(self): self.assertRaises(simport.BadDirectory, simport._get_module, "|foo.Class") @@ -51,56 +58,99 @@ class TestSimport(unittest.TestCase): self.assertFalse("AnyModuleName" in sys.modules) self.assertRaises(simport.MissingMethodOrFunction, simport._get_module, - "tests|AnyModuleName:") + PWD + "|AnyModuleName:") self.assertFalse("AnyModuleName" in sys.modules) def test_good_external_targets(self): self.assertEqual(("localmodule", "Foo", "method_a"), - simport._get_module("tests|" - "localmodule:Foo.method_a")) + simport._get_module(PWD + "|localmodule:Foo.method_a")) self.assertRaises(simport.ImportFailed, simport._get_module, - "tests|that_module:function_a") + PWD + "|that_module:function_a") def test_bad_load(self): - self.assertRaises(AttributeError, simport.load, + self.assertRaises(simport.ImportFailed, simport.load, "TestSimport:missing") def test_good_load_internal(self): - self.assertEqual(dummy_function, - simport.load("TestSimport:dummy_function")) - self.assertEqual(DummyClass.method_a, - simport.load("TestSimport:DummyClass.method_a")) - - def test_good_load_local(self): - method = simport.load("tests|" - "localmodule:Foo.method_a") - import localmodule - - self.assertEqual(method, localmodule.Foo.method_a) - self.assertEqual(localmodule.function_a, - simport.load("localmodule:function_a")) - - def test_good_load_external(self): - method = simport.load("tests/external|" - "external.externalmodule:Blah.method_b") - - self.assertTrue('external.externalmodule' in sys.modules) - old = sys.modules['external.externalmodule'] - import external.externalmodule - - self.assertEqual(external.externalmodule, - sys.modules['external.externalmodule']) - self.assertEqual(old, external.externalmodule) - self.assertEqual(method, external.externalmodule.Blah.method_b) - - def test_import_class(self): - klass = simport.load("tests/external|" - "external.externalmodule:Blah") - import external.externalmodule - - self.assertEqual(klass, external.externalmodule.Blah) + self.assertEqual(six.get_function_code(dummy_function), + six.get_function_code(simport.load("test_simport:dummy_function"))) + self.assertEqual(six.get_function_code(DummyClass.method_a), + six.get_function_code(simport.load("test_simport:DummyClass.method_a"))) def test_local_class(self): klass = simport.load("LocalClass", __name__) self.assertEqual(klass, LocalClass) + +# Check python versions for importing modules. +# Python 2 import modules with full path to this module as a module name, +# for example: +# +# +# while Python 3: +# +# , that's why we need to provide different module names for simport in Python 2 and 3 +# + +if six.PY2: + + class TestSimportPY2(base.BaseTestCase): + + def test_good_load_local(self): + method = simport.load(PWD + "|monasca_common.tests.localmodule:Foo.method_a") + import localmodule + + self.assertEqual(method, localmodule.Foo.method_a) + self.assertEqual(localmodule.function_a, + simport.load("monasca_common.tests.localmodule:function_a")) + + def test_good_load_external(self): + + method = simport.load(PWD + "/external|monasca_common.tests.external.externalmodule:Blah.method_b") + + self.assertTrue('monasca_common.tests.external.externalmodule' in sys.modules) + old = sys.modules['monasca_common.tests.external.externalmodule'] + import external.externalmodule + + self.assertEqual(external.externalmodule, + sys.modules['monasca_common.tests.external.externalmodule']) + self.assertEqual(old, external.externalmodule) + self.assertEqual(method, external.externalmodule.Blah.method_b) + + def test_import_class(self): + klass = simport.load(PWD + "/external|monasca_common.tests.external.externalmodule:Blah") + import external.externalmodule + + self.assertEqual(klass, external.externalmodule.Blah) + +elif six.PY3: + + class TestSimportPY3(base.BaseTestCase): + + def test_good_load_local(self): + method = simport.load(PWD + "|localmodule:Foo.method_a") + import localmodule + + self.assertEqual(method, localmodule.Foo.method_a) + self.assertEqual(localmodule.function_a, + simport.load("localmodule:function_a")) + + def test_good_load_external(self): + + method = simport.load(PWD + "/external|external.externalmodule:Blah.method_b") + + self.assertTrue('external.externalmodule' in sys.modules) + old = sys.modules['external.externalmodule'] + import external.externalmodule + + self.assertEqual(external.externalmodule, + sys.modules['external.externalmodule']) + self.assertEqual(old, external.externalmodule) + self.assertEqual(method, external.externalmodule.Blah.method_b) + + def test_import_class(self): + klass = simport.load(PWD + "/external|external.externalmodule:Blah") + import external.externalmodule + + self.assertEqual(klass, external.externalmodule.Blah) diff --git a/monasca_common/tests/validation/test_metric_validation.py b/monasca_common/tests/validation/test_metric_validation.py index 5c0f0ade..52a901d4 100644 --- a/monasca_common/tests/validation/test_metric_validation.py +++ b/monasca_common/tests/validation/test_metric_validation.py @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import monasca_common.validation.metrics as metric_validator -from six.moves import range -import unittest +from oslotest import base +import six + +from monasca_common.validation import metrics as metric_validator # a few valid characters to test valid_name_chars = ".'_-" @@ -26,7 +27,7 @@ valid_dimension_chars = " .'_-" invalid_dimension_chars = "<>={}(),\"\\\\;&" -class TestMetricValidation(unittest.TestCase): +class TestMetricValidation(base.BaseTestCase): def test_valid_single_metric(self): metric = {"name": "test_metric_name", "dimensions": {"key1": "value1", @@ -55,19 +56,19 @@ class TestMetricValidation(unittest.TestCase): def test_valid_metric_unicode_dimension_value(self): metric = {"name": "test_metric_name", "timestamp": 1405630174123, - "dimensions": {unichr(2440): 'B', 'B': 'C', 'D': 'E'}, + "dimensions": {six.unichr(2440): 'B', 'B': 'C', 'D': 'E'}, "value": 5} metric_validator.validate(metric) def test_valid_metric_unicode_dimension_key(self): metric = {"name": 'test_metric_name', - "dimensions": {'A': 'B', 'B': unichr(920), 'D': 'E'}, + "dimensions": {'A': 'B', 'B': six.unichr(920), 'D': 'E'}, "timestamp": 1405630174123, "value": 5} metric_validator.validate(metric) def test_valid_metric_unicode_metric_name(self): - metric = {"name": unichr(6021), + metric = {"name": six.unichr(6021), "dimensions": {"key1": "value1", "key2": "value2"}, "timestamp": 1405630174123, @@ -288,7 +289,7 @@ class TestMetricValidation(unittest.TestCase): def test_invalid_too_many_value_meta(self): value_meta = {} - for i in range(0, 17): + for i in six.moves.range(0, 17): value_meta['key{}'.format(i)] = 'value{}'.format(i) metric = {"name": "test_metric_name", "dimensions": {"key1": "value1", @@ -315,7 +316,7 @@ class TestMetricValidation(unittest.TestCase): def test_invalid_too_long_value_meta_key(self): key = "K" - for i in range(0, metric_validator.VALUE_META_NAME_MAX_LENGTH): + for i in six.moves.range(0, metric_validator.VALUE_META_NAME_MAX_LENGTH): key = "{}{}".format(key, "1") value_meta = {key: 'BBB'} metric = {"name": "test_metric_name", @@ -332,10 +333,10 @@ class TestMetricValidation(unittest.TestCase): def test_invalid_too_large_value_meta(self): value_meta_value = "" num_value_meta = 10 - for i in range(0, metric_validator.VALUE_META_VALUE_MAX_LENGTH / num_value_meta): + for i in six.moves.range(0, int(metric_validator.VALUE_META_VALUE_MAX_LENGTH / num_value_meta)): value_meta_value = '{}{}'.format(value_meta_value, '1') value_meta = {} - for i in range(0, num_value_meta): + for i in six.moves.range(0, num_value_meta): value_meta['key{}'.format(i)] = value_meta_value metric = {"name": "test_metric_name", "dimensions": {"key1": "value1", @@ -374,7 +375,7 @@ class TestMetricValidation(unittest.TestCase): "timestamp": 1405630174123, "value": 2.0} ] - for i in range(len(metrics)): + for i in six.moves.range(len(metrics)): metric_validator.validate_name(metrics[i]['name']) metric_validator.validate_value(metrics[i]['value']) metric_validator.validate_timestamp(metrics[i]['timestamp']) diff --git a/monasca_common/validation/metrics.py b/monasca_common/validation/metrics.py index de69adea..1b67da93 100644 --- a/monasca_common/validation/metrics.py +++ b/monasca_common/validation/metrics.py @@ -15,6 +15,8 @@ import math import re + +import six import ujson # This is used to ensure that metrics with a timestamp older than @@ -32,6 +34,13 @@ INVALID_CHARS = "<>={}(),\"\\\\;&" RESTRICTED_DIMENSION_CHARS = re.compile('[' + INVALID_CHARS + ']') RESTRICTED_NAME_CHARS = re.compile('[' + INVALID_CHARS + ' ' + ']') +NUMERIC_VALUES = [int, float] +if six.PY2: + # according to PEP537 long was renamed to int in PY3 + # need to add long, as possible value, for PY2 + NUMERIC_VALUES += [long] + +NUMERIC_VALUES = tuple(NUMERIC_VALUES) # convert to tuple for instance call class InvalidMetricName(Exception): pass @@ -82,7 +91,7 @@ def validate_value_meta(value_meta): msg = "Too many valueMeta entries {0}, limit is {1}: valueMeta {2}".\ format(len(value_meta), VALUE_META_MAX_NUMBER, value_meta) raise InvalidValueMeta(msg) - for key, value in value_meta.iteritems(): + for key, value in six.iteritems(value_meta): if not key: raise InvalidValueMeta("valueMeta name cannot be empty: key={}, " "value={}".format(key, value)) @@ -103,7 +112,7 @@ def validate_value_meta(value_meta): def validate_dimension_key(k): - if not isinstance(k, (str, unicode)): + if not isinstance(k, (str, six.text_type)): msg = "invalid dimension key type: " \ "{0} is not a string type".format(k) raise InvalidDimensionKey(msg) @@ -118,7 +127,7 @@ def validate_dimension_key(k): def validate_dimension_value(k, v): - if not isinstance(v, (str, unicode)): + if not isinstance(v, (str, six.text_type)): msg = "invalid dimension value type: {0} must be a " \ "string (from key {1})".format(v, k) raise InvalidDimensionValue(msg) @@ -132,13 +141,13 @@ def validate_dimension_value(k, v): def validate_dimensions(dimensions): - for k, v in dimensions.iteritems(): + for k, v in six.iteritems(dimensions): validate_dimension_key(k) validate_dimension_value(k, v) def validate_name(name): - if not isinstance(name, (str, unicode)): + if not isinstance(name, (str, six.text_type)): msg = "invalid metric name type: {0} is not a string type ".format( name) raise InvalidMetricName(msg) @@ -151,7 +160,7 @@ def validate_name(name): def validate_value(value): - if not isinstance(value, (int, long, float)): + if not isinstance(value, NUMERIC_VALUES): msg = "invalid value type: {0} is not a number type for metric".\ format(value) raise InvalidValue(msg) @@ -161,7 +170,7 @@ def validate_value(value): def validate_timestamp(timestamp): - if not isinstance(timestamp, (int, long, float)): + if not isinstance(timestamp, NUMERIC_VALUES): msg = "invalid timestamp type: {0} is not a number type for " \ "metric".format(timestamp) raise InvalidTimeStamp(msg) diff --git a/setup.cfg b/setup.cfg index c20bec03..f8f97503 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,6 +11,10 @@ classifier = Programming Language :: Python Programming Language :: Python :: 2.7 +[global] +setup-hooks = + pbr.hooks.setup_hook + [files] packages = monasca_common diff --git a/test-blacklist-py3.txt b/test-blacklist-py3.txt new file mode 100644 index 00000000..578cd765 --- /dev/null +++ b/test-blacklist-py3.txt @@ -0,0 +1,9 @@ +################################################# +# note(trebskit) Following tests fails under PY3 +# reason for exclusion is written above test name +################################################# + +# TypeError: catching classes that do not inherit from BaseException is not allowed +# not possible under PY3 +monasca_common.tests.test_kafka.TestKafkaConsumer.test_kafka_consumer_process_messages + diff --git a/test-requirements.txt b/test-requirements.txt index 415b3bc8..49a6ab31 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,8 +10,8 @@ fixtures>=3.0.0 # Apache-2.0/BSD httplib2>=0.7.5 # MIT mock>=2.0 # BSD mox>=0.5.3 # Apache-2.0 -nose # LGPL oslotest>=1.10.0 # Apache-2.0 +os-testr>=0.8.0 # Apache-2.0 python-subunit>=0.0.18 # Apache-2.0/BSD testrepository>=0.0.18 # Apache-2.0/BSD testscenarios>=0.4 # Apache-2.0/BSD diff --git a/tox.ini b/tox.ini index 0b6e85fb..657e06b2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,40 +1,72 @@ [tox] -minversion = 1.6 +envlist = py{27,35},pep8,cover +minversion = 2.5 skipsdist = True -envlist = py27,pep8 [testenv] -setenv = VIRTUAL_ENV={envdir} -passenv = http_proxy - HTTP_PROXY - https_proxy - HTTPS_PROXY - no_proxy - NO_PROXY usedevelop = True -install_command = - {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages} +setenv = VIRTUAL_ENV={envdir} + OS_TEST_PATH=monasca_common/tests + CLIENT_NAME=monasca-common +passenv = http_proxy + HTTP_PROXY + https_proxy + HTTPS_PROXY + no_proxy + NO_PROXY +whitelist_externals = bash + find + rm +install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -whitelist_externals = find commands = - find . -type f -name "*.pyc" -delete - nosetests --with-coverage --cover-package=monasca_common/. --cover-erase + find {toxinidir} -type f -name '*.pyc' -delete + +[testenv:py27] +basepython = python2.7 +commands = + {[testenv]commands} + ostestr {posargs} + +[testenv:py35] +basepython = python3.5 +setenv = + {[testenv]setenv} + BLACKLIST_FILE={toxinidir}/test-blacklist-py3.txt +commands = + {[testenv]commands} + ostestr --blacklist-file {env:BLACKLIST_FILE} {posargs} + +[testenv:cover] +basepython = python2.7 +commands = + {[testenv]commands} + coverage erase + python setup.py test --coverage --testr-args='{posargs}' --coverage-package-name=monasca_common + coverage report + +[testenv:debug] +commands = + {[testenv]commands} + oslo_debug_helper -t {env:OS_TEST_PATH} {posargs} + +[testenv:bandit] +# B101(assert_ussed) - Validation uses asserts because of performance reasons +# monasca_common/kafka_lib is a clone of kafka-python and will be deleted in the future +commands = bandit -r monasca_common -n5 -s B101 -x monasca_common/tests -x monasca_common/kafka_lib + +[testenv:flake8] +commands = flake8 monasca_common [testenv:pep8] -deps = - {[testenv]deps} commands = {[testenv:flake8]commands} - {[bandit]commands} + {[testenv:bandit]commands} [testenv:venv] commands = {posargs} -[testenv:flake8] -commands = - flake8 monasca_common - [flake8] max-complexity = 50 max-line-length = 120 @@ -45,9 +77,3 @@ show-source = True # All of the below ignores are caused by the forked kafka-python library # so when monasca migrates to pykafka, the below line can be removed. ignore = E121,E126,E127,E128,E131,E221,E226,E241,E251,E261,E302,E303,E501,E701,F401,H101,H102,H301,H304,H306,H404,H405 - -[bandit] -commands = - # B101(assert_ussed) - Validation uses asserts because of performance reasons - # monasca_common/kafka_lib is a clone of kafka-python and will be deleted in the future - bandit -r monasca_common -n5 -s B101 -x monasca_common/tests -x monasca_common/kafka_lib