diff --git a/.gitignore b/.gitignore index 551afc29..5c853ecb 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,8 @@ logs/ .idea/ **/*.pyc virtenv/* +.coverage +cover/ +.tox/ +*egg-info/ +.testrepository/ \ No newline at end of file diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 00000000..d851fd21 --- /dev/null +++ b/.testr.conf @@ -0,0 +1,9 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ + OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ + OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-160} \ + ${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./monasca_persister/tests} $LISTOPT $IDOPTION + +test_id_option=--load-list $IDFILE +test_list_option=--list +group_regex=monasca_persister\.tests(?:\.|_)([^_]+) \ No newline at end of file diff --git a/monasca_persister/tests/test_persister_main.py b/monasca_persister/tests/test_persister_main.py new file mode 100644 index 00000000..d71cc2da --- /dev/null +++ b/monasca_persister/tests/test_persister_main.py @@ -0,0 +1,245 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mock import patch +from mock import call +from mock import Mock +import signal + +from oslo_config import cfg +from oslotest import base + +CONF = cfg.CONF + +NUMBER_OF_METRICS_PROCESSES = 2 +NUMBER_OF_ALARM_HIST_PROCESSES = 3 + + +class FakeException(Exception): + pass + + +class TestPersister(base.BaseTestCase): + + def setUp(self): + super(TestPersister, self).setUp() + + from monasca_persister import persister + self.persister = persister + + self._set_patchers() + self._set_mocks() + + def _set_patchers(self): + self.sys_exit_patcher = patch.object(self.persister.sys, 'exit') + self.log_patcher = patch.object(self.persister, 'log') + self.simport_patcher = patch.object(self.persister, 'simport') + self.cfg_patcher = patch.object(self.persister, 'cfg') + self.sleep_patcher = patch.object(self.persister.time, 'sleep') + self.process_patcher = patch.object(self.persister.multiprocessing, 'Process') + + def _set_mocks(self): + self.mock_sys_exit = self.sys_exit_patcher.start() + self.mock_log = self.log_patcher.start() + self.mock_simport = self.simport_patcher.start() + self.mock_cfg = self.cfg_patcher.start() + self.mock_sleep = self.sleep_patcher.start() + self.mock_process_class = self.process_patcher.start() + + self.mock_cfg.CONF.kafka_metrics.num_processors = NUMBER_OF_METRICS_PROCESSES + self.mock_cfg.CONF.kafka_alarm_history.num_processors = NUMBER_OF_ALARM_HIST_PROCESSES + self.mock_cfg.CONF.zookeeper = 'zookeeper' + + self.mock_sleep.side_effect = [FakeException, None] + + self.process_pool = _get_process_list( + NUMBER_OF_METRICS_PROCESSES + NUMBER_OF_ALARM_HIST_PROCESSES + 1) + self.mock_process_class.side_effect = self.process_pool + + def tearDown(self): + super(TestPersister, self).tearDown() + + self.mock_sleep.side_effect = None + + self.mock_sys_exit.side_effect = None + self.mock_sys_exit.reset_mock() + + self.sys_exit_patcher.stop() + self.log_patcher.stop() + self.simport_patcher.stop() + self.cfg_patcher.stop() + self.sleep_patcher.stop() + self.process_patcher.stop() + + self.persister.processors = [] + self.persister.exiting = False + + def _run_persister(self): + self.persister.main() + self.assertTrue(self.mock_process_class.called) + + def _assert_clean_exit_handler_terminates_with_expected_signal( + self, signum, expected): + # in order to really exit, an exception is thrown + + self.mock_sys_exit.side_effect = FakeException + self.assertRaises(FakeException, self.persister.clean_exit, signum) + + self.mock_sys_exit.assert_called_once_with(expected) + + def _assert_correct_number_of_processes_created(self): + + for p in self.process_pool[:-1]: + self.assertTrue(p.start.called) + + self.assertFalse(self.process_pool[-1].called) + + def _assert_process_alive_status_called(self): + + for p in self.process_pool[:-1]: + self.assertTrue(p.is_alive.called) + + def _assert_process_terminate_called(self): + + for p in self.process_pool[:-1]: + self.assertTrue(p.terminate.called) + + def test_active_children_are_killed_during_exit(self): + + with patch.object(self.persister.multiprocessing, 'active_children') as active_children,\ + patch.object(self.persister.os, 'kill') as mock_kill: + + active_children.return_value = [Mock(name='child-1', pid=1), + Mock(name='child-2', pid=2)] + + self.persister.clean_exit(0) + + mock_kill.assert_has_calls([call(1, signal.SIGKILL), call(2, signal.SIGKILL)]) + + def test_active_children_kill_exception_is_ignored(self): + + with patch.object(self.persister.multiprocessing, + 'active_children') as active_children, \ + patch.object(self.persister.os, 'kill') as mock_kill: + + active_children.return_value = [Mock()] + mock_kill.side_effect = FakeException + + self.persister.clean_exit(0) + + self.assertTrue(mock_kill.called) + + def test_clean_exit_does_nothing_when_exiting_is_true(self): + self.persister.exiting = True + + self.assertEqual(None, self.persister.clean_exit(0)) + + self.assertFalse(self.mock_sys_exit.called) + + def test_exception_during_process_termination_is_ignored(self): + dead_process = _get_process('raises_when_terminated') + dead_process.terminate.side_effect = FakeException + self.process_pool[0] = dead_process + + self._run_persister() + + self.assertTrue(dead_process.terminate.called) + + def test_if_not_sigterm_then_clean_exit_handler_terminates_with_signal(self): + + self._assert_clean_exit_handler_terminates_with_expected_signal( + signal.SIGINT, signal.SIGINT) + + def test_if_sigterm_then_clean_exit_handler_terminates_with_zero(self): + + self._assert_clean_exit_handler_terminates_with_expected_signal( + signal.SIGTERM, 0) + + def test_non_running_process_not_terminated(self): + dead_process = _get_process('dead_process', is_alive=False) + self.process_pool[0] = dead_process + + self._run_persister() + + self.assertTrue(dead_process.is_alive.called) + self.assertFalse(dead_process.terminate.called) + + def test_running_processes_are_created_and_terminated(self): + + self._run_persister() + + self._assert_correct_number_of_processes_created() + self._assert_process_alive_status_called() + self._assert_process_terminate_called() + + def test_start_process_handler_creates_and_runs_persister(self): + fake_kafka_config = Mock() + fake_repository = Mock() + + with patch('monasca_persister.repositories.persister.Persister') as mock_persister_class: + self.persister.start_process(fake_repository, fake_kafka_config) + + mock_persister_class.assert_called_once_with( + fake_kafka_config, 'zookeeper', fake_repository) + + +class TestPersisterConfig(base.BaseTestCase): + + def setUp(self): + super(TestPersisterConfig, self).setUp() + + from monasca_persister import persister + + self.assertIsNotNone(persister) + + def _test_zookeeper_options(self): + str_opts = ['uri'] + int_opts = ['partition_interval_recheck_seconds'] + + self._assert_cfg_registered('zookeeper', str_opts, int_opts) + + def _assert_cfg_registered(self, group_name, str_opts, int_opts): + options = {group_name: str_opts + int_opts} + + for key, values in options.items(): + for v in values: + self.assertIsNone(CONF[key][v]) + + def _test_kafka_options(self): + str_opts = ['uri', 'group_id', 'topic', 'consumer_id', 'client_id', 'zookeeper_path'] + int_opts = ['database_batch_size', 'max_wait_time_seconds', 'fetch_size_bytes', + 'buffer_size', 'max_buffer_size', 'num_processors'] + + self._assert_cfg_registered('kafka_metrics', str_opts, int_opts) + self._assert_cfg_registered('kafka_alarm_history', str_opts, int_opts) + + def _test_repositories_options(self): + str_opts = ['metrics_driver', 'alarm_state_history_driver'] + int_opts = [] + + self._assert_cfg_registered('repositories', str_opts, int_opts) + + def test_correct_config_options_are_registered(self): + self._test_zookeeper_options() + self._test_kafka_options() + self._test_repositories_options() + + +def _get_process(name, is_alive=True): + return Mock(name=name, is_alive=Mock(return_value=is_alive)) + + +def _get_process_list(number_of_processes): + processes = [] + for i in range(number_of_processes): + processes.append(_get_process(name='process_{}'.format(i))) + return processes diff --git a/setup.cfg b/setup.cfg index e49b2973..2876bad3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,6 +12,10 @@ description-file = README.md home-page = https://github.com/openstack/monasca-persister license = Apache +[global] +setup-hooks = + pbr.hooks.setup_hook + [entry_points] console_scripts = monasca-persister = monasca_persister.persister:main @@ -19,15 +23,9 @@ console_scripts = [files] packages = monasca_persister -[flake8] -max-line-length = 120 - [pbr] autodoc_index_modules = True -[pep8] -max-line-length = 120 - [wheel] universal = 1 diff --git a/test-requirements.txt b/test-requirements.txt index 964466aa..88ca5ac8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,10 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking<0.12,>=0.11.0 # Apache-2.0 +flake8>=2.5.4,<2.6.0 # MIT +hacking>=0.12.0,!=0.13.0,<0.14 # Apache-2.0 +coverage>=4.0 # Apache-2.0 nose # LGPL mock>=2.0 # BSD +oslotest>=1.10.0 # Apache-2.0 +os-testr>=0.8.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index 5fec2a15..763a13f3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,19 +1,53 @@ [tox] -envlist = py27,pypy,pep8 -minversion = 2.0 +envlist = py27,py35,pep8 +minversion = 2.1 skipsdist = True [testenv] -setenv = VIRTUAL_ENV={envdir} +setenv = + VIRTUAL_ENV={envdir} + DISCOVER_DIRECTORY=tests + CLIENT_NAME=monasca-persister +passenv = http_proxy + HTTP_PROXY + https_proxy + HTTPS_PROXY + no_proxy + NO_PROXY usedevelop = True +whitelist_externals = bash + find + rm install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -whitelist_externals = find commands = - find . -type f -name "*.pyc" -delete - nosetests + find {toxinidir} -type f -name "*.py[c|o]" -delete + +[testenv:py27] +basepython = python2.7 +commands = + {[testenv]commands} + ostestr {posargs} + +[testenv:py35] +basepython = python3.5 +commands = + {[testenv]commands} + ostestr {posargs} + +[testenv:cover] +commands = + {[testenv]commands} + coverage erase + python setup.py test --coverage --testr-args='{posargs}' --coverage-package-name=monasca_persister + coverage report + +[testenv:debug] +commands = + {[testenv]commands} + oslo_debug_helper -t monasca_persister/tests {posargs} [testenv:pep8] commands = flake8 @@ -23,7 +57,7 @@ commands = {posargs} [flake8] max-line-length = 120 -# TODO: ignored checks should be enabled in the future +# TODO: ignored checks should be enabled in the future # H405 multi line docstring summary not separated with an empty line # H904 Wrap long lines in parentheses instead of a backslash ignore = F821,H405,H904,E126,E125,H306,E302,E122