From 138ac174c4f22735cd5b24371c61cdd5c938a259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tr=C4=99bski?= Date: Tue, 11 Jul 2017 08:55:45 +0200 Subject: [PATCH] Allow to specify CLI arguments Passing any CLI arguments to log-api is not possible for the case where it runs under gunicorn server. Gunicorn's argument parsing processes clashes with oslo's. Effectivelly that means that either of them cannot understand the arguments of another. However log-api is capable of being launched under, for example, apache_mod-wsgi. That permits passing oslo CLI arguments. Added simple method that detect the executable that was used to run log-api. If that is not gunicorn, CLI opts will be enabled. Otherwise log-api will print out warning and proceed as it was. Extra: * reworked to use fixtures Change-Id: I6b2fc386aeb823ab735270ffc1d3f7e15985830f --- .gitignore | 8 +-- monasca_log_api/app/main.py | 2 +- monasca_log_api/config.py | 21 ++++-- monasca_log_api/tests/base.py | 71 ++++++++++++++----- monasca_log_api/tests/test_config.py | 40 +++++++++++ monasca_log_api/tests/test_healthchecks.py | 2 +- monasca_log_api/tests/test_kafka_check.py | 7 +- monasca_log_api/tests/test_log_publisher.py | 25 ++----- monasca_log_api/tests/test_logs.py | 6 +- monasca_log_api/tests/test_logs_v3.py | 3 - monasca_log_api/tests/test_monitoring.py | 4 -- monasca_log_api/tests/test_request.py | 5 -- monasca_log_api/tests/test_service.py | 20 ++---- monasca_log_api/tests/test_versions.py | 4 -- .../notes/cli_args-6dc5e2d13337b871.yaml | 6 ++ 15 files changed, 136 insertions(+), 88 deletions(-) create mode 100644 monasca_log_api/tests/test_config.py create mode 100644 releasenotes/notes/cli_args-6dc5e2d13337b871.yaml diff --git a/.gitignore b/.gitignore index cc590837..0c8f4548 100644 --- a/.gitignore +++ b/.gitignore @@ -3,13 +3,13 @@ doc/build/* dist build -cover -.coverage +cover/ +.coverage.* *.egg *.egg-info .eggs/ -.testrepository -.tox +.testrepository/ +.tox/ ChangeLog MANIFEST AUTHORS diff --git a/monasca_log_api/app/main.py b/monasca_log_api/app/main.py index 0c50480f..16e7a8c7 100644 --- a/monasca_log_api/app/main.py +++ b/monasca_log_api/app/main.py @@ -30,7 +30,7 @@ def get_wsgi_app(): return deploy.loadapp( 'config:%s/log-api-paste.ini' % config_dir, - relative_to='../../', + relative_to='./', name='main' ) diff --git a/monasca_log_api/config.py b/monasca_log_api/config.py index a5a53c48..9bd9d656 100644 --- a/monasca_log_api/config.py +++ b/monasca_log_api/config.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +import sys + from oslo_log import log from monasca_log_api import conf @@ -21,9 +23,17 @@ CONF = conf.CONF LOG = log.getLogger(__name__) _CONF_LOADED = False +_GUNICORN_MARKER = 'gunicorn' -def parse_args(): +def _is_running_under_gunicorn(): + """Evaluates if api runs under gunicorn""" + content = filter(lambda x: x != sys.executable and _GUNICORN_MARKER in x, + sys.argv or []) + return len(list(content) if not isinstance(content, list) else content) > 0 + + +def parse_args(argv=None): global _CONF_LOADED if _CONF_LOADED: LOG.debug('Configuration has been already loaded') @@ -32,11 +42,10 @@ def parse_args(): log.set_defaults() log.register_options(CONF) - CONF(args=[], - # NOTE(trebskit) this disables any oslo.cfg CLI - # opts as gunicorn has some trouble with them - # i.e. gunicorn's argparse clashes with the one - # defined inside oslo.cfg + argv = (argv if argv is not None else sys.argv[1:]) + args = ([] if _is_running_under_gunicorn() else argv or []) + + CONF(args=args, prog='log-api', project='monasca', version=version.version_str, diff --git a/monasca_log_api/tests/base.py b/monasca_log_api/tests/base.py index 63264ddd..716bc7cc 100644 --- a/monasca_log_api/tests/base.py +++ b/monasca_log_api/tests/base.py @@ -20,23 +20,16 @@ import string import falcon from falcon import testing +import fixtures import mock from oslo_config import fixture as oo_cfg from oslo_context import fixture as oo_ctx -from oslotest import base as os_test +from oslotest import base as oslotest_base import six from monasca_log_api.api.core import request from monasca_log_api import conf - - -def mock_config(test): - conf.register_opts() - return test.useFixture(oo_cfg.Config(conf=conf.CONF)) - - -def mock_context(test): - return test.useFixture(oo_ctx.ClearRequestContext()) +from monasca_log_api import config class MockedAPI(falcon.API): @@ -125,19 +118,59 @@ UNICODE_MESSAGES = [ ] -class DisableStatsdMixin(object): +class DisableStatsdFixture(fixtures.Fixture): + def setUp(self): - super(DisableStatsdMixin, self).setUp() - self.statsd_patch = mock.patch('monascastatsd.Connection') - self.statsd_check = self.statsd_patch.start() + super(DisableStatsdFixture, self).setUp() + statsd_patch = mock.patch('monascastatsd.Connection') + statsd_patch.start() + self.addCleanup(statsd_patch.stop) -class BaseTestCase(DisableStatsdMixin, os_test.BaseTestCase): - pass +class ConfigFixture(oo_cfg.Config): + """Mocks configuration""" + + def __init__(self): + super(ConfigFixture, self).__init__(config.CONF) + + def setUp(self): + super(ConfigFixture, self).setUp() + self.addCleanup(self._clean_config_loaded_flag) + conf.register_opts() + self._set_defaults() + config.parse_args(argv=[]) # prevent oslo from parsing test args + + @staticmethod + def _clean_config_loaded_flag(): + config._CONF_LOADED = False + + def _set_defaults(self): + self.conf.set_default('kafka_url', '127.0.0.1', 'kafka_healthcheck') + self.conf.set_default('kafka_url', '127.0.0.1', 'log_publisher') + + +class BaseTestCase(oslotest_base.BaseTestCase): + + def setUp(self): + super(BaseTestCase, self).setUp() + self.useFixture(ConfigFixture()) + self.useFixture(DisableStatsdFixture()) + self.useFixture(oo_ctx.ClearRequestContext()) + + @staticmethod + def conf_override(**kw): + """Override flag variables for a test.""" + group = kw.pop('group', None) + for k, v in kw.items(): + config.CONF.set_override(k, v, group) + + @staticmethod + def conf_default(**kw): + """Override flag variables for a test.""" + group = kw.pop('group', None) + for k, v in kw.items(): + config.CONF.set_default(k, v, group) class BaseApiTestCase(BaseTestCase, testing.TestBase): api_class = MockedAPI - - def before(self): - self.conf = mock_config(self) diff --git a/monasca_log_api/tests/test_config.py b/monasca_log_api/tests/test_config.py new file mode 100644 index 00000000..20f16fe6 --- /dev/null +++ b/monasca_log_api/tests/test_config.py @@ -0,0 +1,40 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from monasca_log_api import config +from monasca_log_api.tests import base + + +class TestConfig(base.BaseTestCase): + + @mock.patch('monasca_log_api.config.sys') + def test_should_return_true_if_runs_under_gunicorn(self, sys_patch): + sys_patch.argv = [ + '/bin/gunicorn', + '--capture-output', + '--paste', + 'etc/monasca/log-api-paste.ini', + '--workers', + '1' + ] + sys_patch.executable = '/bin/python' + self.assertTrue(config._is_running_under_gunicorn()) + + @mock.patch('monasca_log_api.config.sys') + def test_should_return_false_if_runs_without_gunicorn(self, sys_patch): + sys_patch.argv = ['/bin/monasca-log-api'] + sys_patch.executable = '/bin/python' + self.assertFalse(config._is_running_under_gunicorn()) diff --git a/monasca_log_api/tests/test_healthchecks.py b/monasca_log_api/tests/test_healthchecks.py index f8c33f96..50432cf1 100644 --- a/monasca_log_api/tests/test_healthchecks.py +++ b/monasca_log_api/tests/test_healthchecks.py @@ -24,8 +24,8 @@ ENDPOINT = '/healthcheck' class TestApiHealthChecks(base.BaseApiTestCase): + def before(self): - super(TestApiHealthChecks, self).before() self.resource = healthchecks.HealthChecks() self.api.add_route( ENDPOINT, diff --git a/monasca_log_api/tests/test_kafka_check.py b/monasca_log_api/tests/test_kafka_check.py index 2067774d..21f7dbe1 100644 --- a/monasca_log_api/tests/test_kafka_check.py +++ b/monasca_log_api/tests/test_kafka_check.py @@ -29,14 +29,9 @@ class KafkaCheckLogicTest(base.BaseTestCase): 'kafka_topics': mocked_topics } - def __init__(self, *args, **kwargs): - super(KafkaCheckLogicTest, self).__init__(*args, **kwargs) - self._conf = None - def setUp(self): super(KafkaCheckLogicTest, self).setUp() - self._conf = base.mock_config(self) - self._conf.config(group='kafka_healthcheck', **self.mock_config) + self.conf_default(group='kafka_healthcheck', **self.mock_config) @mock.patch('monasca_log_api.healthcheck.kafka_check.client.KafkaClient') def test_should_fail_kafka_unavailable(self, kafka_client): diff --git a/monasca_log_api/tests/test_log_publisher.py b/monasca_log_api/tests/test_log_publisher.py index d588b29a..af3499d9 100644 --- a/monasca_log_api/tests/test_log_publisher.py +++ b/monasca_log_api/tests/test_log_publisher.py @@ -20,6 +20,7 @@ import ujson import unittest import mock +from oslo_config import cfg from oslo_log import log import six @@ -33,10 +34,6 @@ EPOCH_START = datetime.datetime(1970, 1, 1) class TestSendMessage(base.BaseTestCase): - def setUp(self): - self.conf = base.mock_config(self) - return super(TestSendMessage, self).setUp() - @mock.patch('monasca_log_api.reference.common.log_publisher.producer' '.KafkaProducer') def test_should_not_send_empty_message(self, _): @@ -133,16 +130,16 @@ class TestSendMessage(base.BaseTestCase): instance.send_message(msg) instance._kafka_publisher.publish.assert_called_once_with( - self.conf.conf.log_publisher.topics[0], + cfg.CONF.log_publisher.topics[0], [ujson.dumps(msg, ensure_ascii=False).encode('utf-8')]) @mock.patch('monasca_log_api.reference.common.log_publisher.producer' '.KafkaProducer') def test_should_send_message_multiple_topics(self, _): topics = ['logs', 'analyzer', 'tester'] - self.conf.config(topics=topics, - max_message_size=5000, - group='log_publisher') + self.conf_override(topics=topics, + max_message_size=5000, + group='log_publisher') instance = log_publisher.LogPublisher() instance._kafka_publisher = mock.Mock() @@ -210,7 +207,7 @@ class TestSendMessage(base.BaseTestCase): expected_message = expected_message.encode('utf-8') instance._kafka_publisher.publish.assert_called_with( - self.conf.conf.log_publisher.topics[0], + cfg.CONF.log_publisher.topics[0], [expected_message] ) except Exception: @@ -228,14 +225,6 @@ class TestTruncation(base.BaseTestCase): } }), 'utf8')) - 2 - def __init__(self, *args, **kwargs): - super(TestTruncation, self).__init__(*args, **kwargs) - self._conf = None - - def setUp(self): - super(TestTruncation, self).setUp() - self._conf = base.mock_config(self) - def test_should_not_truncate_message_if_size_is_smaller(self, _): diff_size = random.randint(1, 100) self._run_truncate_test(log_size_factor=-diff_size, @@ -269,7 +258,7 @@ class TestTruncation(base.BaseTestCase): expected_log_message_size = log_size - truncate_by - self._conf.config( + self.conf_override( group='log_publisher', max_message_size=max_message_size ) diff --git a/monasca_log_api/tests/test_logs.py b/monasca_log_api/tests/test_logs.py index f68be3f9..95fb9576 100644 --- a/monasca_log_api/tests/test_logs.py +++ b/monasca_log_api/tests/test_logs.py @@ -196,7 +196,7 @@ class TestApiLogs(base.BaseApiTestCase): max_log_size = 1000 content_length = max_log_size - 100 - self.conf.config(max_log_size=max_log_size, group='service') + self.conf_override(max_log_size=max_log_size, group='service') self.simulate_request( '/log/single', @@ -217,7 +217,7 @@ class TestApiLogs(base.BaseApiTestCase): max_log_size = 1000 content_length = max_log_size + 100 - self.conf.config(max_log_size=max_log_size, group='service') + self.conf_override(max_log_size=max_log_size, group='service') self.simulate_request( '/log/single', @@ -238,7 +238,7 @@ class TestApiLogs(base.BaseApiTestCase): max_log_size = 1000 content_length = max_log_size - self.conf.config(max_log_size=max_log_size, group='service') + self.conf_override(max_log_size=max_log_size, group='service') self.simulate_request( '/log/single', diff --git a/monasca_log_api/tests/test_logs_v3.py b/monasca_log_api/tests/test_logs_v3.py index a6e9add6..b45e62ce 100644 --- a/monasca_log_api/tests/test_logs_v3.py +++ b/monasca_log_api/tests/test_logs_v3.py @@ -203,9 +203,6 @@ class TestApiLogsMonitoring(base.BaseApiTestCase): class TestApiLogs(base.BaseApiTestCase): - def before(self): - self.conf = base.mock_config(self) - @mock.patch('monasca_log_api.reference.v3.common.bulk_processor.' 'BulkProcessor') def test_should_pass_cross_tenant_id(self, bulk_processor): diff --git a/monasca_log_api/tests/test_monitoring.py b/monasca_log_api/tests/test_monitoring.py index 055b6cba..1048216c 100644 --- a/monasca_log_api/tests/test_monitoring.py +++ b/monasca_log_api/tests/test_monitoring.py @@ -20,10 +20,6 @@ from monasca_log_api.tests import base class TestMonitoring(base.BaseTestCase): - def setUp(self): - super(TestMonitoring, self).setUp() - base.mock_config(self) - @mock.patch('monasca_log_api.monitoring.client.monascastatsd') def test_should_use_default_dimensions_if_none_specified(self, monascastatsd): diff --git a/monasca_log_api/tests/test_request.py b/monasca_log_api/tests/test_request.py index d969da25..ee97c26e 100644 --- a/monasca_log_api/tests/test_request.py +++ b/monasca_log_api/tests/test_request.py @@ -22,11 +22,6 @@ from monasca_log_api.tests import base class TestRequest(base.BaseTestCase): - def setUp(self): - super(TestRequest, self).setUp() - base.mock_config(self) - base.mock_context(self) - def test_use_context_from_request(self): req = request.Request( testing.create_environ( diff --git a/monasca_log_api/tests/test_service.py b/monasca_log_api/tests/test_service.py index 08f7b9e6..fc04fb61 100644 --- a/monasca_log_api/tests/test_service.py +++ b/monasca_log_api/tests/test_service.py @@ -30,13 +30,8 @@ class IsDelegate(base.BaseTestCase): def __init__(self, *args, **kwargs): super(IsDelegate, self).__init__(*args, **kwargs) - self._conf = None self._roles = ['admin'] - def setUp(self): - super(IsDelegate, self).setUp() - self._conf = base.mock_config(self) - def test_is_delegate_ok_role(self): self.assertTrue(validation.validate_is_delegate(self._roles)) @@ -277,9 +272,6 @@ class ContentTypeValidations(base.BaseTestCase): class PayloadSizeValidations(base.BaseTestCase): - def setUp(self): - super(PayloadSizeValidations, self).setUp() - self.conf = base.mock_config(self) def test_should_fail_missing_header(self): content_length = None @@ -294,8 +286,8 @@ class PayloadSizeValidations(base.BaseTestCase): def test_should_pass_limit_not_exceeded(self): content_length = 120 max_log_size = 240 - self.conf.config(max_log_size=max_log_size, - group='service') + self.conf_override(max_log_size=max_log_size, + group='service') req = mock.Mock() req.content_length = content_length @@ -305,8 +297,8 @@ class PayloadSizeValidations(base.BaseTestCase): def test_should_fail_limit_exceeded(self): content_length = 120 max_log_size = 60 - self.conf.config(max_log_size=max_log_size, - group='service') + self.conf_override(max_log_size=max_log_size, + group='service') req = mock.Mock() req.content_length = content_length @@ -320,8 +312,8 @@ class PayloadSizeValidations(base.BaseTestCase): def test_should_fail_limit_equal(self): content_length = 120 max_log_size = 120 - self.conf.config(max_log_size=max_log_size, - group='service') + self.conf_override(max_log_size=max_log_size, + group='service') req = mock.Mock() req.content_length = content_length diff --git a/monasca_log_api/tests/test_versions.py b/monasca_log_api/tests/test_versions.py index aace7e0b..07b172b9 100644 --- a/monasca_log_api/tests/test_versions.py +++ b/monasca_log_api/tests/test_versions.py @@ -24,12 +24,8 @@ def _get_versioned_url(version_id): class TestApiVersions(base.BaseApiTestCase): - def __init__(self, *args, **kwargs): - self.versions = None - super(TestApiVersions, self).__init__(*args, **kwargs) def before(self): - super(TestApiVersions, self).before() self.versions = versions.Versions() self.api.add_route("/version/", self.versions) self.api.add_route("/version/{version_id}", self.versions) diff --git a/releasenotes/notes/cli_args-6dc5e2d13337b871.yaml b/releasenotes/notes/cli_args-6dc5e2d13337b871.yaml new file mode 100644 index 00000000..b25af046 --- /dev/null +++ b/releasenotes/notes/cli_args-6dc5e2d13337b871.yaml @@ -0,0 +1,6 @@ +--- +other: + - | + Enabled possibility of specifying the CLI arguments, when launching + monasca-log-api, for cases where API is not deployed using Gunicorn + server.