diff --git a/.gitignore b/.gitignore index 3fa76f18..cc590837 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ ChangeLog MANIFEST AUTHORS monasca-log-api.log - +etc/monasca/log-api.conf.sample *.swp *.iml diff --git a/config-generator/README.md b/config-generator/README.md new file mode 100644 index 00000000..0985af25 --- /dev/null +++ b/config-generator/README.md @@ -0,0 +1,7 @@ +# config-generator + +To generate sample configuration execute + +```sh +tox -e genconfig +``` diff --git a/config-generator/monasca-log-api.conf b/config-generator/monasca-log-api.conf new file mode 100644 index 00000000..b889fd96 --- /dev/null +++ b/config-generator/monasca-log-api.conf @@ -0,0 +1,7 @@ +[DEFAULT] +output_file = etc/monasca/log-api.conf.sample +wrap_width = 79 +format = ini +summarize = True +namespace = monasca_log_api +namespace = oslo.log diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 19be0733..1a073699 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -184,17 +184,28 @@ function configure_monasca_log_api { # ensure fresh installation of configuration files rm -rf $MONASCA_LOG_API_CONF $MONASCA_LOG_API_PASTE $MONASCA_LOG_API_LOGGING_CONF - if [[ "$MONASCA_LOG_API_CONF_DIR" != "$MONASCA_LOG_API_DIR/etc/monasca" ]]; then - install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api.conf $MONASCA_LOG_API_CONF - install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api-paste.ini $MONASCA_LOG_API_PASTE - install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api-logging.conf $MONASCA_LOG_API_LOGGING_CONF - fi + $MONASCA_LOG_API_BIN_DIR/oslo-config-generator \ + --config-file $MONASCA_LOG_API_DIR/config-generator/monasca-log-api.conf \ + --output-file /tmp/log-api.conf + + install -m 600 /tmp/log-api.conf $MONASCA_LOG_API_CONF && rm -rf /tmp/log-api.conf + install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api-paste.ini $MONASCA_LOG_API_PASTE + install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api-logging.conf $MONASCA_LOG_API_LOGGING_CONF # configure log-api.conf iniset "$MONASCA_LOG_API_CONF" DEFAULT log_config_append $MONASCA_LOG_API_LOGGING_CONF iniset "$MONASCA_LOG_API_CONF" service region $REGION_NAME + iniset "$MONASCA_LOG_API_CONF" log_publisher kafka_url $KAFKA_SERVICE_HOST:$KAFKA_SERVICE_PORT + iniset "$MONASCA_LOG_API_CONF" log_publisher topics log + iniset "$MONASCA_LOG_API_CONF" kafka_healthcheck kafka_url $KAFKA_SERVICE_HOST:$KAFKA_SERVICE_PORT + iniset "$MONASCA_LOG_API_CONF" kafka_healthcheck kafka_topics log + + iniset "$MONASCA_LOG_API_CONF" roles_middleware path "/v2.0/log,/v3.0/logs" + iniset "$MONASCA_LOG_API_CONF" roles_middleware default_roles monasca-user + iniset "$MONASCA_LOG_API_CONF" roles_middleware agent_roles monasca-agent + iniset "$MONASCA_LOG_API_CONF" roles_middleware delegate_roles admin # configure keystone middleware configure_auth_token_middleware "$MONASCA_LOG_API_CONF" "admin" $MONASCA_LOG_API_CACHE_DIR diff --git a/doc/source/.gitignore b/doc/source/.gitignore new file mode 100644 index 00000000..22124f4a --- /dev/null +++ b/doc/source/.gitignore @@ -0,0 +1 @@ +_static/*.conf.sample diff --git a/doc/source/_static/.gitkeep b/doc/source/_static/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/doc/source/conf.py b/doc/source/conf.py index 085779f9..362e97c3 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -36,9 +36,8 @@ extensions = [ 'sphinx.ext.graphviz', 'sphinx.ext.autodoc', 'sphinx.ext.viewcode', - # TODO(trebskit) enable as soon as we get configgen in place - # 'oslo_config.sphinxconfiggen' - # 'oslo_config.sphinxext', + 'oslo_config.sphinxconfiggen', + 'oslo_config.sphinxext', 'openstackdocstheme', ] @@ -52,6 +51,11 @@ bug_tag = u'doc' copyright = u'2014-present, OpenStack Foundation' author = u'OpenStack Foundation' +# sample config +config_generator_config_file = [ + ('config-generator/monasca-log-api.conf', '_static/log-api') +] + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -124,7 +128,7 @@ html_theme = 'openstackdocs' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = [] +html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst index 6910c2bd..45fc312b 100644 --- a/doc/source/configuration/index.rst +++ b/doc/source/configuration/index.rst @@ -12,7 +12,8 @@ monasca-log-api uses the following configuration files for its various services. .. toctree:: - :glob: :maxdepth: 1 - * + configuring + monasca_log_api + sample diff --git a/doc/source/configuration/monasca_log_api.rst b/doc/source/configuration/monasca_log_api.rst new file mode 100644 index 00000000..a138f8e7 --- /dev/null +++ b/doc/source/configuration/monasca_log_api.rst @@ -0,0 +1,8 @@ +.. _monasca-log-api.conf: + +-------------------- +monasca-log-api.conf +-------------------- + +.. show-options:: + :config-file: config-generator/monasca-log-api.conf diff --git a/doc/source/configuration/sample.rst b/doc/source/configuration/sample.rst new file mode 100644 index 00000000..02807e38 --- /dev/null +++ b/doc/source/configuration/sample.rst @@ -0,0 +1,37 @@ +.. _sample-configuration: + +==================================== +monasca-log-api sample configuration +==================================== + +The following are sample configuration files for monasca-log-api and +utilities. These are generated from code +(apart from logging conf and paste conf) and reflect the current state +of code in the monasca-log-api repository. + + +Sample configuration for monasca-log-api +---------------------------------------- + +This sample configuration can also be viewed in `log-api.conf.sample +<../_static/log-api.conf.sample>`_. + +.. literalinclude:: ../_static/log-api.conf.sample + + +Sample configuration for logging +-------------------------------- + +This sample configuration can also be viewed in `log-api-logging.conf +`_. + +.. literalinclude:: ../../../etc/monasca/log-api-logging.conf + + +Sample configuration for paste +------------------------------ + +This sample configuration can also be viewed in `log-api-paste.ini +`_. + +.. literalinclude:: ../../../etc/monasca/log-api-paste.ini diff --git a/etc/monasca/log-api.conf b/etc/monasca/log-api.conf deleted file mode 100644 index 0712f227..00000000 --- a/etc/monasca/log-api.conf +++ /dev/null @@ -1,56 +0,0 @@ -# -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# note(trebskit) all URLs points to default monasca-log-api devstack setup - -[DEFAULT] -log_config_append=/etc/monasca/log-api-logging.conf - -[monitoring] -statsd_host = 127.0.0.1 -statsd_port = 8125 -statsd_buffer = 50 - -[service] -region = region-one -max_log_size = 1048576 - -[log_publisher] -topics = log -kafka_url = 192.168.10.6:9092 -max_message_size = 1048576 - -[keystone_authtoken] -auth_uri = http://127.0.0.1:5000 -auth_url = http://127.0.0.1:5000 -identity_uri = http://127.0.0.1:35357 -admin_user = admin -admin_password = secretadmin -admin_tenant_name = admin -insecure = True -token_cache_time = 300 -memcached_servers = 127.0.0.1:11211 - -[kafka_healthcheck] -kafka_url = 192.168.10.6:9092 -kafka_topics = log - -[roles_middleware] -path = /v2.0/log,/v3.0/logs -default_roles = user,domainuser,domainadmin,monasca-user -agent_roles = monasca-agent -delegate_roles = admin diff --git a/monasca_log_api/app/api.py b/monasca_log_api/app/api.py index 19023845..2eef4531 100644 --- a/monasca_log_api/app/api.py +++ b/monasca_log_api/app/api.py @@ -20,21 +20,15 @@ of monasca-log-api import six import falcon -from oslo_config import cfg from oslo_log import log from monasca_log_api.api.core import request +from monasca_log_api import config from monasca_log_api.reference.common import error_handlers from monasca_log_api.reference import healthchecks from monasca_log_api.reference.v2 import logs as v2_logs from monasca_log_api.reference.v3 import logs as v3_logs from monasca_log_api.reference import versions -from monasca_log_api import version - -LOG = log.getLogger(__name__) -CONF = cfg.CONF - -_CONF_LOADED = False def error_trap(app_name): @@ -48,7 +42,8 @@ def error_trap(app_name): try: return func(*args, **kwargs) except Exception: - LOG.exception('Failed to load application \'%s\'', app_name) + logger = log.getLogger(__name__) + logger.exception('Failed to load application \'%s\'', app_name) raise return _inner_wrapper @@ -61,34 +56,9 @@ def singleton_config(func): @six.wraps(singleton_config) def _wrapper(global_config, **local_conf): - _load_config() + config.parse_args() return func(global_config, **local_conf) - def _load_config(): - global _CONF_LOADED - if _CONF_LOADED: - LOG.debug('Configuration has been already loaded') - return - - log.set_defaults() - log.register_options(CONF) - - CONF(args=[], - # NOTE(trebskit) this disables any oslo.cfg CLI - # opts as gunicorn has some trouble with them - # i.e. gunicorn's argparse clashes with the one - # defined inside oslo.cfg - prog='log-api', - project='monasca', - version=version.version_str, - description='REST-ful API to collect log files') - - log.setup(CONF, - product_name='monasca-log-api', - version=version.version_str) - - _CONF_LOADED = True - return _wrapper diff --git a/monasca_log_api/conf/__init__.py b/monasca_log_api/conf/__init__.py new file mode 100644 index 00000000..d4ef831d --- /dev/null +++ b/monasca_log_api/conf/__init__.py @@ -0,0 +1,84 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import pkgutil + +from oslo_config import cfg +from oslo_log import log +from oslo_utils import importutils + +CONF = cfg.CONF +LOG = log.getLogger(__name__) + + +def load_conf_modules(): + """Loads all modules that contain configuration + + Method iterates over modules of :py:module:`monasca_log_api.conf` + and imports only those that contain following methods: + + - list_opts (required by oslo_config.genconfig) + - register_opts (required by :py:currentmodule:) + + """ + imported_modules = [] + for modname in _list_module_names(): + mod = importutils.import_module('monasca_log_api.conf.' + modname) + required_funcs = ['register_opts', 'list_opts'] + for func in required_funcs: + if not hasattr(mod, func): + msg = ("The module 'monasca_log_api.conf.%s' should have a" + " '%s' function which returns" + " the config options." + % (modname, func)) + LOG.warning(msg) + else: + imported_modules.append(mod) + + LOG.debug('Found %d modules that contain configuration', + len(imported_modules)) + + return imported_modules + + +def _list_module_names(): + module_names = [] + package_path = os.path.dirname(os.path.abspath(__file__)) + for _, modname, ispkg in pkgutil.iter_modules(path=[package_path]): + if not (modname == "opts" and ispkg): + module_names.append(modname) + return module_names + + +def register_opts(): + """Registers all conf modules opts + + This method allows different modules to register + opts according to their needs. + + """ + for mod in load_conf_modules(): + mod.register_opts(CONF) + + +def list_opts(): + """Lists all conf modules opts. + + Goes through all conf modules and yields their opts + + """ + for mod in load_conf_modules(): + mod_opts = mod.list_opts() + yield mod_opts[0], mod_opts[1] diff --git a/monasca_log_api/conf/healthcheck.py b/monasca_log_api/conf/healthcheck.py new file mode 100644 index 00000000..00cf215b --- /dev/null +++ b/monasca_log_api/conf/healthcheck.py @@ -0,0 +1,36 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +kafka_check_opts = [ + cfg.StrOpt('kafka_url', + required=True, + help='Url to kafka server'), + cfg.ListOpt('kafka_topics', + required=True, + default=['logs'], + help='Verify existence of configured topics') +] +kafka_check_group = cfg.OptGroup(name='kafka_healthcheck', + title='kafka_healthcheck') + + +def register_opts(conf): + conf.register_group(kafka_check_group) + conf.register_opts(kafka_check_opts, kafka_check_group) + + +def list_opts(): + return kafka_check_group, kafka_check_opts diff --git a/monasca_log_api/conf/log_publisher.py b/monasca_log_api/conf/log_publisher.py new file mode 100644 index 00000000..5a770729 --- /dev/null +++ b/monasca_log_api/conf/log_publisher.py @@ -0,0 +1,41 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +_MAX_MESSAGE_SIZE = 1048576 + +log_publisher_opts = [ + cfg.StrOpt('kafka_url', + required=True, + help='Url to kafka server'), + cfg.MultiStrOpt('topics', + default=['logs'], + help='Consumer topics'), + cfg.IntOpt('max_message_size', + default=_MAX_MESSAGE_SIZE, + required=True, + help=('Message max size that can be sent ' + 'to kafka, default to %d bytes' % _MAX_MESSAGE_SIZE)) +] +log_publisher_group = cfg.OptGroup(name='log_publisher', title='log_publisher') + + +def register_opts(conf): + conf.register_group(log_publisher_group) + conf.register_opts(log_publisher_opts, log_publisher_group) + + +def list_opts(): + return log_publisher_group, log_publisher_opts diff --git a/monasca_log_api/conf/monitoring.py b/monasca_log_api/conf/monitoring.py new file mode 100644 index 00000000..40533777 --- /dev/null +++ b/monasca_log_api/conf/monitoring.py @@ -0,0 +1,46 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +_DEFAULT_HOST = '127.0.0.1' +_DEFAULT_PORT = 8125 +_DEFAULT_BUFFER_SIZE = 50 + +monitoring_opts = [ + cfg.IPOpt('statsd_host', + default=_DEFAULT_HOST, + help=('IP address of statsd server, default to %s' + % _DEFAULT_HOST)), + cfg.PortOpt('statsd_port', + default=_DEFAULT_PORT, + help='Port of statsd server, default to %d' % _DEFAULT_PORT), + cfg.IntOpt('statsd_buffer', + default=_DEFAULT_BUFFER_SIZE, + required=True, + help=('Maximum number of metric to buffer before sending, ' + 'default to %d' % _DEFAULT_BUFFER_SIZE)), + cfg.DictOpt('dimensions', default={}, + required=False, help='Additional dimensions that can be set') +] +monitoring_group = cfg.OptGroup(name='monitoring', title='monitoring') + + +def register_opts(conf): + conf.register_group(monitoring_group) + conf.register_opts(monitoring_opts, monitoring_group) + + +def list_opts(): + return monitoring_group, monitoring_opts diff --git a/monasca_log_api/conf/role_middleware.py b/monasca_log_api/conf/role_middleware.py new file mode 100644 index 00000000..2e01f7f7 --- /dev/null +++ b/monasca_log_api/conf/role_middleware.py @@ -0,0 +1,43 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +role_m_opts = [ + cfg.ListOpt(name='path', + default='/', + help='List of paths where middleware applies to'), + cfg.ListOpt(name='default_roles', + default=None, + help='List of roles allowed to enter api'), + cfg.ListOpt(name='agent_roles', + default=None, + help=('List of roles, that if set, mean that request ' + 'comes from agent, thus is authorized in the same ' + 'time')), + cfg.ListOpt(name='delegate_roles', + default=['admin'], + help=('Roles that are allowed to POST logs on ' + 'behalf of another tenant (project)')) +] +role_m_group = cfg.OptGroup(name='roles_middleware', title='roles_middleware') + + +def register_opts(conf): + conf.register_group(role_m_group) + conf.register_opts(role_m_opts, role_m_group) + + +def list_opts(): + return role_m_group, role_m_opts diff --git a/monasca_log_api/conf/service.py b/monasca_log_api/conf/service.py new file mode 100644 index 00000000..0ce2496d --- /dev/null +++ b/monasca_log_api/conf/service.py @@ -0,0 +1,37 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +_DEFAULT_MAX_LOG_SIZE = 1024 * 1024 + +service_opts = [ + cfg.StrOpt('region', + default=None, + help='Region'), + cfg.IntOpt('max_log_size', + default=_DEFAULT_MAX_LOG_SIZE, + help=('Refers to payload/envelope size. If either is exceeded' + 'API will throw an error')) +] +service_group = cfg.OptGroup(name='service', title='service') + + +def register_opts(conf): + conf.register_group(service_group) + conf.register_opts(service_opts, service_group) + + +def list_opts(): + return service_group, service_opts diff --git a/monasca_log_api/config.py b/monasca_log_api/config.py new file mode 100644 index 00000000..a5a53c48 --- /dev/null +++ b/monasca_log_api/config.py @@ -0,0 +1,51 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log + +from monasca_log_api import conf +from monasca_log_api import version + +CONF = conf.CONF +LOG = log.getLogger(__name__) + +_CONF_LOADED = False + + +def parse_args(): + global _CONF_LOADED + if _CONF_LOADED: + LOG.debug('Configuration has been already loaded') + return + + log.set_defaults() + log.register_options(CONF) + + CONF(args=[], + # NOTE(trebskit) this disables any oslo.cfg CLI + # opts as gunicorn has some trouble with them + # i.e. gunicorn's argparse clashes with the one + # defined inside oslo.cfg + prog='log-api', + project='monasca', + version=version.version_str, + description='RESTful API to collect log files') + + log.setup(CONF, + product_name='monasca-log-api', + version=version.version_str) + + conf.register_opts() + + _CONF_LOADED = True diff --git a/monasca_log_api/healthcheck/kafka_check.py b/monasca_log_api/healthcheck/kafka_check.py index d1dfc17f..82d03292 100644 --- a/monasca_log_api/healthcheck/kafka_check.py +++ b/monasca_log_api/healthcheck/kafka_check.py @@ -1,4 +1,4 @@ -# Copyright 2015 FUJITSU LIMITED +# Copyright 2015-2017 FUJITSU LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -14,28 +14,13 @@ import collections -from oslo_config import cfg +from monasca_common.kafka_lib import client from oslo_log import log -from monasca_common.kafka_lib import client +from monasca_log_api import conf LOG = log.getLogger(__name__) -CONF = cfg.CONF - -kafka_check_opts = [ - cfg.StrOpt('kafka_url', - required=True, - help='Url to kafka server'), - cfg.ListOpt('kafka_topics', - required=True, - default=['logs'], - help='Verify existence of configured topics') -] -kafka_check_group = cfg.OptGroup(name='kafka_healthcheck', - title='kafka_healthcheck') - -cfg.CONF.register_group(kafka_check_group) -cfg.CONF.register_opts(kafka_check_opts, kafka_check_group) +CONF = conf.CONF CheckResult = collections.namedtuple('CheckResult', ['healthy', 'message']) diff --git a/monasca_log_api/middleware/role_middleware.py b/monasca_log_api/middleware/role_middleware.py index 7fd2070b..40e7bee6 100644 --- a/monasca_log_api/middleware/role_middleware.py +++ b/monasca_log_api/middleware/role_middleware.py @@ -1,4 +1,4 @@ -# Copyright 2015 FUJITSU LIMITED +# Copyright 2015-2017 FUJITSU LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -12,36 +12,15 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo_config import cfg from oslo_log import log from oslo_middleware import base as om from webob import response -CONF = cfg.CONF +from monasca_log_api import conf + +CONF = conf.CONF LOG = log.getLogger(__name__) -role_m_opts = [ - cfg.ListOpt(name='path', - default='/', - help='List of paths where middleware applies to'), - cfg.ListOpt(name='default_roles', - default=None, - help='List of roles allowed to enter api'), - cfg.ListOpt(name='agent_roles', - default=None, - help=('List of roles, that if set, mean that request ' - 'comes from agent, thus is authorized in the same ' - 'time')), - cfg.ListOpt(name='delegate_roles', - default=['admin'], - help=('Roles that are allowed to POST logs on ' - 'behalf of another tenant (project)')) -] -role_m_group = cfg.OptGroup(name='roles_middleware', title='roles_middleware') - -CONF.register_group(role_m_group) -CONF.register_opts(role_m_opts, role_m_group) - _X_IDENTITY_STATUS = 'X-Identity-Status' _X_ROLES = 'X-Roles' _X_MONASCA_LOG_AGENT = 'X-MONASCA-LOG-AGENT' diff --git a/monasca_log_api/monitoring/client.py b/monasca_log_api/monitoring/client.py index 0f23829b..0a9174b1 100644 --- a/monasca_log_api/monitoring/client.py +++ b/monasca_log_api/monitoring/client.py @@ -1,4 +1,4 @@ -# Copyright 2016 FUJITSU LIMITED +# Copyright 2016-2017 FUJITSU LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -14,41 +14,18 @@ import monascastatsd -from oslo_config import cfg from oslo_log import log -LOG = log.getLogger(__name__) -CONF = cfg.CONF +from monasca_log_api import conf -_DEFAULT_HOST = '127.0.0.1' -_DEFAULT_PORT = 8125 -_DEFAULT_BUFFER_SIZE = 50 +LOG = log.getLogger(__name__) +CONF = conf.CONF + +_CLIENT_NAME = 'monasca' _DEFAULT_DIMENSIONS = { 'service': 'monitoring', 'component': 'monasca-log-api' } -_CLIENT_NAME = 'monasca' - -monitoring_opts = [ - cfg.IPOpt('statsd_host', - default=_DEFAULT_HOST, - help=('IP address of statsd server, default to %s' - % _DEFAULT_HOST)), - cfg.PortOpt('statsd_port', - default=_DEFAULT_PORT, - help='Port of statsd server, default to %d' % _DEFAULT_PORT), - cfg.IntOpt('statsd_buffer', - default=_DEFAULT_BUFFER_SIZE, - required=True, - help=('Maximum number of metric to buffer before sending, ' - 'default to %d' % _DEFAULT_BUFFER_SIZE)), - cfg.DictOpt('dimensions') -] - -monitoring_group = cfg.OptGroup(name='monitoring', title='monitoring') - -cfg.CONF.register_group(monitoring_group) -cfg.CONF.register_opts(monitoring_opts, monitoring_group) def get_client(dimensions=None): diff --git a/monasca_log_api/reference/common/log_publisher.py b/monasca_log_api/reference/common/log_publisher.py index 1fd21df4..292fb055 100644 --- a/monasca_log_api/reference/common/log_publisher.py +++ b/monasca_log_api/reference/common/log_publisher.py @@ -18,17 +18,16 @@ import time import falcon from monasca_common.kafka import producer from monasca_common.rest import utils as rest_utils -from oslo_config import cfg from oslo_log import log +from monasca_log_api import conf from monasca_log_api.monitoring import client from monasca_log_api.monitoring import metrics from monasca_log_api.reference.common import model LOG = log.getLogger(__name__) -CONF = cfg.CONF +CONF = conf.CONF -_MAX_MESSAGE_SIZE = 1048576 _RETRY_AFTER = 60 _TIMESTAMP_KEY_SIZE = len( bytearray(str(int(time.time() * 1000)).encode('utf-8'))) @@ -37,25 +36,6 @@ _TRUNCATED_PROPERTY_SIZE = len( _KAFKA_META_DATA_SIZE = 32 _TRUNCATION_SAFE_OFFSET = 1 -log_publisher_opts = [ - cfg.StrOpt('kafka_url', - required=True, - help='Url to kafka server'), - cfg.MultiStrOpt('topics', - default=['logs'], - help='Consumer topics'), - cfg.IntOpt('max_message_size', - default=_MAX_MESSAGE_SIZE, - required=True, - help=('Message max size that can be sent ' - 'to kafka, default to %d bytes' % _MAX_MESSAGE_SIZE)) -] - -log_publisher_group = cfg.OptGroup(name='log_publisher', title='log_publisher') - -cfg.CONF.register_group(log_publisher_group) -cfg.CONF.register_opts(log_publisher_opts, log_publisher_group) - class InvalidMessageException(Exception): pass diff --git a/monasca_log_api/reference/common/validation.py b/monasca_log_api/reference/common/validation.py index bb1f668f..afe27455 100644 --- a/monasca_log_api/reference/common/validation.py +++ b/monasca_log_api/reference/common/validation.py @@ -15,14 +15,14 @@ import re import falcon -from oslo_config import cfg from oslo_log import log import six from monasca_log_api.api import exceptions +from monasca_log_api import conf LOG = log.getLogger(__name__) -CONF = cfg.CONF +CONF = conf.CONF APPLICATION_TYPE_CONSTRAINTS = { 'MAX_LENGTH': 255, diff --git a/monasca_log_api/reference/v2/common/service.py b/monasca_log_api/reference/v2/common/service.py index 5704cc13..ecd9ac00 100644 --- a/monasca_log_api/reference/v2/common/service.py +++ b/monasca_log_api/reference/v2/common/service.py @@ -20,27 +20,12 @@ from oslo_config import cfg from oslo_log import log from monasca_log_api.api import exceptions +from monasca_log_api import conf from monasca_log_api.reference.common import model from monasca_log_api.reference.common import validation LOG = log.getLogger(__name__) -CONF = cfg.CONF - -_DEFAULT_MAX_LOG_SIZE = 1024 * 1024 - -service_opts = [ - cfg.StrOpt('region', - default=None, - help='Region'), - cfg.IntOpt('max_log_size', - default=_DEFAULT_MAX_LOG_SIZE, - help=('Refers to payload/envelope size. If either is exceeded' - 'API will throw an error')) -] -service_group = cfg.OptGroup(name='service', title='service') - -CONF.register_group(service_group) -CONF.register_opts(service_opts, service_group) +CONF = conf.CONF EPOCH_START = datetime.datetime(1970, 1, 1) diff --git a/monasca_log_api/reference/v3/common/bulk_processor.py b/monasca_log_api/reference/v3/common/bulk_processor.py index ad50cc3e..1a04ca59 100644 --- a/monasca_log_api/reference/v3/common/bulk_processor.py +++ b/monasca_log_api/reference/v3/common/bulk_processor.py @@ -12,15 +12,15 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo_config import cfg from oslo_log import log +from monasca_log_api import conf from monasca_log_api.reference.common import log_publisher from monasca_log_api.reference.common import model from monasca_log_api.reference.common import validation LOG = log.getLogger(__name__) -CONF = cfg.CONF +CONF = conf.CONF class BulkProcessor(log_publisher.LogPublisher): diff --git a/monasca_log_api/tests/base.py b/monasca_log_api/tests/base.py index a2ab59ce..63264ddd 100644 --- a/monasca_log_api/tests/base.py +++ b/monasca_log_api/tests/base.py @@ -27,10 +27,12 @@ from oslotest import base as os_test import six from monasca_log_api.api.core import request +from monasca_log_api import conf def mock_config(test): - return test.useFixture(oo_cfg.Config()) + conf.register_opts() + return test.useFixture(oo_cfg.Config(conf=conf.CONF)) def mock_context(test): diff --git a/monasca_log_api/tests/test_monitoring.py b/monasca_log_api/tests/test_monitoring.py index 101ad373..055b6cba 100644 --- a/monasca_log_api/tests/test_monitoring.py +++ b/monasca_log_api/tests/test_monitoring.py @@ -19,6 +19,11 @@ from monasca_log_api.tests import base class TestMonitoring(base.BaseTestCase): + + def setUp(self): + super(TestMonitoring, self).setUp() + base.mock_config(self) + @mock.patch('monasca_log_api.monitoring.client.monascastatsd') def test_should_use_default_dimensions_if_none_specified(self, monascastatsd): diff --git a/releasenotes/notes/osloconfiggen-d8a0f0a8d1acb961.yaml b/releasenotes/notes/osloconfiggen-d8a0f0a8d1acb961.yaml new file mode 100644 index 00000000..8415aef8 --- /dev/null +++ b/releasenotes/notes/osloconfiggen-d8a0f0a8d1acb961.yaml @@ -0,0 +1,18 @@ +--- +prelude: > + Matching OpenStack projects in configuration handling. +other: + - Removed configuration file from project tree. It is now generated via + oslo-config-generator utility. + + Moved all the code where configuration options are registered into single + location to mimic the configuration file (i.e. provide single place as it + was with configuration file). + + Removed final place where configuration is duplicated. In other words + person providing and/or modyfying options could have been forced to include + them also in the configuration file held in the tree. Not the only place + where it is required is Python codebase. + + Devstack plugin is also using new utility to provide configuration for + monasca-log-api launched inside the VM or Gate environment. diff --git a/setup.cfg b/setup.cfg index f27af57e..638f0638 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,6 +43,9 @@ wsgi_scripts = tempest.test_plugins = monasca_log_api_tests = monasca_log_api_tempest.plugin:MonascaLogApiTempestPlugin +oslo.config.opts = + monasca_log_api = monasca_log_api.conf:list_opts + [build_sphinx] all_files = 1 build-dir = doc/build diff --git a/tox.ini b/tox.ini index 9efa6dcb..815975dd 100644 --- a/tox.ini +++ b/tox.ini @@ -81,6 +81,10 @@ commands = {[testenv:bashate]commands} {[testenv:checkniceness]commands} +[testenv:genconfig] +description = Generates sample documentation file for monasca-log-api +commands = oslo-config-generator --config-file=config-generator/monasca-log-api.conf + [testenv:docs] description = Builds api-ref, api-guide, releasenotes and devdocs commands = @@ -144,3 +148,5 @@ show-source = True enable-extensions = H203,H106 [hacking] +import_exceptions = + six.moves