From 5f168778c6ff2bf7539fa3d5507593fd1c78d2c0 Mon Sep 17 00:00:00 2001 From: Adrian Czarnecki Date: Tue, 2 May 2017 15:56:11 +0200 Subject: [PATCH] Use oslo-config-generator To generate the sample config file, do what other OpenStack projects do and use the oslo.config provided mechanism. Story: 2001009 Task: 4201 Change-Id: I490de4110843f7dfd618a6a8cc8d6a4e35db65ce --- .gitignore | 3 +- config-generator/README.md | 7 ++ config-generator/persister.conf | 6 ++ etc/monasca/persister.conf | 65 ---------------- monasca_persister/conf/__init__.py | 74 +++++++++++++++++++ monasca_persister/conf/cassandra.py | 37 ++++++++++ monasca_persister/conf/influxdb.py | 45 +++++++++++ monasca_persister/conf/kafka_alarm_history.py | 57 ++++++++++++++ monasca_persister/conf/kafka_common.py | 60 +++++++++++++++ monasca_persister/conf/kafka_metrics.py | 56 ++++++++++++++ monasca_persister/conf/repositories.py | 37 ++++++++++ monasca_persister/conf/types.py | 58 +++++++++++++++ monasca_persister/conf/zookeeper.py | 39 ++++++++++ monasca_persister/config.py | 46 ++++++++++++ monasca_persister/persister.py | 56 +------------- .../repositories/cassandra/__init__.py | 22 ------ .../repositories/influxdb/__init__.py | 25 ------- monasca_persister/tests/test_config_types.py | 47 ++++++++++++ .../tests/test_persister_main.py | 42 ----------- setup.cfg | 4 +- tox.ini | 6 ++ 21 files changed, 584 insertions(+), 208 deletions(-) create mode 100644 config-generator/README.md create mode 100644 config-generator/persister.conf delete mode 100644 etc/monasca/persister.conf create mode 100644 monasca_persister/conf/__init__.py create mode 100644 monasca_persister/conf/cassandra.py create mode 100644 monasca_persister/conf/influxdb.py create mode 100644 monasca_persister/conf/kafka_alarm_history.py create mode 100644 monasca_persister/conf/kafka_common.py create mode 100644 monasca_persister/conf/kafka_metrics.py create mode 100644 monasca_persister/conf/repositories.py create mode 100644 monasca_persister/conf/types.py create mode 100644 monasca_persister/conf/zookeeper.py create mode 100644 monasca_persister/config.py create mode 100644 monasca_persister/tests/test_config_types.py diff --git a/.gitignore b/.gitignore index 5c853ecb..78592723 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ virtenv/* cover/ .tox/ *egg-info/ -.testrepository/ \ No newline at end of file +.testrepository/ +*.sample \ No newline at end of file diff --git a/config-generator/README.md b/config-generator/README.md new file mode 100644 index 00000000..0985af25 --- /dev/null +++ b/config-generator/README.md @@ -0,0 +1,7 @@ +# config-generator + +To generate sample configuration execute + +```sh +tox -e genconfig +``` diff --git a/config-generator/persister.conf b/config-generator/persister.conf new file mode 100644 index 00000000..99d83b5a --- /dev/null +++ b/config-generator/persister.conf @@ -0,0 +1,6 @@ +[DEFAULT] +output_file = etc/monasca/persister.conf.sample +wrap_width = 90 +format = ini +namespace = monasca_persister +namespace = oslo.log \ No newline at end of file diff --git a/etc/monasca/persister.conf b/etc/monasca/persister.conf deleted file mode 100644 index 5aa9bbd2..00000000 --- a/etc/monasca/persister.conf +++ /dev/null @@ -1,65 +0,0 @@ -[DEFAULT] -log_config_append=/etc/monasca/persister-logging.conf - -[repositories] -# The driver to use for the metrics repository -metrics_driver = monasca_persister.repositories.influxdb.metrics_repository:MetricInfluxdbRepository -#metrics_driver = monasca_persister.repositories.cassandra.metrics_repository:MetricCassandraRepository - -# The driver to use for the alarm state history repository -alarm_state_history_driver = monasca_persister.repositories.influxdb.alarm_state_history_repository:AlarmStateHistInfluxdbRepository -#alarm_state_history_driver = monasca_persister.repositories.cassandra.alarm_state_history_repository:AlarmStateHistCassandraRepository - -[zookeeper] -# Comma separated list of host:port -uri = 192.168.10.4:2181 -partition_interval_recheck_seconds = 15 - -[kafka_alarm_history] -# Comma separated list of Kafka broker host:port. -uri = 192.168.10.4:9092 -group_id = 1_alarm-state-transitions -topic = alarm-state-transitions -consumer_id = 1 -client_id = 1 -database_batch_size = 1000 -max_wait_time_seconds = 30 -# The following 3 values are set to the kakfa-python defaults -fetch_size_bytes = 4096 -buffer_size = 4096 -# 8 times buffer size -max_buffer_size = 32768 -# Path in zookeeper for kafka consumer group partitioning algo -zookeeper_path = /persister_partitions/alarm-state-transitions -num_processors = 1 - -[kafka_metrics] -# Comma separated list of Kafka broker host:port -uri = 192.168.10.4:9092 -group_id = 1_metrics -topic = metrics -consumer_id = 1 -client_id = 1 -database_batch_size = 1000 -max_wait_time_seconds = 30 -# The following 3 values are set to the kakfa-python defaults -fetch_size_bytes = 4096 -buffer_size = 4096 -# 8 times buffer size -max_buffer_size = 32768 -# Path in zookeeper for kafka consumer group partitioning algo -zookeeper_path = /persister_partitions/metrics -num_processors = 1 - -[influxdb] -database_name = mon -ip_address = 192.168.10.4 -port = 8086 -user = mon_persister -password = password - -# Uncomment, set cluster_ip_addresses, and change the repositories to point to the cassandra classes -#[cassandra] -# Comma separated list of Cassandra node IP addresses. No spaces. -#cluster_ip_addresses: 192.168.10.6 -#keyspace: monasca diff --git a/monasca_persister/conf/__init__.py b/monasca_persister/conf/__init__.py new file mode 100644 index 00000000..e57fdc58 --- /dev/null +++ b/monasca_persister/conf/__init__.py @@ -0,0 +1,74 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import pkgutil + +from oslo_config import cfg +from oslo_utils import importutils + +CONF = cfg.CONF + + +def load_conf_modules(): + """Load all modules that contain configuration. + + Method iterates over modules of :py:module:`monasca_persister.conf` + and imports only those that contain following methods: + + - list_opts (required by oslo_config.genconfig) + - register_opts (required by :py:currentmodule:) + + """ + for modname in _list_module_names(): + mod = importutils.import_module('monasca_persister.conf.' + modname) + required_funcs = ['register_opts', 'list_opts'] + for func in required_funcs: + if hasattr(mod, func): + yield mod + + +def _list_module_names(): + module_names = [] + package_path = os.path.dirname(os.path.abspath(__file__)) + for _, modname, ispkg in pkgutil.iter_modules(path=[package_path]): + if not (modname == "opts" and ispkg): + module_names.append(modname) + return module_names + + +def register_opts(): + """Register all conf modules opts. + + This method allows different modules to register + opts according to their needs. + + """ + for mod in load_conf_modules(): + mod.register_opts(cfg.CONF) + + +def list_opts(): + """List all conf modules opts. + + Goes through all conf modules and yields their opts. + + """ + for mod in load_conf_modules(): + mod_opts = mod.list_opts() + if type(mod_opts) is list: + for single_mod_opts in mod_opts: + yield single_mod_opts[0], single_mod_opts[1] + else: + yield mod_opts[0], mod_opts[1] diff --git a/monasca_persister/conf/cassandra.py b/monasca_persister/conf/cassandra.py new file mode 100644 index 00000000..93cdc325 --- /dev/null +++ b/monasca_persister/conf/cassandra.py @@ -0,0 +1,37 @@ +# (C) Copyright 2016 Hewlett Packard Enterprise Development Company LP +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +cassandra_opts = [ + cfg.ListOpt('cluster_ip_addresses', + help='Comma separated list of Cassandra node IP addresses', + default=['127.0.0.1'], + item_type=cfg.IPOpt), + cfg.StrOpt('keyspace', + help='keyspace where metric are stored', + default='monasca')] + +cassandra_group = cfg.OptGroup(name='cassandra') + + +def register_opts(conf): + conf.register_group(cassandra_group) + conf.register_opts(cassandra_opts, cassandra_group) + + +def list_opts(): + return cassandra_group, cassandra_opts diff --git a/monasca_persister/conf/influxdb.py b/monasca_persister/conf/influxdb.py new file mode 100644 index 00000000..2d1fb09e --- /dev/null +++ b/monasca_persister/conf/influxdb.py @@ -0,0 +1,45 @@ +# (C) Copyright 2016-2017 Hewlett Packard Enterprise Development LP +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +influxdb_opts = [ + cfg.StrOpt('database_name', + help='database name where metrics are stored', + default='mon'), + cfg.IPOpt('ip_address', + help='ip address to influxdb'), + cfg.PortOpt('port', + help='port to influxdb', + default=8086), + cfg.StrOpt('user', + help='influxdb user ', + default='mon_persister'), + cfg.StrOpt('password', + secret=True, + help='influxdb password')] + +influxdb_group = cfg.OptGroup(name='influxdb', + title='influxdb') + + +def register_opts(conf): + conf.register_group(influxdb_group) + conf.register_opts(influxdb_opts, influxdb_group) + + +def list_opts(): + return influxdb_group, influxdb_opts diff --git a/monasca_persister/conf/kafka_alarm_history.py b/monasca_persister/conf/kafka_alarm_history.py new file mode 100644 index 00000000..51ce9845 --- /dev/null +++ b/monasca_persister/conf/kafka_alarm_history.py @@ -0,0 +1,57 @@ +# (C) Copyright 2016-2017 Hewlett Packard Enterprise Development LP +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from oslo_config import cfg + +from monasca_persister.conf import kafka_common +from monasca_persister.conf import types + +kafka_alarm_history_group = cfg.OptGroup(name='kafka_alarm_history', + title='kafka_alarm_history') +kafka_alarm_history_opts = [ + # NOTE(czarneckia) default by reference does not work with ListOpt + cfg.ListOpt('uri', + help='Comma separated list of Kafka broker host:port', + default=['127.0.0.1:9092'], + item_type=types.HostAddressPortType()), + cfg.StrOpt('group_id', + help='Kafka Group from which persister get data', + default='1_alarm-state-transitions'), + cfg.StrOpt('topic', + help='Kafka Topic from which persister get data', + default='alarm-state-transitions'), + cfg.StrOpt('zookeeper_path', + help='Path in zookeeper for kafka consumer group partitioning algorithm', + default='/persister_partitions/$kafka_alarm_history.topic') +] + + +# Replace Default OPt with reference to kafka group option +kafka_common_opts = copy.deepcopy(kafka_common.kafka_common_opts) +for opt in kafka_common_opts: + opt.default = '$kafka.{}'.format(opt.name) + + +def register_opts(conf): + conf.register_group(kafka_alarm_history_group) + conf.register_opts(kafka_alarm_history_opts + kafka_common_opts, + kafka_alarm_history_group) + + +def list_opts(): + return kafka_alarm_history_group, kafka_alarm_history_opts diff --git a/monasca_persister/conf/kafka_common.py b/monasca_persister/conf/kafka_common.py new file mode 100644 index 00000000..0c2cfc76 --- /dev/null +++ b/monasca_persister/conf/kafka_common.py @@ -0,0 +1,60 @@ +# (C) Copyright 2016-2017 Hewlett Packard Enterprise Development LP +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + + +kafka_common_opts = [ + cfg.StrOpt('consumer_id', + help='Name/id of persister kafka consumer', + advanced=True, + default='monasca-persister'), + cfg.StrOpt('client_id', + help='id of persister kafka client', + advanced=True, + default='monasca-persister'), + cfg.IntOpt('database_batch_size', + help='Maximum number of metric to buffer before writing to database', + default=1000), + cfg.IntOpt('max_wait_time_seconds', + help='Maximum wait time for write batch to database', + default=30), + cfg.IntOpt('fetch_size_bytes', + help='Fetch size, in bytes. This value is set to the kafka-python defaults', + default=4096), + cfg.IntOpt('buffer_size', + help='Buffer size, in bytes. This value is set to the kafka-python defaults', + default='$kafka.fetch_size_bytes'), + cfg.IntOpt('max_buffer_size', + help='Maximum buffer size, in bytes, default value is 8 time buffer_size.' + 'This value is set to the kafka-python defaults. ', + default=32768), + cfg.IntOpt('num_processors', + help='Number of processes spawned by persister', + default=0) +] + +kafka_common_group = cfg.OptGroup(name='kafka', + title='kafka') + + +def register_opts(conf): + conf.register_group(kafka_common_group) + conf.register_opts(kafka_common_opts, kafka_common_group) + + +def list_opts(): + return kafka_common_group, kafka_common_opts diff --git a/monasca_persister/conf/kafka_metrics.py b/monasca_persister/conf/kafka_metrics.py new file mode 100644 index 00000000..df900445 --- /dev/null +++ b/monasca_persister/conf/kafka_metrics.py @@ -0,0 +1,56 @@ +# (C) Copyright 2016-2017 Hewlett Packard Enterprise Development LP +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from oslo_config import cfg + +from monasca_persister.conf import kafka_common +from monasca_persister.conf import types + +kafka_metrics_group = cfg.OptGroup(name='kafka_metrics', + title='kafka_metrics') +kafka_metrics_opts = [ + # NOTE(czarneckia) default by reference does not work with ListOpt + cfg.ListOpt('uri', + help='Comma separated list of Kafka broker host:port', + default=['127.0.0.1:9092'], + item_type=types.HostAddressPortType()), + cfg.StrOpt('group_id', + help='Kafka Group from which persister get data', + default='1_metrics'), + cfg.StrOpt('topic', + help='Kafka Topic from which persister get data', + default='metrics'), + cfg.StrOpt('zookeeper_path', + help='Path in zookeeper for kafka consumer group partitioning algorithm', + default='/persister_partitions/$kafka_metrics.topic'), +] + +# Replace Default OPt with reference to kafka group option +kafka_common_opts = copy.deepcopy(kafka_common.kafka_common_opts) +for opt in kafka_common_opts: + opt.default = '$kafka.{}'.format(opt.name) + + +def register_opts(conf): + conf.register_group(kafka_metrics_group) + conf.register_opts(kafka_metrics_opts + kafka_common_opts, + kafka_metrics_group) + + +def list_opts(): + return kafka_metrics_group, kafka_metrics_opts diff --git a/monasca_persister/conf/repositories.py b/monasca_persister/conf/repositories.py new file mode 100644 index 00000000..eac796d3 --- /dev/null +++ b/monasca_persister/conf/repositories.py @@ -0,0 +1,37 @@ +# (C) Copyright 2016-2017 Hewlett Packard Enterprise Development LP +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +repositories_opts = [ + cfg.StrOpt(name='metrics_driver', + help='The repository driver to use for metrics', + default='monasca_persister.repositories.influxdb.metrics_repository:MetricInfluxdbRepository'), + cfg.StrOpt(name='alarm_state_history_driver', + help='The repository driver to use for alarm state history', + default='monasca_persister.repositories.influxdb.metrics_repository:MetricInfluxdbRepository')] + +repositories_group = cfg.OptGroup(name='repositories', + title='repositories') + + +def register_opts(conf): + conf.register_group(repositories_group) + conf.register_opts(repositories_opts, repositories_group) + + +def list_opts(): + return repositories_group, repositories_opts diff --git a/monasca_persister/conf/types.py b/monasca_persister/conf/types.py new file mode 100644 index 00000000..5edc56b7 --- /dev/null +++ b/monasca_persister/conf/types.py @@ -0,0 +1,58 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_config import types + + +class HostAddressPortOpt(cfg.Opt): + """Option for HostAddressPortType. + + Accept hostname or ip address with TCP/IP port number. + """ + def __init__(self, name, **kwargs): + ip_port_type = HostAddressPortType() + super(HostAddressPortOpt, self).__init__(name, + type=ip_port_type, + **kwargs) + + +class HostAddressPortType(types.HostAddress): + """HostAddress with additional port.""" + + def __init__(self, version=None): + type_name = 'ip and port value' + super(HostAddressPortType, self).__init__(version, type_name=type_name) + + def __call__(self, value): + addr, port = value.split(':') + addr = self.validate_addr(addr) + port = self._validate_port(port) + if not addr and not port: + raise ValueError('%s is not valid ip with optional port') + return '%s:%d' % (addr, port) + + @staticmethod + def _validate_port(port): + return types.Port()(port) + + def validate_addr(self, addr): + try: + addr = self.ip_address(addr) + except ValueError: + try: + addr = self.hostname(addr) + except ValueError: + raise ValueError("%s is not a valid host address", addr) + return addr diff --git a/monasca_persister/conf/zookeeper.py b/monasca_persister/conf/zookeeper.py new file mode 100644 index 00000000..360e8e90 --- /dev/null +++ b/monasca_persister/conf/zookeeper.py @@ -0,0 +1,39 @@ +# (C) Copyright 2016-2017 Hewlett Packard Enterprise Development LP +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +from monasca_persister.conf import types + +zookeeper_opts = [ + cfg.ListOpt('uri', + help='Comma separated list of zookeper instance host:port', + default=['127.0.0.1:2181'], + item_type=types.HostAddressPortType()), + cfg.IntOpt('partition_interval_recheck_seconds', + help='Time between rechecking if partition is available', + default=15)] + +zookeeper_group = cfg.OptGroup(name='zookeeper', title='zookeeper') + + +def register_opts(conf): + conf.register_group(zookeeper_group) + conf.register_opts(zookeeper_opts, zookeeper_group) + + +def list_opts(): + return zookeeper_group, zookeeper_opts diff --git a/monasca_persister/config.py b/monasca_persister/config.py new file mode 100644 index 00000000..788e2e74 --- /dev/null +++ b/monasca_persister/config.py @@ -0,0 +1,46 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log + +from monasca_persister import conf +from monasca_persister import version + +CONF = conf.CONF +LOG = log.getLogger(__name__) + +_CONF_LOADED = False + + +def parse_args(): + global _CONF_LOADED + if _CONF_LOADED: + LOG.debug('Configuration has been already loaded') + return + + log.set_defaults() + log.register_options(CONF) + + CONF(prog='persister', + project='monasca', + version=version.version_str, + description='Persists metrics & alarm history in TSDB') + + log.setup(CONF, + product_name='persister', + version=version.version_str) + + conf.register_opts() + + _CONF_LOADED = True diff --git a/monasca_persister/persister.py b/monasca_persister/persister.py index 9e0b9eec..70729167 100644 --- a/monasca_persister/persister.py +++ b/monasca_persister/persister.py @@ -1,4 +1,5 @@ # (C) Copyright 2014-2017 Hewlett Packard Enterprise Development LP +# Copyright 2017 FUJITSU LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -31,51 +32,12 @@ from monasca_common.simport import simport from oslo_config import cfg from oslo_log import log +from monasca_persister import config from monasca_persister.repositories import persister -from monasca_persister import version + LOG = log.getLogger(__name__) -zookeeper_opts = [cfg.StrOpt('uri'), - cfg.IntOpt('partition_interval_recheck_seconds')] - -zookeeper_group = cfg.OptGroup(name='zookeeper', title='zookeeper') -cfg.CONF.register_group(zookeeper_group) -cfg.CONF.register_opts(zookeeper_opts, zookeeper_group) - -kafka_common_opts = [cfg.StrOpt('uri'), - cfg.StrOpt('group_id'), - cfg.StrOpt('topic'), - cfg.StrOpt('consumer_id'), - cfg.StrOpt('client_id'), - cfg.IntOpt('database_batch_size'), - cfg.IntOpt('max_wait_time_seconds'), - cfg.IntOpt('fetch_size_bytes'), - cfg.IntOpt('buffer_size'), - cfg.IntOpt('max_buffer_size'), - cfg.StrOpt('zookeeper_path'), - cfg.IntOpt('num_processors')] - -kafka_metrics_opts = kafka_common_opts -kafka_alarm_history_opts = kafka_common_opts - -kafka_metrics_group = cfg.OptGroup(name='kafka_metrics', title='kafka_metrics') -kafka_alarm_history_group = cfg.OptGroup(name='kafka_alarm_history', - title='kafka_alarm_history') - -cfg.CONF.register_group(kafka_metrics_group) -cfg.CONF.register_group(kafka_alarm_history_group) -cfg.CONF.register_opts(kafka_metrics_opts, kafka_metrics_group) -cfg.CONF.register_opts(kafka_alarm_history_opts, kafka_alarm_history_group) - -repositories_opts = [ - cfg.StrOpt('metrics_driver', help='The repository driver to use for metrics'), - cfg.StrOpt('alarm_state_history_driver', help='The repository driver to use for alarm state history')] - -repositories_group = cfg.OptGroup(name='repositories', title='repositories') -cfg.CONF.register_group(repositories_group) -cfg.CONF.register_opts(repositories_opts, repositories_group) - processors = [] # global list to facilitate clean signal handling exiting = False @@ -133,20 +95,10 @@ def start_process(respository, kafka_config): m_persister.run() -def _init_config(): - log.register_options(cfg.CONF) - log.set_defaults() - cfg.CONF(prog='persister', - project='monasca', - version=version.version_str, - description='Persists metrics & alarm history in TSDB') - log.setup(cfg.CONF, 'monasca-persister', version=version.version_str) - - def main(): """Start persister.""" - _init_config() + config.parse_args() metric_repository = simport.load(cfg.CONF.repositories.metrics_driver) alarm_state_history_repository = simport.load(cfg.CONF.repositories.alarm_state_history_driver) diff --git a/monasca_persister/repositories/cassandra/__init__.py b/monasca_persister/repositories/cassandra/__init__.py index a9d04897..e69de29b 100644 --- a/monasca_persister/repositories/cassandra/__init__.py +++ b/monasca_persister/repositories/cassandra/__init__.py @@ -1,22 +0,0 @@ -# (C) Copyright 2016 Hewlett Packard Enterprise Development Company LP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from oslo_config import cfg - -cassandra_opts = [cfg.StrOpt('cluster_ip_addresses'), - cfg.StrOpt('keyspace')] - -cassandra_group = cfg.OptGroup(name='cassandra') -cfg.CONF.register_group(cassandra_group) -cfg.CONF.register_opts(cassandra_opts, cassandra_group) diff --git a/monasca_persister/repositories/influxdb/__init__.py b/monasca_persister/repositories/influxdb/__init__.py index e0f773a3..e69de29b 100644 --- a/monasca_persister/repositories/influxdb/__init__.py +++ b/monasca_persister/repositories/influxdb/__init__.py @@ -1,25 +0,0 @@ -# (C) Copyright 2016-2017 Hewlett Packard Enterprise Development LP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from oslo_config import cfg - -influxdb_opts = [cfg.StrOpt('database_name'), - cfg.StrOpt('ip_address'), - cfg.StrOpt('port'), - cfg.StrOpt('user'), - cfg.StrOpt('password', secret=True)] - -influxdb_group = cfg.OptGroup(name='influxdb', title='influxdb') -cfg.CONF.register_group(influxdb_group) -cfg.CONF.register_opts(influxdb_opts, influxdb_group) diff --git a/monasca_persister/tests/test_config_types.py b/monasca_persister/tests/test_config_types.py new file mode 100644 index 00000000..1c947f36 --- /dev/null +++ b/monasca_persister/tests/test_config_types.py @@ -0,0 +1,47 @@ +# (C) Copyright 2016-2017 Hewlett Packard Enterprise Development LP +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslotest import base + +from monasca_persister.conf import types + + +class TestHostAddressPortType(base.BaseTestCase): + + def setUp(self): + super(TestHostAddressPortType, self).setUp() + self.types = types.HostAddressPortType() + + def test_ip_address(self): + + self.assertEqual('127.0.0.1:2121', self.types('127.0.0.1:2121')) + + def test_hostname(self): + self.assertEqual('localhost:2121', self.types('localhost:2121')) + + # failure scenario + def test_missing_port(self): + self.assertRaises(ValueError, self.types, '127.0.0.1') + + def test_missing_address(self): + self.assertRaises(ValueError, self.types, ':123') + + def test_incorrect_ip(self): + self.assertRaises(ValueError, self.types, '127.surprise.0.1:2121') + + def test_incorrect_port(self): + self.assertRaises(ValueError, self.types, '127.0.0.1:65536') + self.assertRaises(ValueError, self.types, '127.0.0.1:sample') diff --git a/monasca_persister/tests/test_persister_main.py b/monasca_persister/tests/test_persister_main.py index 63cd51f9..478d47b9 100644 --- a/monasca_persister/tests/test_persister_main.py +++ b/monasca_persister/tests/test_persister_main.py @@ -192,48 +192,6 @@ class TestPersister(base.BaseTestCase): fake_kafka_config, 'zookeeper', fake_repository) -class TestPersisterConfig(base.BaseTestCase): - - def setUp(self): - super(TestPersisterConfig, self).setUp() - - from monasca_persister import persister - - self.assertIsNotNone(persister) - - def _test_zookeeper_options(self): - str_opts = ['uri'] - int_opts = ['partition_interval_recheck_seconds'] - - self._assert_cfg_registered('zookeeper', str_opts, int_opts) - - def _assert_cfg_registered(self, group_name, str_opts, int_opts): - options = {group_name: str_opts + int_opts} - - for key, values in options.items(): - for v in values: - self.assertIsNone(CONF[key][v]) - - def _test_kafka_options(self): - str_opts = ['uri', 'group_id', 'topic', 'consumer_id', 'client_id', 'zookeeper_path'] - int_opts = ['database_batch_size', 'max_wait_time_seconds', 'fetch_size_bytes', - 'buffer_size', 'max_buffer_size', 'num_processors'] - - self._assert_cfg_registered('kafka_metrics', str_opts, int_opts) - self._assert_cfg_registered('kafka_alarm_history', str_opts, int_opts) - - def _test_repositories_options(self): - str_opts = ['metrics_driver', 'alarm_state_history_driver'] - int_opts = [] - - self._assert_cfg_registered('repositories', str_opts, int_opts) - - def test_correct_config_options_are_registered(self): - self._test_zookeeper_options() - self._test_kafka_options() - self._test_repositories_options() - - def _get_process(name, is_alive=True): return Mock(name=name, is_alive=Mock(return_value=is_alive)) diff --git a/setup.cfg b/setup.cfg index f7ead8d0..9b818381 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,12 +20,14 @@ setup-hooks = console_scripts = monasca-persister = monasca_persister.persister:main +oslo.config.opts = + monasca_persister = monasca_persister.conf:list_opts + [files] packages = monasca_persister data_files = /etc/monasca = - etc/monasca/persister.conf etc/monasca/persister-logging.conf [extras] diff --git a/tox.ini b/tox.ini index 99686ef4..a61a5b5b 100644 --- a/tox.ini +++ b/tox.ini @@ -58,6 +58,12 @@ commands = {[testenv:flake8]commands} {[bandit]commands} +[testenv:genconfig] +description = Generates sample configuration file for monasca-persister +whitelist_externals = bash +commands = + oslo-config-generator --config-file=config-generator/persister.conf + [testenv:venv] commands = {posargs}