diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 4809020ab8..aa125662f5 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -460,6 +460,9 @@ monasca [monasca-log-metrics:children] monasca +[monasca-thresh:children] +monasca + # Storm [storm-worker:children] storm diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index 10c7cb49d7..a45b04374a 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -469,6 +469,9 @@ monasca [monasca-log-metrics:children] monasca +[monasca-thresh:children] +monasca + # Storm [storm-worker:children] storm diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml index ee71601227..cb987ef9b8 100644 --- a/ansible/roles/monasca/defaults/main.yml +++ b/ansible/roles/monasca/defaults/main.yml @@ -72,6 +72,17 @@ monasca_services: - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla" dimensions: "{{ monasca_log_metrics_dimensions }}" + monasca-thresh: + container_name: monasca_thresh + group: monasca-thresh + enabled: true + image: "{{ monasca_thresh_image_full }}" + volumes: + - "{{ node_config_directory }}/monasca-thresh/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "monasca_thresh:/var/lib/monasca-thresh/" + - "kolla_logs:/var/log/kolla" + dimensions: "{{ monasca_thresh_dimensions }}" #################### # Databases @@ -91,10 +102,14 @@ monasca_kafka_servers: "{% for host in groups['kafka'] %}{{ hostvars[host]['ansi monasca_zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ zookeeper_client_port }}{% if not loop.last %},{% endif %}{% endfor %}" monasca_memcached_servers: "{% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}" monasca_elasticsearch_servers: "{% for host in groups['elasticsearch'] %}'{{ internal_protocol }}://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ elasticsearch_port }}'{% if not loop.last %},{% endif %}{% endfor %}" +monasca_storm_nimbus_servers: "{% for host in groups['storm-nimbus'] %}'{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}'{% if not loop.last %},{% endif %}{% endfor %}" +# Kafka topics used by Monasca services monasca_metrics_topic: "metrics" monasca_raw_logs_topic: "logs" monasca_transformed_logs_topic: "transformed-logs" +monasca_events_topic: "events" +monasca_alarm_state_transitions_topic: "alarm-state-transitions" # Processing pipeline threads. In a large scale deployment you will likely # want to tune these with finer precision. For example, if you have a very @@ -122,11 +137,16 @@ monasca_logstash_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{ monasca_logstash_tag: "{{ monasca_tag }}" monasca_logstash_image_full: "{{ monasca_logstash_image }}:{{ monasca_logstash_tag }}" +monasca_thresh_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-monasca-thresh" +monasca_thresh_tag: "{{ monasca_tag }}" +monasca_thresh_image_full: "{{ monasca_thresh_image }}:{{ monasca_thresh_tag }}" + monasca_api_dimensions: "{{ default_container_dimensions }}" monasca_log_api_dimensions: "{{ default_container_dimensions }}" monasca_log_transformer_dimensions: "{{ default_container_dimensions }}" monasca_log_persister_dimensions: "{{ default_container_dimensions }}" monasca_log_metrics_dimensions: "{{ default_container_dimensions }}" +monasca_thresh_dimensions: "{{ default_container_dimensions }}" #################### diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml index e47955b1e0..e02ad4aad1 100644 --- a/ansible/roles/monasca/handlers/main.yml +++ b/ansible/roles/monasca/handlers/main.yml @@ -108,3 +108,25 @@ - config_json.changed | bool or monasca_log_metrics_confs.changed | bool or monasca_log_metrics_container.changed | bool + +- name: Restart monasca-thresh container + vars: + service_name: "monasca-thresh" + service: "{{ monasca_services[service_name] }}" + config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + monasca_thresh_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + dimensions: "{{ service.dimensions }}" + when: + - kolla_action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or monasca_thresh_confs.changed | bool + or monasca_thresh_storm_conf.changed | bool + or monasca_thresh_container.changed | bool diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml index 4c07bd31b7..dc070b4481 100644 --- a/ansible/roles/monasca/tasks/config.yml +++ b/ansible/roles/monasca/tasks/config.yml @@ -175,6 +175,47 @@ notify: - Restart monasca-log-metrics container +- name: Copying over monasca-thresh config + vars: + service: "{{ monasca_services['monasca-thresh'] }}" + # NOTE(dszumski): We can't use merge_yaml since it replaces empty values + # with `null`. This breaks the thresholder config file parsing (which should + # probably be more robust). + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/monasca-thresh/thresh-config.yml" + mode: "0660" + become: true + register: monasca_thresh_confs + with_first_found: + - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/thresh-config.yml" + - "{{ node_custom_config }}/monasca/thresh-config.yml" + - "{{ role_path }}/templates/monasca-thresh/thresh-config.yml.j2" + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart monasca-thresh container + +- name: Copying over monasca-thresh storm config + vars: + service: "{{ monasca_services['monasca-thresh'] }}" + template: + src: "{{ item }}" + dest: "{{ node_config_directory }}/monasca-thresh/storm.yml" + mode: "0660" + become: true + register: monasca_thresh_storm_conf + with_first_found: + - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/storm.yml" + - "{{ node_custom_config }}/monasca/storm.yml" + - "{{ role_path }}/templates/monasca-thresh/storm.yml.j2" + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart monasca-thresh container + - name: Check monasca containers become: true kolla_docker: diff --git a/ansible/roles/monasca/tasks/deploy.yml b/ansible/roles/monasca/tasks/deploy.yml index aa233dc2c5..3732a6418d 100644 --- a/ansible/roles/monasca/tasks/deploy.yml +++ b/ansible/roles/monasca/tasks/deploy.yml @@ -8,7 +8,8 @@ inventory_hostname in groups['monasca-log-api'] or inventory_hostname in groups['monasca-log-transformer'] or inventory_hostname in groups['monasca-log-persister'] or - inventory_hostname in groups['monasca-log-metrics'] + inventory_hostname in groups['monasca-log-metrics'] or + inventory_hostname in groups['monasca-thresh'] - include_tasks: bootstrap.yml when: inventory_hostname in groups['monasca-api'] @@ -21,4 +22,5 @@ inventory_hostname in groups['monasca-log-api'] or inventory_hostname in groups['monasca-log-transformer'] or inventory_hostname in groups['monasca-log-persister'] or - inventory_hostname in groups['monasca-log-metrics'] + inventory_hostname in groups['monasca-log-metrics'] or + inventory_hostname in groups['monasca-thresh'] diff --git a/ansible/roles/monasca/templates/monasca-thresh/monasca-thresh.json.j2 b/ansible/roles/monasca/templates/monasca-thresh/monasca-thresh.json.j2 new file mode 100644 index 0000000000..b733ddf698 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-thresh/monasca-thresh.json.j2 @@ -0,0 +1,29 @@ +{ + "command": "/opt/storm/bin/storm jar /monasca-thresh-source/monasca-thresh-*/thresh/target/monasca-thresh-*-SNAPSHOT-shaded.jar monasca.thresh.ThresholdingEngine /etc/monasca/thresh-config.yml monasca-thresh local", + "config_files": [ + { + "source": "{{ container_config_directory }}/thresh-config.yml", + "dest": "/etc/monasca/thresh-config.yml", + "owner": "monasca", + "perm": "0600" + }, + { + "source": "/var/lib/kolla/config_files/storm.yml", + "dest": "/opt/storm/conf/storm.yaml", + "owner": "monasca", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/monasca", + "owner": "monasca:kolla", + "recurse": true + }, + { + "path": "/var/lib/monasca-thresh", + "owner": "monasca:kolla", + "recurse": true + } + ] +} diff --git a/ansible/roles/monasca/templates/monasca-thresh/storm.yml.j2 b/ansible/roles/monasca/templates/monasca-thresh/storm.yml.j2 new file mode 100644 index 0000000000..ee48f60e72 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-thresh/storm.yml.j2 @@ -0,0 +1,9 @@ +storm.local.dir: "/var/lib/monasca-thresh/data" +storm.log.dir: "/var/log/kolla/storm" +storm.workers.artifacts.dir: "/var/lib/monasca-thresh/worker-artifacts" +nimbus.seeds: [{{ monasca_storm_nimbus_servers }}] +storm.zookeeper.port: {{ zookeeper_client_port }} +storm.zookeeper.servers: +{% for host in groups['zookeeper'] %} + - "{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}" +{% endfor %} diff --git a/ansible/roles/monasca/templates/monasca-thresh/thresh-config.yml.j2 b/ansible/roles/monasca/templates/monasca-thresh/thresh-config.yml.j2 new file mode 100644 index 0000000000..7a35efa9fe --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-thresh/thresh-config.yml.j2 @@ -0,0 +1,132 @@ +# +# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP +# Copyright 2017 Fujitsu LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#    http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +metricSpoutThreads: 2 +metricSpoutTasks: 2 + +#statsdConfig: +# host: "127.0.0.1" +# port: %MONASCA_STATSD_PORT% +# prefix: monasca.storm. +# dimensions: !!map +# service : monitoring +# component : storm + + +metricSpoutConfig: + kafkaConsumerConfiguration: + # See http://kafka.apache.org/documentation.html#api for semantics and defaults. + topic: "{{ monasca_metrics_topic }}" + numThreads: 1 + groupId: "thresh-metric" + zookeeperConnect: "{{ monasca_zookeeper_servers }}" + consumerId: 1 + socketTimeoutMs: 30000 + socketReceiveBufferBytes: 65536 + fetchMessageMaxBytes: 1048576 + autoCommitEnable: true + autoCommitIntervalMs: 60000 + queuedMaxMessageChunks: 10 + rebalanceMaxRetries: 4 + fetchMinBytes: 1 + fetchWaitMaxMs: 100 + rebalanceBackoffMs: 2000 + refreshLeaderBackoffMs: 200 + autoOffsetReset: largest + consumerTimeoutMs: -1 + clientId: 1 + zookeeperSessionTimeoutMs: 60000 + zookeeperConnectionTimeoutMs: 60000 + zookeeperSyncTimeMs: 2000 + + +eventSpoutConfig: + kafkaConsumerConfiguration: + # See http://kafka.apache.org/documentation.html#api for semantics and defaults. + topic: "{{ monasca_events_topic }}" + numThreads: 1 + groupId: "thresh-event" + zookeeperConnect: "{{ monasca_zookeeper_servers }}" + consumerId: 1 + socketTimeoutMs: 30000 + socketReceiveBufferBytes: 65536 + fetchMessageMaxBytes: 1048576 + autoCommitEnable: true + autoCommitIntervalMs: 60000 + queuedMaxMessageChunks: 10 + rebalanceMaxRetries: 4 + fetchMinBytes: 1 + fetchWaitMaxMs: 100 + rebalanceBackoffMs: 2000 + refreshLeaderBackoffMs: 200 + autoOffsetReset: largest + consumerTimeoutMs: -1 + clientId: 1 + zookeeperSessionTimeoutMs: 60000 + zookeeperConnectionTimeoutMs: 60000 + zookeeperSyncTimeMs: 2000 + + +kafkaProducerConfig: + # See http://kafka.apache.org/documentation.html#api for semantics and defaults. + topic: "{{ monasca_alarm_state_transitions_topic }}" + metadataBrokerList: "{{ monasca_kafka_servers }}" + serializerClass: kafka.serializer.StringEncoder + partitionerClass: + requestRequiredAcks: 1 + requestTimeoutMs: 10000 + producerType: sync + keySerializerClass: + compressionCodec: none + compressedTopics: + messageSendMaxRetries: 3 + retryBackoffMs: 100 + topicMetadataRefreshIntervalMs: 600000 + queueBufferingMaxMs: 5000 + queueBufferingMaxMessages: 10000 + queueEnqueueTimeoutMs: -1 + batchNumMessages: 200 + sendBufferBytes: 102400 + clientId: Threshold_Engine + + +sporadicMetricNamespaces: + - foo + +database: + driverClass: org.drizzle.jdbc.DrizzleDriver + url: "jdbc:drizzle://{{ monasca_database_address }}/{{ monasca_database_name }}" + user: "{{ monasca_database_user }}" + password: "{{ monasca_database_password }}" + properties: + ssl: false + # the maximum amount of time to wait on an empty pool before throwing an exception + maxWaitForConnection: 1s + # the SQL query to run when validating a connection's liveness TODO FIXME + validationQuery: "/* MyService Health Check */ SELECT 1" + # the minimum number of connections to keep open + minSize: 8 + # the maximum number of connections to keep open + maxSize: 41 + hibernateSupport: false + # hibernate provider class + providerClass: com.zaxxer.hikari.hibernate.HikariConnectionProvider + databaseName: "{{ monasca_database_name }}" + serverName: "{{ database_address }}" + portNumber: "{{ database_port }}" + # hibernate auto configuration parameter + autoConfig: validate diff --git a/releasenotes/notes/add-monasca-thresh-f7a860ce996684f9.yaml b/releasenotes/notes/add-monasca-thresh-f7a860ce996684f9.yaml new file mode 100644 index 0000000000..e4cbf47083 --- /dev/null +++ b/releasenotes/notes/add-monasca-thresh-f7a860ce996684f9.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add support for deploying the Monasca thresh service, an Apache Storm + topology for alerting.