Update metricbeat config for the latest release

Change-Id: I312a0c272143973050f81f34867471098cec3286
Signed-off-by: Kevin Carter <kevin@cloudnull.com>
This commit is contained in:
Kevin Carter 2019-02-26 22:22:46 -06:00
parent 4490ed3dea
commit 2d3c0d55f4
2 changed files with 297 additions and 117 deletions

View File

@ -192,12 +192,25 @@ metricbeat.modules:
##---------------------------- Elasticsearch Module ---------------------------
{% if inventory_hostname in (groups['elastic-logstash'] | union(groups['kibana'])) %}
- module: elasticsearch
metricsets: ["node", "node_stats"]
metricsets:
- node
- node_stats
- index
- index_recovery
- index_summary
- shard
- ml_job
enabled: true
period: 30s
hosts: ["localhost:{{ elastic_port }}"]
{% endif %}
#
##----------------------------- envoyproxy Module -----------------------------
- module: envoyproxy
metricsets: ["server"]
period: 10s
hosts: ["localhost:9901"]
##-------------------------------- Etcd Module --------------------------------
{% if etcd_enabled | default(false) | bool %}
- module: etcd
@ -300,46 +313,78 @@ metricbeat.modules:
{% endif %}
#
##----------------------------- Kubernetes Module -----------------------------
## Node metrics, from kubelet:
# Node metrics, from kubelet:
#- module: kubernetes
# metricsets:
# - node
# - system
# - pod
# - container
# - node
# - pod
# - system
# - volume
# period: 30s
# period: 10s
# hosts: ["localhost:10255"]
# enabled: true
# #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
# #ssl.certificate_authorities:
# # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
# #ssl.certificate: "/etc/pki/client/cert.pem"
# #ssl.key: "/etc/pki/client/cert.key"
#
# # Enriching parameters:
# add_metadata: true
# in_cluster: true
# # When used outside the cluster:
# #host: node_name
# #kube_config: ~/.kube/config
#
## State metrics from kube-state-metrics service:
#- module: kubernetes
# enabled: false
# enabled: true
# metricsets:
# - state_node
# - state_deployment
# - state_replicaset
# - state_statefulset
# - state_pod
# - state_container
# period: 30s
# period: 10s
# hosts: ["kube-state-metrics:8080"]
#
# # Enriching parameters:
# add_metadata: true
# in_cluster: true
# # When used outside the cluster:
# #host: node_name
# #kube_config: ~/.kube/config
#
## Kubernetes events
#- module: kubernetes
# enabled: false
# enabled: true
# metricsets:
# - event
#
#--------------------------------- kvm Module --------------------------------
## Kubernetes API server
#- module: kubernetes
# enabled: true
# metricsets:
# - apiserver
# hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"]
##--------------------------------- kvm Module --------------------------------
{% if kvm_enabled | default(false) | bool %}
- module: kvm
metricsets: ["dommemstat"]
enabled: true
period: 10s
hosts: ["unix:///var/run/libvirt/libvirt-sock"]
# For remote hosts, setup network access in libvirtd.conf
# and use the tcp scheme:
# hosts: [ "tcp://<host>:16509" ]
# Timeout to connect to Libvirt server
#timeout: 1s
{% endif %}
##------------------------------ Logstash Module ------------------------------
{% if inventory_hostname in groups['elastic-logstash'] | default([]) %}
- module: logstash
@ -519,6 +564,13 @@ metricbeat.modules:
# # Redis AUTH password. Empty by default.
# #password: foobared
#
##------------------------------- traefik Module ------------------------------
- module: traefik
metricsets: ["health"]
period: 10s
hosts: ["localhost:8080"]
##-------------------------------- uwsgi Module -------------------------------
{% if uwsgi_enabled | default(false) | bool %}
- module: uwsgi
@ -564,121 +616,225 @@ metricbeat.modules:
#
#
#
##================================ General ======================================
#================================ General ======================================
# The name of the shipper that publishes the network data. It can be used to group
# all the transactions sent by a single shipper in the web interface.
# If this options is not defined, the hostname is used.
#name:
# The tags of the shipper are included in their own field with each
# transaction published. Tags make it easy to group servers by different
# logical properties.
#tags: ["service-X", "web-tier"]
# Optional fields that you can specify to add additional information to the
# output. Fields can be scalar values, arrays, dictionaries, or any nested
# combination of these.
#fields:
# env: staging
# If this option is set to true, the custom fields are stored as top-level
# fields in the output document instead of being grouped under a fields
# sub-dictionary. Default is false.
#fields_under_root: false
# Internal queue configuration for buffering events to be published.
#queue:
# Queue type by name (default 'mem')
# The memory queue will present all available events (up to the outputs
# bulk_max_size) to the output, the moment the output is ready to server
# another batch of events.
#mem:
# Max number of events the queue can buffer.
#events: 4096
# Hints the minimum number of events stored in the queue,
# before providing a batch of events to the outputs.
# The default value is set to 2048.
# A value of 0 ensures events are immediately available
# to be sent to the outputs.
#flush.min_events: 2048
# Maximum duration after which events are available to the outputs,
# if the number of events stored in the queue is < min_flush_events.
#flush.timeout: 1s
# The spool queue will store events in a local spool file, before
# forwarding the events to the outputs.
#
# Beta: spooling to disk is currently a beta feature. Use with care.
#
# The spool file is a circular buffer, which blocks once the file/buffer is full.
# Events are put into a write buffer and flushed once the write buffer
# is full or the flush_timeout is triggered.
# Once ACKed by the output, events are removed immediately from the queue,
# making space for new events to be persisted.
#spool:
# The file namespace configures the file path and the file creation settings.
# Once the file exists, the `size`, `page_size` and `prealloc` settings
# will have no more effect.
#file:
# Location of spool file. The default value is ${path.data}/spool.dat.
#path: "${path.data}/spool.dat"
# Configure file permissions if file is created. The default value is 0600.
#permissions: 0600
# File size hint. The spool blocks, once this limit is reached. The default value is 100 MiB.
#size: 100MiB
# The files page size. A file is split into multiple pages of the same size. The default value is 4KiB.
#page_size: 4KiB
# If prealloc is set, the required space for the file is reserved using
# truncate. The default value is true.
#prealloc: true
# Spool writer settings
# Events are serialized into a write buffer. The write buffer is flushed if:
# - The buffer limit has been reached.
# - The configured limit of buffered events is reached.
# - The flush timeout is triggered.
#write:
# Sets the write buffer size.
#buffer_size: 1MiB
# Maximum duration after which events are flushed, if the write buffer
# is not full yet. The default value is 1s.
#flush.timeout: 1s
# Number of maximum buffered events. The write buffer is flushed once the
# limit is reached.
#flush.events: 16384
# Configure the on-disk event encoding. The encoding can be changed
# between restarts.
# Valid encodings are: json, ubjson, and cbor.
#codec: cbor
#read:
# Reader flush timeout, waiting for more events to become available, so
# to fill a complete batch, as required by the outputs.
# If flush_timeout is 0, all available events are forwarded to the
# outputs immediately.
# The default value is 0s.
#flush.timeout: 0s
# Sets the maximum number of CPUs that can be executing simultaneously. The
# default is the number of logical CPUs available in the system.
#max_procs:
#================================ Processors ===================================
# Processors are used to reduce the number of fields in the exported event or to
# enhance the event with external metadata. This section defines a list of
# processors that are applied one by one and the first one receives the initial
# event:
#
## The name of the shipper that publishes the network data. It can be used to group
## all the transactions sent by a single shipper in the web interface.
## If this options is not defined, the hostname is used.
##name:
# event -> filter1 -> event1 -> filter2 ->event2 ...
#
## The tags of the shipper are included in their own field with each
## transaction published. Tags make it easy to group servers by different
## logical properties.
##tags: ["service-X", "web-tier"]
# The supported processors are drop_fields, drop_event, include_fields,
# decode_json_fields, and add_cloud_metadata.
#
## Optional fields that you can specify to add additional information to the
## output. Fields can be scalar values, arrays, dictionaries, or any nested
## combination of these.
##fields:
## env: staging
# For example, you can use the following processors to keep the fields that
# contain CPU load percentages, but remove the fields that contain CPU ticks
# values:
#
## If this option is set to true, the custom fields are stored as top-level
## fields in the output document instead of being grouped under a fields
## sub-dictionary. Default is false.
##fields_under_root: false
#processors:
#- include_fields:
# fields: ["cpu"]
#- drop_fields:
# fields: ["cpu.user", "cpu.system"]
#
## Internal queue configuration for buffering events to be published.
##queue:
# # Queue type by name (default 'mem')
# # The memory queue will present all available events (up to the outputs
# # bulk_max_size) to the output, the moment the output is ready to server
# # another batch of events.
# #mem:
# # Max number of events the queue can buffer.
# #events: 4096
# The following example drops the events that have the HTTP response code 200:
#
# # Hints the minimum number of events stored in the queue,
# # before providing a batch of events to the outputs.
# # A value of 0 (the default) ensures events are immediately available
# # to be sent to the outputs.
# #flush.min_events: 2048
#processors:
#- drop_event:
# when:
# equals:
# http.code: 200
#
# # Maximum duration after which events are available to the outputs,
# # if the number of events stored in the queue is < min_flush_events.
# #flush.timeout: 1s
# The following example renames the field a to b:
#
## Sets the maximum number of CPUs that can be executing simultaneously. The
## default is the number of logical CPUs available in the system.
##max_procs:
#processors:
#- rename:
# fields:
# - from: "a"
# to: "b"
#
##================================ Processors ===================================
# The following example tokenizes the string into fields:
#
## Processors are used to reduce the number of fields in the exported event or to
## enhance the event with external metadata. This section defines a list of
## processors that are applied one by one and the first one receives the initial
## event:
##
## event -> filter1 -> event1 -> filter2 ->event2 ...
##
## The supported processors are drop_fields, drop_event, include_fields, and
## add_cloud_metadata.
##
## For example, you can use the following processors to keep the fields that
## contain CPU load percentages, but remove the fields that contain CPU ticks
## values:
##
##processors:
##- include_fields:
## fields: ["cpu"]
##- drop_fields:
## fields: ["cpu.user", "cpu.system"]
##
## The following example drops the events that have the HTTP response code 200:
##
##processors:
##- drop_event:
## when:
## equals:
## http.code: 200
##
## The following example enriches each event with metadata from the cloud
## provider about the host machine. It works on EC2, GCE, DigitalOcean,
## Tencent Cloud, and Alibaba Cloud.
##
##processors:
##- add_cloud_metadata: ~
##
## The following example enriches each event with the machine's local time zone
## offset from UTC.
##
##processors:
##- add_locale:
## format: offset
##
## The following example enriches each event with docker metadata, it matches
## given fields to an existing container id and adds info from that container:
##
##processors:
##- add_docker_metadata:
## host: "unix:///var/run/docker.sock"
## match_fields: ["system.process.cgroup.id"]
## match_pids: ["process.pid", "process.ppid"]
## match_source: true
## match_source_index: 4
## cleanup_timeout: 60
## # To connect to Docker over TLS you must specify a client and CA certificate.
## #ssl:
## # certificate_authority: "/etc/pki/root/ca.pem"
## # certificate: "/etc/pki/client/cert.pem"
## # key: "/etc/pki/client/cert.key"
##
## The following example enriches each event with docker metadata, it matches
## container id from log path available in `source` field (by default it expects
## it to be /var/lib/docker/containers/*/*.log).
##
##processors:
##- add_docker_metadata: ~
#processors:
#- dissect:
# tokenizer: "%{key1} - %{key2}"
# field: "message"
# target_prefix: "dissect"
#
# The following example enriches each event with metadata from the cloud
# provider about the host machine. It works on EC2, GCE, DigitalOcean,
# Tencent Cloud, and Alibaba Cloud.
#
#processors:
#- add_cloud_metadata: ~
#
# The following example enriches each event with the machine's local time zone
# offset from UTC.
#
#processors:
#- add_locale:
# format: offset
#
# The following example enriches each event with docker metadata, it matches
# given fields to an existing container id and adds info from that container:
#
#processors:
#- add_docker_metadata:
# host: "unix:///var/run/docker.sock"
# match_fields: ["system.process.cgroup.id"]
# match_pids: ["process.pid", "process.ppid"]
# match_source: true
# match_source_index: 4
# match_short_id: false
# cleanup_timeout: 60
# labels.dedot: false
# # To connect to Docker over TLS you must specify a client and CA certificate.
# #ssl:
# # certificate_authority: "/etc/pki/root/ca.pem"
# # certificate: "/etc/pki/client/cert.pem"
# # key: "/etc/pki/client/cert.key"
#
# The following example enriches each event with docker metadata, it matches
# container id from log path available in `source` field (by default it expects
# it to be /var/lib/docker/containers/*/*.log).
#
#processors:
#- add_docker_metadata: ~
#
# The following example enriches each event with host metadata.
#
#processors:
#- add_host_metadata:
# netinfo.enabled: false
#
# The following example enriches each event with process metadata using
# process IDs included in the event.
#
#processors:
#- add_process_metadata:
# match_pids: ["system.process.ppid"]
# target: system.process.parent
#
# The following example decodes fields containing JSON strings
# and replaces the strings with valid JSON objects.
#
#processors:
#- decode_json_fields:
# fields: ["field1", "field2", ...]
# process_array: false
# max_depth: 1
# target: ""
# overwrite_keys: false
processors:
- add_host_metadata: ~
@ -1065,8 +1221,13 @@ processors:
# Boolean flag to enable or disable the output module.
#enabled: true
# Pretty print json event
#pretty: false
# Configure JSON encoding
#codec.json:
# Pretty print json event
#pretty: false
# Configure escaping html symbols in strings.
#escape_html: true
#================================= Paths ======================================
@ -1094,6 +1255,10 @@ processors:
# the default for the logs path is a logs subdirectory inside the home path.
#path.logs: ${path.home}/logs
#================================ Keystore ==========================================
# Location of the Keystore containing the keys and their sensitive values.
#keystore.path: "${path.config}/beats.keystore"
#============================== Dashboards =====================================
{{ elk_macros.setup_dashboards('metricbeat') }}
@ -1125,3 +1290,8 @@ processors:
# Port on which the HTTP endpoint will bind. Default is 5066.
#http.port: 5066
#============================= Process Security ================================
# Enable or disable seccomp system call filtering on Linux. Default is enabled.
#seccomp.enabled: true

View File

@ -222,6 +222,16 @@ setup.dashboards.enabled: false
# Always use the Kibana API for loading the dashboards instead of autodetecting
# how to install the dashboards by first querying Elasticsearch.
#setup.dashboards.always_kibana: false
# If true and Kibana is not reachable at the time when dashboards are loaded,
# it will retry to reconnect to Kibana instead of exiting with an error.
#setup.dashboards.retry.enabled: false
# Duration interval between Kibana connection retries.
#setup.dashboards.retry.interval: 1s
# Maximum number of retries before exiting with an error, 0 for unlimited retrying.
#setup.dashboards.retry.maximum: 0
{%- endmacro %}
{% macro setup_template(beat_name, host, data_nodes, elasticsearch_replicas) -%}