openstack
/
openstack-ansible-ops
Code Issues Proposed changes

Add the ability to enable or disable rollups / indexes 

This change creates a new option to enable or disbale rollup jobs. This
is also providing the default basic index patterns for kibana index
patterns and elastic indexes.

Change-Id: I60e96a2cdbe27de760b54c4d9d43bcde4d09bbf5
Signed-off-by: cloudnull <kevin@cloudnull.com>
This commit is contained in:
cloudnull 2019-02-10 23:53:23 -06:00
parent a7e03710a5
commit 0a0a4a0880
No known key found for this signature in database
GPG Key ID: 9443251A787B9FB3
12 changed files with 160 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

141
elk_metrics_6x/createElasticIndexes.yml
View File

@ -28,6 +28,43 @@
- role: elastic_retention
post_tasks:
- name: Create beat indexes
uri:
url: http://127.0.0.1:9200/{{ item.name }}
method: PUT
body: "{{ item.index_options | to_json }}"
status_code: 200,400
body_format: json
register: elk_indexes
until: elk_indexes is success
retries: 3
delay: 30
with_items: |-
{% set beat_indexes = [] %}
{% for key, value in elastic_beat_retention_policy_hosts.items() %}
{% if ((value.hosts | length) > 0) and (value.make_index | default(false) | bool) %}
{%
set _index = {
'name': key,
'index_options': {
'settings': {
'index': {
'codec': 'best_compression',
'mapping': {
'total_fields': {
'limit': '10000'
}
},
'refresh_interval': elastic_refresh_interval
}
}
}
}
%}
{% set _ = beat_indexes.append(_index) %}
{% endif %}
{% endfor %}
{{ beat_indexes }}
- name: Create basic indexes
uri:
url: http://127.0.0.1:9200/{{ item.name }}
@ -40,15 +77,6 @@
retries: 3
delay: 30
with_items:
- name: "osprofiler-notifications"
index_options:
settings:
index:
codec: "best_compression"
mapping:
total_fields:
limit: "10000"
refresh_interval: "{{ elastic_refresh_interval }}"
- name: "_all/_settings?preserve_existing=true"
index_options:
index.queries.cache.enabled: "true"
@ -120,8 +148,101 @@
delay: 30
vars:
index_option:
template: ".monitoring-*"
template: ".monitoring*"
order: 1
settings:
number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}"
number_of_shards: "{{ ((elasticsearch_number_of_replicas | int) * 2) + 1 }}"
- name: Create custom skydive index template
uri:
url: http://127.0.0.1:9200/_template/skydive
method: PUT
body: "{{ index_option | to_json }}"
status_code: 200
body_format: json
register: create_basicIndexTemplate
until: create_basicIndexTemplate is success
retries: 3
delay: 30
vars:
index_option:
template: "skydive*"
order: 1
settings:
number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}"
number_of_shards: "{{ ((elasticsearch_number_of_replicas | int) * 2) + 1 }}"
- name: Create/Setup known indexes in Kibana
hosts: kibana
become: true
vars_files:
- vars/variables.yml
environment: "{{ deployment_environment_variables | default({}) }}"
roles:
- role: elastic_retention
post_tasks:
- name: Create kibana indexe patterns
uri:
url: "http://127.0.0.1:5601/api/saved_objects/index-pattern/{{ item.name }}"
method: POST
body: "{{ item.index_options | to_json }}"
status_code: 200,409
body_format: json
headers:
Content-Type: "application/json"
kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
with_items: |-
{% set beat_indexes = [] %}
{% for key, value in elastic_beat_retention_policy_hosts.items() %}
{% if (value.hosts | length) > 0 %}
{%
set _index = {
'name': key,
'index_options': {
'attributes': {
'title': (key ~ '*')
}
}
}
%}
{% if value.timeFieldName is defined %}
{% set _ = _index.index_options.attributes.__setitem__('timeFieldName', (value.timeFieldName | string)) %}
{% endif %}
{% set _ = beat_indexes.append(_index) %}
{% endif %}
{% endfor %}
{% set _ = beat_indexes.append({'name': 'default', 'index_options': {'attributes': {'title': '*'}}}) %}
{{ beat_indexes }}
register: kibana_indexes
until: kibana_indexes is success
retries: 6
delay: 30
run_once: true
- name: Create basic indexes
uri:
url: "http://127.0.0.1:5601/api/kibana/settings/defaultIndex"
method: POST
body: "{{ item.index_options | to_json }}"
status_code: 200
body_format: json
headers:
Content-Type: "application/json"
kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
with_items:
- name: "default"
index_options:
value: "default"
register: kibana_indexes
until: kibana_indexes is success
retries: 6
delay: 30
run_once: true
tags:
- server-install

6
elk_metrics_6x/installAPMserver.yml
View File

@ -26,7 +26,8 @@
- role: elastic_apm_server
- role: elastic_rollup
index_name: apm
when:
- elastic_create_rollup | bool
tags:
- apm-server
@ -44,6 +45,7 @@
roles:
- role: elastic_rollup
index_name: apm
when:
- elastic_create_rollup | bool
tags:
- apm-server

3
elk_metrics_6x/installAuditbeat.yml
View File

@ -42,7 +42,8 @@
roles:
- role: elastic_rollup
index_name: auditbeat
when:
- elastic_create_rollup | bool
tags:
- auditbeat

3
elk_metrics_6x/installFilebeat.yml
View File

@ -42,7 +42,8 @@
roles:
- role: elastic_rollup
index_name: filebeat
when:
- elastic_create_rollup | bool
tags:
- filebeat

3
elk_metrics_6x/installHeartbeat.yml
View File

@ -56,7 +56,8 @@
roles:
- role: elastic_rollup
index_name: heartbeat
when:
- elastic_create_rollup | bool
tags:
- heartbeat

3
elk_metrics_6x/installJournalbeat.yml
View File

@ -96,6 +96,7 @@
roles:
- role: elastic_rollup
index_name: journalbeat
when:
- elastic_create_rollup | bool
tags:
- journalbeat

43
elk_metrics_6x/installKibana.yml
View File

@ -22,48 +22,5 @@
roles:
- role: elastic_kibana
post_tasks:
- name: Create basic indexes
uri:
url: "http://127.0.0.1:5601/api/saved_objects/index-pattern/{{ item.name }}"
method: POST
body: "{{ item.index_options | to_json }}"
status_code: 200,409
body_format: json
headers:
Content-Type: "application/json"
kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
with_items:
- name: "*"
index_options:
attributes:
title: "*"
timeFieldName: "@timestamp"
register: kibana_indexes
until: kibana_indexes is success
retries: 6
delay: 30
run_once: true
- name: Create basic indexes
uri:
url: "http://127.0.0.1:5601/api/kibana/settings/defaultIndex"
method: POST
body: "{{ item.index_options | to_json }}"
status_code: 200
body_format: json
headers:
Content-Type: "application/json"
kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
with_items:
- name: "*"
index_options:
value: "*"
register: kibana_indexes
until: kibana_indexes is success
retries: 6
delay: 30
run_once: true
tags:
- server-install

3
elk_metrics_6x/installMetricbeat.yml
View File

@ -42,7 +42,8 @@
roles:
- role: elastic_rollup
index_name: metricbeat
when:
- elastic_create_rollup | bool
tags:
- metricbeat

3
elk_metrics_6x/installPacketbeat.yml
View File

@ -42,7 +42,8 @@
roles:
- role: elastic_rollup
index_name: packetbeat
when:
- elastic_create_rollup | bool
tags:
- packetbeat

1
elk_metrics_6x/roles/elastic_journalbeat/tasks/systemd.general-overrides.conf.j2
View File

@ -1 +0,0 @@
../../../templates/systemd.general-overrides.conf.j2

14
elk_metrics_6x/roles/elastic_retention/defaults/main.yml
View File

@ -70,35 +70,45 @@ elastic_index_retention_algorithm: default
elastic_beat_retention_policy_hosts:
logstash:
make_index: true
weight: 1
hosts: "{{ groups['elastic-logstash'] | default([]) }}"
apm:
make_index: true
timeFieldName: '@timestamp'
weight: 1
hosts: "{{ groups['apm-server'] | default([]) }}"
auditbeat:
timeFieldName: '@timestamp'
weight: 10
hosts: "{{ groups['hosts'] | default([]) }}"
filebeat:
timeFieldName: '@timestamp'
weight: 10
hosts: "{{ groups['hosts'] | default([]) }}"
syslog:
make_index: true
weight: 1
hosts: "{{ groups['hosts'] | default([]) }}"
heartbeat:
timeFieldName: '@timestamp'
weight: 1
hosts: "{{ groups['kibana'][:3] | default([]) }}"
journalbeat:
timeFieldName: '@timestamp'
weight: 3
hosts: "{{ groups['all'] | default([]) }}"
hosts: "{{ groups['hosts'] | default([]) }}"
metricbeat:
timeFieldName: '@timestamp'
weight: 2
hosts: "{{ groups['all'] | default([]) }}"
packetbeat:
timeFieldName: '@timestamp'
weight: 1
hosts: "{{ groups['hosts'] | default([]) }}"
skydive:
weight: 1
hosts: "{{ groups['skydive_analyzers'] | default([]) }}"
hosts: "{{ (((groups['skydive_analyzers'] | default([])) | length) > 0) | ternary((groups['hosts'] | default([])), []) }}"
# Refresh the elasticsearch retention policy local facts.
elastic_retention_refresh: false

1
elk_metrics_6x/vars/variables.yml
View File

@ -11,6 +11,7 @@ q_storage: "{{ (ansible_processor_count | int) * (ansible_processor_threads_per_
apm_port: 8200
elastic_port: 9200
elastic_hap_port: 9201
elastic_create_rollup: false
logstash_beat_input_port: 5044
logstash_syslog_input_port: 5140
logstash_syslog_input_mode: udp