From 0a0a4a0880b2839eff8496d57c88ee3b955895fd Mon Sep 17 00:00:00 2001 From: cloudnull Date: Sun, 10 Feb 2019 23:53:23 -0600 Subject: [PATCH] Add the ability to enable or disable rollups / indexes This change creates a new option to enable or disbale rollup jobs. This is also providing the default basic index patterns for kibana index patterns and elastic indexes. Change-Id: I60e96a2cdbe27de760b54c4d9d43bcde4d09bbf5 Signed-off-by: cloudnull --- elk_metrics_6x/createElasticIndexes.yml | 141 ++++++++++++++++-- elk_metrics_6x/installAPMserver.yml | 6 +- elk_metrics_6x/installAuditbeat.yml | 3 +- elk_metrics_6x/installFilebeat.yml | 3 +- elk_metrics_6x/installHeartbeat.yml | 3 +- elk_metrics_6x/installJournalbeat.yml | 3 +- elk_metrics_6x/installKibana.yml | 43 ------ elk_metrics_6x/installMetricbeat.yml | 3 +- elk_metrics_6x/installPacketbeat.yml | 3 +- .../tasks/systemd.general-overrides.conf.j2 | 1 - .../roles/elastic_retention/defaults/main.yml | 14 +- elk_metrics_6x/vars/variables.yml | 1 + 12 files changed, 160 insertions(+), 64 deletions(-) delete mode 120000 elk_metrics_6x/roles/elastic_journalbeat/tasks/systemd.general-overrides.conf.j2 diff --git a/elk_metrics_6x/createElasticIndexes.yml b/elk_metrics_6x/createElasticIndexes.yml index d9030587..875c68f9 100644 --- a/elk_metrics_6x/createElasticIndexes.yml +++ b/elk_metrics_6x/createElasticIndexes.yml @@ -28,6 +28,43 @@ - role: elastic_retention post_tasks: + - name: Create beat indexes + uri: + url: http://127.0.0.1:9200/{{ item.name }} + method: PUT + body: "{{ item.index_options | to_json }}" + status_code: 200,400 + body_format: json + register: elk_indexes + until: elk_indexes is success + retries: 3 + delay: 30 + with_items: |- + {% set beat_indexes = [] %} + {% for key, value in elastic_beat_retention_policy_hosts.items() %} + {% if ((value.hosts | length) > 0) and (value.make_index | default(false) | bool) %} + {% + set _index = { + 'name': key, + 'index_options': { + 'settings': { + 'index': { + 'codec': 'best_compression', + 'mapping': { + 'total_fields': { + 'limit': '10000' + } + }, + 'refresh_interval': elastic_refresh_interval + } + } + } + } + %} + {% set _ = beat_indexes.append(_index) %} + {% endif %} + {% endfor %} + {{ beat_indexes }} - name: Create basic indexes uri: url: http://127.0.0.1:9200/{{ item.name }} @@ -40,15 +77,6 @@ retries: 3 delay: 30 with_items: - - name: "osprofiler-notifications" - index_options: - settings: - index: - codec: "best_compression" - mapping: - total_fields: - limit: "10000" - refresh_interval: "{{ elastic_refresh_interval }}" - name: "_all/_settings?preserve_existing=true" index_options: index.queries.cache.enabled: "true" @@ -120,8 +148,101 @@ delay: 30 vars: index_option: - template: ".monitoring-*" + template: ".monitoring*" order: 1 settings: number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}" number_of_shards: "{{ ((elasticsearch_number_of_replicas | int) * 2) + 1 }}" + + - name: Create custom skydive index template + uri: + url: http://127.0.0.1:9200/_template/skydive + method: PUT + body: "{{ index_option | to_json }}" + status_code: 200 + body_format: json + register: create_basicIndexTemplate + until: create_basicIndexTemplate is success + retries: 3 + delay: 30 + vars: + index_option: + template: "skydive*" + order: 1 + settings: + number_of_replicas: "{{ elasticsearch_number_of_replicas | int }}" + number_of_shards: "{{ ((elasticsearch_number_of_replicas | int) * 2) + 1 }}" + + +- name: Create/Setup known indexes in Kibana + hosts: kibana + become: true + vars_files: + - vars/variables.yml + + environment: "{{ deployment_environment_variables | default({}) }}" + + roles: + - role: elastic_retention + + post_tasks: + - name: Create kibana indexe patterns + uri: + url: "http://127.0.0.1:5601/api/saved_objects/index-pattern/{{ item.name }}" + method: POST + body: "{{ item.index_options | to_json }}" + status_code: 200,409 + body_format: json + headers: + Content-Type: "application/json" + kbn-xsrf: "{{ inventory_hostname | to_uuid }}" + with_items: |- + {% set beat_indexes = [] %} + {% for key, value in elastic_beat_retention_policy_hosts.items() %} + {% if (value.hosts | length) > 0 %} + {% + set _index = { + 'name': key, + 'index_options': { + 'attributes': { + 'title': (key ~ '*') + } + } + } + %} + {% if value.timeFieldName is defined %} + {% set _ = _index.index_options.attributes.__setitem__('timeFieldName', (value.timeFieldName | string)) %} + {% endif %} + {% set _ = beat_indexes.append(_index) %} + {% endif %} + {% endfor %} + {% set _ = beat_indexes.append({'name': 'default', 'index_options': {'attributes': {'title': '*'}}}) %} + {{ beat_indexes }} + register: kibana_indexes + until: kibana_indexes is success + retries: 6 + delay: 30 + run_once: true + + - name: Create basic indexes + uri: + url: "http://127.0.0.1:5601/api/kibana/settings/defaultIndex" + method: POST + body: "{{ item.index_options | to_json }}" + status_code: 200 + body_format: json + headers: + Content-Type: "application/json" + kbn-xsrf: "{{ inventory_hostname | to_uuid }}" + with_items: + - name: "default" + index_options: + value: "default" + register: kibana_indexes + until: kibana_indexes is success + retries: 6 + delay: 30 + run_once: true + + tags: + - server-install diff --git a/elk_metrics_6x/installAPMserver.yml b/elk_metrics_6x/installAPMserver.yml index 2a2a92bb..53c4b645 100644 --- a/elk_metrics_6x/installAPMserver.yml +++ b/elk_metrics_6x/installAPMserver.yml @@ -26,7 +26,8 @@ - role: elastic_apm_server - role: elastic_rollup index_name: apm - + when: + - elastic_create_rollup | bool tags: - apm-server @@ -44,6 +45,7 @@ roles: - role: elastic_rollup index_name: apm - + when: + - elastic_create_rollup | bool tags: - apm-server diff --git a/elk_metrics_6x/installAuditbeat.yml b/elk_metrics_6x/installAuditbeat.yml index dec43dab..957ce0ae 100644 --- a/elk_metrics_6x/installAuditbeat.yml +++ b/elk_metrics_6x/installAuditbeat.yml @@ -42,7 +42,8 @@ roles: - role: elastic_rollup index_name: auditbeat - + when: + - elastic_create_rollup | bool tags: - auditbeat diff --git a/elk_metrics_6x/installFilebeat.yml b/elk_metrics_6x/installFilebeat.yml index d0d6d906..d40fb154 100644 --- a/elk_metrics_6x/installFilebeat.yml +++ b/elk_metrics_6x/installFilebeat.yml @@ -42,7 +42,8 @@ roles: - role: elastic_rollup index_name: filebeat - + when: + - elastic_create_rollup | bool tags: - filebeat diff --git a/elk_metrics_6x/installHeartbeat.yml b/elk_metrics_6x/installHeartbeat.yml index c79febf9..d7228fa6 100644 --- a/elk_metrics_6x/installHeartbeat.yml +++ b/elk_metrics_6x/installHeartbeat.yml @@ -56,7 +56,8 @@ roles: - role: elastic_rollup index_name: heartbeat - + when: + - elastic_create_rollup | bool tags: - heartbeat diff --git a/elk_metrics_6x/installJournalbeat.yml b/elk_metrics_6x/installJournalbeat.yml index 71574c5c..9d239795 100644 --- a/elk_metrics_6x/installJournalbeat.yml +++ b/elk_metrics_6x/installJournalbeat.yml @@ -96,6 +96,7 @@ roles: - role: elastic_rollup index_name: journalbeat - + when: + - elastic_create_rollup | bool tags: - journalbeat diff --git a/elk_metrics_6x/installKibana.yml b/elk_metrics_6x/installKibana.yml index 9d6bec1f..087160c1 100644 --- a/elk_metrics_6x/installKibana.yml +++ b/elk_metrics_6x/installKibana.yml @@ -22,48 +22,5 @@ roles: - role: elastic_kibana - post_tasks: - - name: Create basic indexes - uri: - url: "http://127.0.0.1:5601/api/saved_objects/index-pattern/{{ item.name }}" - method: POST - body: "{{ item.index_options | to_json }}" - status_code: 200,409 - body_format: json - headers: - Content-Type: "application/json" - kbn-xsrf: "{{ inventory_hostname | to_uuid }}" - with_items: - - name: "*" - index_options: - attributes: - title: "*" - timeFieldName: "@timestamp" - register: kibana_indexes - until: kibana_indexes is success - retries: 6 - delay: 30 - run_once: true - - - name: Create basic indexes - uri: - url: "http://127.0.0.1:5601/api/kibana/settings/defaultIndex" - method: POST - body: "{{ item.index_options | to_json }}" - status_code: 200 - body_format: json - headers: - Content-Type: "application/json" - kbn-xsrf: "{{ inventory_hostname | to_uuid }}" - with_items: - - name: "*" - index_options: - value: "*" - register: kibana_indexes - until: kibana_indexes is success - retries: 6 - delay: 30 - run_once: true - tags: - server-install diff --git a/elk_metrics_6x/installMetricbeat.yml b/elk_metrics_6x/installMetricbeat.yml index 8082fd59..66e2df43 100644 --- a/elk_metrics_6x/installMetricbeat.yml +++ b/elk_metrics_6x/installMetricbeat.yml @@ -42,7 +42,8 @@ roles: - role: elastic_rollup index_name: metricbeat - + when: + - elastic_create_rollup | bool tags: - metricbeat diff --git a/elk_metrics_6x/installPacketbeat.yml b/elk_metrics_6x/installPacketbeat.yml index 2264c8e6..b83b2b7f 100644 --- a/elk_metrics_6x/installPacketbeat.yml +++ b/elk_metrics_6x/installPacketbeat.yml @@ -42,7 +42,8 @@ roles: - role: elastic_rollup index_name: packetbeat - + when: + - elastic_create_rollup | bool tags: - packetbeat diff --git a/elk_metrics_6x/roles/elastic_journalbeat/tasks/systemd.general-overrides.conf.j2 b/elk_metrics_6x/roles/elastic_journalbeat/tasks/systemd.general-overrides.conf.j2 deleted file mode 120000 index 9ddff7cc..00000000 --- a/elk_metrics_6x/roles/elastic_journalbeat/tasks/systemd.general-overrides.conf.j2 +++ /dev/null @@ -1 +0,0 @@ -../../../templates/systemd.general-overrides.conf.j2 \ No newline at end of file diff --git a/elk_metrics_6x/roles/elastic_retention/defaults/main.yml b/elk_metrics_6x/roles/elastic_retention/defaults/main.yml index e58f8945..00a789d9 100644 --- a/elk_metrics_6x/roles/elastic_retention/defaults/main.yml +++ b/elk_metrics_6x/roles/elastic_retention/defaults/main.yml @@ -70,35 +70,45 @@ elastic_index_retention_algorithm: default elastic_beat_retention_policy_hosts: logstash: + make_index: true weight: 1 hosts: "{{ groups['elastic-logstash'] | default([]) }}" apm: + make_index: true + timeFieldName: '@timestamp' weight: 1 hosts: "{{ groups['apm-server'] | default([]) }}" auditbeat: + timeFieldName: '@timestamp' weight: 10 hosts: "{{ groups['hosts'] | default([]) }}" filebeat: + timeFieldName: '@timestamp' weight: 10 hosts: "{{ groups['hosts'] | default([]) }}" syslog: + make_index: true weight: 1 hosts: "{{ groups['hosts'] | default([]) }}" heartbeat: + timeFieldName: '@timestamp' weight: 1 hosts: "{{ groups['kibana'][:3] | default([]) }}" journalbeat: + timeFieldName: '@timestamp' weight: 3 - hosts: "{{ groups['all'] | default([]) }}" + hosts: "{{ groups['hosts'] | default([]) }}" metricbeat: + timeFieldName: '@timestamp' weight: 2 hosts: "{{ groups['all'] | default([]) }}" packetbeat: + timeFieldName: '@timestamp' weight: 1 hosts: "{{ groups['hosts'] | default([]) }}" skydive: weight: 1 - hosts: "{{ groups['skydive_analyzers'] | default([]) }}" + hosts: "{{ (((groups['skydive_analyzers'] | default([])) | length) > 0) | ternary((groups['hosts'] | default([])), []) }}" # Refresh the elasticsearch retention policy local facts. elastic_retention_refresh: false diff --git a/elk_metrics_6x/vars/variables.yml b/elk_metrics_6x/vars/variables.yml index de367f87..17cb80f4 100644 --- a/elk_metrics_6x/vars/variables.yml +++ b/elk_metrics_6x/vars/variables.yml @@ -11,6 +11,7 @@ q_storage: "{{ (ansible_processor_count | int) * (ansible_processor_threads_per_ apm_port: 8200 elastic_port: 9200 elastic_hap_port: 9201 +elastic_create_rollup: false logstash_beat_input_port: 5044 logstash_syslog_input_port: 5140 logstash_syslog_input_mode: udp