From 5586d8a80fe2ffdb2659b552b01f67cfefc750e3 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Sun, 30 Dec 2018 19:20:04 -0600 Subject: [PATCH] Convert template setup to a role This change reduces code throughout the playbooks thereby speeding up the task execution. * A new role named `elastic_beat_setup` was created to facilitate template setup as needed. * Beats retention policies are now defined on the elastic-logstash nodes instead of on all target hosts. This method will speed-up deployments on massive installations while streamlining all deployments. * Kibana variable assumptions have been fixed. This will allow for deployments without Kibana to be accomplished. Change-Id: I36343264042e81dfcb68bad0f6c3a503e525eceb Signed-off-by: Kevin Carter --- elk_metrics_6x/installAPMserver.yml | 18 ++++++ elk_metrics_6x/installAuditbeat.yml | 22 ++++++- elk_metrics_6x/installFilebeat.yml | 22 ++++++- elk_metrics_6x/installHeartbeat.yml | 22 ++++++- elk_metrics_6x/installJournalbeat.yml | 17 +++++- elk_metrics_6x/installMetricbeat.yml | 22 ++++++- elk_metrics_6x/installPacketbeat.yml | 22 ++++++- .../roles/elastic_apm_server/tasks/main.yml | 11 +++- .../templates/apm-server.yml.j2 | 4 +- .../tasks/auditbeat_setup.yml | 60 ------------------- .../roles/elastic_auditbeat/tasks/main.yml | 11 +++- .../templates/auditbeat.yml.j2 | 4 +- .../elastic_beat_setup/defaults/main.yml | 31 ++++++++++ .../roles/elastic_beat_setup/meta/main.yml | 35 +++++++++++ .../tasks/main.yml} | 35 ++++++----- .../vars/data-node-variables.yml | 9 ++- .../elastic_filebeat/tasks/filebeat_setup.yml | 60 ------------------- .../roles/elastic_filebeat/tasks/main.yml | 11 +++- .../templates/filebeat.yml.j2 | 4 +- .../tasks/heartbeat_setup.yml | 60 ------------------- .../roles/elastic_heartbeat/tasks/main.yml | 11 +++- .../templates/heartbeat.yml.j2 | 4 +- .../tasks/journalbeat_setup.yml | 60 ------------------- .../roles/elastic_journalbeat/tasks/main.yml | 11 +++- .../templates/journalbeat.yml.j2 | 4 +- .../roles/elastic_metricbeat/tasks/main.yml | 11 +++- .../tasks/metricbeat_setup.yml | 60 ------------------- .../templates/metricbeat.yml.j2 | 4 +- .../roles/elastic_packetbeat/tasks/main.yml | 11 +++- .../tasks/packetbeat_setup.yml | 60 ------------------- .../templates/packetbeat.yml.j2 | 4 +- elk_metrics_6x/site-beats-core.yml | 15 ----- 32 files changed, 310 insertions(+), 425 deletions(-) delete mode 100644 elk_metrics_6x/roles/elastic_auditbeat/tasks/auditbeat_setup.yml create mode 100644 elk_metrics_6x/roles/elastic_beat_setup/defaults/main.yml create mode 100644 elk_metrics_6x/roles/elastic_beat_setup/meta/main.yml rename elk_metrics_6x/roles/{elastic_apm_server/tasks/apm_setup.yml => elastic_beat_setup/tasks/main.yml} (59%) delete mode 100644 elk_metrics_6x/roles/elastic_filebeat/tasks/filebeat_setup.yml delete mode 100644 elk_metrics_6x/roles/elastic_heartbeat/tasks/heartbeat_setup.yml delete mode 100644 elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml delete mode 100644 elk_metrics_6x/roles/elastic_metricbeat/tasks/metricbeat_setup.yml delete mode 100644 elk_metrics_6x/roles/elastic_packetbeat/tasks/packetbeat_setup.yml diff --git a/elk_metrics_6x/installAPMserver.yml b/elk_metrics_6x/installAPMserver.yml index 92145116..2a2a92bb 100644 --- a/elk_metrics_6x/installAPMserver.yml +++ b/elk_metrics_6x/installAPMserver.yml @@ -29,3 +29,21 @@ tags: - apm-server + +- name: Setup apm-server rollup + hosts: elastic-logstash[0] + become: true + vars: + haproxy_ssl: false + + vars_files: + - vars/variables.yml + + environment: "{{ deployment_environment_variables | default({}) }}" + + roles: + - role: elastic_rollup + index_name: apm + + tags: + - apm-server diff --git a/elk_metrics_6x/installAuditbeat.yml b/elk_metrics_6x/installAuditbeat.yml index 121e94f4..dec43dab 100644 --- a/elk_metrics_6x/installAuditbeat.yml +++ b/elk_metrics_6x/installAuditbeat.yml @@ -24,8 +24,28 @@ roles: - role: elastic_auditbeat + + tags: + - beat-install + +- name: Setup auditbeat rollup + hosts: elastic-logstash[0] + become: true + vars: + haproxy_ssl: false + + vars_files: + - vars/variables.yml + + environment: "{{ deployment_environment_variables | default({}) }}" + + roles: - role: elastic_rollup index_name: auditbeat tags: - - beat-install + - auditbeat + +- import_playbook: fieldRefresh.yml + vars: + index_pattern: auditbeat-* diff --git a/elk_metrics_6x/installFilebeat.yml b/elk_metrics_6x/installFilebeat.yml index 5b349fde..d0d6d906 100644 --- a/elk_metrics_6x/installFilebeat.yml +++ b/elk_metrics_6x/installFilebeat.yml @@ -24,8 +24,28 @@ roles: - role: elastic_filebeat + + tags: + - beat-install + +- name: Setup filebeat rollup + hosts: elastic-logstash[0] + become: true + vars: + haproxy_ssl: false + + vars_files: + - vars/variables.yml + + environment: "{{ deployment_environment_variables | default({}) }}" + + roles: - role: elastic_rollup index_name: filebeat tags: - - beat-install + - filebeat + +- import_playbook: fieldRefresh.yml + vars: + index_pattern: filebeat-* diff --git a/elk_metrics_6x/installHeartbeat.yml b/elk_metrics_6x/installHeartbeat.yml index 593458b3..c79febf9 100644 --- a/elk_metrics_6x/installHeartbeat.yml +++ b/elk_metrics_6x/installHeartbeat.yml @@ -38,8 +38,28 @@ roles: - role: elastic_heartbeat + + tags: + - beat-install + +- name: Setup heartbeat rollup + hosts: elastic-logstash[0] + become: true + vars: + haproxy_ssl: false + + vars_files: + - vars/variables.yml + + environment: "{{ deployment_environment_variables | default({}) }}" + + roles: - role: elastic_rollup index_name: heartbeat tags: - - beat-install + - heartbeat + +- import_playbook: fieldRefresh.yml + vars: + index_pattern: heartbeat-* diff --git a/elk_metrics_6x/installJournalbeat.yml b/elk_metrics_6x/installJournalbeat.yml index f3a63793..cc379729 100644 --- a/elk_metrics_6x/installJournalbeat.yml +++ b/elk_metrics_6x/installJournalbeat.yml @@ -62,6 +62,21 @@ - journal_dir.stat.exists | bool - ansible_service_mgr == 'systemd' + tags: + - beat-install + +- name: Setup journalbeat rollup + hosts: elastic-logstash[0] + become: true + vars: + haproxy_ssl: false + + vars_files: + - vars/variables.yml + + environment: "{{ deployment_environment_variables | default({}) }}" + + roles: - role: elastic_rollup when: - journal_dir.stat.exists | bool @@ -69,4 +84,4 @@ index_name: journalbeat tags: - - beat-install + - journalbeat diff --git a/elk_metrics_6x/installMetricbeat.yml b/elk_metrics_6x/installMetricbeat.yml index d174c1ec..8082fd59 100644 --- a/elk_metrics_6x/installMetricbeat.yml +++ b/elk_metrics_6x/installMetricbeat.yml @@ -24,8 +24,28 @@ roles: - role: elastic_metricbeat + + tags: + - beat-install + +- name: Setup metricbeat rollup + hosts: elastic-logstash[0] + become: true + vars: + haproxy_ssl: false + + vars_files: + - vars/variables.yml + + environment: "{{ deployment_environment_variables | default({}) }}" + + roles: - role: elastic_rollup index_name: metricbeat tags: - - beat-install + - metricbeat + +- import_playbook: fieldRefresh.yml + vars: + index_pattern: metricbeat-* diff --git a/elk_metrics_6x/installPacketbeat.yml b/elk_metrics_6x/installPacketbeat.yml index 15086c38..2264c8e6 100644 --- a/elk_metrics_6x/installPacketbeat.yml +++ b/elk_metrics_6x/installPacketbeat.yml @@ -24,8 +24,28 @@ roles: - role: elastic_packetbeat + + tags: + - beat-install + +- name: Setup packetbeat rollup + hosts: elastic-logstash[0] + become: true + vars: + haproxy_ssl: false + + vars_files: + - vars/variables.yml + + environment: "{{ deployment_environment_variables | default({}) }}" + + roles: - role: elastic_rollup index_name: packetbeat tags: - - beat-install + - packetbeat + +- import_playbook: fieldRefresh.yml + vars: + index_pattern: packetbeat-* diff --git a/elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml b/elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml index 7675df81..d0f5d2be 100644 --- a/elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml @@ -69,9 +69,14 @@ notify: - Enable and restart apm server -- include_tasks: apm_setup.yml - tags: - - setup +- name: Run the beat setup role + include_role: + name: elastic_beat_setup + private: true + when: + - (groups['kibana'] | length) > 0 + vars: + elastic_beat_name: "apm-server" - name: Force beat handlers meta: flush_handlers diff --git a/elk_metrics_6x/roles/elastic_apm_server/templates/apm-server.yml.j2 b/elk_metrics_6x/roles/elastic_apm_server/templates/apm-server.yml.j2 index 5050e8c8..b0dfaf3f 100644 --- a/elk_metrics_6x/roles/elastic_apm_server/templates/apm-server.yml.j2 +++ b/elk_metrics_6x/roles/elastic_apm_server/templates/apm-server.yml.j2 @@ -137,7 +137,9 @@ apm-server: {{ elk_macros.setup_template('apm', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} #============================== Kibana ===================================== -{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] + ':' + kibana_port|string) }} +{% if (groups['kibana'] | length) > 0 %} +{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} +{% endif %} #================================ Logging ====================================== {{ elk_macros.beat_logging('apm-server') }} diff --git a/elk_metrics_6x/roles/elastic_auditbeat/tasks/auditbeat_setup.yml b/elk_metrics_6x/roles/elastic_auditbeat/tasks/auditbeat_setup.yml deleted file mode 100644 index 69929d04..00000000 --- a/elk_metrics_6x/roles/elastic_auditbeat/tasks/auditbeat_setup.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Refresh local facts - setup: - filter: ansible_local - gather_subset: "!all" - tags: - - always - -- name: Load templates - shell: >- - auditbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -E 'setup.template.enabled=true' - -E 'setup.template.overwrite=true' - -e -v - with_items: - - "--template" - - "--pipelines" - - "--machine-learning" - - "--dashboards" - register: templates - environment: - no_proxy: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}" - until: templates is success - retries: 5 - delay: 5 - run_once: true - when: - - ((ansible_local['elastic']['setup']['auditbeat_loaded_templates'] is undefined) or - (not (ansible_local['elastic']['setup']['auditbeat_loaded_templates'] | bool))) or - (elk_package_state | default('present')) == "latest" - tags: - - setup - -- name: Set template fact - ini_file: - dest: "/etc/ansible/facts.d/elastic.fact" - section: "setup" - option: auditbeat_loaded_templates - value: true - when: - - templates is changed - tags: - - setup diff --git a/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml index 8589ed8d..aed5986f 100644 --- a/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml @@ -82,9 +82,14 @@ notify: - Enable and restart auditbeat -- include_tasks: auditbeat_setup.yml - tags: - - setup +- name: Run the beat setup role + include_role: + name: elastic_beat_setup + private: true + when: + - (groups['kibana'] | length) > 0 + vars: + elastic_beat_name: "auditbeat" - name: Force beat handlers meta: flush_handlers diff --git a/elk_metrics_6x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 b/elk_metrics_6x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 index 55621948..11e61221 100644 --- a/elk_metrics_6x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 +++ b/elk_metrics_6x/roles/elastic_auditbeat/templates/auditbeat.yml.j2 @@ -666,7 +666,9 @@ processors: {{ elk_macros.setup_template('auditbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} #============================== Kibana ===================================== -{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] + ':' + kibana_port|string) }} +{% if (groups['kibana'] | length) > 0 %} +{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} +{% endif %} #================================ Logging ====================================== {{ elk_macros.beat_logging('auditbeat') }} diff --git a/elk_metrics_6x/roles/elastic_beat_setup/defaults/main.yml b/elk_metrics_6x/roles/elastic_beat_setup/defaults/main.yml new file mode 100644 index 00000000..eb4bb352 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_beat_setup/defaults/main.yml @@ -0,0 +1,31 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Each setup flag is run one at a time. +elastic_setup_flags: + - "--template" + - "--pipelines" + - "--machine-learning" + - "--dashboards" + +# Setup options are cast as a string with, one option per line. +elastic_beat_setup_options: >- + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -E 'setup.template.enabled=true' + -E 'setup.template.overwrite=true' + +# The node defined here will be used with the environment variable, "no_proxy". +elastic_beat_kibana_host: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}" diff --git a/elk_metrics_6x/roles/elastic_beat_setup/meta/main.yml b/elk_metrics_6x/roles/elastic_beat_setup/meta/main.yml new file mode 100644 index 00000000..c529fd00 --- /dev/null +++ b/elk_metrics_6x/roles/elastic_beat_setup/meta/main.yml @@ -0,0 +1,35 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +galaxy_info: + author: OpenStack + description: Elastic v6.x repositories role + company: Rackspace + license: Apache2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - trusty + - xenial + - bionic + categories: + - cloud + - development + - beats + - elastic-beats + - elasticsearch + - elastic-stack +dependencies: [] diff --git a/elk_metrics_6x/roles/elastic_apm_server/tasks/apm_setup.yml b/elk_metrics_6x/roles/elastic_beat_setup/tasks/main.yml similarity index 59% rename from elk_metrics_6x/roles/elastic_apm_server/tasks/apm_setup.yml rename to elk_metrics_6x/roles/elastic_beat_setup/tasks/main.yml index aa770006..df4d9fc4 100644 --- a/elk_metrics_6x/roles/elastic_apm_server/tasks/apm_setup.yml +++ b/elk_metrics_6x/roles/elastic_beat_setup/tasks/main.yml @@ -13,6 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +- name: Check for coordination_nodes var + fail: + msg: >- + To use this role the variable `coordination_nodes` must be defined. + when: + - coordination_nodes is undefined + +- name: Check for elastic_beat_name var + fail: + msg: >- + To use this role the variable `elastic_beat_name` must be defined. + when: + - elastic_beat_name is undefined + - name: Refresh local facts setup: filter: ansible_local @@ -22,28 +36,21 @@ - name: Load templates shell: >- - apm-server setup + {{ elastic_beat_name }} setup {{ item }} - -E 'apm-server.host=localhost:8200' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -E 'setup.template.enabled=true' - -E 'setup.template.overwrite=true' + {{ elastic_beat_setup_options }} -e -v - with_items: - - "--template" - - "--pipelines" - - "--machine-learning" - - "--dashboards" + with_items: "{{ elastic_setup_flags }}" register: templates environment: - no_proxy: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}" + no_proxy: "{{ elastic_beat_kibana_host }}" until: templates is success retries: 5 delay: 5 run_once: true when: - - ((ansible_local['elastic']['setup']['apm_loaded_templates'] is undefined) or - (not (ansible_local['elastic']['setup']['apm_loaded_templates'] | bool))) or + - ((ansible_local['elastic']['setup'][elastic_beat_name + '_loaded_templates'] is undefined) or + (not (ansible_local['elastic']['setup'][elastic_beat_name + '_loaded_templates'] | bool))) or (elk_package_state | default('present')) == "latest" tags: - setup @@ -52,7 +59,7 @@ ini_file: dest: "/etc/ansible/facts.d/elastic.fact" section: "setup" - option: apm_loaded_templates + option: "{{ elastic_beat_name + '_loaded_templates' }}" value: true when: - templates is changed diff --git a/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml b/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml index 0cb7de96..2354daaa 100644 --- a/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml +++ b/elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml @@ -121,9 +121,14 @@ master_nodes: |- {{ nodes }} master_node_count: "{{ master_nodes | length }}" -coordination_nodes: >- +coordination_nodes: |- + {% if (groups['kibana'] | length) > 0 %} + {% set c_nodes = groups['kibana'] %} + {% else %} + {% set c_nodes = groups['elastic-logstash'] %} + {% endif %} {{ - (groups['kibana'] | map('extract', hostvars, 'ansible_host') | list) + (c_nodes | map('extract', hostvars, 'ansible_host') | list) | map('regex_replace', '(.*)' ,'\1:' ~ elastic_port) | list }} diff --git a/elk_metrics_6x/roles/elastic_filebeat/tasks/filebeat_setup.yml b/elk_metrics_6x/roles/elastic_filebeat/tasks/filebeat_setup.yml deleted file mode 100644 index 331e26ac..00000000 --- a/elk_metrics_6x/roles/elastic_filebeat/tasks/filebeat_setup.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Refresh local facts - setup: - filter: ansible_local - gather_subset: "!all" - tags: - - always - -- name: Load templates - shell: >- - filebeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -E 'setup.template.enabled=true' - -E 'setup.template.overwrite=true' - -e -v - with_items: - - "--template" - - "--pipelines" - - "--machine-learning" - - "--dashboards" - register: templates - environment: - no_proxy: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}" - until: templates is success - retries: 5 - delay: 5 - run_once: true - when: - - ((ansible_local['elastic']['setup']['filebeat_loaded_templates'] is undefined) or - (not (ansible_local['elastic']['setup']['filebeat_loaded_templates'] | bool))) or - (elk_package_state | default('present')) == "latest" - tags: - - setup - -- name: Set template fact - ini_file: - dest: "/etc/ansible/facts.d/elastic.fact" - section: "setup" - option: filebeat_loaded_templates - value: true - when: - - templates is changed - tags: - - setup diff --git a/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml index 01aeb6d3..7fb6daff 100644 --- a/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml @@ -82,9 +82,14 @@ notify: - Enable and restart filebeat -- include_tasks: filebeat_setup.yml - tags: - - setup +- name: Run the beat setup role + include_role: + name: elastic_beat_setup + private: true + when: + - (groups['kibana'] | length) > 0 + vars: + elastic_beat_name: "filebeat" - name: Force beat handlers meta: flush_handlers diff --git a/elk_metrics_6x/roles/elastic_filebeat/templates/filebeat.yml.j2 b/elk_metrics_6x/roles/elastic_filebeat/templates/filebeat.yml.j2 index e1be6736..49ab5525 100644 --- a/elk_metrics_6x/roles/elastic_filebeat/templates/filebeat.yml.j2 +++ b/elk_metrics_6x/roles/elastic_filebeat/templates/filebeat.yml.j2 @@ -912,7 +912,9 @@ processors: {{ elk_macros.setup_template('filebeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} #============================== Kibana ===================================== -{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] + ':' + kibana_port|string) }} +{% if (groups['kibana'] | length) > 0 %} +{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} +{% endif %} #================================ Logging ====================================== {{ elk_macros.beat_logging('filebeat') }} diff --git a/elk_metrics_6x/roles/elastic_heartbeat/tasks/heartbeat_setup.yml b/elk_metrics_6x/roles/elastic_heartbeat/tasks/heartbeat_setup.yml deleted file mode 100644 index c5856793..00000000 --- a/elk_metrics_6x/roles/elastic_heartbeat/tasks/heartbeat_setup.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Refresh local facts - setup: - filter: ansible_local - gather_subset: "!all" - tags: - - always - -- name: Load templates - shell: >- - heartbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -E 'setup.template.enabled=true' - -E 'setup.template.overwrite=true' - -e -v - with_items: - - "--template" - - "--pipelines" - - "--machine-learning" - - "--dashboards" - register: templates - environment: - no_proxy: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}" - until: templates is success - retries: 5 - delay: 5 - run_once: true - when: - - ((ansible_local['elastic']['setup']['heartbeat_loaded_templates'] is undefined) or - (not (ansible_local['elastic']['setup']['heartbeat_loaded_templates'] | bool))) or - (elk_package_state | default('present')) == "latest" - tags: - - setup - -- name: Set template fact - ini_file: - dest: "/etc/ansible/facts.d/elastic.fact" - section: "setup" - option: heartbeat_loaded_templates - value: true - when: - - templates is changed - tags: - - setup diff --git a/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml index e3352553..c1e9f19a 100644 --- a/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml @@ -88,9 +88,14 @@ notify: - Enable and restart heartbeat -- include_tasks: heartbeat_setup.yml - tags: - - setup +- name: Run the beat setup role + include_role: + name: elastic_beat_setup + private: true + when: + - (groups['kibana'] | length) > 0 + vars: + elastic_beat_name: "heartbeat" - name: Force beat handlers meta: flush_handlers diff --git a/elk_metrics_6x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 b/elk_metrics_6x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 index 49324b97..24466e5e 100644 --- a/elk_metrics_6x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 +++ b/elk_metrics_6x/roles/elastic_heartbeat/templates/heartbeat.yml.j2 @@ -780,7 +780,9 @@ processors: {{ elk_macros.setup_template('heartbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} #============================== Kibana ===================================== -{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] + ':' + kibana_port|string) }} +{% if (groups['kibana'] | length) > 0 %} +{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} +{% endif %} #================================ Logging ====================================== {{ elk_macros.beat_logging('heartbeat') }} diff --git a/elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml b/elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml deleted file mode 100644 index 172153e4..00000000 --- a/elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Refresh local facts - setup: - filter: ansible_local - gather_subset: "!all" - tags: - - always - -- name: Load templates - shell: >- - journalbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -E 'setup.template.enabled=true' - -E 'setup.template.overwrite=true' - -e -v - with_items: - - "--template" - - "--pipelines" - - "--machine-learning" - - "--dashboards" - register: templates - environment: - no_proxy: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}" - until: templates is success - retries: 5 - delay: 5 - run_once: true - when: - - ((ansible_local['elastic']['setup']['journalbeat_loaded_templates'] is undefined) or - (not (ansible_local['elastic']['setup']['journalbeat_loaded_templates'] | bool))) or - (elk_package_state | default('present')) == "latest" - tags: - - setup - -- name: Set template fact - ini_file: - dest: "/etc/ansible/facts.d/elastic.fact" - section: "setup" - option: journalbeat_loaded_templates - value: true - when: - - templates is changed - tags: - - setup diff --git a/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml index bfca73a3..0d453815 100644 --- a/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml @@ -88,9 +88,14 @@ notify: - Enable and restart journalbeat -- include_tasks: journalbeat_setup.yml - tags: - - setup +- name: Run the beat setup role + include_role: + name: elastic_beat_setup + private: true + when: + - (groups['kibana'] | length) > 0 + vars: + elastic_beat_name: "journalbeat" - name: Force beat handlers meta: flush_handlers diff --git a/elk_metrics_6x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 b/elk_metrics_6x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 index bcd3a1ad..0ca51c7a 100644 --- a/elk_metrics_6x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 +++ b/elk_metrics_6x/roles/elastic_journalbeat/templates/journalbeat.yml.j2 @@ -579,7 +579,9 @@ tags: {{ elk_macros.setup_template('journalbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} #============================== Kibana ===================================== -{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] + ':' + kibana_port|string) }} +{% if (groups['kibana'] | length) > 0 %} +{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} +{% endif %} #================================ Logging ====================================== {{ elk_macros.beat_logging('journalbeat') }} diff --git a/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml index abd66be0..07319c7d 100644 --- a/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml @@ -305,9 +305,14 @@ notify: - Enable and restart metricbeat -- include_tasks: metricbeat_setup.yml - tags: - - setup +- name: Run the beat setup role + include_role: + name: elastic_beat_setup + private: true + when: + - (groups['kibana'] | length) > 0 + vars: + elastic_beat_name: "metricbeat" - name: Force beat handlers meta: flush_handlers diff --git a/elk_metrics_6x/roles/elastic_metricbeat/tasks/metricbeat_setup.yml b/elk_metrics_6x/roles/elastic_metricbeat/tasks/metricbeat_setup.yml deleted file mode 100644 index 7aee47a9..00000000 --- a/elk_metrics_6x/roles/elastic_metricbeat/tasks/metricbeat_setup.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Refresh local facts - setup: - filter: ansible_local - gather_subset: "!all" - tags: - - always - -- name: Load templates - shell: >- - metricbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -E 'setup.template.enabled=true' - -E 'setup.template.overwrite=true' - -e -v - with_items: - - "--template" - - "--pipelines" - - "--machine-learning" - - "--dashboards" - register: templates - environment: - no_proxy: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}" - until: templates is success - retries: 5 - delay: 5 - run_once: true - when: - - ((ansible_local['elastic']['setup']['metricbeat_loaded_templates'] is undefined) or - (not (ansible_local['elastic']['setup']['metricbeat_loaded_templates'] | bool))) or - (elk_package_state | default('present')) == "latest" - tags: - - setup - -- name: Set template fact - ini_file: - dest: "/etc/ansible/facts.d/elastic.fact" - section: "setup" - option: metricbeat_loaded_templates - value: true - when: - - templates is changed - tags: - - setup diff --git a/elk_metrics_6x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 b/elk_metrics_6x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 index 9fdf54bd..a48abdf5 100644 --- a/elk_metrics_6x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 +++ b/elk_metrics_6x/roles/elastic_metricbeat/templates/metricbeat.yml.j2 @@ -1102,7 +1102,9 @@ processors: {{ elk_macros.setup_template('metricbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} #================================ Kibana ======================================= -{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] + ':' + kibana_port|string) }} +{% if (groups['kibana'] | length) > 0 %} +{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} +{% endif %} #================================ Logging ====================================== {{ elk_macros.beat_logging('metricbeat') }} diff --git a/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml index e9c492c3..89acee01 100644 --- a/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml @@ -73,9 +73,14 @@ notify: - Enable and restart packetbeat -- include_tasks: packetbeat_setup.yml - tags: - - setup +- name: Run the beat setup role + include_role: + name: elastic_beat_setup + private: true + when: + - (groups['kibana'] | length) > 0 + vars: + elastic_beat_name: "packetbeat" - name: Force beat handlers meta: flush_handlers diff --git a/elk_metrics_6x/roles/elastic_packetbeat/tasks/packetbeat_setup.yml b/elk_metrics_6x/roles/elastic_packetbeat/tasks/packetbeat_setup.yml deleted file mode 100644 index 21d875db..00000000 --- a/elk_metrics_6x/roles/elastic_packetbeat/tasks/packetbeat_setup.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Refresh local facts - setup: - filter: ansible_local - gather_subset: "!all" - tags: - - always - -- name: Load templates - shell: >- - packetbeat setup - {{ item }} - -E 'output.logstash.enabled=false' - -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' - -E 'setup.template.enabled=true' - -E 'setup.template.overwrite=true' - -e -v - with_items: - - "--template" - - "--pipelines" - - "--machine-learning" - - "--dashboards" - register: templates - environment: - no_proxy: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}" - until: templates is success - retries: 5 - delay: 5 - run_once: true - when: - - ((ansible_local['elastic']['setup']['packetbeat_loaded_templates'] is undefined) or - (not (ansible_local['elastic']['setup']['packetbeat_loaded_templates'] | bool))) or - (elk_package_state | default('present')) == "latest" - tags: - - setup - -- name: Set template fact - ini_file: - dest: "/etc/ansible/facts.d/elastic.fact" - section: "setup" - option: packetbeat_loaded_templates - value: true - when: - - templates is changed - tags: - - setup diff --git a/elk_metrics_6x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 b/elk_metrics_6x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 index 3f779a8d..ab434f6f 100644 --- a/elk_metrics_6x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 +++ b/elk_metrics_6x/roles/elastic_packetbeat/templates/packetbeat.yml.j2 @@ -1056,7 +1056,9 @@ processors: {{ elk_macros.setup_template('packetbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }} #================================ Kibana ======================================= -{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] + ':' + kibana_port|string) }} +{% if (groups['kibana'] | length) > 0 %} +{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }} +{% endif %} #================================ Logging ====================================== {{ elk_macros.beat_logging('packetbeat') }} diff --git a/elk_metrics_6x/site-beats-core.yml b/elk_metrics_6x/site-beats-core.yml index 14d63c8d..e91f2294 100644 --- a/elk_metrics_6x/site-beats-core.yml +++ b/elk_metrics_6x/site-beats-core.yml @@ -12,28 +12,13 @@ # limitations under the License. - import_playbook: installMetricbeat.yml -- import_playbook: fieldRefresh.yml - vars: - index_pattern: metricbeat-* - import_playbook: installPacketbeat.yml -- import_playbook: fieldRefresh.yml - vars: - index_pattern: packetbeat-* - import_playbook: installAuditbeat.yml -- import_playbook: fieldRefresh.yml - vars: - index_pattern: auditbeat-* - import_playbook: installHeartbeat.yml -- import_playbook: fieldRefresh.yml - vars: - index_pattern: heartbeat-* - import_playbook: installFilebeat.yml -- import_playbook: fieldRefresh.yml - vars: - index_pattern: filebeat-* - import_playbook: installJournalbeat.yml