From 3a69a1c43dad679fd53be87a12381a99be47eade Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 22 Nov 2018 00:47:14 -0600 Subject: [PATCH] Update elk_6x for 6.5.x This change updates the roles / playbooks to begin using Elasticsearch relesae 6.5.x. Core to this change is the conversion of the journalbeat role from custom compiled go, to simple package install which was made possible by the folks at elastic within this release. Because of the conversion the "beats-community" playbook has been removed given its now empty. A change to the bootstrap script was made allowing it to parse an OS id with a "-" in it, like "opensuse-tumbleweed". Change-Id: Ic9b80234d6a6ce876bff885f3223874602d55dd6 Signed-off-by: Kevin Carter --- .../bootstrap-embedded-ansible.sh | 2 + elk_metrics_6x/installKibana.yml | 5 -- .../roles/elastic_auditbeat/tasks/main.yml | 2 +- .../roles/elastic_filebeat/tasks/main.yml | 2 +- .../roles/elastic_heartbeat/tasks/main.yml | 2 +- .../roles/elastic_journalbeat/meta/main.yml | 2 +- .../tasks/journalbeat_setup.yml | 50 +++++------- .../roles/elastic_journalbeat/tasks/main.yml | 78 +++++-------------- .../roles/elastic_journalbeat/vars/redhat.yml | 4 +- .../roles/elastic_journalbeat/vars/suse.yml | 4 +- .../roles/elastic_journalbeat/vars/ubuntu.yml | 4 +- .../roles/elastic_metricbeat/tasks/main.yml | 2 +- elk_metrics_6x/site-beats-community.yml | 14 ---- elk_metrics_6x/site-beats-core.yml | 2 + elk_metrics_6x/site.yml | 1 - 15 files changed, 51 insertions(+), 123 deletions(-) delete mode 100644 elk_metrics_6x/site-beats-community.yml diff --git a/bootstrap-embedded-ansible/bootstrap-embedded-ansible.sh b/bootstrap-embedded-ansible/bootstrap-embedded-ansible.sh index 961d6534..6a53a760 100755 --- a/bootstrap-embedded-ansible/bootstrap-embedded-ansible.sh +++ b/bootstrap-embedded-ansible/bootstrap-embedded-ansible.sh @@ -19,6 +19,8 @@ export ANSIBLE_EMBED_HOME="${HOME}/ansible_venv" OPTS+=('ANSIBLE_EMBED_HOME') source /etc/os-release +export ID="$(echo ${ID} | awk -F'-' '{print $1}')" + if [[ ! -e "${ANSIBLE_EMBED_HOME}/bin/ansible" ]]; then if [ ${ID} = "ubuntu" ]; then apt-get update diff --git a/elk_metrics_6x/installKibana.yml b/elk_metrics_6x/installKibana.yml index 4d9d200d..9d6bec1f 100644 --- a/elk_metrics_6x/installKibana.yml +++ b/elk_metrics_6x/installKibana.yml @@ -39,11 +39,6 @@ attributes: title: "*" timeFieldName: "@timestamp" - - name: "journalbeat-*" - index_options: - attributes: - title: "journalbeat-*" - timeFieldName: "@timestamp" register: kibana_indexes until: kibana_indexes is success retries: 6 diff --git a/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml index d02fc902..8589ed8d 100644 --- a/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml @@ -43,7 +43,7 @@ - name: Ensure beat is installed (aarch64) apt: - deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/auditbeat-6.4.1-arm64.deb' + deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/auditbeat-6.5.0-arm64.deb' when: - ansible_pkg_mgr == 'apt' - ansible_architecture == 'aarch64' diff --git a/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml index c4a2f71e..01aeb6d3 100644 --- a/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml @@ -43,7 +43,7 @@ - name: Ensure beat is installed (aarch64) apt: - deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/filebeat-6.4.1-arm64.deb' + deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/filebeat-6.5.0-arm64.deb' when: - ansible_pkg_mgr == 'apt' - ansible_architecture == 'aarch64' diff --git a/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml index 7176e14e..e3352553 100644 --- a/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml @@ -43,7 +43,7 @@ - name: Ensure beat is installed (aarch64) apt: - deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/heartbeat-6.4.1-arm64.deb' + deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/heartbeat-6.5.0-arm64.deb' when: - ansible_pkg_mgr == 'apt' - ansible_architecture == 'aarch64' diff --git a/elk_metrics_6x/roles/elastic_journalbeat/meta/main.yml b/elk_metrics_6x/roles/elastic_journalbeat/meta/main.yml index 568404b6..00b05be8 100644 --- a/elk_metrics_6x/roles/elastic_journalbeat/meta/main.yml +++ b/elk_metrics_6x/roles/elastic_journalbeat/meta/main.yml @@ -32,4 +32,4 @@ galaxy_info: - elastic-stack dependencies: - role: elastic_data_hosts - - role: go_install_1.10 + - role: elastic_repositories diff --git a/elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml b/elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml index a6dea322..849968f8 100644 --- a/elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml +++ b/elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml @@ -13,32 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Notice - debug: - msg: >- - The setup task for templates and dashboards is broken due to missing - configuration. Once the following issue - [ https://github.com/mheese/journalbeat/issues/136 ] is - resolved, this should be enabled. - -# - name: Load templates -# shell: >- -# /usr/local/bin/journalbeat -setup -# {{ item }} -# -E 'output.logstash.enabled=false' -# -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' -# -E 'setup.template.enabled=true' -# -E 'setup.template.overwrite=true' -# -e -v -# with_items: -# - "--template" -# - "--pipelines" -# - "--machine-learning" -# - "--dashboards" -# register: templates -# until: templates is success -# retries: 5 -# delay: 5 -# run_once: true -# tags: -# - setup +- name: Load templates + shell: >- + journalbeat setup + {{ item }} + -E 'output.logstash.enabled=false' + -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}' + -E 'setup.template.enabled=true' + -E 'setup.template.overwrite=true' + -e -v + with_items: + - "--template" + - "--pipelines" + - "--machine-learning" + - "--dashboards" + register: templates + until: templates is success + retries: 5 + delay: 5 + run_once: true + tags: + - setup diff --git a/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml index 19694767..bfca73a3 100644 --- a/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml @@ -25,6 +25,14 @@ tags: - always +- name: Uninstall legacy journalbeat + file: + path: "{{ item }}" + state: absent + with_items: + - /etc/systemd/system/journalbeat.service + - /usr/local/bin/journalbeat + - name: Ensure beat is installed package: name: "{{ journalbeat_distro_packages }}" @@ -39,42 +47,16 @@ tags: - package_install -- name: create the system group - group: - name: "journalbeat" - state: "present" - system: "yes" - -- name: Create the journalbeat user - user: - name: "journalbeat" - group: "journalbeat" - comment: "journalbeat user" - shell: "/bin/false" - createhome: "yes" - home: "/usr/share/journalbeat" - -- name: Create journalbeat data path - file: - path: "{{ item }}" - state: directory - owner: "journalbeat" - group: "journalbeat" - mode: "0755" - with_items: - - "/usr/share/journalbeat" - - "/var/lib/journalbeat" - - "/etc/journalbeat" - -- name: Install journalbeat - shell: /opt/go1.10.1/go/bin/go get -v github.com/mheese/journalbeat - args: - creates: /usr/local/bin/journalbeat - environment: - PATH: "{{ ansible_env.PATH }}:/opt/go1.10.1/go/bin" - GOPATH: /usr/local +- name: Ensure beat is installed (aarch64) + apt: + deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/journalbeat-6.5.0-arm64.deb' + when: + - ansible_pkg_mgr == 'apt' + - ansible_architecture == 'aarch64' notify: - - Enable and restart journalbeat + - Enable and restart heartbeat + tags: + - package_install - name: Create journalbeat systemd service config dir file: @@ -95,7 +77,7 @@ notify: - Enable and restart journalbeat -- name: Drop journalbeat conf files +- name: Drop journalbeat configs template: src: "{{ item.src }}" dest: "{{ item.dest }}" @@ -106,30 +88,6 @@ notify: - Enable and restart journalbeat -- name: Run the systemd service role - include_role: - name: systemd_service - private: true - vars: - systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}" - systemd_service_restart_changed: false - systemd_services: - - service_name: "journalbeat" - execstarts: - - /usr/local/bin/journalbeat - -c /etc/journalbeat/journalbeat.yml - -path.home /usr/share/journalbeat - -path.config /etc/journalbeat - -path.data /var/lib/journalbeat - -path.logs /var/log/beats - config_overrides: - Service: - EnvironmentFile: "-/etc/default/go1.10.1" - Unit: - Documentation: https://github.com/mheese/journalbeat/blob/master/README.md - Wants: network-online.target - After: network-online.target - - include_tasks: journalbeat_setup.yml tags: - setup diff --git a/elk_metrics_6x/roles/elastic_journalbeat/vars/redhat.yml b/elk_metrics_6x/roles/elastic_journalbeat/vars/redhat.yml index 77c33669..ac55767d 100644 --- a/elk_metrics_6x/roles/elastic_journalbeat/vars/redhat.yml +++ b/elk_metrics_6x/roles/elastic_journalbeat/vars/redhat.yml @@ -14,6 +14,4 @@ # limitations under the License. journalbeat_distro_packages: - - gcc - - git - - systemd-devel + - journalbeat diff --git a/elk_metrics_6x/roles/elastic_journalbeat/vars/suse.yml b/elk_metrics_6x/roles/elastic_journalbeat/vars/suse.yml index 77c33669..ac55767d 100644 --- a/elk_metrics_6x/roles/elastic_journalbeat/vars/suse.yml +++ b/elk_metrics_6x/roles/elastic_journalbeat/vars/suse.yml @@ -14,6 +14,4 @@ # limitations under the License. journalbeat_distro_packages: - - gcc - - git - - systemd-devel + - journalbeat diff --git a/elk_metrics_6x/roles/elastic_journalbeat/vars/ubuntu.yml b/elk_metrics_6x/roles/elastic_journalbeat/vars/ubuntu.yml index bc1ee7bb..4b271059 100644 --- a/elk_metrics_6x/roles/elastic_journalbeat/vars/ubuntu.yml +++ b/elk_metrics_6x/roles/elastic_journalbeat/vars/ubuntu.yml @@ -14,6 +14,4 @@ # limitations under the License. journalbeat_distro_packages: - - gcc - - git - - libsystemd-dev + - journalbeat \ No newline at end of file diff --git a/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml b/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml index 3e7abc97..22cd8a06 100644 --- a/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml @@ -52,7 +52,7 @@ - name: Ensure beat is installed (aarch64) apt: - deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/metricbeat-6.4.1-arm64.deb' + deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/metricbeat-6.5.0-arm64.deb' when: - ansible_pkg_mgr == 'apt' - ansible_architecture == 'aarch64' diff --git a/elk_metrics_6x/site-beats-community.yml b/elk_metrics_6x/site-beats-community.yml deleted file mode 100644 index 86891276..00000000 --- a/elk_metrics_6x/site-beats-community.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- import_playbook: installJournalbeat.yml diff --git a/elk_metrics_6x/site-beats-core.yml b/elk_metrics_6x/site-beats-core.yml index e5670d2b..14d63c8d 100644 --- a/elk_metrics_6x/site-beats-core.yml +++ b/elk_metrics_6x/site-beats-core.yml @@ -35,3 +35,5 @@ - import_playbook: fieldRefresh.yml vars: index_pattern: filebeat-* + +- import_playbook: installJournalbeat.yml diff --git a/elk_metrics_6x/site.yml b/elk_metrics_6x/site.yml index 01b23aa1..92fbe28b 100644 --- a/elk_metrics_6x/site.yml +++ b/elk_metrics_6x/site.yml @@ -13,5 +13,4 @@ - import_playbook: site-elka.yml - import_playbook: site-beats-core.yml -- import_playbook: site-beats-community.yml - import_playbook: createElasticIndexes.yml