Merge "Update elk_6x for 6.5.x"

bootstrap-embedded-ansible/bootstrap-embedded-ansible.sh

@@ -19,6 +19,8 @@ export ANSIBLE_EMBED_HOME="${HOME}/ansible_venv"
 OPTS+=('ANSIBLE_EMBED_HOME')
 
 source /etc/os-release
+export ID="$(echo ${ID} | awk -F'-' '{print $1}')"
+
 if [[ ! -e "${ANSIBLE_EMBED_HOME}/bin/ansible" ]]; then
   if [  ${ID} = "ubuntu" ]; then
     apt-get update

elk_metrics_6x/installKibana.yml

@@ -39,11 +39,6 @@
             attributes:
               title: "*"
               timeFieldName: "@timestamp"
-        - name: "journalbeat-*"
-          index_options:
-            attributes:
-              title: "journalbeat-*"
-              timeFieldName: "@timestamp"
       register: kibana_indexes
       until: kibana_indexes is success
       retries: 6

elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml

@@ -43,7 +43,7 @@
 
 - name: Ensure beat is installed (aarch64)
   apt:
-    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/auditbeat-6.4.1-arm64.deb'
+    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/auditbeat-6.5.0-arm64.deb'
   when:
     - ansible_pkg_mgr == 'apt'
     - ansible_architecture == 'aarch64'

elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml

@@ -43,7 +43,7 @@
 
 - name: Ensure beat is installed (aarch64)
   apt:
-    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/filebeat-6.4.1-arm64.deb'
+    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/filebeat-6.5.0-arm64.deb'
   when:
     - ansible_pkg_mgr == 'apt'
     - ansible_architecture == 'aarch64'

elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml

@@ -43,7 +43,7 @@
 
 - name: Ensure beat is installed (aarch64)
   apt:
-    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/heartbeat-6.4.1-arm64.deb'
+    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/heartbeat-6.5.0-arm64.deb'
   when:
     - ansible_pkg_mgr == 'apt'
     - ansible_architecture == 'aarch64'

elk_metrics_6x/roles/elastic_journalbeat/meta/main.yml

@@ -32,4 +32,4 @@ galaxy_info:
     - elastic-stack
 dependencies:
   - role: elastic_data_hosts
-  - role: go_install_1.10
+  - role: elastic_repositories

elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml

@@ -13,32 +13,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Notice
+- name: Load templates
-  debug:
+  shell: >-
-    msg: >-
+    journalbeat setup
-      The setup task for templates and dashboards is broken due to missing
+    {{ item }}
-      configuration. Once the following issue
+    -E 'output.logstash.enabled=false'
-      [ https://github.com/mheese/journalbeat/issues/136 ] is
+    -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-      resolved, this should be enabled.
+    -E 'setup.template.enabled=true'
-
+    -E 'setup.template.overwrite=true'
-# - name: Load templates
+    -e -v
-#   shell: >-
+  with_items:
-#     /usr/local/bin/journalbeat -setup
+    - "--template"
-#     {{ item }}
+    - "--pipelines"
-#     -E 'output.logstash.enabled=false'
+    - "--machine-learning"
-#     -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
+    - "--dashboards"
-#     -E 'setup.template.enabled=true'
+  register: templates
-#     -E 'setup.template.overwrite=true'
+  until: templates is success
-#     -e -v
+  retries: 5
-#   with_items:
+  delay: 5
-#     - "--template"
+  run_once: true
-#     - "--pipelines"
+  tags:
-#     - "--machine-learning"
+    - setup
-#     - "--dashboards"
-#   register: templates
-#   until: templates is success
-#   retries: 5
-#   delay: 5
-#   run_once: true
-#   tags:
-#     - setup

elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml

@@ -25,6 +25,14 @@
   tags:
     - always
 
+- name: Uninstall legacy journalbeat
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - /etc/systemd/system/journalbeat.service
+    - /usr/local/bin/journalbeat
+
 - name: Ensure beat is installed
   package:
     name: "{{ journalbeat_distro_packages }}"
@@ -39,42 +47,16 @@
   tags:
     - package_install
 
-- name: create the system group
+- name: Ensure beat is installed (aarch64)
-  group:
+  apt:
-    name: "journalbeat"
+    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/journalbeat-6.5.0-arm64.deb'
-    state: "present"
+  when:
-    system: "yes"
+    - ansible_pkg_mgr == 'apt'
-
+    - ansible_architecture == 'aarch64'
-- name: Create the journalbeat user
-  user:
-    name: "journalbeat"
-    group: "journalbeat"
-    comment: "journalbeat user"
-    shell: "/bin/false"
-    createhome: "yes"
-    home: "/usr/share/journalbeat"
-
-- name: Create journalbeat data path
-  file:
-    path: "{{ item }}"
-    state: directory
-    owner: "journalbeat"
-    group: "journalbeat"
-    mode: "0755"
-  with_items:
-    - "/usr/share/journalbeat"
-    - "/var/lib/journalbeat"
-    - "/etc/journalbeat"
-
-- name: Install journalbeat
-  shell: /opt/go1.10.1/go/bin/go get -v github.com/mheese/journalbeat
-  args:
-    creates: /usr/local/bin/journalbeat
-  environment:
-    PATH: "{{ ansible_env.PATH }}:/opt/go1.10.1/go/bin"
-    GOPATH: /usr/local
   notify:
-    - Enable and restart journalbeat
+    - Enable and restart heartbeat
+  tags:
+    - package_install
 
 - name: Create journalbeat systemd service config dir
   file:
@@ -95,7 +77,7 @@
   notify:
     - Enable and restart journalbeat
 
-- name: Drop journalbeat conf files
+- name: Drop journalbeat configs
   template:
     src: "{{ item.src }}"
     dest: "{{ item.dest }}"
@@ -106,30 +88,6 @@
   notify:
     - Enable and restart journalbeat
 
-- name: Run the systemd service role
-  include_role:
-    name: systemd_service
-    private: true
-  vars:
-    systemd_service_enabled: "{{ ((elk_package_state | default('present')) != 'absent') | ternary(true, false) }}"
-    systemd_service_restart_changed: false
-    systemd_services:
-      - service_name: "journalbeat"
-        execstarts:
-          - /usr/local/bin/journalbeat
-            -c /etc/journalbeat/journalbeat.yml
-            -path.home /usr/share/journalbeat
-            -path.config /etc/journalbeat
-            -path.data /var/lib/journalbeat
-            -path.logs /var/log/beats
-        config_overrides:
-          Service:
-            EnvironmentFile: "-/etc/default/go1.10.1"
-          Unit:
-            Documentation: https://github.com/mheese/journalbeat/blob/master/README.md
-            Wants: network-online.target
-            After: network-online.target
-
 - include_tasks: journalbeat_setup.yml
   tags:
     - setup

elk_metrics_6x/roles/elastic_journalbeat/vars/redhat.yml

@@ -14,6 +14,4 @@
 # limitations under the License.
 
 journalbeat_distro_packages:
-  - gcc
+  - journalbeat
-  - git
-  - systemd-devel

elk_metrics_6x/roles/elastic_journalbeat/vars/suse.yml

@@ -14,6 +14,4 @@
 # limitations under the License.
 
 journalbeat_distro_packages:
-  - gcc
+  - journalbeat
-  - git
-  - systemd-devel

elk_metrics_6x/roles/elastic_journalbeat/vars/ubuntu.yml

@@ -14,6 +14,4 @@
 # limitations under the License.
 
 journalbeat_distro_packages:
-  - gcc
+  - journalbeat
-  - git
-  - libsystemd-dev

elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml

@@ -52,7 +52,7 @@
 
 - name: Ensure beat is installed (aarch64)
   apt:
-    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/metricbeat-6.4.1-arm64.deb'
+    deb: 'https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/8709ca2640344a4ba85cba0a1d6eea69/aarch64/metricbeat-6.5.0-arm64.deb'
   when:
     - ansible_pkg_mgr == 'apt'
     - ansible_architecture == 'aarch64'

elk_metrics_6x/site-beats-community.yml

@@ -1,14 +0,0 @@
----
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- import_playbook: installJournalbeat.yml

elk_metrics_6x/site-beats-core.yml

@@ -35,3 +35,5 @@
 - import_playbook: fieldRefresh.yml
   vars:
     index_pattern: filebeat-*
+
+- import_playbook: installJournalbeat.yml

elk_metrics_6x/site.yml

@@ -13,5 +13,4 @@
 
 - import_playbook: site-elka.yml
 - import_playbook: site-beats-core.yml
-- import_playbook: site-beats-community.yml
 - import_playbook: createElasticIndexes.yml