From 25f74fb943d76f46d5b166ba8c0506a4a2ce06ca Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Thu, 18 Jun 2020 17:09:24 +0100 Subject: [PATCH] Upgrade to Logstash 6 Elasticsearch and Kibana 6 are used on all distros, but Logstash is still pinned at 2 which is not compatible. See the product compatibility matrix [1]. [1] https://www.elastic.co/support/matrix#matrix_compatibility Partial-Bug: #1884090 Change-Id: I1579e762bcfb07f0baa73507eb9955ae2f83c4ec --- docker/base/Dockerfile.j2 | 5 +- docker/logstash/Dockerfile.j2 | 58 +++++-------------- kolla/template/repos.yaml | 7 +++ ...logstash6-everywhere-7f9027ff17c6022d.yaml | 4 ++ 4 files changed, 25 insertions(+), 49 deletions(-) create mode 100644 releasenotes/notes/add-logstash6-everywhere-7f9027ff17c6022d.yaml diff --git a/docker/base/Dockerfile.j2 b/docker/base/Dockerfile.j2 index fd2ced8901..f761da6dd7 100644 --- a/docker/base/Dockerfile.j2 +++ b/docker/base/Dockerfile.j2 @@ -104,14 +104,11 @@ RUN {{ macros.install_packages(base_centos_language_packages | customizable("cen ] %} {% elif base_arch == 'aarch64' %} {% set base_yum_repo_files = [ + 'elasticsearch.repo', 'grafana.repo', 'rabbitmq_rabbitmq-server.repo', 'rabbitmq_rabbitmq-erlang-aarch64.repo', ] %} - # FIXME(mgoddard): Not available for CentOS 8 yet. - # - # 'elasticsearch.repo', - # {% set base_yum_repo_keys = [ 'https://packages.grafana.com/gpg.key', diff --git a/docker/logstash/Dockerfile.j2 b/docker/logstash/Dockerfile.j2 index 3e8bcdb1f4..57a5a8460b 100644 --- a/docker/logstash/Dockerfile.j2 +++ b/docker/logstash/Dockerfile.j2 @@ -9,58 +9,26 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build {{ macros.configure_user(name='logstash', shell='/bin/bash', homedir='/usr/share/logstash') }} +{{ macros.enable_extra_repos(['logstash']) }} + {% if base_package_type == 'rpm' %} - {% set logstash_packages = [ - 'java-1.8.0-openjdk-headless' - ] %} - -ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk/ - + {% set elk_java_package = 'java-11-openjdk-headless' %} +ENV JAVA_HOME /usr/lib/jvm/jre-11-openjdk/ {% elif base_package_type == 'deb' %} - - {% if base_distro == 'debian' %} - {% set java_version = '11' %} - {% elif base_distro == 'ubuntu' %} - {% set java_version = '8' %} - {% endif %} - - {% set logstash_packages = [ - 'openjdk-' + java_version + '-jre-headless', - 'logrotate' - ] %} - -ENV JAVA_HOME /usr/lib/jvm/java-{{java_version}}-openjdk-{{debian_arch}}/ - + {% set elk_java_package = 'openjdk-11-jre-headless' %} +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-{{debian_arch}}/ {% endif %} +{% set logstash_packages = [ + 'logstash-oss', +] %} + +# Post-install script of Logstash fails when trying to install Logstash and +# Java together. +{{ macros.install_packages([elk_java_package]) }} {{ macros.install_packages(logstash_packages | customizable("packages")) }} COPY extend_start.sh /usr/local/bin/kolla_extend_start -# FIXME(dszumski): Logstash was added to Kolla for Monasca. Monasca is -# currently tied to an old messaging version in Kafka (0.9) and doesn't -# work with Logstash 5.x or above. When Monasca is updated to use a more -# recent messaging version this patch should be reverted and the Monasca -# Logstash configuration files upgraded to use the new format. -{% if base_package_type == 'rpm' %} - -ENV logstash_rpm_url=https://download.elastic.co/logstash/logstash/packages/centos/logstash-2.4.1.noarch.rpm -ENV logstash_rpm_sha1sum=68c5e78c3d941f06f77d638c0756b6aea715bec5 -RUN curl -sSL -o /tmp/logstash.rpm ${logstash_rpm_url} \ - && echo "${logstash_rpm_sha1sum} /tmp/logstash.rpm" | sha1sum -c \ - && dnf -y install /tmp/logstash.rpm \ - && rm -f /tmp/logstash.rpm - -{% elif base_package_type == 'deb' %} - -ENV logstash_deb_url=https://download.elastic.co/logstash/logstash/packages/debian/logstash-2.4.1_all.deb -ENV logstash_deb_sha1sum=7ba3b174a3ef48a7d0945d9b5c7f12c5005abb47 -RUN curl -sSL -o /tmp/logstash.deb ${logstash_deb_url} \ - && echo "${logstash_deb_sha1sum} /tmp/logstash.deb" | sha1sum -c \ - && apt install /tmp/logstash.deb \ - && rm -f /tmp/logstash.deb - -{% endif %} - RUN chmod 755 /usr/local/bin/kolla_extend_start {% block logstash_footer %}{% endblock %} diff --git a/kolla/template/repos.yaml b/kolla/template/repos.yaml index 29434a529a..52fb7565dd 100644 --- a/kolla/template/repos.yaml +++ b/kolla/template/repos.yaml @@ -10,6 +10,7 @@ centos: grafana: "grafana" hacluster: "HighAvailability" influxdb: "influxdb" + logstash: "elasticsearch-kibana-logstash-6.x" kibana: "elasticsearch-kibana-logstash-6.x" opstools: "centos-opstools" rabbitmq: "rabbitmq_rabbitmq-server" @@ -24,6 +25,7 @@ centos-aarch64: extras: "extras" grafana: "grafana" hacluster: "HighAvailability" + logstash: "elasticsearch-kibana-logstash-6.x" opstools: "centos-opstools" rabbitmq: "rabbitmq_rabbitmq-server" @@ -34,6 +36,7 @@ centos-ppc64le: epel-modular: "epel-modular" extras: "extras" hacluster: "HighAvailability" + logstash: "elasticsearch-kibana-logstash-6.x" opstools: "centos-opstools" rabbitmq: "rabbitmq_rabbitmq-server" @@ -48,6 +51,7 @@ debian: erlang: "deb https://dl.bintray.com/rabbitmq-erlang/debian/ buster erlang" grafana: "deb https://packages.grafana.com/oss/deb stable main" influxdb: "deb https://repos.influxdata.com/debian buster stable" + logstash: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main" kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main" rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ buster main" td-agent: "deb http://packages.treasuredata.com/3/debian/buster buster contrib" @@ -57,6 +61,7 @@ debian-aarch64: erlang: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./" grafana: "deb https://packages.grafana.com/oss/deb stable main" influxdb: "deb https://repos.influxdata.com/debian buster stable" + logstash: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main" kibana: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main" libvirt: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/debian-buster ./" rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ buster main" @@ -66,6 +71,7 @@ ubuntu: erlang: "deb https://dl.bintray.com/rabbitmq-erlang/debian/ bionic erlang" grafana: "deb https://packages.grafana.com/oss/deb stable main" influxdb: "deb https://repos.influxdata.com/ubuntu bionic stable" + logstash: "deb https://artifacts.elastic.co/packages/oss-6.x/apt stable main" kibana: "deb https://artifacts.elastic.co/packages/oss-6.x/apt stable main" mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main" opendaylight: "deb http://ppa.launchpad.net/odl-team/carbon/ubuntu xenial main" @@ -79,6 +85,7 @@ ubuntu-aarch64: erlang: "deb https://obs.linaro.org/repos/home:/marcin.juszkiewicz/ubuntu-bionic ./" grafana: "deb https://packages.grafana.com/oss/deb stable main" influxdb: "deb https://repos.influxdata.com/ubuntu bionic stable" + logstash: "deb [arch=amd64] https://artifacts.elastic.co/packages/oss-6.x/apt stable main" mariadb: "deb http://downloads.mariadb.com/MariaDB/mariadb-10.3/repo/ubuntu bionic main" rabbitmq: "deb https://dl.bintray.com/rabbitmq/debian/ bionic main" diff --git a/releasenotes/notes/add-logstash6-everywhere-7f9027ff17c6022d.yaml b/releasenotes/notes/add-logstash6-everywhere-7f9027ff17c6022d.yaml new file mode 100644 index 0000000000..e0b6a28d07 --- /dev/null +++ b/releasenotes/notes/add-logstash6-everywhere-7f9027ff17c6022d.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + The Logstash image has been upgraded from Logstash 2 to Logstash 6.