From c0794133d68bc7a9031c0432ba95098ba5ffced2 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Thu, 25 Aug 2016 11:50:55 +0200 Subject: [PATCH] Fix Elasticsearch curator job This change upgrades the version of elasticsearch-curator because the previous version (3.3.0) doesn't support Elasticsearch 2.x. As a consequence, data older than the defined retention period was never removed from Elasticsearch. The curator is now installed on all the Elasticsearch nodes (previously only on the primary node) and by configuration, it will only be executed on the ES cluster master node. Change-Id: I9da9e67fa4d353e78bd752456a9b01ca1fbae704 Closes-Bug: #1616765 Related-Bug: #1602719 --- .../puppet/manifests/elasticsearch.pp | 20 +++++++++++- .../puppet/manifests/provision_services.pp | 7 ----- .../manifests/curator.pp | 22 +++++++++---- .../lma_logging_analytics_curator_spec.rb | 14 ++++++--- .../templates/curator.yaml.erb | 23 ++++++++++++++ .../templates/delete_indices.yaml.erb | 23 ++++++++++++++ functions.sh | 31 ++++++++++++------- pre_build_hook | 15 +++++---- 8 files changed, 119 insertions(+), 36 deletions(-) create mode 100644 deployment_scripts/puppet/modules/lma_logging_analytics/templates/curator.yaml.erb create mode 100644 deployment_scripts/puppet/modules/lma_logging_analytics/templates/delete_indices.yaml.erb diff --git a/deployment_scripts/puppet/manifests/elasticsearch.pp b/deployment_scripts/puppet/manifests/elasticsearch.pp index b7eaa57..6000194 100644 --- a/deployment_scripts/puppet/manifests/elasticsearch.pp +++ b/deployment_scripts/puppet/manifests/elasticsearch.pp @@ -14,7 +14,6 @@ notice('fuel-plugin-elasticsearch-kibana: elasticsearch.pp') - # Java $java = $::operatingsystem ? { CentOS => 'java-1.8.0-openjdk-headless', @@ -41,3 +40,22 @@ class { 'lma_logging_analytics::elasticsearch': version => '2.3.3', require => Package[$java], } + +# The plugin's packages used to have a higher priority but this isn't the case +# anymore on MOS 9. Since MOS ships an older version of python-elasticsearch +# (incompatible with python-elasticsearch-curator), we need to force the +# installation of python-elasticsearch before installing the curator package. +package { 'python-elasticsearch': + ensure => '2.3.0', +} + +# The curator is installed on all the nodes but by configuration, it will only +# be executed on the ES cluster master node +class { 'lma_logging_analytics::curator': + host => hiera('lma::elasticsearch::listen_address'), + port => hiera('lma::elasticsearch::rest_port'), + retention_period => hiera('lma::elasticsearch::retention_period'), + prefixes => ['log', 'notification'], + package_version => '4.0.6', + require => Package['python-elasticsearch'], +} diff --git a/deployment_scripts/puppet/manifests/provision_services.pp b/deployment_scripts/puppet/manifests/provision_services.pp index c18933f..0cb803c 100644 --- a/deployment_scripts/puppet/manifests/provision_services.pp +++ b/deployment_scripts/puppet/manifests/provision_services.pp @@ -64,12 +64,6 @@ lma_logging_analytics::es_template { ['log', 'notification']: number_of_replicas => $number_of_replicas, host => $es_vip, port => $es_port, -} -> -class { 'lma_logging_analytics::curator': - host => $es_vip, - port => $es_port, - retention_period => hiera('lma::elasticsearch::retention_period'), - prefixes => ['log', 'notification'], } $kibana_link_created_file = '/var/cache/kibana_link_created' @@ -79,7 +73,6 @@ exec { 'notify_kibana_url': -H 'Content-Type: application/json' -X POST -d '${kibana_link_data}' \ http://${master_ip}:8000/api/clusters/${deployment_id}/plugin_links \ -o /dev/null | /bin/grep 201 && touch ${kibana_link_created_file}", - require => Class['lma_logging_analytics::curator'], } if $two_links { diff --git a/deployment_scripts/puppet/modules/lma_logging_analytics/manifests/curator.pp b/deployment_scripts/puppet/modules/lma_logging_analytics/manifests/curator.pp index 9f6ac17..2771875 100644 --- a/deployment_scripts/puppet/modules/lma_logging_analytics/manifests/curator.pp +++ b/deployment_scripts/puppet/modules/lma_logging_analytics/manifests/curator.pp @@ -17,13 +17,14 @@ class lma_logging_analytics::curator ( $port = '9200', $retention_period = $lma_logging_analytics::params::retention_period, $prefixes = $lma_logging_analytics::params::indexes_prefixes, + $package_version = 'latest', ) inherits lma_logging_analytics::params { validate_integer($retention_period) validate_array($prefixes) package { 'python-elasticsearch-curator': - ensure => installed, + ensure => $package_version, } if size($prefixes) > 0 and $retention_period > 0 { @@ -32,13 +33,22 @@ class lma_logging_analytics::curator ( # drop indices too early. $real_retention_period = 1 + $retention_period $regex = join($prefixes, '|') + + file { '/etc/elasticsearch/curator.yaml': + ensure => present, + # This template uses $host and $port + content => template('lma_logging_analytics/curator.yaml.erb'), + } + + file { '/etc/elasticsearch/delete_indices.yaml': + ensure => present, + # This template uses $regex and $real_retention_period + content => template('lma_logging_analytics/delete_indices.yaml.erb'), + } + cron { 'es-curator': ensure => present, - command => join([ - "/usr/local/bin/curator --host ${host} --port ${port} ", - "--debug delete indices --regex '^(${regex})-.*$' --time-unit days ", - "--older-than ${real_retention_period} ", - "--timestring \"\\%Y.\\%m.\\%d\""], ''), + command => '/usr/local/bin/curator --config /etc/elasticsearch/curator.yaml /etc/elasticsearch/delete_indices.yaml', minute => '0', hour => '2', month => '*', diff --git a/deployment_scripts/puppet/modules/lma_logging_analytics/spec/classes/lma_logging_analytics_curator_spec.rb b/deployment_scripts/puppet/modules/lma_logging_analytics/spec/classes/lma_logging_analytics_curator_spec.rb index cf6254c..25b5752 100644 --- a/deployment_scripts/puppet/modules/lma_logging_analytics/spec/classes/lma_logging_analytics_curator_spec.rb +++ b/deployment_scripts/puppet/modules/lma_logging_analytics/spec/classes/lma_logging_analytics_curator_spec.rb @@ -20,10 +20,12 @@ describe 'lma_logging_analytics::curator', :type => :class do describe 'with defaults' do it { is_expected.to contain_package('python-elasticsearch-curator').with( - :ensure => 'installed' + :ensure => 'latest' )} it { is_expected.not_to contain_cron('es-curator') } + it { is_expected.not_to contain_file('/etc/elasticsearch/curator.yaml') } + it { is_expected.not_to contain_file('/etc/elasticsearch/delete_indices.yaml') } end describe 'with index prefixes and retention period > 0' do @@ -32,10 +34,13 @@ describe 'lma_logging_analytics::curator', :type => :class do end it { is_expected.to contain_package('python-elasticsearch-curator').with( - :ensure => 'installed' + :ensure => 'latest' )} - it { is_expected.to contain_cron('es-curator') } + it { is_expected.to contain_cron('es-curator').with_command( + '/usr/local/bin/curator --config /etc/elasticsearch/curator.yaml /etc/elasticsearch/delete_indices.yaml') } + it { is_expected.to contain_file('/etc/elasticsearch/curator.yaml') } + it { is_expected.to contain_file('/etc/elasticsearch/delete_indices.yaml') } end describe 'with index prefixes and retention period > 0 and host' do @@ -43,6 +48,7 @@ describe 'lma_logging_analytics::curator', :type => :class do {:retention_period => 10, :prefixes => ['foo'], :host => 'foo.org'} end - it { is_expected.to contain_cron('es-curator').with_command(/--host foo.org/) } + it { is_expected.to contain_cron('es-curator').with_command( + '/usr/local/bin/curator --config /etc/elasticsearch/curator.yaml /etc/elasticsearch/delete_indices.yaml') } end end diff --git a/deployment_scripts/puppet/modules/lma_logging_analytics/templates/curator.yaml.erb b/deployment_scripts/puppet/modules/lma_logging_analytics/templates/curator.yaml.erb new file mode 100644 index 0000000..3ef2895 --- /dev/null +++ b/deployment_scripts/puppet/modules/lma_logging_analytics/templates/curator.yaml.erb @@ -0,0 +1,23 @@ +--- +client: + hosts: + - <%= @host %> + port: <%= @port %> + url_prefix: + use_ssl: False + certificate: + client_cert: + client_key: + aws_key: + aws_secret_key: + aws_region: + ssl_no_validate: False + http_auth: + timeout: 30 + master_only: True + +logging: + loglevel: INFO + logfile: + logformat: default + blacklist: ['elasticsearch', 'urllib3'] diff --git a/deployment_scripts/puppet/modules/lma_logging_analytics/templates/delete_indices.yaml.erb b/deployment_scripts/puppet/modules/lma_logging_analytics/templates/delete_indices.yaml.erb new file mode 100644 index 0000000..5256810 --- /dev/null +++ b/deployment_scripts/puppet/modules/lma_logging_analytics/templates/delete_indices.yaml.erb @@ -0,0 +1,23 @@ +--- +actions: + 1: + action: delete_indices + description: >- + Delete indices older than <%= @real_retention_period %> days (based on index name). + options: + ignore_empty_list: True + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: "^<%= @regex %>-.*$" + exclude: + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: <%= @real_retention_period %> + exclude: diff --git a/functions.sh b/functions.sh index cf9e407..6268da3 100644 --- a/functions.sh +++ b/functions.sh @@ -20,19 +20,22 @@ MODULES_DIR="${ROOT}"/deployment_scripts/puppet/modules RPM_REPO="${ROOT}"/repositories/centos/ DEB_REPO="${ROOT}"/repositories/ubuntu/ -# Download RPM or DEB packages and store them in the local repository directory -function download_package { - while [ $# -gt 0 ]; do - if [[ "$1" == *.deb ]]; then - REPO=$DEB_REPO - elif [[ "$1" == *.rpm ]]; then - REPO=$RPM_REPO - else - echo "Invalid URL for download_package(): $1" - fi +function get_package_path { + FILE=$(basename "$1") + if [[ "$1" == *.deb ]]; then + echo "$DEB_REPO"/"$FILE" + elif [[ "$1" == *.rpm ]]; then + echo "$RPM_REPO"/"$FILE" + else + echo "Invalid URL for $1" + exit 1 + fi +} - FILE=$(basename "$1") - wget -qO - "$1" > "$REPO"/"$FILE" +# Download RPM or DEB packages and store them in the local repository directory +function download_packages { + while [ $# -gt 0 ]; do + wget -qO - "$1" > "$(get_package_path "$1")" shift done } @@ -44,3 +47,7 @@ function download_puppet_module { wget -qO- "$2" | tar -C "${MODULES_DIR}/$1" --strip-components=1 -xz } +function check_md5sum { + FILE="$(get_package_path "$1")" + echo "$2 $FILE" | md5sum --check --strict +} diff --git a/pre_build_hook b/pre_build_hook index 47fa077..b4a202b 100755 --- a/pre_build_hook +++ b/pre_build_hook @@ -16,7 +16,7 @@ HTPASSWD_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/leinaddm-htpasswd TZ_VERSION="2016f-0ubuntu0.14.04_all" # Packages needed to install JRE headless -download_package http://mirrors.kernel.org/ubuntu/pool/main/p/pcsc-lite/libpcsclite1_1.8.10-1ubuntu1_amd64.deb \ +download_packages http://mirrors.kernel.org/ubuntu/pool/main/p/pcsc-lite/libpcsclite1_1.8.10-1ubuntu1_amd64.deb \ http://mirrors.kernel.org/ubuntu/pool/main/j/java-common/java-common_0.51_all.deb \ "http://security.ubuntu.com/ubuntu/pool/main/t/tzdata/tzdata-java_${TZ_VERSION}.deb" \ "http://security.ubuntu.com/ubuntu/pool/main/t/tzdata/tzdata_${TZ_VERSION}.deb" \ @@ -28,10 +28,14 @@ download_package http://mirrors.kernel.org/ubuntu/pool/main/p/pcsc-lite/libpcscl https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.3.3/elasticsearch-2.3.3.deb # Download curator and dependencies -download_package http://packages.elasticsearch.org/curator/3/debian/pool/main/p/python/python-elasticsearch-curator_3.3.0_all.deb \ - http://packages.elasticsearch.org/curator/3/debian/pool/main/p/python/python-elasticsearch_1.6.0_all.deb \ - http://packages.elasticsearch.org/curator/3/debian/pool/main/p/python/python-click_4.0_all.deb \ - http://packages.elasticsearch.org/curator/3/debian/pool/main/p/python/python-urllib3_1.10.4_all.deb +download_packages http://packages.elastic.co/curator/4/debian/pool/main/p/python/python-elasticsearch-curator_4.0.6_all.deb \ + http://packages.elastic.co/curator/4/debian/pool/main/p/python/python-elasticsearch_2.3.0_all.deb \ + http://packages.elastic.co/curator/4/debian/pool/main/p/python/python-click_6.6_all.deb \ + http://packages.elastic.co/curator/4/debian/pool/main/p/python/python-pyyaml_3.11_amd64.deb +check_md5sum python-elasticsearch-curator_4.0.6_all.deb eb69388edf54aafa2445423f615c0817 +check_md5sum python-click_6.6_all.deb f508745fcd6f64a317cc94603b7f4f9d +check_md5sum python-pyyaml_3.11_amd64.deb 9f4ef75d24f6bc92879e0c57dc4f4b71 +check_md5sum python-elasticsearch_2.3.0_all.deb 4d17c82d397fc57034d78e9d09ae400d # Install puppet manifests download_puppet_module "concat" "$CONCAT_TARBALL_URL" @@ -42,7 +46,6 @@ download_puppet_module "datacat" "$DATACAT_TARBALL_URL" download_puppet_module "apache" "${APACHE_TARBALL_URL}" download_puppet_module "htpasswd" "${HTPASSWD_TARBALL_URL}" - # Update tzdata-java package to remove strict dependency on tzdata (see LP bug #1560436) ROOT="$(dirname "$(readlink -f "$0")")" DEB_REPO="${ROOT}"/repositories/ubuntu/