Fix Elasticsearch curator job

This change upgrades the version of elasticsearch-curator because the
previous version (3.3.0) doesn't support Elasticsearch 2.x. As a
consequence, data older than the defined retention period was never
removed from Elasticsearch.

The curator is now installed on all the Elasticsearch nodes (previously
only on the primary node) and by configuration, it will only be
executed on the ES cluster master node.

Change-Id: I9da9e67fa4d353e78bd752456a9b01ca1fbae704
Closes-Bug: #1616765
Related-Bug: #1602719
This commit is contained in:
Simon Pasquier 2016-08-25 11:50:55 +02:00
parent d0e0a8d09a
commit c0794133d6
8 changed files with 119 additions and 36 deletions

View File

@ -14,7 +14,6 @@
notice('fuel-plugin-elasticsearch-kibana: elasticsearch.pp')
# Java
$java = $::operatingsystem ? {
CentOS => 'java-1.8.0-openjdk-headless',
@ -41,3 +40,22 @@ class { 'lma_logging_analytics::elasticsearch':
version => '2.3.3',
require => Package[$java],
}
# The plugin's packages used to have a higher priority but this isn't the case
# anymore on MOS 9. Since MOS ships an older version of python-elasticsearch
# (incompatible with python-elasticsearch-curator), we need to force the
# installation of python-elasticsearch before installing the curator package.
package { 'python-elasticsearch':
ensure => '2.3.0',
}
# The curator is installed on all the nodes but by configuration, it will only
# be executed on the ES cluster master node
class { 'lma_logging_analytics::curator':
host => hiera('lma::elasticsearch::listen_address'),
port => hiera('lma::elasticsearch::rest_port'),
retention_period => hiera('lma::elasticsearch::retention_period'),
prefixes => ['log', 'notification'],
package_version => '4.0.6',
require => Package['python-elasticsearch'],
}

View File

@ -64,12 +64,6 @@ lma_logging_analytics::es_template { ['log', 'notification']:
number_of_replicas => $number_of_replicas,
host => $es_vip,
port => $es_port,
} ->
class { 'lma_logging_analytics::curator':
host => $es_vip,
port => $es_port,
retention_period => hiera('lma::elasticsearch::retention_period'),
prefixes => ['log', 'notification'],
}
$kibana_link_created_file = '/var/cache/kibana_link_created'
@ -79,7 +73,6 @@ exec { 'notify_kibana_url':
-H 'Content-Type: application/json' -X POST -d '${kibana_link_data}' \
http://${master_ip}:8000/api/clusters/${deployment_id}/plugin_links \
-o /dev/null | /bin/grep 201 && touch ${kibana_link_created_file}",
require => Class['lma_logging_analytics::curator'],
}
if $two_links {

View File

@ -17,13 +17,14 @@ class lma_logging_analytics::curator (
$port = '9200',
$retention_period = $lma_logging_analytics::params::retention_period,
$prefixes = $lma_logging_analytics::params::indexes_prefixes,
$package_version = 'latest',
) inherits lma_logging_analytics::params {
validate_integer($retention_period)
validate_array($prefixes)
package { 'python-elasticsearch-curator':
ensure => installed,
ensure => $package_version,
}
if size($prefixes) > 0 and $retention_period > 0 {
@ -32,13 +33,22 @@ class lma_logging_analytics::curator (
# drop indices too early.
$real_retention_period = 1 + $retention_period
$regex = join($prefixes, '|')
file { '/etc/elasticsearch/curator.yaml':
ensure => present,
# This template uses $host and $port
content => template('lma_logging_analytics/curator.yaml.erb'),
}
file { '/etc/elasticsearch/delete_indices.yaml':
ensure => present,
# This template uses $regex and $real_retention_period
content => template('lma_logging_analytics/delete_indices.yaml.erb'),
}
cron { 'es-curator':
ensure => present,
command => join([
"/usr/local/bin/curator --host ${host} --port ${port} ",
"--debug delete indices --regex '^(${regex})-.*$' --time-unit days ",
"--older-than ${real_retention_period} ",
"--timestring \"\\%Y.\\%m.\\%d\""], ''),
command => '/usr/local/bin/curator --config /etc/elasticsearch/curator.yaml /etc/elasticsearch/delete_indices.yaml',
minute => '0',
hour => '2',
month => '*',

View File

@ -20,10 +20,12 @@ describe 'lma_logging_analytics::curator', :type => :class do
describe 'with defaults' do
it { is_expected.to contain_package('python-elasticsearch-curator').with(
:ensure => 'installed'
:ensure => 'latest'
)}
it { is_expected.not_to contain_cron('es-curator') }
it { is_expected.not_to contain_file('/etc/elasticsearch/curator.yaml') }
it { is_expected.not_to contain_file('/etc/elasticsearch/delete_indices.yaml') }
end
describe 'with index prefixes and retention period > 0' do
@ -32,10 +34,13 @@ describe 'lma_logging_analytics::curator', :type => :class do
end
it { is_expected.to contain_package('python-elasticsearch-curator').with(
:ensure => 'installed'
:ensure => 'latest'
)}
it { is_expected.to contain_cron('es-curator') }
it { is_expected.to contain_cron('es-curator').with_command(
'/usr/local/bin/curator --config /etc/elasticsearch/curator.yaml /etc/elasticsearch/delete_indices.yaml') }
it { is_expected.to contain_file('/etc/elasticsearch/curator.yaml') }
it { is_expected.to contain_file('/etc/elasticsearch/delete_indices.yaml') }
end
describe 'with index prefixes and retention period > 0 and host' do
@ -43,6 +48,7 @@ describe 'lma_logging_analytics::curator', :type => :class do
{:retention_period => 10, :prefixes => ['foo'], :host => 'foo.org'}
end
it { is_expected.to contain_cron('es-curator').with_command(/--host foo.org/) }
it { is_expected.to contain_cron('es-curator').with_command(
'/usr/local/bin/curator --config /etc/elasticsearch/curator.yaml /etc/elasticsearch/delete_indices.yaml') }
end
end

View File

@ -0,0 +1,23 @@
---
client:
hosts:
- <%= @host %>
port: <%= @port %>
url_prefix:
use_ssl: False
certificate:
client_cert:
client_key:
aws_key:
aws_secret_key:
aws_region:
ssl_no_validate: False
http_auth:
timeout: 30
master_only: True
logging:
loglevel: INFO
logfile:
logformat: default
blacklist: ['elasticsearch', 'urllib3']

View File

@ -0,0 +1,23 @@
---
actions:
1:
action: delete_indices
description: >-
Delete indices older than <%= @real_retention_period %> days (based on index name).
options:
ignore_empty_list: True
timeout_override:
continue_if_exception: False
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: "^<%= @regex %>-.*$"
exclude:
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: <%= @real_retention_period %>
exclude:

View File

@ -20,19 +20,22 @@ MODULES_DIR="${ROOT}"/deployment_scripts/puppet/modules
RPM_REPO="${ROOT}"/repositories/centos/
DEB_REPO="${ROOT}"/repositories/ubuntu/
# Download RPM or DEB packages and store them in the local repository directory
function download_package {
while [ $# -gt 0 ]; do
if [[ "$1" == *.deb ]]; then
REPO=$DEB_REPO
elif [[ "$1" == *.rpm ]]; then
REPO=$RPM_REPO
else
echo "Invalid URL for download_package(): $1"
fi
function get_package_path {
FILE=$(basename "$1")
if [[ "$1" == *.deb ]]; then
echo "$DEB_REPO"/"$FILE"
elif [[ "$1" == *.rpm ]]; then
echo "$RPM_REPO"/"$FILE"
else
echo "Invalid URL for $1"
exit 1
fi
}
FILE=$(basename "$1")
wget -qO - "$1" > "$REPO"/"$FILE"
# Download RPM or DEB packages and store them in the local repository directory
function download_packages {
while [ $# -gt 0 ]; do
wget -qO - "$1" > "$(get_package_path "$1")"
shift
done
}
@ -44,3 +47,7 @@ function download_puppet_module {
wget -qO- "$2" | tar -C "${MODULES_DIR}/$1" --strip-components=1 -xz
}
function check_md5sum {
FILE="$(get_package_path "$1")"
echo "$2 $FILE" | md5sum --check --strict
}

View File

@ -16,7 +16,7 @@ HTPASSWD_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/leinaddm-htpasswd
TZ_VERSION="2016f-0ubuntu0.14.04_all"
# Packages needed to install JRE headless
download_package http://mirrors.kernel.org/ubuntu/pool/main/p/pcsc-lite/libpcsclite1_1.8.10-1ubuntu1_amd64.deb \
download_packages http://mirrors.kernel.org/ubuntu/pool/main/p/pcsc-lite/libpcsclite1_1.8.10-1ubuntu1_amd64.deb \
http://mirrors.kernel.org/ubuntu/pool/main/j/java-common/java-common_0.51_all.deb \
"http://security.ubuntu.com/ubuntu/pool/main/t/tzdata/tzdata-java_${TZ_VERSION}.deb" \
"http://security.ubuntu.com/ubuntu/pool/main/t/tzdata/tzdata_${TZ_VERSION}.deb" \
@ -28,10 +28,14 @@ download_package http://mirrors.kernel.org/ubuntu/pool/main/p/pcsc-lite/libpcscl
https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.3.3/elasticsearch-2.3.3.deb
# Download curator and dependencies
download_package http://packages.elasticsearch.org/curator/3/debian/pool/main/p/python/python-elasticsearch-curator_3.3.0_all.deb \
http://packages.elasticsearch.org/curator/3/debian/pool/main/p/python/python-elasticsearch_1.6.0_all.deb \
http://packages.elasticsearch.org/curator/3/debian/pool/main/p/python/python-click_4.0_all.deb \
http://packages.elasticsearch.org/curator/3/debian/pool/main/p/python/python-urllib3_1.10.4_all.deb
download_packages http://packages.elastic.co/curator/4/debian/pool/main/p/python/python-elasticsearch-curator_4.0.6_all.deb \
http://packages.elastic.co/curator/4/debian/pool/main/p/python/python-elasticsearch_2.3.0_all.deb \
http://packages.elastic.co/curator/4/debian/pool/main/p/python/python-click_6.6_all.deb \
http://packages.elastic.co/curator/4/debian/pool/main/p/python/python-pyyaml_3.11_amd64.deb
check_md5sum python-elasticsearch-curator_4.0.6_all.deb eb69388edf54aafa2445423f615c0817
check_md5sum python-click_6.6_all.deb f508745fcd6f64a317cc94603b7f4f9d
check_md5sum python-pyyaml_3.11_amd64.deb 9f4ef75d24f6bc92879e0c57dc4f4b71
check_md5sum python-elasticsearch_2.3.0_all.deb 4d17c82d397fc57034d78e9d09ae400d
# Install puppet manifests
download_puppet_module "concat" "$CONCAT_TARBALL_URL"
@ -42,7 +46,6 @@ download_puppet_module "datacat" "$DATACAT_TARBALL_URL"
download_puppet_module "apache" "${APACHE_TARBALL_URL}"
download_puppet_module "htpasswd" "${HTPASSWD_TARBALL_URL}"
# Update tzdata-java package to remove strict dependency on tzdata (see LP bug #1560436)
ROOT="$(dirname "$(readlink -f "$0")")"
DEB_REPO="${ROOT}"/repositories/ubuntu/