Fix Elasticsearch curator job
This change upgrades the version of elasticsearch-curator because the previous version (3.3.0) doesn't support Elasticsearch 2.x. As a consequence, data older than the defined retention period was never removed from Elasticsearch. The curator is now installed on all the Elasticsearch nodes (previously only on the primary node) and by configuration, it will only be executed on the ES cluster master node. Change-Id: I9da9e67fa4d353e78bd752456a9b01ca1fbae704 Closes-Bug: #1616765 Related-Bug: #1602719
This commit is contained in:
parent
d0e0a8d09a
commit
c0794133d6
|
@ -14,7 +14,6 @@
|
|||
|
||||
notice('fuel-plugin-elasticsearch-kibana: elasticsearch.pp')
|
||||
|
||||
|
||||
# Java
|
||||
$java = $::operatingsystem ? {
|
||||
CentOS => 'java-1.8.0-openjdk-headless',
|
||||
|
@ -41,3 +40,22 @@ class { 'lma_logging_analytics::elasticsearch':
|
|||
version => '2.3.3',
|
||||
require => Package[$java],
|
||||
}
|
||||
|
||||
# The plugin's packages used to have a higher priority but this isn't the case
|
||||
# anymore on MOS 9. Since MOS ships an older version of python-elasticsearch
|
||||
# (incompatible with python-elasticsearch-curator), we need to force the
|
||||
# installation of python-elasticsearch before installing the curator package.
|
||||
package { 'python-elasticsearch':
|
||||
ensure => '2.3.0',
|
||||
}
|
||||
|
||||
# The curator is installed on all the nodes but by configuration, it will only
|
||||
# be executed on the ES cluster master node
|
||||
class { 'lma_logging_analytics::curator':
|
||||
host => hiera('lma::elasticsearch::listen_address'),
|
||||
port => hiera('lma::elasticsearch::rest_port'),
|
||||
retention_period => hiera('lma::elasticsearch::retention_period'),
|
||||
prefixes => ['log', 'notification'],
|
||||
package_version => '4.0.6',
|
||||
require => Package['python-elasticsearch'],
|
||||
}
|
||||
|
|
|
@ -64,12 +64,6 @@ lma_logging_analytics::es_template { ['log', 'notification']:
|
|||
number_of_replicas => $number_of_replicas,
|
||||
host => $es_vip,
|
||||
port => $es_port,
|
||||
} ->
|
||||
class { 'lma_logging_analytics::curator':
|
||||
host => $es_vip,
|
||||
port => $es_port,
|
||||
retention_period => hiera('lma::elasticsearch::retention_period'),
|
||||
prefixes => ['log', 'notification'],
|
||||
}
|
||||
|
||||
$kibana_link_created_file = '/var/cache/kibana_link_created'
|
||||
|
@ -79,7 +73,6 @@ exec { 'notify_kibana_url':
|
|||
-H 'Content-Type: application/json' -X POST -d '${kibana_link_data}' \
|
||||
http://${master_ip}:8000/api/clusters/${deployment_id}/plugin_links \
|
||||
-o /dev/null | /bin/grep 201 && touch ${kibana_link_created_file}",
|
||||
require => Class['lma_logging_analytics::curator'],
|
||||
}
|
||||
|
||||
if $two_links {
|
||||
|
|
|
@ -17,13 +17,14 @@ class lma_logging_analytics::curator (
|
|||
$port = '9200',
|
||||
$retention_period = $lma_logging_analytics::params::retention_period,
|
||||
$prefixes = $lma_logging_analytics::params::indexes_prefixes,
|
||||
$package_version = 'latest',
|
||||
) inherits lma_logging_analytics::params {
|
||||
|
||||
validate_integer($retention_period)
|
||||
validate_array($prefixes)
|
||||
|
||||
package { 'python-elasticsearch-curator':
|
||||
ensure => installed,
|
||||
ensure => $package_version,
|
||||
}
|
||||
|
||||
if size($prefixes) > 0 and $retention_period > 0 {
|
||||
|
@ -32,13 +33,22 @@ class lma_logging_analytics::curator (
|
|||
# drop indices too early.
|
||||
$real_retention_period = 1 + $retention_period
|
||||
$regex = join($prefixes, '|')
|
||||
|
||||
file { '/etc/elasticsearch/curator.yaml':
|
||||
ensure => present,
|
||||
# This template uses $host and $port
|
||||
content => template('lma_logging_analytics/curator.yaml.erb'),
|
||||
}
|
||||
|
||||
file { '/etc/elasticsearch/delete_indices.yaml':
|
||||
ensure => present,
|
||||
# This template uses $regex and $real_retention_period
|
||||
content => template('lma_logging_analytics/delete_indices.yaml.erb'),
|
||||
}
|
||||
|
||||
cron { 'es-curator':
|
||||
ensure => present,
|
||||
command => join([
|
||||
"/usr/local/bin/curator --host ${host} --port ${port} ",
|
||||
"--debug delete indices --regex '^(${regex})-.*$' --time-unit days ",
|
||||
"--older-than ${real_retention_period} ",
|
||||
"--timestring \"\\%Y.\\%m.\\%d\""], ''),
|
||||
command => '/usr/local/bin/curator --config /etc/elasticsearch/curator.yaml /etc/elasticsearch/delete_indices.yaml',
|
||||
minute => '0',
|
||||
hour => '2',
|
||||
month => '*',
|
||||
|
|
|
@ -20,10 +20,12 @@ describe 'lma_logging_analytics::curator', :type => :class do
|
|||
|
||||
describe 'with defaults' do
|
||||
it { is_expected.to contain_package('python-elasticsearch-curator').with(
|
||||
:ensure => 'installed'
|
||||
:ensure => 'latest'
|
||||
)}
|
||||
|
||||
it { is_expected.not_to contain_cron('es-curator') }
|
||||
it { is_expected.not_to contain_file('/etc/elasticsearch/curator.yaml') }
|
||||
it { is_expected.not_to contain_file('/etc/elasticsearch/delete_indices.yaml') }
|
||||
end
|
||||
|
||||
describe 'with index prefixes and retention period > 0' do
|
||||
|
@ -32,10 +34,13 @@ describe 'lma_logging_analytics::curator', :type => :class do
|
|||
end
|
||||
|
||||
it { is_expected.to contain_package('python-elasticsearch-curator').with(
|
||||
:ensure => 'installed'
|
||||
:ensure => 'latest'
|
||||
)}
|
||||
|
||||
it { is_expected.to contain_cron('es-curator') }
|
||||
it { is_expected.to contain_cron('es-curator').with_command(
|
||||
'/usr/local/bin/curator --config /etc/elasticsearch/curator.yaml /etc/elasticsearch/delete_indices.yaml') }
|
||||
it { is_expected.to contain_file('/etc/elasticsearch/curator.yaml') }
|
||||
it { is_expected.to contain_file('/etc/elasticsearch/delete_indices.yaml') }
|
||||
end
|
||||
|
||||
describe 'with index prefixes and retention period > 0 and host' do
|
||||
|
@ -43,6 +48,7 @@ describe 'lma_logging_analytics::curator', :type => :class do
|
|||
{:retention_period => 10, :prefixes => ['foo'], :host => 'foo.org'}
|
||||
end
|
||||
|
||||
it { is_expected.to contain_cron('es-curator').with_command(/--host foo.org/) }
|
||||
it { is_expected.to contain_cron('es-curator').with_command(
|
||||
'/usr/local/bin/curator --config /etc/elasticsearch/curator.yaml /etc/elasticsearch/delete_indices.yaml') }
|
||||
end
|
||||
end
|
||||
|
|
|
@ -0,0 +1,23 @@
|
|||
---
|
||||
client:
|
||||
hosts:
|
||||
- <%= @host %>
|
||||
port: <%= @port %>
|
||||
url_prefix:
|
||||
use_ssl: False
|
||||
certificate:
|
||||
client_cert:
|
||||
client_key:
|
||||
aws_key:
|
||||
aws_secret_key:
|
||||
aws_region:
|
||||
ssl_no_validate: False
|
||||
http_auth:
|
||||
timeout: 30
|
||||
master_only: True
|
||||
|
||||
logging:
|
||||
loglevel: INFO
|
||||
logfile:
|
||||
logformat: default
|
||||
blacklist: ['elasticsearch', 'urllib3']
|
|
@ -0,0 +1,23 @@
|
|||
---
|
||||
actions:
|
||||
1:
|
||||
action: delete_indices
|
||||
description: >-
|
||||
Delete indices older than <%= @real_retention_period %> days (based on index name).
|
||||
options:
|
||||
ignore_empty_list: True
|
||||
timeout_override:
|
||||
continue_if_exception: False
|
||||
disable_action: False
|
||||
filters:
|
||||
- filtertype: pattern
|
||||
kind: regex
|
||||
value: "^<%= @regex %>-.*$"
|
||||
exclude:
|
||||
- filtertype: age
|
||||
source: name
|
||||
direction: older
|
||||
timestring: '%Y.%m.%d'
|
||||
unit: days
|
||||
unit_count: <%= @real_retention_period %>
|
||||
exclude:
|
31
functions.sh
31
functions.sh
|
@ -20,19 +20,22 @@ MODULES_DIR="${ROOT}"/deployment_scripts/puppet/modules
|
|||
RPM_REPO="${ROOT}"/repositories/centos/
|
||||
DEB_REPO="${ROOT}"/repositories/ubuntu/
|
||||
|
||||
# Download RPM or DEB packages and store them in the local repository directory
|
||||
function download_package {
|
||||
while [ $# -gt 0 ]; do
|
||||
if [[ "$1" == *.deb ]]; then
|
||||
REPO=$DEB_REPO
|
||||
elif [[ "$1" == *.rpm ]]; then
|
||||
REPO=$RPM_REPO
|
||||
else
|
||||
echo "Invalid URL for download_package(): $1"
|
||||
fi
|
||||
|
||||
function get_package_path {
|
||||
FILE=$(basename "$1")
|
||||
wget -qO - "$1" > "$REPO"/"$FILE"
|
||||
if [[ "$1" == *.deb ]]; then
|
||||
echo "$DEB_REPO"/"$FILE"
|
||||
elif [[ "$1" == *.rpm ]]; then
|
||||
echo "$RPM_REPO"/"$FILE"
|
||||
else
|
||||
echo "Invalid URL for $1"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Download RPM or DEB packages and store them in the local repository directory
|
||||
function download_packages {
|
||||
while [ $# -gt 0 ]; do
|
||||
wget -qO - "$1" > "$(get_package_path "$1")"
|
||||
shift
|
||||
done
|
||||
}
|
||||
|
@ -44,3 +47,7 @@ function download_puppet_module {
|
|||
wget -qO- "$2" | tar -C "${MODULES_DIR}/$1" --strip-components=1 -xz
|
||||
}
|
||||
|
||||
function check_md5sum {
|
||||
FILE="$(get_package_path "$1")"
|
||||
echo "$2 $FILE" | md5sum --check --strict
|
||||
}
|
||||
|
|
|
@ -16,7 +16,7 @@ HTPASSWD_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/leinaddm-htpasswd
|
|||
TZ_VERSION="2016f-0ubuntu0.14.04_all"
|
||||
|
||||
# Packages needed to install JRE headless
|
||||
download_package http://mirrors.kernel.org/ubuntu/pool/main/p/pcsc-lite/libpcsclite1_1.8.10-1ubuntu1_amd64.deb \
|
||||
download_packages http://mirrors.kernel.org/ubuntu/pool/main/p/pcsc-lite/libpcsclite1_1.8.10-1ubuntu1_amd64.deb \
|
||||
http://mirrors.kernel.org/ubuntu/pool/main/j/java-common/java-common_0.51_all.deb \
|
||||
"http://security.ubuntu.com/ubuntu/pool/main/t/tzdata/tzdata-java_${TZ_VERSION}.deb" \
|
||||
"http://security.ubuntu.com/ubuntu/pool/main/t/tzdata/tzdata_${TZ_VERSION}.deb" \
|
||||
|
@ -28,10 +28,14 @@ download_package http://mirrors.kernel.org/ubuntu/pool/main/p/pcsc-lite/libpcscl
|
|||
https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.3.3/elasticsearch-2.3.3.deb
|
||||
|
||||
# Download curator and dependencies
|
||||
download_package http://packages.elasticsearch.org/curator/3/debian/pool/main/p/python/python-elasticsearch-curator_3.3.0_all.deb \
|
||||
http://packages.elasticsearch.org/curator/3/debian/pool/main/p/python/python-elasticsearch_1.6.0_all.deb \
|
||||
http://packages.elasticsearch.org/curator/3/debian/pool/main/p/python/python-click_4.0_all.deb \
|
||||
http://packages.elasticsearch.org/curator/3/debian/pool/main/p/python/python-urllib3_1.10.4_all.deb
|
||||
download_packages http://packages.elastic.co/curator/4/debian/pool/main/p/python/python-elasticsearch-curator_4.0.6_all.deb \
|
||||
http://packages.elastic.co/curator/4/debian/pool/main/p/python/python-elasticsearch_2.3.0_all.deb \
|
||||
http://packages.elastic.co/curator/4/debian/pool/main/p/python/python-click_6.6_all.deb \
|
||||
http://packages.elastic.co/curator/4/debian/pool/main/p/python/python-pyyaml_3.11_amd64.deb
|
||||
check_md5sum python-elasticsearch-curator_4.0.6_all.deb eb69388edf54aafa2445423f615c0817
|
||||
check_md5sum python-click_6.6_all.deb f508745fcd6f64a317cc94603b7f4f9d
|
||||
check_md5sum python-pyyaml_3.11_amd64.deb 9f4ef75d24f6bc92879e0c57dc4f4b71
|
||||
check_md5sum python-elasticsearch_2.3.0_all.deb 4d17c82d397fc57034d78e9d09ae400d
|
||||
|
||||
# Install puppet manifests
|
||||
download_puppet_module "concat" "$CONCAT_TARBALL_URL"
|
||||
|
@ -42,7 +46,6 @@ download_puppet_module "datacat" "$DATACAT_TARBALL_URL"
|
|||
download_puppet_module "apache" "${APACHE_TARBALL_URL}"
|
||||
download_puppet_module "htpasswd" "${HTPASSWD_TARBALL_URL}"
|
||||
|
||||
|
||||
# Update tzdata-java package to remove strict dependency on tzdata (see LP bug #1560436)
|
||||
ROOT="$(dirname "$(readlink -f "$0")")"
|
||||
DEB_REPO="${ROOT}"/repositories/ubuntu/
|
||||
|
|
Loading…
Reference in New Issue