Merge "Rotate Elasticsearch logs"
This commit is contained in:
commit
fbe120735a
|
@ -14,11 +14,6 @@
|
|||
|
||||
notice('fuel-plugin-elasticsearch-kibana: elasticsearch.pp')
|
||||
|
||||
$listen_address = hiera('lma::elasticsearch::listen_address')
|
||||
$es_nodes = hiera('lma::elasticsearch::nodes')
|
||||
$es_dir = hiera('lma::elasticsearch::data_dir')
|
||||
$es_instance = hiera('lma::elasticsearch::instance_name')
|
||||
$es_heap_size = hiera('lma::elasticsearch::jvm_size')
|
||||
|
||||
# Java
|
||||
$java = $::operatingsystem ? {
|
||||
|
@ -31,39 +26,17 @@ package { $java:
|
|||
ensure => installed,
|
||||
}
|
||||
|
||||
file { $es_dir:
|
||||
ensure => 'directory',
|
||||
}
|
||||
|
||||
# Install elasticsearch
|
||||
class { 'elasticsearch':
|
||||
datadir => "${es_dir}/elasticsearch_data",
|
||||
init_defaults => {
|
||||
'MAX_LOCKED_MEMORY' => 'unlimited',
|
||||
'ES_HEAP_SIZE' => "${es_heap_size}g"
|
||||
},
|
||||
require => [File[$es_dir], Package[$java]],
|
||||
}
|
||||
|
||||
# Start an instance of elasticsearch
|
||||
elasticsearch::instance { $es_instance:
|
||||
config => {
|
||||
'threadpool.bulk.queue_size' => '1000',
|
||||
'bootstrap.mlockall' => true,
|
||||
'http.cors.allow-origin' => '/.*/',
|
||||
'http.cors.enabled' => true,
|
||||
'cluster.name' => hiera('lma::elasticsearch::cluster_name'),
|
||||
'node.name' => hiera('lma::elasticsearch::node_name'),
|
||||
'node.master' => true,
|
||||
'node.data' => true,
|
||||
'discovery.zen.ping.multicast' => {'enabled' => false},
|
||||
'discovery.zen.ping.unicast.hosts' => $es_nodes,
|
||||
'discovery.zen.minimum_master_nodes' => hiera('lma::elasticsearch::minimum_master_nodes'),
|
||||
'gateway.recover_after_time' => hiera('lma::elasticsearch::recover_after_time'),
|
||||
'gateway.recover_after_nodes' => hiera('lma::elasticsearch::recover_after_nodes'),
|
||||
'gateway.expected_nodes' => size($es_nodes),
|
||||
'http.bind_host' => $listen_address,
|
||||
'transport.bind_host' => $listen_address,
|
||||
'transport.publish_host' => $listen_address,
|
||||
}
|
||||
class { 'lma_logging_analytics::elasticsearch':
|
||||
listen_address => hiera('lma::elasticsearch::listen_address'),
|
||||
node_name => hiera('lma::elasticsearch::node_name'),
|
||||
nodes => hiera('lma::elasticsearch::nodes'),
|
||||
data_dir => hiera('lma::elasticsearch::data_dir'),
|
||||
instance_name => hiera('lma::elasticsearch::instance_name'),
|
||||
heap_size => hiera('lma::elasticsearch::jvm_size'),
|
||||
cluster_name => hiera('lma::elasticsearch::cluster_name'),
|
||||
logs_dir => hiera('lma::elasticsearch::logs_dir'),
|
||||
minimum_master_nodes => hiera('lma::elasticsearch::minimum_master_nodes'),
|
||||
recover_after_time => hiera('lma::elasticsearch::recover_after_time'),
|
||||
recover_after_nodes => hiera('lma::elasticsearch::recover_after_nodes'),
|
||||
require => Package[$java],
|
||||
}
|
||||
|
|
|
@ -61,6 +61,9 @@ if is_integer($elasticsearch_kibana['recover_after_nodes']) and $elasticsearch_k
|
|||
notice("Set recover_after_nodes to ${recover_after_nodes}")
|
||||
}
|
||||
|
||||
$instance_name = 'es-01'
|
||||
$logs_dir = "/var/log/elasticsearch/${instance_name}"
|
||||
|
||||
$calculated_content = inline_template('
|
||||
---
|
||||
lma::corosync_roles:
|
||||
|
@ -80,11 +83,12 @@ lma::elasticsearch::nodes:
|
|||
<% end -%>
|
||||
lma::elasticsearch::number_of_replicas: <%= @number_of_replicas %>
|
||||
lma::elasticsearch::minimum_master_nodes: <%= @minimum_master_nodes %>
|
||||
lma::elasticsearch::recover_after_time: <%= @recover_after_time %>m
|
||||
lma::elasticsearch::recover_after_time: <%= @recover_after_time %>
|
||||
lma::elasticsearch::recover_after_nodes: <%= @recover_after_nodes %>
|
||||
lma::elasticsearch::data_dir: "<%= @elasticsearch_kibana["data_dir"] %>"
|
||||
lma::elasticsearch::logs_dir: "<%= @logs_dir %>"
|
||||
lma::elasticsearch::jvm_size: <%= @elasticsearch_kibana["jvm_heap_size"] %>
|
||||
lma::elasticsearch::instance_name: es-01
|
||||
lma::elasticsearch::instance_name: <%= @instance_name %>
|
||||
lma::elasticsearch::node_name: "<%= @fqdn %>_es-01"
|
||||
lma::elasticsearch::cluster_name: lma
|
||||
')
|
||||
|
|
|
@ -10,6 +10,16 @@ Usage
|
|||
-----
|
||||
|
||||
```puppet
|
||||
class {'lma_logging_analytics::elasticsearch':
|
||||
listen_address => 'localhost',
|
||||
node_name => $::fqdn,
|
||||
nodes => [$::fqdn, 'node-x', 'node-y']
|
||||
data_dir => '/opt/es-data'
|
||||
instance_name => 'es-01',
|
||||
heap_size => 16,
|
||||
cluster_name => 'my_cluster',
|
||||
}
|
||||
|
||||
class {'lma_logging_analytics::kibana': }
|
||||
```
|
||||
|
||||
|
|
|
@ -0,0 +1,34 @@
|
|||
# This file is managed by Puppet, do not edit manually, your changes *will* be overwritten!
|
||||
|
||||
es.logger.level: INFO
|
||||
rootLogger: INFO, file
|
||||
|
||||
logger.action: DEBUG
|
||||
logger.index.indexing.slowlog: TRACE, index_indexing_slow_log_file
|
||||
logger.index.search.slowlog: TRACE, index_search_slow_log_file
|
||||
|
||||
additivity:
|
||||
index.search.slowlog: false
|
||||
index.indexing.slowlog: false
|
||||
|
||||
appender:
|
||||
file:
|
||||
type: file
|
||||
file: ${path.logs}/${cluster.name}.log
|
||||
layout:
|
||||
type: pattern
|
||||
conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
|
||||
|
||||
index_search_slow_log_file:
|
||||
type: file
|
||||
file: ${path.logs}/${cluster.name}_index_search_slowlog.log
|
||||
layout:
|
||||
type: pattern
|
||||
conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
|
||||
|
||||
index_indexing_slow_log_file:
|
||||
type: file
|
||||
file: ${path.logs}/${cluster.name}_index_indexing_slowlog.log
|
||||
layout:
|
||||
type: pattern
|
||||
conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
|
|
@ -0,0 +1,89 @@
|
|||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
class lma_logging_analytics::elasticsearch (
|
||||
$listen_address,
|
||||
$data_dir,
|
||||
$node_name,
|
||||
$instance_name,
|
||||
$cluster_name,
|
||||
$nodes = [],
|
||||
$logs_dir = '/var/log/elasticsearch',
|
||||
$minimum_master_nodes = 1,
|
||||
$recover_after_time = 5,
|
||||
$recover_after_nodes = 1,
|
||||
$is_master = true,
|
||||
$is_data = true,
|
||||
$heap_size = 1,
|
||||
$listen_port = 9200,
|
||||
){
|
||||
|
||||
validate_bool($is_master)
|
||||
validate_bool($is_data)
|
||||
validate_integer($minimum_master_nodes)
|
||||
validate_integer($recover_after_time)
|
||||
validate_integer($recover_after_nodes)
|
||||
validate_integer($heap_size)
|
||||
validate_integer($listen_port)
|
||||
|
||||
file { $data_dir:
|
||||
ensure => 'directory',
|
||||
}
|
||||
|
||||
# Install elasticsearch
|
||||
class { '::elasticsearch':
|
||||
datadir => "${data_dir}/elasticsearch_data",
|
||||
init_defaults => {
|
||||
'MAX_LOCKED_MEMORY' => 'unlimited',
|
||||
'ES_HEAP_SIZE' => "${heap_size}g",
|
||||
'LOG_DIR' => $logs_dir,
|
||||
},
|
||||
require => File[$data_dir],
|
||||
}
|
||||
|
||||
$config = {
|
||||
'threadpool.bulk.queue_size' => '1000',
|
||||
'bootstrap.mlockall' => true,
|
||||
'http.cors.allow-origin' => '/.*/',
|
||||
'http.cors.enabled' => true,
|
||||
'cluster.name' => $cluster_name,
|
||||
'path.logs' => $logs_dir,
|
||||
'node.name' => $node_name,
|
||||
'node.master' => $is_master,
|
||||
'node.data' => $is_data,
|
||||
'discovery.zen.ping.multicast' => {'enabled' => false},
|
||||
'discovery.zen.ping.unicast.hosts' => $nodes,
|
||||
'discovery.zen.minimum_master_nodes' => $minimum_master_nodes,
|
||||
'gateway.recover_after_time' => "${recover_after_time}m",
|
||||
'gateway.recover_after_nodes' => $recover_after_nodes,
|
||||
'gateway.expected_nodes' => size($nodes),
|
||||
'http.bind_host' => $listen_address,
|
||||
'transport.bind_host' => $listen_address,
|
||||
'transport.publish_host' => $listen_address,
|
||||
}
|
||||
# Start an instance of elasticsearch
|
||||
::elasticsearch::instance { $instance_name:
|
||||
logging_file => 'puppet:///modules/lma_logging_analytics/elasticsearch_logging.yaml',
|
||||
config => $config,
|
||||
}
|
||||
|
||||
# This template uses $logs_dir variable
|
||||
file { "/etc/logrotate.d/elasticsearch-${instance_name}.conf":
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0644',
|
||||
content => template('lma_logging_analytics/elasticsearch_logrotate.conf.erb'),
|
||||
}
|
||||
}
|
|
@ -0,0 +1,40 @@
|
|||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
require 'spec_helper'
|
||||
|
||||
describe 'lma_logging_analytics::elasticsearch', :type => :class do
|
||||
let(:facts) do
|
||||
{:kernel => 'Linux', :operatingsystem => 'Ubuntu'}
|
||||
end
|
||||
|
||||
describe 'minmal parameter' do
|
||||
let(:params) do
|
||||
{:listen_address => '127.0.0.1',
|
||||
:data_dir => '/tmp/es-data',
|
||||
:node_name => 'foo-host',
|
||||
:cluster_name => 'es-cluster',
|
||||
:instance_name => 'es-42',
|
||||
:nodes => ['node-1', 'node-2', 'foo-host'],
|
||||
}
|
||||
end
|
||||
|
||||
it { is_expected.to contain_class('elasticsearch') }
|
||||
it { is_expected.to contain_elasticsearch__instance('es-42') }
|
||||
it { is_expected.to contain_file('/tmp/es-data').with(
|
||||
:ensure => 'directory'
|
||||
)}
|
||||
it { is_expected.to contain_file('/etc/logrotate.d/elasticsearch-es-42.conf') }
|
||||
end
|
||||
|
||||
end
|
|
@ -0,0 +1,22 @@
|
|||
# managed by puppet
|
||||
<%= @logs_dir %>/*.log {
|
||||
# Elasticsearch cannot be told to close its log file and re-open it so we need to
|
||||
# use the copytruncate option
|
||||
copytruncate
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
# logrotate allows to use only year, month, day and unix epoch
|
||||
dateext
|
||||
dateformat -%Y%m%d-%s
|
||||
# number of rotated files to keep
|
||||
rotate 10
|
||||
# do not rotate files unless both size and time conditions are met
|
||||
hourly
|
||||
minsize 20M
|
||||
# force rotate if filesize exceeded 100M
|
||||
maxsize 100M
|
||||
# this must map the /var/log directory group ownership
|
||||
su root syslog
|
||||
}
|
Loading…
Reference in New Issue