x
/
fuel-plugin-lma-collector
Code Issues Proposed changes

Merge "Configure HAProxy to forward to the aggregator"

This commit is contained in:
Jenkins 2015-08-26 12:44:28 +00:00 committed by Gerrit Code Review
parent 936943de76 746b6f0f78
commit a2704f9281
13 changed files with 248 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
deployment_scripts/puppet/manifests/aggregator.pp Normal file
View File

@ -0,0 +1,68 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
$lma_collector = hiera('lma_collector')
$roles = node_roles(hiera('nodes'), hiera('uid'))
$is_controller = member($roles, 'controller') or member($roles, 'primary-controller')
$aggregator_address = hiera('management_vip')
$internal_address = hiera('internal_address')
$aggregator_port = 5565
$check_port = 5566
class { 'lma_collector::aggregator::client':
address => $aggregator_address,
port => $aggregator_port,
}
if $is_controller {
class { 'lma_collector::aggregator::server':
listen_address => $internal_address,
listen_port => $aggregator_port,
http_check_port => $check_port,
}
# Hacks needed to leverage the haproxy_service defined type
include haproxy::params
Haproxy::Service { use_include => true }
Haproxy::Balancermember { use_include => true }
# HAProxy configuration
openstack::ha::haproxy_service { 'lma':
order => '999',
listen_port => $aggregator_port,
balancermember_port => $aggregator_port,
haproxy_config_options => {
'option' => ['httpchk', 'tcplog'],
'balance' => 'roundrobin',
'mode' => 'tcp',
},
balancermember_options => "check port ${check_port}",
internal => true,
internal_virtual_ip => $aggregator_address,
public => false,
public_virtual_ip => undef,
ipaddresses => [ $internal_address ],
server_names => [ $::hostname ],
}
# Allow traffic from HAProxy to the local LMA collector
firewall { '998 lma':
port => [$aggregator_port, $check_port],
source => $aggregator_address,
destination => $internal_address,
proto => 'tcp',
action => 'accept',
}
}

10
deployment_scripts/puppet/manifests/base.pp
View File

@ -94,12 +94,10 @@ if $is_controller{
}
class { 'lma_collector':
tags => merge($tags, $additional_tags),
groups => $additional_groups,
# pre_script => $pre_script,
aggregator_address => hiera('management_vip'),
pacemaker_managed => $pacemaker_managed,
rabbitmq_resource => $rabbitmq_resource,
tags => merge($tags, $additional_tags),
groups => $additional_groups,
pacemaker_managed => $pacemaker_managed,
rabbitmq_resource => $rabbitmq_resource,
}
if $elasticsearch_mode != 'disabled' {

7
deployment_scripts/puppet/manifests/controller.pp
View File

@ -17,6 +17,7 @@ include lma_collector::params
$ceilometer = hiera('ceilometer')
$lma_collector = hiera('lma_collector')
$rabbit = hiera('rabbit')
$management_vip = hiera('management_vip')
if $ceilometer['enabled'] {
$notification_topics = [$lma_collector::params::openstack_topic, $lma_collector::params::lma_topic]
@ -38,11 +39,6 @@ if hiera('deployment_mode') =~ /^ha_/ {
$ha_deployment = false
}
class { 'lma_collector::aggregator':
listen_address => hiera('internal_address'),
listen_port => 5565,
}
# OpenStack notifcations are always useful for indexation and metrics collection
class { 'lma_collector::notifications::controller':
host => '127.0.0.1',
@ -74,7 +70,6 @@ if $lma_collector['influxdb_mode'] != 'disabled' {
$nova = hiera('nova')
$neutron = hiera('quantum_settings')
$cinder = hiera('cinder')
$management_vip = hiera('management_vip')
if $ha_deployment {
$haproxy_socket = '/var/lib/haproxy/stats'

28
deployment_scripts/puppet/modules/lma_collector/files/plugins/decoders/noop.lua Normal file
View File

@ -0,0 +1,28 @@
-- Copyright 2015 Mirantis, Inc.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
local msg_type = read_config('msg_type') or error('msg_type must be defined')
local msg = {
Type = msg_type,
Severity = 7, -- debug
Payload = nil,
Fields = nil,
}
function process_message ()
inject_message(msg)
return 0
end

27
deployment_scripts/puppet/modules/lma_collector/manifests/aggregator.pp → deployment_scripts/puppet/modules/lma_collector/manifests/aggregator/client.pp
View File

@ -11,20 +11,23 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
class lma_collector::aggregator (
$listen_address = $lma_collector::params::aggregator_address,
$listen_port = $lma_collector::params::aggregator_port,
class lma_collector::aggregator::client (
$address = undef,
$port = $lma_collector::params::aggregator_port,
) inherits lma_collector::params {
include lma_collector::service
validate_string($listen_address)
validate_integer($listen_port)
if $address == undef {
fail('address parameter should be defined!')
}
heka::input::tcp { 'aggregator':
config_dir => $lma_collector::params::config_dir,
address => $listen_address,
port => $listen_port,
notify => Class['lma_collector::service'],
validate_string($address)
$config_dir = $lma_collector::params::config_dir
heka::output::tcp { 'aggregator':
config_dir => $config_dir,
address => $address,
port => $port,
max_file_size => $lma_collector::params::buffering_max_file_size,
}
}

51
deployment_scripts/puppet/modules/lma_collector/manifests/aggregator/server.pp Normal file
View File

@ -0,0 +1,51 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
class lma_collector::aggregator::server (
$listen_address = $lma_collector::params::aggregator_address,
$listen_port = $lma_collector::params::aggregator_port,
$http_check_port = undef,
) inherits lma_collector::params {
include lma_collector::service
validate_string($listen_address)
validate_integer($listen_port)
heka::input::tcp { 'aggregator':
config_dir => $lma_collector::params::config_dir,
address => $listen_address,
port => $listen_port,
notify => Class['lma_collector::service'],
}
if $http_check_port {
heka::decoder::sandbox { 'http-check':
config_dir => $lma_collector::params::config_dir,
filename => "${lma_collector::params::plugins_dir}/decoders/noop.lua" ,
config => {
msg_type => 'lma.http-check',
},
notify => Class['lma_collector::service'],
}
heka::input::httplisten { 'http-check':
config_dir => $lma_collector::params::config_dir,
address => $listen_address,
port => $http_check_port,
decoder => 'http-check',
require => Heka::Decoder::Sandbox['http-check'],
notify => Class['lma_collector::service'],
}
}
}

4
deployment_scripts/puppet/modules/lma_collector/manifests/collectd/controller.pp
View File

@ -96,8 +96,8 @@ class lma_collector::collectd::controller (
$modules['haproxy'] = {
'Socket' => $haproxy_socket,
'Mapping' => $haproxy_names_mapping,
# Ignore internal proxy
'ProxyIgnore' => 'Stats',
# Ignore internal stats and lma proxies
'ProxyIgnore' => ['Stats', 'lma'],
}
}

9
deployment_scripts/puppet/modules/lma_collector/manifests/init.pp
View File

@ -193,13 +193,4 @@ class lma_collector (
ensure => present,
}
}
if $aggregator_address {
heka::output::tcp { 'aggregator':
config_dir => $config_dir,
address => $aggregator_address,
port => $aggregator_port,
max_file_size => $lma_collector::params::buffering_max_file_size,
}
}
}

2
deployment_scripts/puppet/modules/lma_collector/manifests/params.pp
View File

@ -24,7 +24,7 @@ class lma_collector::params {
$dashboard_port = '4352'
$aggregator_address = '127.0.0.1'
$aggregator_port = '5565'
$aggregator_port = 5565
$tags = {}

32
deployment_scripts/puppet/modules/lma_collector/spec/classes/lma_collector_aggregator_client_spec.rb Normal file
View File

@ -0,0 +1,32 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
require 'spec_helper'
describe 'lma_collector::aggregator::client' do
let(:facts) do
{:kernel => 'Linux', :operatingsystem => 'Ubuntu',
:osfamily => 'Debian'}
end
describe 'with defaults' do
it { is_expected.to raise_error(Puppet::Error) }
end
describe 'with address = 127.0.0.2' do
let(:params) do
{:address => '127.0.0.2'}
end
it { is_expected.to contain_heka__output__tcp('aggregator').with_address('127.0.0.2') }
end
end

34
deployment_scripts/puppet/modules/lma_collector/spec/classes/lma_collector_aggregator_server_spec.rb Normal file
View File

@ -0,0 +1,34 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
require 'spec_helper'
describe 'lma_collector::aggregator::server' do
let(:facts) do
{:kernel => 'Linux', :operatingsystem => 'Ubuntu',
:osfamily => 'Debian'}
end
describe 'with defaults' do
it { is_expected.to contain_heka__input__tcp('aggregator') }
it { is_expected.not_to contain_heka__input__http_listen('check-http') }
end
describe 'with http_check_port = 3000' do
let(:params) do
{:http_check_port => 3000}
end
it { is_expected.to contain_heka__input__tcp('aggregator') }
it { is_expected.to contain_heka__input__httplisten('http-check').with_port(3000) }
end
end

6
pre_build_hook
View File

@ -10,6 +10,7 @@ COLLECTD_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/puppet-collectd-4
APACHE_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/puppetlabs-apache-1.4.0.tar.gz"
STDLIB_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/puppetlabs-stdlib-4.7.0.tar.gz"
CONCAT_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/puppetlabs-concat-1.2.4.tar.gz"
FIREWALL_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/puppetlabs-firewall-1.7.0.tar.gz"
download_packages \
https://github.com/mozilla-services/heka/releases/download/v${HEKA_VERSION}/heka_${HEKA_VERSION}_amd64.deb \
@ -47,12 +48,13 @@ download_packages \
# Extract dependent manifests from fuel-library
rm -rf "${MODULES_DIR:?}"/{cinder,glance,heat,inifile,keystone,neutron,nova,openstack,pacemaker}
rm -rf "${MODULES_DIR:?}"/{cinder,glance,haproxy,heat,inifile,keystone,neutron,nova,openstack,pacemaker}
wget -qO- "${FUEL_LIB_TARBALL_URL}" | \
tar -C "${MODULES_DIR}" --strip-components=3 -zxvf - \
fuel-library-${FUEL_LIB_COMMIT}/deployment/puppet/{cinder,glance,heat,inifile,keystone,neutron,nova,openstack,pacemaker}
fuel-library-${FUEL_LIB_COMMIT}/deployment/puppet/{cinder,glance,haproxy,heat,inifile,keystone,neutron,nova,openstack,pacemaker}
download_puppet_module "collectd" "${COLLECTD_TARBALL_URL}"
download_puppet_module "apache" "${APACHE_TARBALL_URL}"
download_puppet_module "stdlib" "${STDLIB_TARBALL_URL}"
download_puppet_module "concat" "${CONCAT_TARBALL_URL}"
download_puppet_module "firewall" "${FIREWALL_TARBALL_URL}"

8
tasks.yaml
View File

@ -16,6 +16,14 @@
puppet_modules: puppet/modules
timeout: 600
- role: '*'
stage: post_deployment/8200
type: puppet
parameters:
puppet_manifest: puppet/manifests/aggregator.pp
puppet_modules: puppet/modules
timeout: 600
- role: ['controller', 'primary-controller']
stage: post_deployment/8200
type: puppet