diff --git a/deployment_scripts/puppet/manifests/aggregator.pp b/deployment_scripts/puppet/manifests/aggregator.pp new file mode 100644 index 000000000..574921305 --- /dev/null +++ b/deployment_scripts/puppet/manifests/aggregator.pp @@ -0,0 +1,68 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +$lma_collector = hiera('lma_collector') +$roles = node_roles(hiera('nodes'), hiera('uid')) +$is_controller = member($roles, 'controller') or member($roles, 'primary-controller') + +$aggregator_address = hiera('management_vip') +$internal_address = hiera('internal_address') +$aggregator_port = 5565 +$check_port = 5566 + +class { 'lma_collector::aggregator::client': + address => $aggregator_address, + port => $aggregator_port, +} + +if $is_controller { + class { 'lma_collector::aggregator::server': + listen_address => $internal_address, + listen_port => $aggregator_port, + http_check_port => $check_port, + } + + # Hacks needed to leverage the haproxy_service defined type + include haproxy::params + Haproxy::Service { use_include => true } + Haproxy::Balancermember { use_include => true } + + # HAProxy configuration + openstack::ha::haproxy_service { 'lma': + order => '999', + listen_port => $aggregator_port, + balancermember_port => $aggregator_port, + haproxy_config_options => { + 'option' => ['httpchk', 'tcplog'], + 'balance' => 'roundrobin', + 'mode' => 'tcp', + }, + balancermember_options => "check port ${check_port}", + internal => true, + internal_virtual_ip => $aggregator_address, + public => false, + public_virtual_ip => undef, + ipaddresses => [ $internal_address ], + server_names => [ $::hostname ], + } + + # Allow traffic from HAProxy to the local LMA collector + firewall { '998 lma': + port => [$aggregator_port, $check_port], + source => $aggregator_address, + destination => $internal_address, + proto => 'tcp', + action => 'accept', + } +} diff --git a/deployment_scripts/puppet/manifests/base.pp b/deployment_scripts/puppet/manifests/base.pp index 83e1398f2..d9918b857 100644 --- a/deployment_scripts/puppet/manifests/base.pp +++ b/deployment_scripts/puppet/manifests/base.pp @@ -94,12 +94,10 @@ if $is_controller{ } class { 'lma_collector': - tags => merge($tags, $additional_tags), - groups => $additional_groups, - # pre_script => $pre_script, - aggregator_address => hiera('management_vip'), - pacemaker_managed => $pacemaker_managed, - rabbitmq_resource => $rabbitmq_resource, + tags => merge($tags, $additional_tags), + groups => $additional_groups, + pacemaker_managed => $pacemaker_managed, + rabbitmq_resource => $rabbitmq_resource, } if $elasticsearch_mode != 'disabled' { diff --git a/deployment_scripts/puppet/manifests/controller.pp b/deployment_scripts/puppet/manifests/controller.pp index f9c54187a..46edd1fb5 100644 --- a/deployment_scripts/puppet/manifests/controller.pp +++ b/deployment_scripts/puppet/manifests/controller.pp @@ -17,6 +17,7 @@ include lma_collector::params $ceilometer = hiera('ceilometer') $lma_collector = hiera('lma_collector') $rabbit = hiera('rabbit') +$management_vip = hiera('management_vip') if $ceilometer['enabled'] { $notification_topics = [$lma_collector::params::openstack_topic, $lma_collector::params::lma_topic] @@ -38,11 +39,6 @@ if hiera('deployment_mode') =~ /^ha_/ { $ha_deployment = false } -class { 'lma_collector::aggregator': - listen_address => hiera('internal_address'), - listen_port => 5565, -} - # OpenStack notifcations are always useful for indexation and metrics collection class { 'lma_collector::notifications::controller': host => '127.0.0.1', @@ -74,7 +70,6 @@ if $lma_collector['influxdb_mode'] != 'disabled' { $nova = hiera('nova') $neutron = hiera('quantum_settings') $cinder = hiera('cinder') - $management_vip = hiera('management_vip') if $ha_deployment { $haproxy_socket = '/var/lib/haproxy/stats' diff --git a/deployment_scripts/puppet/modules/lma_collector/files/plugins/decoders/noop.lua b/deployment_scripts/puppet/modules/lma_collector/files/plugins/decoders/noop.lua new file mode 100644 index 000000000..be9a9ddc7 --- /dev/null +++ b/deployment_scripts/puppet/modules/lma_collector/files/plugins/decoders/noop.lua @@ -0,0 +1,28 @@ +-- Copyright 2015 Mirantis, Inc. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +local msg_type = read_config('msg_type') or error('msg_type must be defined') + +local msg = { + Type = msg_type, + Severity = 7, -- debug + Payload = nil, + Fields = nil, +} + +function process_message () + inject_message(msg) + + return 0 +end diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/aggregator.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/aggregator/client.pp similarity index 57% rename from deployment_scripts/puppet/modules/lma_collector/manifests/aggregator.pp rename to deployment_scripts/puppet/modules/lma_collector/manifests/aggregator/client.pp index 960baf53f..ca2193137 100644 --- a/deployment_scripts/puppet/modules/lma_collector/manifests/aggregator.pp +++ b/deployment_scripts/puppet/modules/lma_collector/manifests/aggregator/client.pp @@ -11,20 +11,23 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -# -class lma_collector::aggregator ( - $listen_address = $lma_collector::params::aggregator_address, - $listen_port = $lma_collector::params::aggregator_port, +class lma_collector::aggregator::client ( + $address = undef, + $port = $lma_collector::params::aggregator_port, ) inherits lma_collector::params { - include lma_collector::service - validate_string($listen_address) - validate_integer($listen_port) + if $address == undef { + fail('address parameter should be defined!') + } - heka::input::tcp { 'aggregator': - config_dir => $lma_collector::params::config_dir, - address => $listen_address, - port => $listen_port, - notify => Class['lma_collector::service'], + validate_string($address) + + $config_dir = $lma_collector::params::config_dir + + heka::output::tcp { 'aggregator': + config_dir => $config_dir, + address => $address, + port => $port, + max_file_size => $lma_collector::params::buffering_max_file_size, } } diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/aggregator/server.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/aggregator/server.pp new file mode 100644 index 000000000..2e5b9ca0f --- /dev/null +++ b/deployment_scripts/puppet/modules/lma_collector/manifests/aggregator/server.pp @@ -0,0 +1,51 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +class lma_collector::aggregator::server ( + $listen_address = $lma_collector::params::aggregator_address, + $listen_port = $lma_collector::params::aggregator_port, + $http_check_port = undef, +) inherits lma_collector::params { + include lma_collector::service + + validate_string($listen_address) + validate_integer($listen_port) + + heka::input::tcp { 'aggregator': + config_dir => $lma_collector::params::config_dir, + address => $listen_address, + port => $listen_port, + notify => Class['lma_collector::service'], + } + + if $http_check_port { + heka::decoder::sandbox { 'http-check': + config_dir => $lma_collector::params::config_dir, + filename => "${lma_collector::params::plugins_dir}/decoders/noop.lua" , + config => { + msg_type => 'lma.http-check', + }, + notify => Class['lma_collector::service'], + } + + heka::input::httplisten { 'http-check': + config_dir => $lma_collector::params::config_dir, + address => $listen_address, + port => $http_check_port, + decoder => 'http-check', + require => Heka::Decoder::Sandbox['http-check'], + notify => Class['lma_collector::service'], + } + } +} diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/controller.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/controller.pp index 95325d3fe..2762ccb6b 100644 --- a/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/controller.pp +++ b/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/controller.pp @@ -96,8 +96,8 @@ class lma_collector::collectd::controller ( $modules['haproxy'] = { 'Socket' => $haproxy_socket, 'Mapping' => $haproxy_names_mapping, - # Ignore internal proxy - 'ProxyIgnore' => 'Stats', + # Ignore internal stats and lma proxies + 'ProxyIgnore' => ['Stats', 'lma'], } } diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp index b62749b8f..161c27f9c 100644 --- a/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp +++ b/deployment_scripts/puppet/modules/lma_collector/manifests/init.pp @@ -193,13 +193,4 @@ class lma_collector ( ensure => present, } } - - if $aggregator_address { - heka::output::tcp { 'aggregator': - config_dir => $config_dir, - address => $aggregator_address, - port => $aggregator_port, - max_file_size => $lma_collector::params::buffering_max_file_size, - } - } } diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/params.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/params.pp index ce536a1cc..8aa78053d 100644 --- a/deployment_scripts/puppet/modules/lma_collector/manifests/params.pp +++ b/deployment_scripts/puppet/modules/lma_collector/manifests/params.pp @@ -24,7 +24,7 @@ class lma_collector::params { $dashboard_port = '4352' $aggregator_address = '127.0.0.1' - $aggregator_port = '5565' + $aggregator_port = 5565 $tags = {} diff --git a/deployment_scripts/puppet/modules/lma_collector/spec/classes/lma_collector_aggregator_client_spec.rb b/deployment_scripts/puppet/modules/lma_collector/spec/classes/lma_collector_aggregator_client_spec.rb new file mode 100644 index 000000000..fe34ab105 --- /dev/null +++ b/deployment_scripts/puppet/modules/lma_collector/spec/classes/lma_collector_aggregator_client_spec.rb @@ -0,0 +1,32 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +require 'spec_helper' + +describe 'lma_collector::aggregator::client' do + let(:facts) do + {:kernel => 'Linux', :operatingsystem => 'Ubuntu', + :osfamily => 'Debian'} + end + + describe 'with defaults' do + it { is_expected.to raise_error(Puppet::Error) } + end + + describe 'with address = 127.0.0.2' do + let(:params) do + {:address => '127.0.0.2'} + end + it { is_expected.to contain_heka__output__tcp('aggregator').with_address('127.0.0.2') } + end +end diff --git a/deployment_scripts/puppet/modules/lma_collector/spec/classes/lma_collector_aggregator_server_spec.rb b/deployment_scripts/puppet/modules/lma_collector/spec/classes/lma_collector_aggregator_server_spec.rb new file mode 100644 index 000000000..e87313348 --- /dev/null +++ b/deployment_scripts/puppet/modules/lma_collector/spec/classes/lma_collector_aggregator_server_spec.rb @@ -0,0 +1,34 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +require 'spec_helper' + +describe 'lma_collector::aggregator::server' do + let(:facts) do + {:kernel => 'Linux', :operatingsystem => 'Ubuntu', + :osfamily => 'Debian'} + end + + describe 'with defaults' do + it { is_expected.to contain_heka__input__tcp('aggregator') } + it { is_expected.not_to contain_heka__input__http_listen('check-http') } + end + + describe 'with http_check_port = 3000' do + let(:params) do + {:http_check_port => 3000} + end + it { is_expected.to contain_heka__input__tcp('aggregator') } + it { is_expected.to contain_heka__input__httplisten('http-check').with_port(3000) } + end +end diff --git a/pre_build_hook b/pre_build_hook index c73381896..deb020fcc 100755 --- a/pre_build_hook +++ b/pre_build_hook @@ -10,6 +10,7 @@ COLLECTD_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/puppet-collectd-4 APACHE_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/puppetlabs-apache-1.4.0.tar.gz" STDLIB_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/puppetlabs-stdlib-4.7.0.tar.gz" CONCAT_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/puppetlabs-concat-1.2.4.tar.gz" +FIREWALL_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/puppetlabs-firewall-1.7.0.tar.gz" download_packages \ https://github.com/mozilla-services/heka/releases/download/v${HEKA_VERSION}/heka_${HEKA_VERSION}_amd64.deb \ @@ -47,12 +48,13 @@ download_packages \ # Extract dependent manifests from fuel-library -rm -rf "${MODULES_DIR:?}"/{cinder,glance,heat,inifile,keystone,neutron,nova,openstack,pacemaker} +rm -rf "${MODULES_DIR:?}"/{cinder,glance,haproxy,heat,inifile,keystone,neutron,nova,openstack,pacemaker} wget -qO- "${FUEL_LIB_TARBALL_URL}" | \ tar -C "${MODULES_DIR}" --strip-components=3 -zxvf - \ - fuel-library-${FUEL_LIB_COMMIT}/deployment/puppet/{cinder,glance,heat,inifile,keystone,neutron,nova,openstack,pacemaker} + fuel-library-${FUEL_LIB_COMMIT}/deployment/puppet/{cinder,glance,haproxy,heat,inifile,keystone,neutron,nova,openstack,pacemaker} download_puppet_module "collectd" "${COLLECTD_TARBALL_URL}" download_puppet_module "apache" "${APACHE_TARBALL_URL}" download_puppet_module "stdlib" "${STDLIB_TARBALL_URL}" download_puppet_module "concat" "${CONCAT_TARBALL_URL}" +download_puppet_module "firewall" "${FIREWALL_TARBALL_URL}" diff --git a/tasks.yaml b/tasks.yaml index b8834132a..3eae6b125 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -16,6 +16,14 @@ puppet_modules: puppet/modules timeout: 600 +- role: '*' + stage: post_deployment/8200 + type: puppet + parameters: + puppet_manifest: puppet/manifests/aggregator.pp + puppet_modules: puppet/modules + timeout: 600 + - role: ['controller', 'primary-controller'] stage: post_deployment/8200 type: puppet