Kafka support
- get kafka IPs - configure ceilometer.conf - configure hindsight configs - deploy hindsight on Kafka nodes - disable heka when Kafka detected Change-Id: I3df403affab6f8427141c19a07a3312af7e92630
This commit is contained in:
Volodymyr Kornylyuk
parent
621d57e8a3
commit
a14b61fcb1
|
|
@ -75,6 +75,24 @@ service { 'ceilometer-collector':
|
|||
hasstatus => true,
|
||||
}
|
||||
|
||||
# Kafka integration
|
||||
|
||||
if hiera('telemetry::kafka::enabled') {
|
||||
|
||||
ceilometer_config { 'oslo_messaging_kafka/consumer_group': value => 'ceilometer' }
|
||||
|
||||
$kafka_ips = hiera('telemetry::kafka::broker_list')
|
||||
$kafka_url = "kafka://${kafka_ips}"
|
||||
$rabbit_url = hiera('telemetry::rabbit::url')
|
||||
|
||||
ceilometer_config { 'DEFAULT/transport_url': value => $kafka_url }
|
||||
ceilometer_config { 'notification/messaging_urls': value => [$kafka_url,$rabbit_url] }
|
||||
ceilometer_config { 'oslo_messaging_notifications/transport_url': value => $kafka_url }
|
||||
|
||||
ceilometer_config { 'compute/resource_update_interval': value => 600 }
|
||||
|
||||
}
|
||||
|
||||
# TODO validate values before proceed
|
||||
|
||||
ceilometer_config { 'database/metering_connection': value => $metering_connection }
|
||||
|
|
@ -91,7 +109,7 @@ service { 'ceilometer-service':
|
|||
hasstatus => true,
|
||||
hasrestart => true,
|
||||
tag => 'ceilometer-service',
|
||||
}
|
||||
}
|
||||
|
||||
Ceilometer_config<||> ~> Service['ceilometer-service']
|
||||
|
||||
|
|
|
|||
|
|
@ -1,35 +1,40 @@
|
|||
|
||||
## Get values
|
||||
notice('MODULAR: fuel-plugin-telemetry: heka.pp')
|
||||
|
||||
$config_dir = hiera('telemetry::heka::config_dir')
|
||||
$amqp_url = hiera('telemetry::rabbit::url')
|
||||
$metadata_fields = hiera('telemetry::metadata_fields')
|
||||
if !hiera('telemetry::kafka::enabled') {
|
||||
|
||||
if hiera('telemetry::elasticsearch::server',false) {
|
||||
## Get values
|
||||
|
||||
$config_dir = hiera('telemetry::heka::config_dir')
|
||||
$amqp_url = hiera('telemetry::rabbit::url')
|
||||
$metadata_fields = hiera('telemetry::metadata_fields')
|
||||
|
||||
|
||||
if hiera('telemetry::elasticsearch::server',false) {
|
||||
$ip = hiera('telemetry::elasticsearch::server')
|
||||
$port = hiera('telemetry::elasticsearch::server')
|
||||
$elasticsearch_url = "http://${ip}:${port}"
|
||||
} else {
|
||||
} else {
|
||||
#no Elasticsearch
|
||||
#heka failed to start if url schemma is not valid, so we set http here
|
||||
$elasticsearch_url = 'http://'
|
||||
}
|
||||
}
|
||||
|
||||
$influxdb_address = hiera('telemetry::influxdb::address')
|
||||
$influxdb_port = hiera('telemetry::influxdb::port')
|
||||
$influxdb_database = hiera('telemetry::influxdb::database')
|
||||
$influxdb_user = hiera('telemetry::influxdb::user')
|
||||
$influxdb_password = hiera('telemetry::influxdb::password')
|
||||
$influxdb_address = hiera('telemetry::influxdb::address')
|
||||
$influxdb_port = hiera('telemetry::influxdb::port')
|
||||
$influxdb_database = hiera('telemetry::influxdb::database')
|
||||
$influxdb_user = hiera('telemetry::influxdb::user')
|
||||
$influxdb_password = hiera('telemetry::influxdb::password')
|
||||
|
||||
$modules_dir = hiera('telemetry::lua::modules_dir')
|
||||
$modules_dir = hiera('telemetry::lua::modules_dir')
|
||||
|
||||
### Heka configuration
|
||||
### Heka configuration
|
||||
|
||||
File {
|
||||
File {
|
||||
before => Service['telemetry-collector']
|
||||
}
|
||||
}
|
||||
|
||||
file {
|
||||
file {
|
||||
"${config_dir}/amqp-openstack_sample.toml": content => template( 'telemetry/heka/amqp-openstack_sample.toml.erb' );
|
||||
"${config_dir}/decoder-sample.toml": content => template( 'telemetry/heka/decoder-sample.toml.erb' );
|
||||
"${config_dir}/encoder-influxdb.toml": content => template( 'telemetry/heka/encoder-influxdb.toml.erb' );
|
||||
|
|
@ -40,25 +45,25 @@ file {
|
|||
# TODO disable config when Elasticsearch not in use
|
||||
"${config_dir}/output-elasticsearch-resource2.toml": content => template( 'telemetry/heka/output-elasticsearch-resource2.toml.erb' );
|
||||
"${config_dir}/output-influxdb-samples.toml": content => template( 'telemetry/heka/output-influxdb-samples.toml.erb' );
|
||||
}
|
||||
}
|
||||
|
||||
# Heka Installation
|
||||
# Heka Installation
|
||||
|
||||
$version = hiera('telemetry::heka::version')
|
||||
$max_message_size = hiera('telemetry::heka::max_message_size')
|
||||
$max_process_inject = hiera('telemetry::heka::max_process_inject')
|
||||
$max_timer_inject = hiera('telemetry::heka::max_timer_inject')
|
||||
$poolsize = hiera('telemetry::heka::poolsize')
|
||||
$version = hiera('telemetry::heka::version')
|
||||
$max_message_size = hiera('telemetry::heka::max_message_size')
|
||||
$max_process_inject = hiera('telemetry::heka::max_process_inject')
|
||||
$max_timer_inject = hiera('telemetry::heka::max_timer_inject')
|
||||
$poolsize = hiera('telemetry::heka::poolsize')
|
||||
|
||||
# TODO we dont't need them on controller
|
||||
$install_init_script = false
|
||||
# TODO we dont't need them on controller
|
||||
$install_init_script = false
|
||||
|
||||
# Workaround for heka module
|
||||
# to prevent changing home of root user
|
||||
$user='root'
|
||||
user { $user: }
|
||||
# Workaround for heka module
|
||||
# to prevent changing home of root user
|
||||
$user='root'
|
||||
user { $user: }
|
||||
|
||||
::heka { 'telemetry-collector':
|
||||
::heka { 'telemetry-collector':
|
||||
config_dir => '/etc/telemetry-collector',
|
||||
user => $user,
|
||||
#additional_groups => $additional_groups,
|
||||
|
|
@ -69,11 +74,11 @@ user { $user: }
|
|||
poolsize => $poolsize,
|
||||
install_init_script => $install_init_script,
|
||||
version => $version,
|
||||
}
|
||||
}
|
||||
|
||||
# Heka pacemaker config
|
||||
# Heka pacemaker config
|
||||
|
||||
pacemaker::service { 'telemetry-collector':
|
||||
pacemaker::service { 'telemetry-collector':
|
||||
ensure => present,
|
||||
prefix => false,
|
||||
primitive_class => 'ocf',
|
||||
|
|
@ -109,10 +114,12 @@ pacemaker::service { 'telemetry-collector':
|
|||
'timeout' => '30',
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
service { 'telemetry-collector':
|
||||
service { 'telemetry-collector':
|
||||
ensure => 'running',
|
||||
enable => true,
|
||||
provider => 'pacemaker',
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -98,7 +98,8 @@ if $telemetry['influxdb_address'] {
|
|||
]))
|
||||
}
|
||||
|
||||
# TODO test for multiple inxlixdb nodes !!!
|
||||
# TODO test for multiple influxdb nodes !!!
|
||||
# TODO use vip?
|
||||
$influxdb_address = $nodes_array[0]['network_roles']['management']
|
||||
|
||||
$retention_period = $influxdb_grafana['retention_period']
|
||||
|
|
@ -123,6 +124,23 @@ $metadata_fields = join(['status deleted container_format min_ram updated_at '
|
|||
'min_disk is_public size checksum created_at disk_format protected instance_host ',
|
||||
'host display_name instance_id instance_type status state'])
|
||||
|
||||
# Kafka
|
||||
$kafka_port = 9092
|
||||
$kafka_nodes = get_nodes_hash_by_roles($network_metadata, ['kafka', 'primary-kafka'])
|
||||
$kafka_ip_map = get_node_to_ipaddr_map_by_network_role($kafka_nodes, 'management')
|
||||
|
||||
if count($kafka_ip_map)>0 {
|
||||
notice('Kafka nodes found')
|
||||
$kafka_enabled = true
|
||||
$kafka_ips = sort(values($kafka_ip_map))
|
||||
# Format: host:port,host:port for ceiolmeter.conf
|
||||
$tmp_list = join($kafka_ips,":${kafka_port},")
|
||||
$broker_list = join([$tmp_list,":${kafka_port}"])
|
||||
} else {
|
||||
notice('No Kafka nodes found')
|
||||
$kafka_enabled = false
|
||||
}
|
||||
|
||||
$calculated_content = inline_template('
|
||||
---
|
||||
ceilometer:
|
||||
|
|
@ -161,6 +179,16 @@ telemetry::heka::max_timer_inject: 10
|
|||
telemetry::heka::poolsize: 100
|
||||
telemetry::heka::config_dir: "/etc/telemetry-collector"
|
||||
|
||||
<% if @kafka_enabled -%>
|
||||
telemetry::kafka::broker_list: "<%= @broker_list %>"
|
||||
telemetry::kafka::nodes_list:
|
||||
<% @kafka_ips.each do |s| -%>
|
||||
- "<%= s %>"
|
||||
<% end -%>
|
||||
<% end -%>
|
||||
telemetry::kafka::enabled: <%= @kafka_enabled %>
|
||||
telemetry::kafka::port: <%= @kafka_port %>
|
||||
|
||||
telemetry::rabbit::url: "<%= @amqp_url %>"
|
||||
|
||||
telemetry::metadata_fields: "<%= @metadata_fields %>"
|
||||
|
|
|
|||
|
|
@ -12,8 +12,9 @@ $metadata_fields = hiera('telemetry::metadata_fields')
|
|||
# TODO settings/hiera
|
||||
$topics = 'metering.sample'
|
||||
|
||||
#TODO kafka integration
|
||||
$brokerlist = '"broker1:9092"'
|
||||
# Kafka integration
|
||||
$brokerlist = hiera('telemetry::kafka::nodes_list')
|
||||
$kafka_port = hiera('telemetry::kafka::port')
|
||||
|
||||
# Install packages
|
||||
|
||||
|
|
@ -23,6 +24,12 @@ package { 'hindsight': }
|
|||
package { 'librdkafka1': }
|
||||
package { 'lua-sandbox-extensions': }
|
||||
package { 'python-oslo.messaging': }
|
||||
package { 'python-pip': }
|
||||
|
||||
package { 'kafka-python':
|
||||
ensure => '1.2.5',
|
||||
provider => 'pip'
|
||||
}
|
||||
|
||||
# User/group
|
||||
|
||||
|
|
@ -81,12 +88,13 @@ file { '/etc/telemetry_hindsight/hindsight.cfg':
|
|||
}
|
||||
|
||||
# Templates
|
||||
# TODO unhardkode kafka port
|
||||
|
||||
$configs = {
|
||||
"${run_dir}/output/influxdb_ceilometer.cfg" => {
|
||||
content => template( "${templates}/output/influxdb_ceilometer.cfg.erb"),
|
||||
},
|
||||
"${run_dir}/input/ceilometer_kafka.cfg" => {
|
||||
"${run_dir}/input/kafka_input.cfg" => {
|
||||
content => template( "${templates}/input/kafka_input.cfg.erb"),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
filename = "kafka_input.lua"
|
||||
brokerlist = <%= @brokerlist %>
|
||||
<%
|
||||
brokers = @brokerlist.join(":#{@kafka_port}\", \"")
|
||||
brokers = '"'+brokers+":#{@kafka_port}\""
|
||||
-%>
|
||||
brokerlist = {<%= brokers %>}
|
||||
topics = {"<%= @topics %>"}
|
||||
consumer_conf = {
|
||||
["group.id"] = "telemetry_collector",
|
||||
|
|
|
|||
|
|
@ -197,7 +197,7 @@
|
|||
type: puppet
|
||||
version: 2.1.0
|
||||
groups: [primary-controller, controller]
|
||||
requires: [post_deployment_start,telemetry-ceilometer-controller]
|
||||
requires: [post_deployment_start,telemetry-hiera,telemetry-ceilometer-controller]
|
||||
required_for: [post_deployment_end]
|
||||
# required_for: [deploy_end]
|
||||
# requires: [deploy_start, telemetry-ceilometer-controller]
|
||||
|
|
@ -233,8 +233,9 @@
|
|||
type: puppet
|
||||
version: 2.1.0
|
||||
groups: [primary-controller, controller]
|
||||
role: [primary-controller, controller, kafka, primary-kafka]
|
||||
required_for: [post_deployment_end]
|
||||
requires: [telemetry-influxdb-create-db]
|
||||
requires: [telemetry-hiera, telemetry-influxdb-create-db]
|
||||
parameters:
|
||||
puppet_manifest: puppet/manifests/lua-scripts.pp
|
||||
puppet_modules: puppet/modules:/etc/puppet/modules
|
||||
|
|
@ -245,7 +246,7 @@
|
|||
version: 2.1.0
|
||||
groups: [primary-controller, controller]
|
||||
required_for: [post_deployment_end]
|
||||
requires: [telemetry-lua-scripts]
|
||||
requires: [telemetry-influxdb-create-db]
|
||||
parameters:
|
||||
puppet_manifest: puppet/manifests/heka.pp
|
||||
puppet_modules: puppet/modules:/etc/puppet/modules
|
||||
|
|
@ -256,7 +257,8 @@
|
|||
version: 2.1.0
|
||||
groups: [primary-controller, controller]
|
||||
required_for: [post_deployment_end]
|
||||
requires: [telemetry-heka]
|
||||
role: [primary-kafka, kafka]
|
||||
requires: [telemetry-influxdb-create-db, telemetry-hiera]
|
||||
parameters:
|
||||
puppet_manifest: puppet/manifests/hindsight.pp
|
||||
puppet_modules: puppet/modules:/etc/puppet/modules
|
||||
|
|
|
|||
Loading…
Reference in New Issue