diff --git a/deployment_scripts/puppet/manifests/configure.pp b/deployment_scripts/puppet/manifests/configure.pp index b2f5614..ac851e0 100644 --- a/deployment_scripts/puppet/manifests/configure.pp +++ b/deployment_scripts/puppet/manifests/configure.pp @@ -70,9 +70,27 @@ create_resources(package, $packages) # Stop not needed any more service service { 'ceilometer-collector': - ensure => stopped, - enable => false, - hasstatus => true, + ensure => stopped, + enable => false, + hasstatus => true, +} + +# Kafka integration + +if hiera('telemetry::kafka::enabled') { + + ceilometer_config { 'oslo_messaging_kafka/consumer_group': value => 'ceilometer' } + + $kafka_ips = hiera('telemetry::kafka::broker_list') + $kafka_url = "kafka://${kafka_ips}" + $rabbit_url = hiera('telemetry::rabbit::url') + + ceilometer_config { 'DEFAULT/transport_url': value => $kafka_url } + ceilometer_config { 'notification/messaging_urls': value => [$kafka_url,$rabbit_url] } + ceilometer_config { 'oslo_messaging_notifications/transport_url': value => $kafka_url } + + ceilometer_config { 'compute/resource_update_interval': value => 600 } + } # TODO validate values before proceed @@ -85,13 +103,13 @@ ceilometer_config { 'notification/store_events': value => false } service { 'ceilometer-service': - ensure => $service_ensure, - name => $::ceilometer::params::api_service_name, - enable => $enabled, - hasstatus => true, - hasrestart => true, - tag => 'ceilometer-service', - } + ensure => $service_ensure, + name => $::ceilometer::params::api_service_name, + enable => $enabled, + hasstatus => true, + hasrestart => true, + tag => 'ceilometer-service', +} Ceilometer_config<||> ~> Service['ceilometer-service'] diff --git a/deployment_scripts/puppet/manifests/heka.pp b/deployment_scripts/puppet/manifests/heka.pp index 89eb18f..5410e48 100644 --- a/deployment_scripts/puppet/manifests/heka.pp +++ b/deployment_scripts/puppet/manifests/heka.pp @@ -1,118 +1,125 @@ -## Get values +notice('MODULAR: fuel-plugin-telemetry: heka.pp') -$config_dir = hiera('telemetry::heka::config_dir') -$amqp_url = hiera('telemetry::rabbit::url') -$metadata_fields = hiera('telemetry::metadata_fields') +if !hiera('telemetry::kafka::enabled') { -if hiera('telemetry::elasticsearch::server',false) { - $ip = hiera('telemetry::elasticsearch::server') - $port = hiera('telemetry::elasticsearch::server') - $elasticsearch_url = "http://${ip}:${port}" -} else { - #no Elasticsearch - #heka failed to start if url schemma is not valid, so we set http here - $elasticsearch_url = 'http://' -} + ## Get values -$influxdb_address = hiera('telemetry::influxdb::address') -$influxdb_port = hiera('telemetry::influxdb::port') -$influxdb_database = hiera('telemetry::influxdb::database') -$influxdb_user = hiera('telemetry::influxdb::user') -$influxdb_password = hiera('telemetry::influxdb::password') + $config_dir = hiera('telemetry::heka::config_dir') + $amqp_url = hiera('telemetry::rabbit::url') + $metadata_fields = hiera('telemetry::metadata_fields') -$modules_dir = hiera('telemetry::lua::modules_dir') -### Heka configuration + if hiera('telemetry::elasticsearch::server',false) { + $ip = hiera('telemetry::elasticsearch::server') + $port = hiera('telemetry::elasticsearch::server') + $elasticsearch_url = "http://${ip}:${port}" + } else { + #no Elasticsearch + #heka failed to start if url schemma is not valid, so we set http here + $elasticsearch_url = 'http://' + } -File { - before => Service['telemetry-collector'] -} + $influxdb_address = hiera('telemetry::influxdb::address') + $influxdb_port = hiera('telemetry::influxdb::port') + $influxdb_database = hiera('telemetry::influxdb::database') + $influxdb_user = hiera('telemetry::influxdb::user') + $influxdb_password = hiera('telemetry::influxdb::password') -file { - "${config_dir}/amqp-openstack_sample.toml": content => template( 'telemetry/heka/amqp-openstack_sample.toml.erb' ); - "${config_dir}/decoder-sample.toml": content => template( 'telemetry/heka/decoder-sample.toml.erb' ); - "${config_dir}/encoder-influxdb.toml": content => template( 'telemetry/heka/encoder-influxdb.toml.erb' ); - "${config_dir}/encoder-resource-elasticsearch.toml": content => template( 'telemetry/heka/encoder-resource-elasticsearch.toml.erb' ); - "${config_dir}/file-output-resource.toml": content => template( 'telemetry/heka/file-output-resource.toml.erb' ); - "${config_dir}/file-output.toml": content => template( 'telemetry/heka/file-output.toml.erb' ); - "${config_dir}/filter-influxdb_accumulator_sample.toml": content => template( 'telemetry/heka/filter-influxdb_accumulator_sample.toml.erb' ); - # TODO disable config when Elasticsearch not in use - "${config_dir}/output-elasticsearch-resource2.toml": content => template( 'telemetry/heka/output-elasticsearch-resource2.toml.erb' ); - "${config_dir}/output-influxdb-samples.toml": content => template( 'telemetry/heka/output-influxdb-samples.toml.erb' ); -} + $modules_dir = hiera('telemetry::lua::modules_dir') -# Heka Installation + ### Heka configuration -$version = hiera('telemetry::heka::version') -$max_message_size = hiera('telemetry::heka::max_message_size') -$max_process_inject = hiera('telemetry::heka::max_process_inject') -$max_timer_inject = hiera('telemetry::heka::max_timer_inject') -$poolsize = hiera('telemetry::heka::poolsize') + File { + before => Service['telemetry-collector'] + } -# TODO we dont't need them on controller -$install_init_script = false + file { + "${config_dir}/amqp-openstack_sample.toml": content => template( 'telemetry/heka/amqp-openstack_sample.toml.erb' ); + "${config_dir}/decoder-sample.toml": content => template( 'telemetry/heka/decoder-sample.toml.erb' ); + "${config_dir}/encoder-influxdb.toml": content => template( 'telemetry/heka/encoder-influxdb.toml.erb' ); + "${config_dir}/encoder-resource-elasticsearch.toml": content => template( 'telemetry/heka/encoder-resource-elasticsearch.toml.erb' ); + "${config_dir}/file-output-resource.toml": content => template( 'telemetry/heka/file-output-resource.toml.erb' ); + "${config_dir}/file-output.toml": content => template( 'telemetry/heka/file-output.toml.erb' ); + "${config_dir}/filter-influxdb_accumulator_sample.toml": content => template( 'telemetry/heka/filter-influxdb_accumulator_sample.toml.erb' ); + # TODO disable config when Elasticsearch not in use + "${config_dir}/output-elasticsearch-resource2.toml": content => template( 'telemetry/heka/output-elasticsearch-resource2.toml.erb' ); + "${config_dir}/output-influxdb-samples.toml": content => template( 'telemetry/heka/output-influxdb-samples.toml.erb' ); + } -# Workaround for heka module -# to prevent changing home of root user -$user='root' -user { $user: } + # Heka Installation -::heka { 'telemetry-collector': - config_dir => '/etc/telemetry-collector', - user => $user, - #additional_groups => $additional_groups, - hostname => $::hostname, - max_message_size => $max_message_size, - max_process_inject => $max_process_inject, - max_timer_inject => $max_timer_inject, - poolsize => $poolsize, - install_init_script => $install_init_script, - version => $version, -} + $version = hiera('telemetry::heka::version') + $max_message_size = hiera('telemetry::heka::max_message_size') + $max_process_inject = hiera('telemetry::heka::max_process_inject') + $max_timer_inject = hiera('telemetry::heka::max_timer_inject') + $poolsize = hiera('telemetry::heka::poolsize') -# Heka pacemaker config + # TODO we dont't need them on controller + $install_init_script = false -pacemaker::service { 'telemetry-collector': - ensure => present, - prefix => false, - primitive_class => 'ocf', - primitive_type => 'ocf-telemetry', - use_handler => false, - complex_type => 'clone', - complex_metadata => { - # the resource should start as soon as the dependent resources - # (eg RabbitMQ) are running *locally* - 'interleave' => true, - }, - metadata => { - # Make sure that Pacemaker tries to restart the resource if it fails - # too many times - 'failure-timeout' => '120s', - 'migration-threshold' => '3', - }, - parameters => { - 'service_name' => 'telemetry-collector', - 'config' => '/etc/telemetry-collector', - 'log_file' => '/var/log/telemetry-collector.log', - 'user' => $user, - }, - operations => { - 'monitor' => { - 'interval' => '20', - 'timeout' => '10', + # Workaround for heka module + # to prevent changing home of root user + $user='root' + user { $user: } + + ::heka { 'telemetry-collector': + config_dir => '/etc/telemetry-collector', + user => $user, + #additional_groups => $additional_groups, + hostname => $::hostname, + max_message_size => $max_message_size, + max_process_inject => $max_process_inject, + max_timer_inject => $max_timer_inject, + poolsize => $poolsize, + install_init_script => $install_init_script, + version => $version, + } + + # Heka pacemaker config + + pacemaker::service { 'telemetry-collector': + ensure => present, + prefix => false, + primitive_class => 'ocf', + primitive_type => 'ocf-telemetry', + use_handler => false, + complex_type => 'clone', + complex_metadata => { + # the resource should start as soon as the dependent resources + # (eg RabbitMQ) are running *locally* + 'interleave' => true, }, - 'start' => { - 'timeout' => '30', + metadata => { + # Make sure that Pacemaker tries to restart the resource if it fails + # too many times + 'failure-timeout' => '120s', + 'migration-threshold' => '3', }, - 'stop' => { - 'timeout' => '30', + parameters => { + 'service_name' => 'telemetry-collector', + 'config' => '/etc/telemetry-collector', + 'log_file' => '/var/log/telemetry-collector.log', + 'user' => $user, }, - }, -} + operations => { + 'monitor' => { + 'interval' => '20', + 'timeout' => '10', + }, + 'start' => { + 'timeout' => '30', + }, + 'stop' => { + 'timeout' => '30', + }, + }, + } + + service { 'telemetry-collector': + ensure => 'running', + enable => true, + provider => 'pacemaker', + } -service { 'telemetry-collector': - ensure => 'running', - enable => true, - provider => 'pacemaker', } diff --git a/deployment_scripts/puppet/manifests/hiera.pp b/deployment_scripts/puppet/manifests/hiera.pp index 2f8def9..bc72f61 100644 --- a/deployment_scripts/puppet/manifests/hiera.pp +++ b/deployment_scripts/puppet/manifests/hiera.pp @@ -98,7 +98,8 @@ if $telemetry['influxdb_address'] { ])) } - # TODO test for multiple inxlixdb nodes !!! + # TODO test for multiple influxdb nodes !!! + # TODO use vip? $influxdb_address = $nodes_array[0]['network_roles']['management'] $retention_period = $influxdb_grafana['retention_period'] @@ -112,17 +113,34 @@ if $telemetry['influxdb_address'] { # Rabbit -$rabbit_info = hiera('rabbit') +$rabbit_info = hiera('rabbit') $rabbit_password = $rabbit_info['password'] -$rabbit_user = $rabbit_info['user'] +$rabbit_user = $rabbit_info['user'] # TODO take one? -$amqp_host = hiera('amqp_hosts') -$amqp_url = "amqp://${rabbit_user}:${rabbit_password}@${amqp_host}/" +$amqp_host = hiera('amqp_hosts') +$amqp_url = "amqp://${rabbit_user}:${rabbit_password}@${amqp_host}/" $metadata_fields = join(['status deleted container_format min_ram updated_at ', 'min_disk is_public size checksum created_at disk_format protected instance_host ', 'host display_name instance_id instance_type status state']) +# Kafka +$kafka_port = 9092 +$kafka_nodes = get_nodes_hash_by_roles($network_metadata, ['kafka', 'primary-kafka']) +$kafka_ip_map = get_node_to_ipaddr_map_by_network_role($kafka_nodes, 'management') + +if count($kafka_ip_map)>0 { + notice('Kafka nodes found') + $kafka_enabled = true + $kafka_ips = sort(values($kafka_ip_map)) + # Format: host:port,host:port for ceiolmeter.conf + $tmp_list = join($kafka_ips,":${kafka_port},") + $broker_list = join([$tmp_list,":${kafka_port}"]) +} else { + notice('No Kafka nodes found') + $kafka_enabled = false +} + $calculated_content = inline_template(' --- ceilometer: @@ -161,6 +179,16 @@ telemetry::heka::max_timer_inject: 10 telemetry::heka::poolsize: 100 telemetry::heka::config_dir: "/etc/telemetry-collector" +<% if @kafka_enabled -%> +telemetry::kafka::broker_list: "<%= @broker_list %>" +telemetry::kafka::nodes_list: +<% @kafka_ips.each do |s| -%> + - "<%= s %>" +<% end -%> +<% end -%> +telemetry::kafka::enabled: <%= @kafka_enabled %> +telemetry::kafka::port: <%= @kafka_port %> + telemetry::rabbit::url: "<%= @amqp_url %>" telemetry::metadata_fields: "<%= @metadata_fields %>" diff --git a/deployment_scripts/puppet/manifests/hindsight.pp b/deployment_scripts/puppet/manifests/hindsight.pp index 19e618e..33b9e37 100644 --- a/deployment_scripts/puppet/manifests/hindsight.pp +++ b/deployment_scripts/puppet/manifests/hindsight.pp @@ -12,8 +12,9 @@ $metadata_fields = hiera('telemetry::metadata_fields') # TODO settings/hiera $topics = 'metering.sample' -#TODO kafka integration -$brokerlist = '"broker1:9092"' +# Kafka integration +$brokerlist = hiera('telemetry::kafka::nodes_list') +$kafka_port = hiera('telemetry::kafka::port') # Install packages @@ -23,6 +24,12 @@ package { 'hindsight': } package { 'librdkafka1': } package { 'lua-sandbox-extensions': } package { 'python-oslo.messaging': } +package { 'python-pip': } + +package { 'kafka-python': + ensure => '1.2.5', + provider => 'pip' +} # User/group @@ -81,12 +88,13 @@ file { '/etc/telemetry_hindsight/hindsight.cfg': } # Templates +# TODO unhardkode kafka port $configs = { "${run_dir}/output/influxdb_ceilometer.cfg" => { content => template( "${templates}/output/influxdb_ceilometer.cfg.erb"), }, - "${run_dir}/input/ceilometer_kafka.cfg" => { + "${run_dir}/input/kafka_input.cfg" => { content => template( "${templates}/input/kafka_input.cfg.erb"), } } diff --git a/deployment_scripts/puppet/modules/telemetry/templates/hindsight/input/kafka_input.cfg.erb b/deployment_scripts/puppet/modules/telemetry/templates/hindsight/input/kafka_input.cfg.erb index 57e123b..96d27d7 100644 --- a/deployment_scripts/puppet/modules/telemetry/templates/hindsight/input/kafka_input.cfg.erb +++ b/deployment_scripts/puppet/modules/telemetry/templates/hindsight/input/kafka_input.cfg.erb @@ -1,9 +1,13 @@ filename = "kafka_input.lua" -brokerlist = <%= @brokerlist %> +<% + brokers = @brokerlist.join(":#{@kafka_port}\", \"") + brokers = '"'+brokers+":#{@kafka_port}\"" +-%> +brokerlist = {<%= brokers %>} topics = {"<%= @topics %>"} consumer_conf = { - ["group.id"] = "telemetry_collector", - ["message.max.bytes"] = 16024, + ["group.id"] = "telemetry_collector", + ["message.max.bytes"] = 16024, } topic_conf = {} metadata_fields = "<%= @metadata_fields %>" diff --git a/deployment_tasks.yaml b/deployment_tasks.yaml index 4ebfb5d..0296ffe 100644 --- a/deployment_tasks.yaml +++ b/deployment_tasks.yaml @@ -197,7 +197,7 @@ type: puppet version: 2.1.0 groups: [primary-controller, controller] - requires: [post_deployment_start,telemetry-ceilometer-controller] + requires: [post_deployment_start,telemetry-hiera,telemetry-ceilometer-controller] required_for: [post_deployment_end] # required_for: [deploy_end] # requires: [deploy_start, telemetry-ceilometer-controller] @@ -233,8 +233,9 @@ type: puppet version: 2.1.0 groups: [primary-controller, controller] + role: [primary-controller, controller, kafka, primary-kafka] required_for: [post_deployment_end] - requires: [telemetry-influxdb-create-db] + requires: [telemetry-hiera, telemetry-influxdb-create-db] parameters: puppet_manifest: puppet/manifests/lua-scripts.pp puppet_modules: puppet/modules:/etc/puppet/modules @@ -245,7 +246,7 @@ version: 2.1.0 groups: [primary-controller, controller] required_for: [post_deployment_end] - requires: [telemetry-lua-scripts] + requires: [telemetry-influxdb-create-db] parameters: puppet_manifest: puppet/manifests/heka.pp puppet_modules: puppet/modules:/etc/puppet/modules @@ -256,7 +257,8 @@ version: 2.1.0 groups: [primary-controller, controller] required_for: [post_deployment_end] - requires: [telemetry-heka] + role: [primary-kafka, kafka] + requires: [telemetry-influxdb-create-db, telemetry-hiera] parameters: puppet_manifest: puppet/manifests/hindsight.pp puppet_modules: puppet/modules:/etc/puppet/modules