Allow ability to configure pipeline publishers

This adds a template for managing the pipeline.yaml file which allows one to
configure the pipeline publishers such as gnocchi or prometheus. The template
file was taken from the latest Rocky tarball for ceilometer.

Change-Id: I938219bfdcfa69a910cae4b72a40c19d5124d99e

attributes/default.rb

@@ -71,6 +71,9 @@ default['openstack']['aodh']['service_role'] = 'admin'
 
 default['openstack']['telemetry_metric']['gnocchi-upgrade-options'] = ''
 
+# Configuration for /etc/ceilometer/pipeline.yaml
+default['openstack']['telemetry']['pipeline']['publishers'] = ['gnocchi://']
+
 %w(telemetry telemetry_metric aodh).each do |ts|
   # specify whether to enable SSL for ceilometer API endpoint
   default['openstack'][ts]['ssl']['enabled'] = false

recipes/agent-central.rb

@@ -32,5 +32,6 @@ end
 service 'ceilometer-agent-central' do
   service_name platform['agent_central_service']
   subscribes :restart, "template[#{node['openstack']['telemetry']['conf_file']}]"
+  subscribes :restart, "template[#{::File.join(node['openstack']['telemetry']['conf_dir'], 'pipeline.yaml')}]"
   action [:enable, :start]
 end

recipes/agent-compute.rb

@@ -32,5 +32,6 @@ end
 service 'ceilometer-agent-compute' do
   service_name platform['agent_compute_service']
   subscribes :restart, "template[#{node['openstack']['telemetry']['conf_file']}]"
+  subscribes :restart, "template[#{::File.join(node['openstack']['telemetry']['conf_dir'], 'pipeline.yaml')}]"
   action [:enable, :start]
 end

recipes/agent-notification.rb

@@ -31,5 +31,6 @@ end
 service 'ceilometer-agent-notification' do
   service_name platform['agent_notification_service']
   subscribes :restart, "template[#{node['openstack']['telemetry']['conf_file']}]"
+  subscribes :restart, "template[#{::File.join(node['openstack']['telemetry']['conf_dir'], 'pipeline.yaml')}]"
   action [:enable, :start]
 end

recipes/common.rb

@@ -103,3 +103,13 @@ template node['openstack']['telemetry']['conf_file'] do
     service_config: ceilometer_conf_options
   )
 end
+
+template ::File.join(node['openstack']['telemetry']['conf_dir'], 'pipeline.yaml') do
+  source 'pipeline.yaml.erb'
+  owner node['openstack']['telemetry']['user']
+  group node['openstack']['telemetry']['group']
+  mode 0o0640
+  variables(
+    publishers: node['openstack']['telemetry']['pipeline']['publishers']
+  )
+end

spec/agent-central_spec.rb

@@ -24,6 +24,9 @@ describe 'openstack-telemetry::agent-central' do
       it 'subscribes to its config file' do
         expect(chef_run.service('ceilometer-agent-central')).to subscribe_to('template[/etc/ceilometer/ceilometer.conf]').delayed
       end
+      it 'subscribes to /etc/ceilometer/pipeline.yaml' do
+        expect(chef_run.service('ceilometer-agent-central')).to subscribe_to('template[/etc/ceilometer/pipeline.yaml]').delayed
+      end
     end
   end
 end

spec/agent-compute_spec.rb

@@ -24,6 +24,9 @@ describe 'openstack-telemetry::agent-compute' do
       it 'subscribes to its config file' do
         expect(chef_run.service('ceilometer-agent-compute')).to subscribe_to('template[/etc/ceilometer/ceilometer.conf]').delayed
       end
+      it 'subscribes to /etc/ceilometer/pipeline.yaml' do
+        expect(chef_run.service('ceilometer-agent-compute')).to subscribe_to('template[/etc/ceilometer/pipeline.yaml]').delayed
+      end
     end
   end
 end

spec/agent-notification_spec.rb

@@ -24,6 +24,9 @@ describe 'openstack-telemetry::agent-notification' do
       it 'subscribes to its config file' do
         expect(chef_run.service('ceilometer-agent-notification')).to subscribe_to('template[/etc/ceilometer/ceilometer.conf]').delayed
       end
+      it 'subscribes to /etc/ceilometer/pipeline.yaml' do
+        expect(chef_run.service('ceilometer-agent-notification')).to subscribe_to('template[/etc/ceilometer/pipeline.yaml]').delayed
+      end
     end
   end
 end

spec/common_spec.rb

@@ -28,6 +28,28 @@ describe 'openstack-telemetry::common' do
       expect(chef_run).to upgrade_package 'ceilometer-common'
     end
 
+    describe '/etc/ceilometer/pipeline.yaml' do
+      pipeline_yaml = <<-EOL
+      publishers:
+          - gnocchi://
+
+EOL
+
+      it do
+        expect(chef_run).to create_template('/etc/ceilometer/pipeline.yaml')
+          .with(
+            source: 'pipeline.yaml.erb',
+            owner: 'ceilometer',
+            group: 'ceilometer',
+            mode: 0o0640
+          )
+      end
+
+      it do
+        expect(chef_run).to render_file('/etc/ceilometer/pipeline.yaml').with_content(pipeline_yaml)
+      end
+    end
+
     describe '/etc/ceilometer' do
       let(:dir) { chef_run.directory('/etc/ceilometer') }
 

templates/pipeline.yaml.erb

@@ -0,0 +1,107 @@
+<%= node['openstack']['common']['custom_template_banner'] %>
+---
+sources:
+    - name: meter_source
+      meters:
+          - "*"
+      sinks:
+          - meter_sink
+    - name: cpu_source
+      meters:
+          - "cpu"
+      sinks:
+          - cpu_sink
+          - cpu_delta_sink
+    - name: disk_source
+      meters:
+          - "disk.read.bytes"
+          - "disk.read.requests"
+          - "disk.write.bytes"
+          - "disk.write.requests"
+          - "disk.device.read.bytes"
+          - "disk.device.read.requests"
+          - "disk.device.write.bytes"
+          - "disk.device.write.requests"
+      sinks:
+          - disk_sink
+    - name: network_source
+      meters:
+          - "network.incoming.bytes"
+          - "network.incoming.packets"
+          - "network.outgoing.bytes"
+          - "network.outgoing.packets"
+      sinks:
+          - network_sink
+sinks:
+    - name: meter_sink
+      publishers:
+          <% @publishers.each do |publisher| %>
+          - <%= publisher %>
+          <% end -%>
+
+    # All these transformers are deprecated, and will be removed in the future, don't use them.
+    - name: cpu_sink
+      transformers:
+          - name: "rate_of_change"
+            parameters:
+                target:
+                    name: "cpu_util"
+                    unit: "%"
+                    type: "gauge"
+                    max: 100
+                    scale: "100.0 / (10**9 * (resource_metadata.cpu_number or 1))"
+      publishers:
+          <% @publishers.each do |publisher| %>
+          - <%= publisher %>
+          <% end -%>
+
+    # All these transformers are deprecated, and will be removed in the future, don't use them.
+    - name: cpu_delta_sink
+      transformers:
+          - name: "delta"
+            parameters:
+                target:
+                    name: "cpu.delta"
+                growth_only: True
+      publishers:
+          <% @publishers.each do |publisher| %>
+          - <%= publisher %>
+          <% end -%>
+
+    # All these transformers are deprecated, and will be removed in the future, don't use them.
+    - name: disk_sink
+      transformers:
+          - name: "rate_of_change"
+            parameters:
+                source:
+                    map_from:
+                        name: "(disk\\.device|disk)\\.(read|write)\\.(bytes|requests)"
+                        unit: "(B|request)"
+                target:
+                    map_to:
+                        name: "\\1.\\2.\\3.rate"
+                        unit: "\\1/s"
+                    type: "gauge"
+      publishers:
+          <% @publishers.each do |publisher| %>
+          - <%= publisher %>
+          <% end -%>
+
+    # All these transformers are deprecated, and will be removed in the future, don't use them.
+    - name: network_sink
+      transformers:
+          - name: "rate_of_change"
+            parameters:
+                source:
+                   map_from:
+                       name: "network\\.(incoming|outgoing)\\.(bytes|packets)"
+                       unit: "(B|packet)"
+                target:
+                    map_to:
+                        name: "network.\\1.\\2.rate"
+                        unit: "\\1/s"
+                    type: "gauge"
+      publishers:
+          <% @publishers.each do |publisher| %>
+          - <%= publisher %>
+          <% end -%>