Adding logrotate configuration for ceilometer

Closes-bug: #1635316

(cherry picked from commit f3996f1657)

 Conflicts:
	ansible/roles/common/templates/heka-openstack.toml.j2

Change-Id: I3195518f1ee52249bc6259d3cab0f479f5522df1

ansible/roles/common/tasks/config.yml

@@ -72,6 +72,7 @@
     - { name: "ansible", enabled: "yes" }
     - { name: "aodh", enabled: "{{ enable_aodh }}" }
     - { name: "barbican", enabled: "{{ enable_barbican }}" }
+    - { name: "ceilometer", enabled: "{{ enable_ceilometer }}" }
     - { name: "cinder", enabled: "{{ enable_cinder }}" }
     - { name: "cloudkitty", enabled: "{{ enable_cloudkitty }}" }
     - { name: "glance", enabled: "{{ enable_glance }}" }

ansible/roles/common/templates/cron-logrotate-ceilometer.conf.j2

@@ -0,0 +1,3 @@
+"/var/log/kolla/ceilometer/*.log"
+{
+}

ansible/roles/common/templates/cron.json.j2

@@ -3,6 +3,7 @@
     ( 'ansible', 'yes' ),
     ( 'aodh', enable_aodh ),
     ( 'barbican', enable_barbican ),
+    ( 'ceilometer', enable_ceilometer ),
     ( 'cinder', enable_cinder ),
     ( 'cloudkitty', enable_cloudkitty ),
     ( 'glance', enable_glance ),

ansible/roles/common/templates/heka-ceilometer.toml.j2

@@ -0,0 +1,13 @@
+[ceilometer_apache_log_decoder]
+type = "SandboxDecoder"
+filename = "lua_decoders/os_ceilometer_apache_log.lua"
+    [ceilometer_apache_log_decoder.config]
+    apache_log_pattern = '%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"'
+
+[ceilometer_apache_logstreamer_input]
+type = "LogstreamerInput"
+decoder = "ceilometer_apache_log_decoder"
+log_directory = "/var/log/kolla"
+file_match = 'ceilometer/(?P<Service>ceilometer-api.*)\.log\.?(?P<Seq>\d*)$'
+priority = ["^Seq"]
+differentiator = ["Service"]

ansible/roles/common/templates/heka-openstack.toml.j2

@@ -6,6 +6,6 @@ filename = "lua_decoders/os_openstack_log.lua"
 type = "LogstreamerInput"
 decoder = "openstack_log_decoder"
 log_directory = "/var/log/kolla"
-file_match = '(?P<Service>cloudkitty|nova|glance|keystone|neutron|ceph|cinder|heat|murano|magnum|mistral|manila|senlin|sahara)/(?P<Program>.*)\.log\.?(?P<Seq>\d*)$'
+file_match = '(?P<Service>cloudkitty|nova|glance|keystone|neutron|ceilometer|ceph|cinder|heat|murano|magnum|mistral|manila|searchlight|senlin|sahara)/(?P<Program>.*)\.log\.?(?P<Seq>\d*)$'
 priority = ["^Seq"]
 differentiator = ["Service", "_", "Program"]

docker/heka/plugins/decoders/os_ceilometer_apache_log.lua

@@ -0,0 +1,72 @@
+-- Copyright 2015 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+local l      = require 'lpeg'
+l.locale(l)
+
+local common_log_format = require 'common_log_format'
+local patt = require 'os_patterns'
+local utils  = require 'os_utils'
+
+local msg = {
+    Timestamp   = nil,
+    Type        = 'log',
+    Hostname    = nil,
+    Payload     = nil,
+    Pid         = nil,
+    Fields      = nil,
+    Severity    = 6,
+}
+
+local severity_label = utils.severity_to_label_map[msg.Severity]
+
+local apache_log_pattern = read_config("apache_log_pattern") or error(
+    "apache_log_pattern configuration must be specificed")
+local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern)
+local request_grammar = l.Ct(patt.http_request)
+
+function process_message ()
+
+    -- logger is either "ceilometer-apache-public" or "ceilometer-apache-admin"
+    local logger = read_message("Logger")
+
+    local log = read_message("Payload")
+
+    local m
+
+    m = apache_grammar:match(log)
+    if m then
+        msg.Logger = 'openstack.ceilometer'
+        msg.Payload = log
+        msg.Timestamp = m.time
+
+        msg.Fields = {}
+        msg.Fields.http_status = m.status
+        msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec
+        msg.Fields.programname = logger
+        msg.Fields.severity_label = severity_label
+
+        local request = m.request
+        m = request_grammar:match(request)
+        if m then
+            msg.Fields.http_method = m.http_method
+            msg.Fields.http_url = m.http_url
+            msg.Fields.http_version = m.http_version
+        end
+
+        return utils.safe_inject_message(msg)
+    end
+
+    return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+end