From a6f365dfa55245f244d069ec533eae5f12de4013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Lemoine?= Date: Tue, 13 Sep 2016 12:09:37 +0000 Subject: [PATCH] Add support for multi-value metrics This commit adds support for metrics with multiple values. Multi-value metrics will for example be needed for alarming. Change-Id: I496fa1925c389f2638cf9b99243fbf45d7d2dad7 --- docker/hindsight/Dockerfile.j2 | 5 +- docker/hindsight/modules/message.lua | 88 ++++++++++++++++++++++++ docker/hindsight/output/influxdb_tcp.lua | 49 ++++++------- service/files/hindsight.cfg.j2 | 2 +- 4 files changed, 114 insertions(+), 30 deletions(-) create mode 100644 docker/hindsight/modules/message.lua diff --git a/docker/hindsight/Dockerfile.j2 b/docker/hindsight/Dockerfile.j2 index 81514a1..1014dc3 100644 --- a/docker/hindsight/Dockerfile.j2 +++ b/docker/hindsight/Dockerfile.j2 @@ -17,8 +17,9 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 1FA22B08 \ /usr/share/luasandbox/sandboxes/heka/input/heka_tcp.lua \ /var/lib/hindsight/run/input/ -ADD output/influxdb_tcp.lua /var/lib/hindsight/run/output/ -ADD input/kubelet_stats.lua /var/lib/hindsight/run/input/ +ADD output/*.lua /var/lib/hindsight/run/output/ +ADD input/*.lua /var/lib/hindsight/run/input/ +ADD modules/*.lua /opt/ccp/lua/modules/stacklight/ RUN useradd --user-group hindsight \ && usermod -a -G microservices hindsight \ diff --git a/docker/hindsight/modules/message.lua b/docker/hindsight/modules/message.lua new file mode 100644 index 0000000..6cc9e5c --- /dev/null +++ b/docker/hindsight/modules/message.lua @@ -0,0 +1,88 @@ +-- Copyright 2015-2016 Mirantis, Inc. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +local cjson = require 'cjson' + +local inject_message = inject_message +local read_message = read_message +local string = string +local pcall = pcall + +local M = {} +setfenv(1, M) -- Remove external access to contain everything in the module + +-- Return the value and index of the last field with a given name. +function read_field(name) + local i = -1 + local value = nil + local variable_name = string.format('Fields[%s]', name) + repeat + local tmp = read_message(variable_name, i + 1) + if tmp == nil then + break + end + value = tmp + i = i + 1 + until false + return value, i +end + + +-- Extract value(s) from the message. The value can be either a scalar value +-- or a table for multi-value metrics.Return nil and an error message on +-- failure. The argument "tags" is optional, it's used for sanity checks. +function read_values(tags) + if not tags then + tags = {} + end + local value + local value_fields, value_fields_index = read_field('value_fields') + if value_fields ~= nil then + if tags['value_fields'] ~= nil and value_fields_index == 0 then + return nil, 'index of field "value_fields" should not be 0' + end + local i = 0 + value = {} + repeat + local value_key = read_message( + 'Fields[value_fields]', value_fields_index, i) + if value_key == nil then + break + end + local value_val, value_index = read_field(value_key) + if value_val == nil then + return nil, string.format('field "%s" is missing', value_key) + end + if tags[value_key] ~= nil and value_index == 0 then + return nil, string.format( + 'index of field "%s" should not be 0', value_key) + end + value[value_key] = value_val + i = i + 1 + until false + else + local value_index + value, value_index = read_field('value') + if value == nil then + -- "value" is a required field + return nil, 'field "value" is missing' + end + if tags['value'] ~= nil and value_index == 0 then + return nil, 'index of field "value" should not be 0' + end + end + return value, '' +end + +return M diff --git a/docker/hindsight/output/influxdb_tcp.lua b/docker/hindsight/output/influxdb_tcp.lua index b16e758..c89ea8e 100644 --- a/docker/hindsight/output/influxdb_tcp.lua +++ b/docker/hindsight/output/influxdb_tcp.lua @@ -5,6 +5,7 @@ local os = require 'os' local http = require 'socket.http' +local message = require 'stacklight.message' --local write = require 'io'.write --local flush = require 'io'.flush @@ -30,7 +31,7 @@ local function escape_string(str) return tostring(str):gsub("([ ,])", "\\%1") end -local function encode_value(value) +local function encode_scalar_value(value) if type(value) == "number" then -- Always send numbers as formatted floats, so InfluxDB will accept -- them if they happen to change from ints to floats between @@ -44,6 +45,21 @@ local function encode_value(value) end end +local function encode_value(value) + if type(value) == "table" then + local values = {} + for k, v in pairs(value) do + table.insert( + values, + string.format("%s=%s", escape_string(k), encode_scalar_value(v)) + ) + end + return tablec.concat(values, ',') + else + return "value=" .. encode_scalar_value(value) + end +end + local function write_batch() assert(buffer_len > 0) local body = table.concat(buffer, '\n') @@ -84,29 +100,12 @@ local function create_database() end --- return the value and index of the last field with a given name -local function read_field(name) - local i = -1 - local value = nil - local variable_name = string.format('Fields[%s]', name) - repeat - local tmp = read_message(variable_name, i + 1) - if tmp == nil then - break - end - value = tmp - i = i + 1 - until false - return value, i -end - - -- create a line for the current message, return nil and an error string -- if the message is invalid local function create_line() local tags = {} - local dimensions, dimensions_index = read_field('dimensions') + local dimensions, dimensions_index = message.read_field('dimensions') if dimensions then local i = 0 repeat @@ -134,7 +133,7 @@ local function create_line() return nil, 'index of field "dimensions" should not be 0' end - local name, name_index = read_field('name') + local name, name_index = message.read_field('name') if name == nil then -- "name" is a required field return nil, 'field "name" is missing' @@ -143,13 +142,9 @@ local function create_line() return nil, 'index of field "name" should not be 0' end - local value, value_index = read_field('value') + local value, err_msg = message.read_values(tags) if value == nil then - -- "value" is a required field - return nil, 'field "value" is missing' - end - if tags['value'] ~= nil and value_index == 0 then - return nil, 'index of field "value" should not be 0' + return nil, err_msg end local tags_array = {} @@ -157,7 +152,7 @@ local function create_line() table.insert(tags_array, string.format('%s=%s', tag_key, tag_val)) end - return string.format('%s,%s value=%s %d', + return string.format('%s,%s %s %d', escape_string(name), table.concat(tags_array, ','), encode_value(value), diff --git a/service/files/hindsight.cfg.j2 b/service/files/hindsight.cfg.j2 index 3086ee6..bacc36b 100644 --- a/service/files/hindsight.cfg.j2 +++ b/service/files/hindsight.cfg.j2 @@ -1,7 +1,7 @@ output_path = "/var/lib/hindsight/output" sandbox_load_path = "/var/lib/hindsight/load" sandbox_run_path = "/var/lib/hindsight/run" -analysis_lua_path = "/usr/lib/x86_64-linux-gnu/luasandbox/modules/?.lua" +analysis_lua_path = "/usr/lib/x86_64-linux-gnu/luasandbox/modules/?.lua;/opt/ccp/lua/modules/?.lua" analysis_lua_cpath = "/usr/lib/x86_64-linux-gnu/luasandbox/modules/?.so" io_lua_path = analysis_lua_path .. ";/usr/lib/x86_64-linux-gnu/luasandbox/io_modules/?.lua" io_lua_cpath = analysis_lua_cpath .. ";/usr/lib/x86_64-linux-gnu/luasandbox/io_modules/?.so"