Add monitoring agent service

This commit adds Snap and Hindsight Dockerfiles, and an
application definition to deploy Snap and Hindsight in pods
controlled by a DaemonSet.

This commit covers the "collector" part of the
"introduce-monitoring" specification [*].

[*] <https://review.openstack.org/gitweb?p=openstack/fuel-ccp-specs.git;a=blob;f=specs/introduce-monitoring.rst;h=9e50a1a91cbd851ba6d5b5a30ae2c6a2388dc1ba;hb=c8c111f10ab104d0f508d17906b6bd30e94bdfd6>

Change-Id: Ib65499d266c8c19c93efc288294a2f56e71485d8

docker/hindsight/Dockerfile.j2

@@ -0,0 +1,17 @@
+FROM {{ namespace }}/base-tools:{{ tag }}
+MAINTAINER {{ maintainer }}
+
+# NOTE(elemoine) hindsight and lua_sandbox are built from sources (specific
+# commits). In the future we may want to create and host Debian packages.
+
+ADD install.sh /tmp/
+RUN bash /tmp/install.sh \
+    && rm /tmp/install.sh
+
+ADD output/influxdb_tcp.lua /var/lib/hindsight/run/output/
+
+RUN useradd --user-group hindsight \
+    && usermod -a -G microservices hindsight \
+    && chown -R hindsight: /var/lib/hindsight /etc/hindsight
+
+USER hindsight

docker/hindsight/install.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+set -e
+
+#
+# Install build dependencies
+#
+
+BUILD_DEPS="git g++ make cmake"
+apt-get update
+apt-get install -y --no-install-recommends $BUILD_DEPS
+
+#
+# Build lua_sandbox
+#
+
+cd /tmp
+git clone https://github.com/mozilla-services/lua_sandbox.git
+cd lua_sandbox
+# last tested commit
+# https://github.com/mozilla-services/lua_sandbox/commit/f1ee9eb19f4d237b78b585a8b1c6d056e3b3c9fb
+git checkout f1ee9eb19f4d237b78b585a8b1c6d056e3b3c9fb
+mkdir release
+cd release
+cmake -DCMAKE_BUILD_TYPE=release ..
+make
+make install
+
+#
+# Build Hindsight
+#
+
+cd /tmp
+git clone https://github.com/trink/hindsight
+cd hindsight
+# last tested commit
+# https://github.com/trink/hindsight/commit/056321863d4e03f843bbac48ace3ed5e88e4b8fc
+git checkout 056321863d4e03f843bbac48ace3ed5e88e4b8fc
+mkdir release
+cd release
+cmake -DCMAKE_BUILD_TYPE=release ..
+make
+make install
+
+#
+# Create Hindsight dirs and copy lua_sandbox and Hindsight plugins
+# into Hindsight's "run" directory
+#
+
+mkdir -p \
+ /etc/hindsight \
+ /var/lib/hindsight/output \
+ /var/lib/hindsight/load/analysis \
+ /var/lib/hindsight/load/input \
+ /var/lib/hindsight/load/output \
+ /var/lib/hindsight/run/analysis \
+ /var/lib/hindsight/run/input \
+ /var/lib/hindsight/run/output
+
+cp /tmp/lua_sandbox/sandboxes/heka/input/heka_tcp.lua /var/lib/hindsight/run/input/
+cp /tmp/lua_sandbox/sandboxes/heka/input/syslog_udp.lua /var/lib/hindsight/run/input/
+cp /tmp/hindsight/sandboxes/input/prune_input.lua /var/lib/hindsight/run/input/
+
+#
+# Remove build directories
+#
+rm -rf /tmp/lua_sandbox /tmp/hindsight
+apt-get purge -y --auto-remove $BUILD_DEPS
+apt-get clean
+rm -rf /var/lib/apt/lists/*
+
+exit 0

docker/hindsight/output/influxdb_tcp.lua

@@ -0,0 +1,211 @@
+--
+-- Inspired from the lua_sandbox Postgres Output Example
+-- https://github.com/mozilla-services/lua_sandbox/blob/f1ee9eb/docs/heka/output.md#example-postgres-output
+--
+
+local os = require 'os'
+local http = require 'socket.http'
+
+--local write  = require 'io'.write
+--local flush  = require 'io'.flush
+
+local influxdb_host = read_config('host') or error('influxdb host is required')
+local influxdb_port = read_config('port') or error('influxdb port is required')
+
+local batch_max_lines = read_config('batch_max_lines') or 3000
+assert(batch_max_lines > 0, 'batch_max_lines must be greater than zero')
+
+local db = read_config("database") or error("database config is required")
+
+local write_url = string.format('http://%s:%d/write?db=%s', influxdb_host, influxdb_port, db)
+local query_url = string.format('http://%s:%s/query', influxdb_host, influxdb_port)
+
+local database_created = false
+
+local buffer = {}
+local buffer_len = 0
+
+
+local function escape_string(str)
+    return tostring(str):gsub("([ ,])", "\\%1")
+end
+
+local function encode_value(value)
+    if type(value) == "number" then
+        -- Always send numbers as formatted floats, so InfluxDB will accept
+        -- them if they happen to change from ints to floats between
+        -- points in time.  Forcing them to always be floats avoids this.
+        return string.format("%.6f", value)
+    elseif type(value) == "string" then
+        -- string values need to be double quoted
+        return '"' .. value:gsub('"', '\\"') .. '"'
+    elseif type(value) == "boolean" then
+        return '"' .. tostring(value) .. '"'
+    end
+end
+
+local function write_batch()
+    assert(buffer_len > 0)
+    local body = table.concat(buffer, '\n')
+    local resp_body, resp_status = http.request(write_url, body)
+    if resp_body and resp_status == 204 then
+        -- success
+        buffer = {}
+        buffer_len = 0
+        return resp_body, ''
+    else
+        -- error
+        local err_msg = resp_status
+        if resp_body then
+            err_msg = string.format('influxdb write error: [%s] %s',
+                resp_status, resp_body)
+        end
+        return nil, err_msg
+    end
+end
+
+
+local function create_database()
+    -- query won't fail if database already exists
+    local body = string.format('q=CREATE DATABASE %s', db)
+    local resp_body, resp_status = http.request(query_url, body)
+    if resp_body and resp_status == 200 then
+        -- success
+        return resp_body, ''
+    else
+        -- error
+        local err_msg = resp_status
+        if resp_body then
+            err_msg = string.format('influxdb create database error [%s] %s',
+                resp_status, resp_body)
+        end
+        return nil, err_msg
+    end
+end
+
+
+-- return the value and index of the last field with a given name
+local function read_field(name)
+    local i = -1
+    local value = nil
+    local variable_name = string.format('Fields[%s]', name)
+    repeat
+        local tmp = read_message(variable_name, i + 1)
+        if tmp == nil then
+            break
+        end
+        value = tmp
+        i = i + 1
+    until false
+    return value, i
+end
+
+
+-- create a line for the current message, return nil and an error string
+-- if the message is invalid
+local function create_line()
+
+    -- FIXME(elemoine) get hostname from Field[Hostname]
+    local tags = {
+        hostname = escape_string(read_message('Hostname'))
+    }
+    local dimensions, dimensions_index = read_field('dimensions')
+    if dimensions then
+        local i = 0
+        repeat
+            local tag_key = read_message('Fields[dimensions]', dimensions_index, i)
+            if tag_key == nil then
+                break
+            end
+            -- skip the plugin_running_on dimension
+            if tag_key ~= 'plugin_running_on' then
+                local variable_name = string.format('Fields[%s]', tag_key)
+                local tag_val = read_message(variable_name, 0)
+                if tag_val == nil then
+                    -- the dimension is advertized in the "dimensions" field
+                    -- but there is no field for it, so we consider the
+                    -- entire message as invalid
+                    return nil, string.format('dimension "%s" is missing', tag_key)
+                end
+                tags[escape_string(tag_key)] = escape_string(tag_val)
+            end
+            i = i + 1
+        until false
+    end
+
+    if tags['dimensions'] ~= nil and dimensions_index == 0 then
+        return nil, 'index of field "dimensions" should not be 0'
+    end
+
+    local name, name_index = read_field('name')
+    if name == nil then
+        -- "name" is a required field
+        return nil, 'field "name" is missing'
+    end
+    if tags['name'] ~= nil and name_index == 0 then
+        return nil, 'index of field "name" should not be 0'
+    end
+
+    local value, value_index = read_field('value')
+    if value == nil then
+        -- "value" is a required field
+        return nil, 'field "value" is missing'
+    end
+    if tags['value'] ~= nil and value_index == 0 then
+        return nil, 'index of field "value" should not be 0'
+    end
+
+    local tags_array = {}
+    for tag_key, tag_val in pairs(tags) do
+        table.insert(tags_array, string.format('%s=%s', tag_key, tag_val))
+    end
+
+    return string.format('%s,%s value=%s %d',
+        escape_string(name),
+        table.concat(tags_array, ','),
+        encode_value(value),
+        string.format('%d', read_message('Timestamp'))), ''
+end
+
+
+function process_message()
+
+    if not database_created then
+        local ok, err_msg = create_database()
+        if not ok then
+            return -3, err_msg  -- retry
+        end
+        database_created = true
+    end
+
+    local line, err_msg = create_line()
+    if line == nil then
+        -- the message is not valid, skip it
+        return -2, err_msg  -- skip
+    end
+
+    buffer_len = buffer_len + 1
+    buffer[buffer_len] = line
+
+    if buffer_len > batch_max_lines then
+        local ok, err_msg = write_batch()
+        if not ok then
+            buffer[buffer_len] = nil
+            buffer_len = buffer_len - 1
+            return -3, err_msg  -- retry
+        end
+        return 0
+    end
+
+    return -4 -- batching
+end
+
+
+function timer_event(ns)
+    if buffer_len > 0 then
+        local ok, _ = write_batch()
+        if ok then
+            update_checkpoint()
+        end
+    end
+end

docker/snap/Dockerfile.j2

@@ -0,0 +1,18 @@
+FROM {{ namespace }}/base-tools:{{ tag }}
+MAINTAINER {{ maintainer }}
+
+RUN mkdir -p /etc/snap/auto
+
+# install Go and Snap
+ADD install.sh /tmp/
+RUN apt-get -y -t jessie-backports --no-install-recommends install golang \
+    && bash /tmp/install.sh /etc/snap/auto \
+    && rm /tmp/install.sh \
+    && useradd --user-group snap \
+    && usermod -a -G microservices snap \
+    && chown -R snap: /etc/snap \
+    && apt-get purge -y --auto-remove golang \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+USER snap

docker/snap/install.sh

@@ -0,0 +1,181 @@
+#!/bin/bash
+
+set -e
+
+AUTO_DISCOVERY_PATH="$1"
+
+#
+# Install build dependencies
+#
+
+BUILD_DEPS="debhelper fakeroot g++ git libc6-dev make cmake"
+apt-get update
+apt-get install -y --no-install-recommends $BUILD_DEPS
+
+#
+# Install Snap and Snap plugins
+#
+
+export GOPATH="/go"
+GOPATH_ORIG=$GOPATH
+
+mkdir -p "$GOPATH/src" "$GOPATH/bin"
+chmod -R 777 "$GOPATH"
+
+export PATH=/usr/local/go/bin:$GOPATH/bin/:$PATH
+
+go get github.com/tools/godep
+
+# Get Snap
+go get -d github.com/intelsdi-x/snap
+
+# Get Snap plugins
+
+# https://github.com/intelsdi-x/snap-plugin-collector-cpu/commit/c4a90ddf785835a3d6d830eb0292aa418b2bcd9e
+REFSPEC="c4a90ddf785835a3d6d830eb0292aa418b2bcd9e"
+go get -d github.com/intelsdi-x/snap-plugin-collector-cpu
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-cpu
+git checkout ${REFSPEC}
+make deps
+
+# https://github.com/intelsdi-x/snap-plugin-collector-df/commit/411e248ccf2c3897548a08336470b6390e9f6e68
+REFSPEC="411e248ccf2c3897548a08336470b6390e9f6e68"
+go get -d github.com/intelsdi-x/snap-plugin-collector-df
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-df
+git checkout ${REFSPEC}
+make deps
+
+# https://github.com/intelsdi-x/snap-plugin-collector-disk/commit/8af1b89502c584aba37c18da1d12313cee53f8a0
+REFSPEC="8af1b89502c584aba37c18da1d12313cee53f8a0"
+go get -d github.com/intelsdi-x/snap-plugin-collector-disk
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-disk
+git checkout ${REFSPEC}
+make deps
+
+# https://github.com/intelsdi-x/snap-plugin-collector-interface/commit/d2c8ff6bdf6277b4d2f3c653d603fe6b65d3196a
+REFSPEC="d2c8ff6bdf6277b4d2f3c653d603fe6b65d3196a"
+go get -d github.com/intelsdi-x/snap-plugin-collector-interface
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-interface
+git checkout ${REFSPEC}
+make deps
+
+# https://github.com/intelsdi-x/snap-plugin-collector-load/commit/dc0b214ace415f31093fea2ba89384c7f994102e
+REFSPEC="dc0b214ace415f31093fea2ba89384c7f994102e"
+go get -d github.com/intelsdi-x/snap-plugin-collector-load
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-load
+git checkout ${REFSPEC}
+make deps
+
+# https://github.com/intelsdi-x/snap-plugin-collector-meminfo/commit/983c4ae32ef38cb1acd886309f97a389a264179b
+REFSPEC="983c4ae32ef38cb1acd886309f97a389a264179b"
+go get -d github.com/intelsdi-x/snap-plugin-collector-meminfo
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-meminfo
+git checkout ${REFSPEC}
+make deps
+
+# https://github.com/intelsdi-x/snap-plugin-collector-processes/commit/6a40b01a1fe0953c1c7cfa85d919e89745673f21
+REFSPEC="6a40b01a1fe0953c1c7cfa85d919e89745673f21"
+go get -d github.com/intelsdi-x/snap-plugin-collector-processes
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-processes
+git checkout ${REFSPEC}
+make deps
+
+# https://github.com/intelsdi-x/snap-plugin-collector-swap/commit/4afdd8658cef1bb5744b38c9a77aa4589afd5f8d
+REFSPEC="4afdd8658cef1bb5744b38c9a77aa4589afd5f8d"
+go get -d github.com/intelsdi-x/snap-plugin-collector-swap
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-swap
+git checkout ${REFSPEC}
+make deps
+
+# Build Snap
+
+# https://github.com/intelsdi-x/snap/commit/5137f6cb389b3502b8d79d05aac2d5a6ecec1b41
+REFSPEC="5137f6cb389b3502b8d79d05aac2d5a6ecec1b41"
+cd $GOPATH/src/github.com/intelsdi-x/snap
+git checkout ${REFSPEC}
+make deps all install
+cp build/plugin/* /usr/local/bin/
+ln -s /usr/local/bin/snap-plugin-processor-passthru $AUTO_DISCOVERY_PATH
+# the mock-file publisher plugin may be useful for debugging
+ln -s /usr/local/bin/snap-plugin-publisher-mock-file $AUTO_DISCOVERY_PATH
+
+# Build Snap plugins
+
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-cpu
+make all
+cp build/rootfs/snap-plugin-collector-cpu /usr/local/bin
+ln -s /usr/local/bin/snap-plugin-collector-cpu $AUTO_DISCOVERY_PATH
+
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-df
+make all
+cp build/rootfs/snap-plugin-collector-df /usr/local/bin
+ln -s /usr/local/bin/snap-plugin-collector-df $AUTO_DISCOVERY_PATH
+
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-disk
+make all
+cp build/rootfs/snap-plugin-collector-disk /usr/local/bin
+ln -s /usr/local/bin/snap-plugin-collector-disk $AUTO_DISCOVERY_PATH
+
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-interface
+make all
+cp build/rootfs/snap-plugin-collector-interface /usr/local/bin
+ln -s /usr/local/bin/snap-plugin-collector-interface $AUTO_DISCOVERY_PATH
+
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-load
+make all
+cp build/rootfs/snap-plugin-collector-load /usr/local/bin
+ln -s /usr/local/bin/snap-plugin-collector-load $AUTO_DISCOVERY_PATH
+
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-meminfo
+make all
+cp build/rootfs/snap-plugin-collector-meminfo /usr/local/bin
+ln -s /usr/local/bin/snap-plugin-collector-meminfo $AUTO_DISCOVERY_PATH
+
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-processes
+make all
+cp build/rootfs/snap-plugin-collector-processes /usr/local/bin
+ln -s /usr/local/bin/snap-plugin-collector-processes $AUTO_DISCOVERY_PATH
+# the "processes" plugin accesses files like /proc/1/io which
+# only the "root" can read
+chmod u+s /usr/local/bin/snap-plugin-collector-processes
+
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-collector-swap
+make all
+cp build/rootfs/snap-plugin-collector-swap /usr/local/bin
+ln -s /usr/local/bin/snap-plugin-collector-swap $AUTO_DISCOVERY_PATH
+
+# Get Heka and build the minimun required for the Heka publisher plugin
+# https://github.com/mozilla-services/heka/commit/e8799385d16915a80b69061d05542da6342e58e4
+REFSPEC="e8799385d16915a80b69061d05542da6342e58e4"
+cd /tmp
+git clone -b dev --single-branch https://github.com/mozilla-services/heka
+cd heka
+git checkout ${REFSPEC}
+source env.sh  # changes GOPATH to /tmp/heka/build/heka
+mkdir -p build
+cd build
+cmake ..
+make message_matcher_parser
+
+# Get and build Heka plugin
+# https://github.com/intelsdi-x/snap-plugin-publisher-heka/commit/e95f8cc48edf29fc8fd8ab2fa3f0c6f6ab054674
+REFSPEC="e95f8cc48edf29fc8fd8ab2fa3f0c6f6ab054674"
+go get -d github.com/intelsdi-x/snap-plugin-publisher-heka
+cd $GOPATH/src/github.com/intelsdi-x/snap
+make deps
+cd $GOPATH/src/github.com/intelsdi-x/snap-plugin-publisher-heka
+git checkout ${REFSPEC}
+make deps all
+cp build/rootfs/snap-plugin-publisher-heka /usr/local/bin
+ln -s /usr/local/bin/snap-plugin-publisher-heka $AUTO_DISCOVERY_PATH
+rm -rf /tmp/heka
+
+#
+# Clean up
+#
+apt-get purge -y --auto-remove $BUILD_DEPS
+apt-get clean
+rm -rf /var/lib/apt/lists/*
+rm -rf $GOPATH_ORIG
+
+exit 0

service/files/defaults.yaml

@@ -4,15 +4,17 @@ configs:
   kibana_port: 5601
   heka_max_procs: 2
   heka_service_pattern: "^k8s_(.-)%..*"
+  hindsight_heka_tcp_port: 5565
   grafana_host: "grafana"
   grafana_port: 3000
   grafana_user: "admin"
   grafana_password: "admin"
-  influxdb_database: "mcp"
+  influxdb_database: "ccp"
   influxdb_host: "influxdb"
   influxdb_password: ""
   influxdb_port: 8086
   influxdb_user: ""
+  snap_log_level: 3
 versions:
   influxdb_version: "0.13.0"
   grafana_version: "3.0.3-1463994644"

service/files/hindsight.cfg

@@ -0,0 +1,23 @@
+output_path             = "/var/lib/hindsight/output"
+sandbox_load_path       = "/var/lib/hindsight/load"
+sandbox_run_path        = "/var/lib/hindsight/run"
+analysis_lua_path       = "/usr/local/lib/luasandbox/modules/?.lua"
+analysis_lua_cpath      = "/usr/local/lib/luasandbox/modules/?.so"
+io_lua_path             = analysis_lua_path .. ";/usr/local/lib/luasandbox/io_modules/?.lua"
+io_lua_cpath            = analysis_lua_cpath .. ";/usr/local/lib/luasandbox/io_modules/?.so"
+
+-- FIXME(elemoine) set "hostname"
+-- hostname =
+
+input_defaults = {
+  -- output_limit      = 64 * 1024
+  -- memory_limit      = 8 * 1024 * 1024
+  -- instruction_limit = 1e6
+  -- preserve_data     = false
+  -- ticker_interval   = 0
+}
+analysis_defaults = {
+}
+output_defaults = {
+}
+

service/files/hindsight_heka_tcp.cfg

@@ -0,0 +1,6 @@
+filename = "heka_tcp.lua"
+address = "localhost"
+port = {{ hindsight_heka_tcp_port }}
+-- the heka_tcp plugin is a "Continuous" plugin, so instruction_limit
+-- must be set to zero
+instruction_limit = 0

service/files/hindsight_influxdb_tcp.cfg

@@ -0,0 +1,8 @@
+filename = "influxdb_tcp.lua"
+host = "influxdb"
+port = {{ influxdb_port }}
+database = "{{ influxdb_database }}"
+batch_max_lines = 3000
+message_matcher = "TRUE"
+ticker_interval = 10
+log_level = 7

service/files/hindsight_prune_input.cfg

@@ -0,0 +1,5 @@
+filename = "prune_input.lua"
+ticker_interval = 60
+input = true
+analysis = true
+exit_on_stall = false

service/files/snap-task.json

@@ -0,0 +1,102 @@
+{
+  "version": 1,
+  "schedule": {
+    "type": "simple",
+    "interval": "10s"
+  },
+  "workflow": {
+    "collect": {
+      "config": {
+          "/intel/procfs": {
+              "proc_path":  "/host-proc"
+          }
+      },
+      "metrics": {
+        "/intel/procfs/cpu/all/idle_percentage": {},
+        "/intel/procfs/cpu/all/iowait_percentage": {},
+        "/intel/procfs/cpu/all/irq_percentage": {},
+        "/intel/procfs/cpu/all/nice_percentage": {},
+        "/intel/procfs/cpu/all/softirq_percentage": {},
+        "/intel/procfs/cpu/all/steal_percentage": {},
+        "/intel/procfs/cpu/all/system_percentage": {},
+        "/intel/procfs/cpu/all/user_percentage": {},
+        "/intel/procfs/filesystem/*/inodes_free": {},
+        "/intel/procfs/filesystem/*/inodes_reserved": {},
+        "/intel/procfs/filesystem/*/inodes_used": {},
+        "/intel/procfs/filesystem/*/space_free": {},
+        "/intel/procfs/filesystem/*/space_reserved": {},
+        "/intel/procfs/filesystem/*/space_used": {},
+        "/intel/procfs/filesystem/*/inodes_percent_free": {},
+        "/intel/procfs/filesystem/*/inodes_percent_reserved": {},
+        "/intel/procfs/filesystem/*/inodes_percent_used": {},
+        "/intel/procfs/filesystem/*/space_percent_free": {},
+        "/intel/procfs/filesystem/*/space_percent_reserved": {},
+        "/intel/procfs/filesystem/*/space_percent_used": {},
+        "/intel/procfs/filesystem/*/device_name": {},
+        "/intel/procfs/filesystem/*/device_type": {},
+        "/intel/procfs/disk/*/merged_read": {},
+        "/intel/procfs/disk/*/merged_write": {},
+        "/intel/procfs/disk/*/octets_read": {},
+        "/intel/procfs/disk/*/octets_write": {},
+        "/intel/procfs/disk/*/ops_read": {},
+        "/intel/procfs/disk/*/ops_write": {},
+        "/intel/procfs/disk/*/time_read": {},
+        "/intel/procfs/disk/*/time_write": {},
+        "/intel/procfs/iface/*/bytes_recv": {},
+        "/intel/procfs/iface/*/bytes_sent": {},
+        "/intel/procfs/iface/*/compressed_recv": {},
+        "/intel/procfs/iface/*/compressed_sent": {},
+        "/intel/procfs/iface/*/drop_recv": {},
+        "/intel/procfs/iface/*/drop_sent": {},
+        "/intel/procfs/iface/*/errs_recv": {},
+        "/intel/procfs/iface/*/errs_sent": {},
+        "/intel/procfs/iface/*/fifo_recv": {},
+        "/intel/procfs/iface/*/fifo_sent": {},
+        "/intel/procfs/iface/*/frame_recv": {},
+        "/intel/procfs/iface/*/frame_sent": {},
+        "/intel/procfs/iface/*/multicast_recv": {},
+        "/intel/procfs/iface/*/multicast_sent": {},
+        "/intel/procfs/iface/*/packets_recv": {},
+        "/intel/procfs/iface/*/packets_sent": {},
+        "/intel/procfs/load/min1": {},
+        "/intel/procfs/load/min5": {},
+        "/intel/procfs/load/min15": {},
+        "/intel/procfs/load/min1_rel": {},
+        "/intel/procfs/load/min5_rel": {},
+        "/intel/procfs/load/min15_rel": {},
+        "/intel/procfs/load/runnable_scheduling": {},
+        "/intel/procfs/load/existing_scheduling": {},
+        "/intel/procfs/meminfo/buffers": {},
+        "/intel/procfs/meminfo/cached": {},
+        "/intel/procfs/meminfo/mem_free": {},
+        "/intel/procfs/meminfo/mem_used": {},
+        "/intel/procfs/processes/dead": {},
+        "/intel/procfs/processes/parked": {},
+        "/intel/procfs/processes/running": {},
+        "/intel/procfs/processes/sleeping": {},
+        "/intel/procfs/processes/stopped": {},
+        "/intel/procfs/processes/tracing": {},
+        "/intel/procfs/processes/waiting": {},
+        "/intel/procfs/processes/wakekill": {},
+        "/intel/procfs/processes/waking": {},
+        "/intel/procfs/processes/zombie": {},
+        "/intel/procfs/swap/all/cached_bytes": {},
+        "/intel/procfs/swap/all/free_bytes": {},
+        "/intel/procfs/swap/io/in_pages_per_sec": {},
+        "/intel/procfs/swap/io/out_pages_per_sec": {},
+        "/intel/procfs/swap/all/used_bytes": {}
+      },
+      "process": [{
+        "plugin_name": "passthru",
+        "process": null,
+        "publish": [{
+          "plugin_name": "heka",
+          "config": {
+            "host": "localhost",
+            "port": {{ hindsight_heka_tcp_port }}
+          }
+        }]
+      }]
+    }
+  }
+}

service/files/snap.conf

@@ -0,0 +1,4 @@
+log_level: {{ snap_log_level }}
+control:
+  plugin_trust_level: 0
+  auto_discover_path: /etc/snap/auto

service/stacklight-collector.yaml

@@ -0,0 +1,61 @@
+service:
+  name: stacklight-collector
+  daemonset: true
+  containers:
+    - name: hindsight
+      image: hindsight
+      probes:
+        readiness: "true"
+        liveness: "true"
+      daemon:
+        command: /usr/local/bin/hindsight /etc/hindsight/hindsight.cfg
+        files:
+          - hindsight.cfg
+          - heka-tcp.cfg
+          - prune-input.cfg
+          - influxdb-tcp.cfg
+      volumes:
+        - name: hindsight-output
+          type: empty-dir
+          path: /var/lib/hindsight/output
+    - name: snap
+      image: snap
+      probes:
+        readiness: "true"
+        liveness: "true"
+      privileged: true
+      daemon:
+        command: snapd --config /etc/snap/snap.conf
+        files:
+          - snap.conf
+          - snap-task.json
+      volumes:
+        - name: proc
+          type: host
+          path: /proc
+          mount-path: /host-proc
+files:
+  hindsight.cfg:
+    path: /etc/hindsight/hindsight.cfg
+    content: hindsight.cfg
+    perm: "0600"
+  heka-tcp.cfg:
+    path: /var/lib/hindsight/run/input/heka_tcp.cfg
+    content: hindsight_heka_tcp.cfg
+    perm: "0600"
+  prune-input.cfg:
+    path: /var/lib/hindsight/run/input/prune_input.cfg
+    content: hindsight_prune_input.cfg
+    perm: "0600"
+  influxdb-tcp.cfg:
+    path: /var/lib/hindsight/run/output/influxdb_tcp.cfg
+    content: hindsight_influxdb_tcp.cfg
+    perm: "0600"
+  snap.conf:
+    path: /etc/snap/snap.conf
+    content: snap.conf
+    perm: "0600"
+  snap-task.json:
+    path: /etc/snap/auto/task.json
+    content: snap-task.json
+    perm: "0600"