Initial switch to influxdb

Berksfile

@@ -17,4 +17,5 @@ cookbook 'vertica', git: 'https://github.com/hpcloud-mon/cookbooks-vertica'
 cookbook 'zookeeper', git: 'https://github.com/hpcloud-mon/cookbooks-zookeeper'
 
 # Community cookbooks
+cookbook 'influxdb', git: 'https://github.com/SimpleFinance/chef-influxdb'
 cookbook 'percona', git: 'https://github.com/phlipper/chef-percona'

README.md

@@ -31,15 +31,6 @@ Installs a mini monitoring environment based on Vagrant. Intended for developmen
 ```
 git clone https://github.com/hpcloud-mon/mon-vagrant
 ```
-Vertica must be downloaded from the [Vertica site](https://my.vertica.com/). Download these packages and place in the root of this repository.
-
-- `vertica_7.0.1-0_amd64.deb`
-- `vertica-r-lang_7.0.1-0_amd64.deb`
-
-The `vertica::console` recipe is not enabled by default, but if it is added, this package is also needed.
-
-- `vertica-console_7.0.1-0_amd64.deb`
-
 ## Install Vagrant
 
 ### Install VirtualBox and Vagrant
@@ -131,6 +122,24 @@ VM to use them also.
 vagrant plugin install vagrant-proxyconf
 ```
 
+# Vertica
+Vertica is supported instead of influxdb, this is especially useful for large deployments.
+
+Before using Vertica must be downloaded from the [Vertica site](https://my.vertica.com/). Download these packages and place in the root of this repository.
+
+- `vertica_7.0.1-0_amd64.deb`
+- `vertica-r-lang_7.0.1-0_amd64.deb`
+
+The `vertica::console` recipe is not enabled by default, but if it is added, this package is also needed.
+
+- `vertica-console_7.0.1-0_amd64.deb`
+
+After the vertica packages are installed the configuration must be changed to run Vertica. Roles and data_bags for this setup are provided, copy them
+in place with the following commands then run `vagrant up` as normal.
+- `cp roles/Mini-Mon-Vertica.json roles/Mini-Mon.json`
+- `cp data_bags/mon_persister/mon_persister-Vertica.json data_bags/mon_persister/mon_persister.json`
+- `cp data_bags/mon_api/mon_api-Vertica.json data_bags/mon_api/mon_api.json`
+
 # Alternate Vagrant Configurations
 To run any of these alternate configs, simply run the Vagrant commands from within the subdir.  Note that the Vertica debs must be _copied_
 (not symlinked) into the subdir as well. See the README.md in the subdir for more details.

cookbooks/mini-mon/README.md

@@ -9,6 +9,7 @@ apt cookbook
 Recipes
 ---------
 - default - configures base apt repostories needed for mini-mon as well as option apt cache
+- influxdb - creates an influxdb database and users
 - keystone - simple keystone installation
 - mysql_schema - Defines the mysql_schema used by mini-mon
 - postfix - extremely simple postfix install

cookbooks/mini-mon/metadata.rb

@@ -4,5 +4,6 @@ maintainer_email "hpcs-mon@hp.com"
 license          "All rights reserved"
 description      "Base setup for all vagrant boxes"
 long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
-version          "0.0.8"
+version          "0.0.9"
 depends          "apt"
+depends          "influxdb"

cookbooks/mini-mon/recipes/influxdb.rb

@@ -0,0 +1,14 @@
+# Create an influxdb database and users
+# Leverages the cookbook from https://github.com/SimpleFinance/chef-influxdb
+
+influxdb_database 'mon' do
+  action :create
+end
+
+['mon_api', 'mon_persister'].each do |user|
+  influxdb_user user do
+    action :create
+    password 'password'
+    databases ['mon']
+  end
+end

data_bags/mon_api/mon_api-Vertica.json

@@ -0,0 +1,21 @@
+{
+  "id" : "mon_api",
+  "api_region": "useast",
+  "database-configuration": {
+    "database-type": "vertica"
+  },
+  "vertica" : {
+    "dbname" : "mon",
+    "hostname" : "localhost"
+  },
+  "zookeeper" : {
+    "hostname" : "localhost"
+  },
+  "mysql": {
+   "hostname":"localhost",
+   "schema": "mon"
+  },
+  "kafka": {
+    "hostname": "localhost"
+  }
+}

data_bags/mon_api/mon_api.json

@@ -2,9 +2,9 @@
   "id" : "mon_api",
   "api_region": "useast",
   "database-configuration": {
-    "database-type": "vertica"
+    "database-type": "influxdb"
   },
-  "vertica" : {
+  "influxdb" : {
     "dbname" : "mon",
     "hostname" : "localhost"
   },

data_bags/mon_api/mon_credentials.json

@@ -14,6 +14,11 @@
         "password": "password",
         "schema": "mon"
     },
+    "influxdb": {
+        "hostname": "localhost",
+        "username": "mon_api",
+        "password": "password"
+    },
     "vertica": {
         "hostname": "localhost",
         "username": "mon_api",

data_bags/mon_persister/mon_credentials.json

@@ -1,5 +1,9 @@
 {
   "id" : "mon_credentials",
+  "influxdb": {
+    "user": "root",
+    "password": "root"
+  },
   "vertica" : {
     "user" : "dbadmin",
     "password" : "password"

data_bags/mon_persister/mon_persister-Vertica.json

@@ -0,0 +1,37 @@
+{
+    "id": "mon_persister",
+    "alarm_history": {
+        "topic": "alarm-state-transitions"
+    },
+    "metrics": {
+        "topic": "metrics"
+    },
+    "kafka": {
+        "num_threads": "1",
+        "group_id": "1",
+        "consumer_id": 1
+    },
+    "disruptor": {
+        "buffer_size": "1048576",
+        "num_processors": "1"
+    },
+    "output_processor": {
+        "batch_size": "10000"
+    },
+    "deduper_config": {
+        "dedupe_run_frequencey_seconds": "30"
+    },
+    "database_configuration": {
+        "database_type": "vertica"
+    },
+    "vertica_metric_repository_config": {
+        "max_cache_size": "2000000"
+    },
+    "vertica": {
+        "dbname": "mon",
+        "hostname": "localhost"
+    },
+    "zookeeper": {
+        "hostname": "localhost"
+    }
+}

data_bags/mon_persister/mon_persister.json

@@ -3,6 +3,10 @@
     "alarm_history": {
         "topic": "alarm-state-transitions"
     },
+    "influxdb": {
+      "dbname": "mon",
+      "hostname": "localhost"
+    },
     "metrics": {
         "topic": "metrics"
     },

roles/Mini-Mon-Vertica.json

@@ -0,0 +1,76 @@
+{
+  "name": "Mini-Mon",
+  "description": "Sets up the entire monitoring stack on one box",
+  "json_class": "Chef::Role",
+  "default_attributes": {
+    "apt": {
+      "periodic_update_min_delay": 60
+    },
+    "java": {
+      "install_flavor": "openjdk",
+      "jdk_version": "7"
+    },
+    "kafka": {
+      "listen_address": "127.0.0.1",
+      "topics": {
+        "metrics": { "replicas": 1, "partitions": 4 },
+        "events": { "replicas": 1, "partitions": 4 },
+        "alarm-state-transitions": { "replicas": 1, "partitions": 4 },
+        "alarm-notifications": { "replicas": 1, "partitions": 4 }
+      }
+    },
+    "percona": {
+      "backup": {
+        "password": "password"
+      },
+      "cluster": {
+        "package": "percona-xtradb-cluster-56"
+      },
+      "main_config_file": "/etc/mysql/my.cnf",
+      "server": {
+        "bind_address": "0.0.0.0",
+        "replication": {
+          "password": "password"
+        },
+        "root_password": "password",
+        "skip_name_resolve": true
+      }
+    },
+    "storm": {
+      "nimbus": {
+        "host": "127.0.0.1"
+      },
+      "ui": {
+        "port": "8088"
+      },
+      "zookeeper": {
+        "quorum": [
+          "127.0.0.1"
+        ]
+      }
+    }
+  },
+  "override_attributes": {
+  },
+  "chef_type": "role",
+  "run_list": [
+    "recipe[mini-mon]",
+    "recipe[percona::cluster]",
+    "recipe[mini-mon::mysql_schema]",
+    "recipe[zookeeper]",
+    "recipe[kafka]",
+    "recipe[kafka::create_topics]",
+    "recipe[mini-mon::postfix]",
+    "recipe[mon_notification]",
+    "recipe[vertica]",
+    "recipe[sysctl]",
+    "recipe[mon_api]",
+    "recipe[mon_persister]",
+    "recipe[storm::nimbus]",
+    "recipe[storm::supervisor]",
+    "recipe[mon_thresh]",
+    "recipe[mon_agent]"
+  ],
+  "env_run_lists": {
+  }
+}

roles/Mini-Mon.json

@@ -62,10 +62,11 @@
     "recipe[kafka::create_topics]",
     "recipe[mini-mon::postfix]",
     "recipe[mon_notification]",
-    "recipe[vertica]",
+    "recipe[influxdb]",
+    "recipe[mini-mon::influxdb]",
     "recipe[sysctl]",
-    "recipe[mon_api]",
     "recipe[mon_persister]",
+    "recipe[mon_api]",
     "recipe[storm::nimbus]",
     "recipe[storm::supervisor]",
     "recipe[mon_thresh]",