From dd714a946bcab684410a4e00662bfc11ea4de1d2 Mon Sep 17 00:00:00 2001 From: Tim Kuhlman Date: Mon, 7 Jul 2014 10:51:10 -0600 Subject: [PATCH] Initial switch to influxdb --- Berksfile | 1 + README.md | 27 ++++--- cookbooks/mini-mon/README.md | 1 + cookbooks/mini-mon/metadata.rb | 3 +- cookbooks/mini-mon/recipes/influxdb.rb | 14 ++++ data_bags/mon_api/mon_api-Vertica.json | 21 +++++ data_bags/mon_api/mon_api.json | 4 +- data_bags/mon_api/mon_credentials.json | 5 ++ data_bags/mon_persister/mon_credentials.json | 4 + .../mon_persister/mon_persister-Vertica.json | 37 +++++++++ data_bags/mon_persister/mon_persister.json | 4 + roles/Mini-Mon-Vertica.json | 76 +++++++++++++++++++ roles/Mini-Mon.json | 5 +- 13 files changed, 188 insertions(+), 14 deletions(-) create mode 100644 cookbooks/mini-mon/recipes/influxdb.rb create mode 100644 data_bags/mon_api/mon_api-Vertica.json create mode 100644 data_bags/mon_persister/mon_persister-Vertica.json create mode 100644 roles/Mini-Mon-Vertica.json diff --git a/Berksfile b/Berksfile index dac8929..16c42c1 100644 --- a/Berksfile +++ b/Berksfile @@ -17,4 +17,5 @@ cookbook 'vertica', git: 'https://github.com/hpcloud-mon/cookbooks-vertica' cookbook 'zookeeper', git: 'https://github.com/hpcloud-mon/cookbooks-zookeeper' # Community cookbooks +cookbook 'influxdb', git: 'https://github.com/SimpleFinance/chef-influxdb' cookbook 'percona', git: 'https://github.com/phlipper/chef-percona' diff --git a/README.md b/README.md index 7db7776..518bfd2 100644 --- a/README.md +++ b/README.md @@ -31,15 +31,6 @@ Installs a mini monitoring environment based on Vagrant. Intended for developmen ``` git clone https://github.com/hpcloud-mon/mon-vagrant ``` -Vertica must be downloaded from the [Vertica site](https://my.vertica.com/). Download these packages and place in the root of this repository. - -- `vertica_7.0.1-0_amd64.deb` -- `vertica-r-lang_7.0.1-0_amd64.deb` - -The `vertica::console` recipe is not enabled by default, but if it is added, this package is also needed. - -- `vertica-console_7.0.1-0_amd64.deb` - ## Install Vagrant ### Install VirtualBox and Vagrant @@ -131,6 +122,24 @@ VM to use them also. vagrant plugin install vagrant-proxyconf ``` +# Vertica +Vertica is supported instead of influxdb, this is especially useful for large deployments. + +Before using Vertica must be downloaded from the [Vertica site](https://my.vertica.com/). Download these packages and place in the root of this repository. + +- `vertica_7.0.1-0_amd64.deb` +- `vertica-r-lang_7.0.1-0_amd64.deb` + +The `vertica::console` recipe is not enabled by default, but if it is added, this package is also needed. + +- `vertica-console_7.0.1-0_amd64.deb` + +After the vertica packages are installed the configuration must be changed to run Vertica. Roles and data_bags for this setup are provided, copy them +in place with the following commands then run `vagrant up` as normal. +- `cp roles/Mini-Mon-Vertica.json roles/Mini-Mon.json` +- `cp data_bags/mon_persister/mon_persister-Vertica.json data_bags/mon_persister/mon_persister.json` +- `cp data_bags/mon_api/mon_api-Vertica.json data_bags/mon_api/mon_api.json` + # Alternate Vagrant Configurations To run any of these alternate configs, simply run the Vagrant commands from within the subdir. Note that the Vertica debs must be _copied_ (not symlinked) into the subdir as well. See the README.md in the subdir for more details. diff --git a/cookbooks/mini-mon/README.md b/cookbooks/mini-mon/README.md index af84be3..d60efce 100644 --- a/cookbooks/mini-mon/README.md +++ b/cookbooks/mini-mon/README.md @@ -9,6 +9,7 @@ apt cookbook Recipes --------- - default - configures base apt repostories needed for mini-mon as well as option apt cache +- influxdb - creates an influxdb database and users - keystone - simple keystone installation - mysql_schema - Defines the mysql_schema used by mini-mon - postfix - extremely simple postfix install diff --git a/cookbooks/mini-mon/metadata.rb b/cookbooks/mini-mon/metadata.rb index 64615aa..739977e 100644 --- a/cookbooks/mini-mon/metadata.rb +++ b/cookbooks/mini-mon/metadata.rb @@ -4,5 +4,6 @@ maintainer_email "hpcs-mon@hp.com" license "All rights reserved" description "Base setup for all vagrant boxes" long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) -version "0.0.8" +version "0.0.9" depends "apt" +depends "influxdb" diff --git a/cookbooks/mini-mon/recipes/influxdb.rb b/cookbooks/mini-mon/recipes/influxdb.rb new file mode 100644 index 0000000..80dc85c --- /dev/null +++ b/cookbooks/mini-mon/recipes/influxdb.rb @@ -0,0 +1,14 @@ +# Create an influxdb database and users +# Leverages the cookbook from https://github.com/SimpleFinance/chef-influxdb + +influxdb_database 'mon' do + action :create +end + +['mon_api', 'mon_persister'].each do |user| + influxdb_user user do + action :create + password 'password' + databases ['mon'] + end +end diff --git a/data_bags/mon_api/mon_api-Vertica.json b/data_bags/mon_api/mon_api-Vertica.json new file mode 100644 index 0000000..e7772d7 --- /dev/null +++ b/data_bags/mon_api/mon_api-Vertica.json @@ -0,0 +1,21 @@ +{ + "id" : "mon_api", + "api_region": "useast", + "database-configuration": { + "database-type": "vertica" + }, + "vertica" : { + "dbname" : "mon", + "hostname" : "localhost" + }, + "zookeeper" : { + "hostname" : "localhost" + }, + "mysql": { + "hostname":"localhost", + "schema": "mon" + }, + "kafka": { + "hostname": "localhost" + } +} diff --git a/data_bags/mon_api/mon_api.json b/data_bags/mon_api/mon_api.json index c4c96b2..73722fb 100644 --- a/data_bags/mon_api/mon_api.json +++ b/data_bags/mon_api/mon_api.json @@ -2,9 +2,9 @@ "id" : "mon_api", "api_region": "useast", "database-configuration": { - "database-type": "vertica" + "database-type": "influxdb" }, - "vertica" : { + "influxdb" : { "dbname" : "mon", "hostname" : "localhost" }, diff --git a/data_bags/mon_api/mon_credentials.json b/data_bags/mon_api/mon_credentials.json index ec48d32..12d9c62 100644 --- a/data_bags/mon_api/mon_credentials.json +++ b/data_bags/mon_api/mon_credentials.json @@ -14,6 +14,11 @@ "password": "password", "schema": "mon" }, + "influxdb": { + "hostname": "localhost", + "username": "mon_api", + "password": "password" + }, "vertica": { "hostname": "localhost", "username": "mon_api", diff --git a/data_bags/mon_persister/mon_credentials.json b/data_bags/mon_persister/mon_credentials.json index 338b8db..8d0aa63 100644 --- a/data_bags/mon_persister/mon_credentials.json +++ b/data_bags/mon_persister/mon_credentials.json @@ -1,5 +1,9 @@ { "id" : "mon_credentials", + "influxdb": { + "user": "root", + "password": "root" + }, "vertica" : { "user" : "dbadmin", "password" : "password" diff --git a/data_bags/mon_persister/mon_persister-Vertica.json b/data_bags/mon_persister/mon_persister-Vertica.json new file mode 100644 index 0000000..fb37253 --- /dev/null +++ b/data_bags/mon_persister/mon_persister-Vertica.json @@ -0,0 +1,37 @@ +{ + "id": "mon_persister", + "alarm_history": { + "topic": "alarm-state-transitions" + }, + "metrics": { + "topic": "metrics" + }, + "kafka": { + "num_threads": "1", + "group_id": "1", + "consumer_id": 1 + }, + "disruptor": { + "buffer_size": "1048576", + "num_processors": "1" + }, + "output_processor": { + "batch_size": "10000" + }, + "deduper_config": { + "dedupe_run_frequencey_seconds": "30" + }, + "database_configuration": { + "database_type": "vertica" + }, + "vertica_metric_repository_config": { + "max_cache_size": "2000000" + }, + "vertica": { + "dbname": "mon", + "hostname": "localhost" + }, + "zookeeper": { + "hostname": "localhost" + } +} diff --git a/data_bags/mon_persister/mon_persister.json b/data_bags/mon_persister/mon_persister.json index fb37253..acedcb9 100644 --- a/data_bags/mon_persister/mon_persister.json +++ b/data_bags/mon_persister/mon_persister.json @@ -3,6 +3,10 @@ "alarm_history": { "topic": "alarm-state-transitions" }, + "influxdb": { + "dbname": "mon", + "hostname": "localhost" + }, "metrics": { "topic": "metrics" }, diff --git a/roles/Mini-Mon-Vertica.json b/roles/Mini-Mon-Vertica.json new file mode 100644 index 0000000..4469d3f --- /dev/null +++ b/roles/Mini-Mon-Vertica.json @@ -0,0 +1,76 @@ +{ + "name": "Mini-Mon", + "description": "Sets up the entire monitoring stack on one box", + "json_class": "Chef::Role", + "default_attributes": { + "apt": { + "periodic_update_min_delay": 60 + }, + "java": { + "install_flavor": "openjdk", + "jdk_version": "7" + }, + "kafka": { + "listen_address": "127.0.0.1", + "topics": { + "metrics": { "replicas": 1, "partitions": 4 }, + "events": { "replicas": 1, "partitions": 4 }, + "alarm-state-transitions": { "replicas": 1, "partitions": 4 }, + "alarm-notifications": { "replicas": 1, "partitions": 4 } + } + }, + "percona": { + "backup": { + "password": "password" + }, + "cluster": { + "package": "percona-xtradb-cluster-56" + }, + "main_config_file": "/etc/mysql/my.cnf", + "server": { + "bind_address": "0.0.0.0", + "replication": { + "password": "password" + }, + "root_password": "password", + "skip_name_resolve": true + } + }, + "storm": { + "nimbus": { + "host": "127.0.0.1" + }, + "ui": { + "port": "8088" + }, + "zookeeper": { + "quorum": [ + "127.0.0.1" + ] + } + } + }, + "override_attributes": { + }, + "chef_type": "role", + "run_list": [ + "recipe[mini-mon]", + "recipe[percona::cluster]", + "recipe[mini-mon::mysql_schema]", + "recipe[zookeeper]", + "recipe[kafka]", + "recipe[kafka::create_topics]", + "recipe[mini-mon::postfix]", + "recipe[mon_notification]", + "recipe[vertica]", + "recipe[sysctl]", + "recipe[mon_api]", + "recipe[mon_persister]", + "recipe[storm::nimbus]", + "recipe[storm::supervisor]", + "recipe[mon_thresh]", + "recipe[mon_agent]" + ], + "env_run_lists": { + } +} diff --git a/roles/Mini-Mon.json b/roles/Mini-Mon.json index fc99071..66f2422 100644 --- a/roles/Mini-Mon.json +++ b/roles/Mini-Mon.json @@ -62,10 +62,11 @@ "recipe[kafka::create_topics]", "recipe[mini-mon::postfix]", "recipe[mon_notification]", - "recipe[vertica]", + "recipe[influxdb]", + "recipe[mini-mon::influxdb]", "recipe[sysctl]", - "recipe[mon_api]", "recipe[mon_persister]", + "recipe[mon_api]", "recipe[storm::nimbus]", "recipe[storm::supervisor]", "recipe[mon_thresh]",