Add the option to store the InfluxDB WAL in memory

This change adds a new option to the Fuel UI to store the InfluxDB WAL
files in memory. This greatly improves write performances but it may
lead to data loss in case of server crash.

The WAL partition will use 10% of the total RAM (but no more than 4GB).

DocImpact

Change-Id: I6153cd24322ef3eab53505bf4cb1f72d9594ec04

deployment_scripts/puppet/manifests/hiera.pp

@@ -53,6 +53,14 @@ $influxdb_password = $influxdb_grafana['influxdb_userpass']
 $influxdb_dbname   = $influxdb_grafana['influxdb_dbname']
 
 $retention_period = $influxdb_grafana['retention_period']
+if $influxdb_grafana['influxdb_in_memory_wal'] {
+  $influxdb_wal_storage = 'memory'
+  # Allocate 10% of the total RAM for the WAL partition (but no more than 4GB)
+  $influxdb_wal_size = min(4 * 1024 * 1024 * 1024, $::memorysize_mb * 1024 * 0.1)
+} else {
+  $influxdb_wal_storage = 'disk'
+  $influxdb_wal_size = 0
+}
 
 # Parameters related to MySQL
 $host = $influxdb_grafana['mysql_host']
@@ -138,6 +146,8 @@ lma::corosync_roles:
 # from 1 or 2 nodes to 3 nodes.
 lma::influxdb::replication_factor: 3
 lma::influxdb::retention_period: <%= @retention_period %>
+lma::influxdb::wal::storage: <%= @influxdb_wal_storage %>
+lma::influxdb::wal::size: <%= @influxdb_wal_size %>
 
 lma::influxdb::admin_username: "root"
 lma::influxdb::admin_password: <%= @influxdb_admin_password %>

deployment_scripts/puppet/manifests/influxdb.pp

@@ -15,6 +15,7 @@
 notice('fuel-plugin-influxdb-grafana: influxdb.pp')
 
 $data_directory = hiera('lma::influxdb::data_dir')
+$wal_dir = "${data_directory}/wal"
 
 # We set raft_nodes only for the non-primary node. The primary node will be
 # started as the first node and it will be the leader of the Raft cluster.
@@ -37,11 +38,33 @@ file { $data_directory:
   require => User['influxdb'],
 }
 
+if hiera('lma::influxdb::wal::storage') == 'memory' {
+  $wal_size = hiera('lma::influxdb::wal::size')
+  file { $wal_dir:
+    ensure  => directory,
+    owner   => 'influxdb',
+    group   => 'influxdb',
+    require => File[$data_directory],
+  }
+
+  mount { $wal_dir:
+    ensure   => mounted,
+    device   => 'tmpfs',
+    atboot   => true,
+    options  => "size=${wal_size},rw",
+    fstype   => 'tmpfs',
+    remounts => false,
+    before   => Class['lma_monitoring_analytics::influxdb'],
+    require  => File[$wal_dir],
+  }
+}
+
 # We cannot mix IP addresses and hostnames otherwise the Raft cluster won't
 # start. We have to stick with IP addresses because hostnames map to the
 # managament network space.
 class { 'lma_monitoring_analytics::influxdb':
   base_directory => $data_directory,
+  wal_dir        => $wal_dir,
   hostname       => hiera('lma::influxdb::listen_address'),
   raft_nodes     => $raft_nodes,
   version        => '0.11.1-1',

deployment_scripts/puppet/modules/lma_monitoring_analytics/manifests/influxdb.pp

@@ -16,6 +16,7 @@
 
 class lma_monitoring_analytics::influxdb (
   $base_directory = $lma_monitoring_analytics::params::influxdb_dir,
+  $wal_dir        = undef,
   $hostname       = undef,
   $raft_nodes     = undef,
   $version        = undef,
@@ -25,11 +26,17 @@ class lma_monitoring_analytics::influxdb (
     validate_array($raft_nodes)
   }
 
+  if $wal_dir {
+    $_wal_dir = $wal_dir
+  } else {
+    $_wal_dir = "${base_directory}/wal"
+  }
+
   class { '::influxdb':
     data_dir   => "${base_directory}/data",
     meta_dir   => "${base_directory}/meta",
     hh_dir     => "${base_directory}/hh",
-    wal_dir    => "${base_directory}/wal",
+    wal_dir    => $_wal_dir,
     snapshot   => true,
     hostname   => $hostname,
     raft_nodes => $raft_nodes,

environment_config.yaml

@@ -52,6 +52,13 @@ attributes:
       source: '^[\S]{4,}$'
       error: "You must provide a password with at least 4 characters"
 
+  influxdb_in_memory_wal:
+    value: false
+    label: 'Store WAL files in memory'
+    description: 'Store the Write-Ahead-Log (WAL) files in memory instead of disk. This will improve the write performances but data may be lost in case of server crash.'
+    weight: 61
+    type: "checkbox"
+
   grafana_username:
     value: 'lma'
     label: 'User name'