From f4c9e359fa77d32fa4ed61cc81b58cfe18c5e1ad Mon Sep 17 00:00:00 2001
From: Takashi Kajinami <tkajinam@redhat.com>
Date: Thu, 7 Mar 2019 10:38:54 +0900
Subject: [PATCH] Add sleep in cinder cron jobs

The current cron job for purge is executed at the precisely same time,
if we deploy multiple controller nodes with database clusterd,
for example using Galera cluster, it could cause DB spike.

This patch introduces a randomized sleep before executing purge job,
to avoid such kind of collision.

This commit is a kind of cherry-pick of the commit in puppet-nova
with change id I093a713a4f0ba48686de615aa8cb22b17a56917b .

Co-Authored-By: Rajesh Tailor <ratailor@redhat.com>

Change-Id: I36cf5f1e66fed580786970d9a953d983539a6e43
---
 manifests/cron/db_purge.pp                | 17 +++++++++++++++--
 spec/classes/cinder_cron_db_purge_spec.rb | 21 +++++++++++++++++++++
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/manifests/cron/db_purge.pp b/manifests/cron/db_purge.pp
index 06fb8ef4..0e40c97f 100644
--- a/manifests/cron/db_purge.pp
+++ b/manifests/cron/db_purge.pp
@@ -51,6 +51,12 @@
 #    (optional) Path to file to which rows should be archived
 #    Defaults to '/var/log/cinder/cinder-rowsflush.log'.
 #
+#  [*maxdelay*]
+#    (optional) In Seconds. Should be a positive integer.
+#    Induces a random delay before running the cronjob to avoid running
+#    all cron jobs at the same time on all hosts this job is configured.
+#    Defaults to 0.
+#
 class cinder::cron::db_purge (
   $minute      = 1,
   $hour        = 0,
@@ -59,13 +65,20 @@ class cinder::cron::db_purge (
   $weekday     = '*',
   $user        = 'cinder',
   $age         = 30,
-  $destination = '/var/log/cinder/cinder-rowsflush.log'
+  $destination = '/var/log/cinder/cinder-rowsflush.log',
+  $maxdelay    = 0
 ) {
 
   include ::cinder::deps
 
+  if $maxdelay == 0 {
+    $sleep = ''
+  } else {
+    $sleep = "sleep `expr \${RANDOM} \\% ${maxdelay}`; "
+  }
+
   cron { 'cinder-manage db purge':
-    command     => "cinder-manage db purge ${age} >>${destination} 2>&1",
+    command     => "${sleep}cinder-manage db purge ${age} >>${destination} 2>&1",
     environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh',
     user        => $user,
     minute      => $minute,
diff --git a/spec/classes/cinder_cron_db_purge_spec.rb b/spec/classes/cinder_cron_db_purge_spec.rb
index eb54a7de..cb57eafc 100644
--- a/spec/classes/cinder_cron_db_purge_spec.rb
+++ b/spec/classes/cinder_cron_db_purge_spec.rb
@@ -10,6 +10,7 @@ describe 'cinder::cron::db_purge' do
       :weekday     => '*',
       :user        => 'cinder',
       :age         => '30',
+      :maxdelay    => 0,
       :destination => '/var/log/cinder/cinder-rowsflush.log'
     }
   end
@@ -28,6 +29,26 @@ describe 'cinder::cron::db_purge' do
         :require     => 'Anchor[cinder::install::end]'
       )}
     end
+
+    context 'with required parameters with max delay enabled' do
+      before :each do
+        params.merge!(
+          :maxdelay => 600
+        )
+      end
+
+      it { should contain_cron('cinder-manage db purge').with(
+        :command     => "sleep `expr ${RANDOM} \\% #{params[:maxdelay]}`; cinder-manage db purge #{params[:age]} >>#{params[:destination]} 2>&1",
+        :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh',
+        :user        => params[:user],
+        :minute      => params[:minute],
+        :hour        => params[:hour],
+        :monthday    => params[:monthday],
+        :month       => params[:month],
+        :weekday     => params[:weekday],
+        :require     => 'Anchor[cinder::install::end]'
+      )}
+    end
   end
 
   on_supported_os({