From f7d619731286c4c76dcdadbf61aaac9644fece9a Mon Sep 17 00:00:00 2001
From: Tom Patzig <tom.patzig@sap.com>
Date: Wed, 7 Sep 2016 11:16:49 +0200
Subject: [PATCH] refresh instances_path when shared storage used

When doing Live migration with shared storage, it happens erratically,
that the check for the shared storage test_file fails. Because the shared
volume is under heavy IO (many instances on many compute nodes) the client
does not immediately sees the new content of the folder. This delay
could take up to 30s.
This can be fixed if the client is forced to refresh the directories
content, which can be achieved by 'touch' on the directory. Doing so,
the test_file is visibile instantly, within ms.
The patch adds a 'touch' on instances_path in check_shared_storage_test_file,
before checking the existence of the file.

Change-Id: I16be39142278517f43e6eca3441a56cbc9561113
Closes-Bug: #1617299
(cherry picked from commit 1af73d1fb3169c5b3cce77d94316922496bbaf9a)
---
 nova/tests/unit/virt/libvirt/test_driver.py | 22 +++++++++++++++++++++
 nova/virt/libvirt/driver.py                 |  8 ++++++++
 2 files changed, 30 insertions(+)

diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py
index 49aac4585cba..8488aa6c4fac 100644
--- a/nova/tests/unit/virt/libvirt/test_driver.py
+++ b/nova/tests/unit/virt/libvirt/test_driver.py
@@ -6952,6 +6952,28 @@ class LibvirtConnTestCase(test.NoDBTestCase):
         drvr.cleanup_live_migration_destination_check(self.context,
                                                       dest_check_data)
 
+    @mock.patch('os.path.exists', return_value=True)
+    @mock.patch('os.utime')
+    def test_check_shared_storage_test_file_exists(self, mock_utime,
+                                                   mock_path_exists):
+        tmpfile_path = os.path.join(CONF.instances_path, 'tmp123')
+        drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
+        self.assertTrue(drvr._check_shared_storage_test_file(
+            'tmp123', mock.sentinel.instance))
+        mock_utime.assert_called_once_with(CONF.instances_path, None)
+        mock_path_exists.assert_called_once_with(tmpfile_path)
+
+    @mock.patch('os.path.exists', return_value=False)
+    @mock.patch('os.utime')
+    def test_check_shared_storage_test_file_does_not_exist(self, mock_utime,
+                                                   mock_path_exists):
+        tmpfile_path = os.path.join(CONF.instances_path, 'tmp123')
+        drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
+        self.assertFalse(drvr._check_shared_storage_test_file(
+            'tmp123', mock.sentinel.instance))
+        mock_utime.assert_called_once_with(CONF.instances_path, None)
+        mock_path_exists.assert_called_once_with(tmpfile_path)
+
     def _mock_can_live_migrate_source(self, block_migration=False,
                                       is_shared_block_storage=False,
                                       is_shared_instance_path=False,
diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py
index 19b15e7847ac..00d019b873a5 100644
--- a/nova/virt/libvirt/driver.py
+++ b/nova/virt/libvirt/driver.py
@@ -5659,6 +5659,14 @@ class LibvirtDriver(driver.ComputeDriver):
 
         Cannot confirm tmpfile return False.
         """
+        # NOTE(tpatzig): if instances_path is a shared volume that is
+        # under heavy IO (many instances on many compute nodes),
+        # then checking the existence of the testfile fails,
+        # just because it takes longer until the client refreshes and new
+        # content gets visible.
+        # os.utime (like touch) on the directory forces the client to refresh.
+        os.utime(CONF.instances_path, None)
+
         tmp_file = os.path.join(CONF.instances_path, filename)
         if not os.path.exists(tmp_file):
             exists = False