Delete snasphot on out of space error.

If generating snapshot throws an IOError with 28 error code
(errno.ENOSPC) whole snapshot is deleted to not clutter drive.

Change-Id: I442b6bfe7ea5d3b3661351ed36f973f870d4d95d
Partial-bug: 1529182

shotgun/manager.py

@@ -12,8 +12,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import errno
 import logging
 import os
+import shutil
 
 import fabric.exceptions
 
@@ -31,23 +33,33 @@ class Manager(object):
 
     def snapshot(self):
         logger.debug("Making snapshot")
-        utils.execute("rm -rf {0}".format(os.path.dirname(self.conf.target)))
+        self.clear_target()
         excludes = []
-        for obj_data in self.conf.objects:
-            logger.debug("Dumping: %s", obj_data)
-            self.action_single(obj_data, action='snapshot')
-            if 'exclude' in obj_data:
-                excludes.extend(os.path.join(obj_data['path'], ex)
-                                for ex in obj_data['exclude'])
+        try:
+            for obj_data in self.conf.objects:
+                logger.debug("Dumping: %s", obj_data)
+                self.action_single(obj_data, action='snapshot')
+                if 'exclude' in obj_data:
+                    excludes += (os.path.join(obj_data['path'], ex)
+                                 for ex in obj_data['exclude'])
 
-        logger.debug("Dumping shotgun log and archiving dump directory: %s",
-                     self.conf.target)
-        self.action_single(self.conf.self_log_object, action='snapshot')
+            logger.debug("Dumping shotgun log "
+                         "and archiving dump directory: %s",
+                         self.conf.target)
+            self.action_single(self.conf.self_log_object, action='snapshot')
 
-        utils.compress(self.conf.target, self.conf.compression_level, excludes)
+            utils.compress(self.conf.target, self.conf.compression_level,
+                           excludes)
+
+            with open(self.conf.lastdump, "w") as fo:
+                fo.write("{0}.tar.xz".format(self.conf.target))
+        except IOError as e:
+            if e.errno == errno.ENOSPC:
+                logger.error("Not enough space in "
+                             "{} for snapshot".format(self.conf.target))
+                self.clear_target()
+            raise
 
-        with open(self.conf.lastdump, "w") as fo:
-            fo.write("{0}.tar.xz".format(self.conf.target))
         return "{0}.tar.xz".format(self.conf.target)
 
     def action_single(self, object, action='snapshot'):
@@ -63,3 +75,11 @@ class Manager(object):
             logger.debug("Gathering report for: %s", obj_data)
             for report in self.action_single(obj_data, action='report'):
                 yield report
+
+    def clear_target(self):
+        def on_rmtree_error(function, path, excinfo):
+            msg = "Clearing target failed. function: {}, path: {}, excinfo: {}"
+            logger.error(msg.format(function, path, excinfo))
+
+        shutil.rmtree(os.path.dirname(self.conf.target),
+                      onerror=on_rmtree_error)

shotgun/test/test_manager.py

@@ -26,9 +26,9 @@ from shotgun.test import base
 class TestManager(base.BaseTestCase):
 
     @mock.patch('shotgun.manager.Driver.getDriver')
-    @mock.patch('shotgun.manager.utils.execute')
     @mock.patch('shotgun.manager.utils.compress')
-    def test_snapshot(self, mcompress, mexecute, mget):
+    @mock.patch('shutil.rmtree')
+    def test_snapshot(self, mrmtree, mcompress, mget):
         data = {
             "type": "file",
             "path": "/remote_dir/remote_file",
@@ -45,12 +45,12 @@ class TestManager(base.BaseTestCase):
         manager.snapshot()
         calls = [mock.call(data, conf), mock.call(conf.self_log_object, conf)]
         mget.assert_has_calls(calls, any_order=True)
-        mexecute.assert_called_once_with('rm -rf /target')
+        mrmtree.assert_called_once_with('/target', onerror=mock.ANY)
 
     @mock.patch('shotgun.manager.Driver.getDriver')
-    @mock.patch('shotgun.manager.utils.execute')
+    @mock.patch('shutil.rmtree')
     @mock.patch('shotgun.manager.utils.compress')
-    def test_snapshot_network_error(self, mcompress, mexecute, mget):
+    def test_snapshot_network_error(self, mcompress, mrmtree, mget):
         objs = [
             {"type": "file",
              "path": "/remote_file1",
@@ -92,7 +92,7 @@ class TestManager(base.BaseTestCase):
                                mock.call(processed_obj, conf),
                                mock.call(offline_obj, conf),
                                mock.call(offline_obj, conf)], any_order=True)
-        mexecute.assert_called_once_with('rm -rf /tmp')
+        mrmtree.assert_called_once_with('/tmp', onerror=mock.ANY)
 
     @mock.patch('shotgun.manager.Manager.action_single')
     def test_report(self, mock_action):
@@ -132,3 +132,49 @@ class TestManager(base.BaseTestCase):
         mock_driver_instance.report.mock_reset()
         mock_driver_instance.snapshot.assert_called_once_with()
         self.assertFalse(mock_driver_instance.report.called)
+
+    @mock.patch('shotgun.manager.Manager.action_single')
+    @mock.patch('shutil.rmtree')
+    def test_snapshot_rm_without_disk_space(self, mrmtree, mock_action):
+        mock_action.side_effect = IOError(28, "Not enough space")
+
+        data = {
+            "type": "file",
+            "path": "/remote_dir/remote_file",
+            "host": {
+                "address": "remote_host",
+            },
+        }
+        conf = mock.MagicMock()
+        conf.target = "/target/data"
+        conf.objects = [data]
+        conf.lastdump = tempfile.mkstemp()[1]
+        conf.self_log_object = {"type": "file", "path": "/path"}
+        manager = Manager(conf)
+        with self.assertRaises(IOError):
+            manager.snapshot()
+        calls = [mock.call('/target', onerror=mock.ANY) for _ in range(2)]
+        mrmtree.assert_has_calls(calls)
+
+    @mock.patch('shotgun.manager.Manager.action_single')
+    @mock.patch('shutil.rmtree')
+    def test_snapshot_doesnt_clean_on_generic_ioerror(self, mrmtree,
+                                                      mock_action):
+        mock_action.side_effect = IOError(1, "Generic error")
+
+        data = {
+            "type": "file",
+            "path": "/remote_dir/remote_file",
+            "host": {
+                "address": "remote_host",
+            },
+        }
+        conf = mock.MagicMock()
+        conf.target = "/target/data"
+        conf.objects = [data]
+        conf.lastdump = tempfile.mkstemp()[1]
+        conf.self_log_object = {"type": "file", "path": "/path"}
+        manager = Manager(conf)
+        with self.assertRaises(IOError):
+            manager.snapshot()
+        mrmtree.assert_called_once_with('/target', onerror=mock.ANY)