diff --git a/shotgun/manager.py b/shotgun/manager.py index c9d192f..3440c5a 100644 --- a/shotgun/manager.py +++ b/shotgun/manager.py @@ -12,8 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. +import errno import logging import os +import shutil import fabric.exceptions @@ -31,23 +33,33 @@ class Manager(object): def snapshot(self): logger.debug("Making snapshot") - utils.execute("rm -rf {0}".format(os.path.dirname(self.conf.target))) + self.clear_target() excludes = [] - for obj_data in self.conf.objects: - logger.debug("Dumping: %s", obj_data) - self.action_single(obj_data, action='snapshot') - if 'exclude' in obj_data: - excludes.extend(os.path.join(obj_data['path'], ex) - for ex in obj_data['exclude']) + try: + for obj_data in self.conf.objects: + logger.debug("Dumping: %s", obj_data) + self.action_single(obj_data, action='snapshot') + if 'exclude' in obj_data: + excludes += (os.path.join(obj_data['path'], ex) + for ex in obj_data['exclude']) - logger.debug("Dumping shotgun log and archiving dump directory: %s", - self.conf.target) - self.action_single(self.conf.self_log_object, action='snapshot') + logger.debug("Dumping shotgun log " + "and archiving dump directory: %s", + self.conf.target) + self.action_single(self.conf.self_log_object, action='snapshot') - utils.compress(self.conf.target, self.conf.compression_level, excludes) + utils.compress(self.conf.target, self.conf.compression_level, + excludes) + + with open(self.conf.lastdump, "w") as fo: + fo.write("{0}.tar.xz".format(self.conf.target)) + except IOError as e: + if e.errno == errno.ENOSPC: + logger.error("Not enough space in " + "{} for snapshot".format(self.conf.target)) + self.clear_target() + raise - with open(self.conf.lastdump, "w") as fo: - fo.write("{0}.tar.xz".format(self.conf.target)) return "{0}.tar.xz".format(self.conf.target) def action_single(self, object, action='snapshot'): @@ -63,3 +75,11 @@ class Manager(object): logger.debug("Gathering report for: %s", obj_data) for report in self.action_single(obj_data, action='report'): yield report + + def clear_target(self): + def on_rmtree_error(function, path, excinfo): + msg = "Clearing target failed. function: {}, path: {}, excinfo: {}" + logger.error(msg.format(function, path, excinfo)) + + shutil.rmtree(os.path.dirname(self.conf.target), + onerror=on_rmtree_error) diff --git a/shotgun/test/test_manager.py b/shotgun/test/test_manager.py index 19e57b8..83dd6ab 100644 --- a/shotgun/test/test_manager.py +++ b/shotgun/test/test_manager.py @@ -26,9 +26,9 @@ from shotgun.test import base class TestManager(base.BaseTestCase): @mock.patch('shotgun.manager.Driver.getDriver') - @mock.patch('shotgun.manager.utils.execute') @mock.patch('shotgun.manager.utils.compress') - def test_snapshot(self, mcompress, mexecute, mget): + @mock.patch('shutil.rmtree') + def test_snapshot(self, mrmtree, mcompress, mget): data = { "type": "file", "path": "/remote_dir/remote_file", @@ -45,12 +45,12 @@ class TestManager(base.BaseTestCase): manager.snapshot() calls = [mock.call(data, conf), mock.call(conf.self_log_object, conf)] mget.assert_has_calls(calls, any_order=True) - mexecute.assert_called_once_with('rm -rf /target') + mrmtree.assert_called_once_with('/target', onerror=mock.ANY) @mock.patch('shotgun.manager.Driver.getDriver') - @mock.patch('shotgun.manager.utils.execute') + @mock.patch('shutil.rmtree') @mock.patch('shotgun.manager.utils.compress') - def test_snapshot_network_error(self, mcompress, mexecute, mget): + def test_snapshot_network_error(self, mcompress, mrmtree, mget): objs = [ {"type": "file", "path": "/remote_file1", @@ -92,7 +92,7 @@ class TestManager(base.BaseTestCase): mock.call(processed_obj, conf), mock.call(offline_obj, conf), mock.call(offline_obj, conf)], any_order=True) - mexecute.assert_called_once_with('rm -rf /tmp') + mrmtree.assert_called_once_with('/tmp', onerror=mock.ANY) @mock.patch('shotgun.manager.Manager.action_single') def test_report(self, mock_action): @@ -132,3 +132,49 @@ class TestManager(base.BaseTestCase): mock_driver_instance.report.mock_reset() mock_driver_instance.snapshot.assert_called_once_with() self.assertFalse(mock_driver_instance.report.called) + + @mock.patch('shotgun.manager.Manager.action_single') + @mock.patch('shutil.rmtree') + def test_snapshot_rm_without_disk_space(self, mrmtree, mock_action): + mock_action.side_effect = IOError(28, "Not enough space") + + data = { + "type": "file", + "path": "/remote_dir/remote_file", + "host": { + "address": "remote_host", + }, + } + conf = mock.MagicMock() + conf.target = "/target/data" + conf.objects = [data] + conf.lastdump = tempfile.mkstemp()[1] + conf.self_log_object = {"type": "file", "path": "/path"} + manager = Manager(conf) + with self.assertRaises(IOError): + manager.snapshot() + calls = [mock.call('/target', onerror=mock.ANY) for _ in range(2)] + mrmtree.assert_has_calls(calls) + + @mock.patch('shotgun.manager.Manager.action_single') + @mock.patch('shutil.rmtree') + def test_snapshot_doesnt_clean_on_generic_ioerror(self, mrmtree, + mock_action): + mock_action.side_effect = IOError(1, "Generic error") + + data = { + "type": "file", + "path": "/remote_dir/remote_file", + "host": { + "address": "remote_host", + }, + } + conf = mock.MagicMock() + conf.target = "/target/data" + conf.objects = [data] + conf.lastdump = tempfile.mkstemp()[1] + conf.self_log_object = {"type": "file", "path": "/path"} + manager = Manager(conf) + with self.assertRaises(IOError): + manager.snapshot() + mrmtree.assert_called_once_with('/target', onerror=mock.ANY)