Out of space detection during compression

Since compression command is actually a pipeline
we cannot use return code to detect out of space condition
so, the only way is to parse stderr

Change-Id: I2a95333bcabb73d0716d9790e14851cfefd47478
Partial-Bug: #1543491

shotgun/manager.py

@@ -34,14 +34,16 @@ class Manager(object):
     def snapshot(self):
         logger.debug("Making snapshot")
         self.clear_target()
-        excludes = []
+        exclusions = []
         try:
             for obj_data in self.conf.objects:
                 logger.debug("Dumping: %s", obj_data)
                 self.action_single(obj_data, action='snapshot')
                 if 'exclude' in obj_data:
-                    excludes += (os.path.join(obj_data['path'], ex)
-                                 for ex in obj_data['exclude'])
+                    exclusions += (
+                        os.path.join(obj_data['path'], ex).lstrip('/')
+                        for ex in obj_data['exclude']
+                    )
 
             logger.debug("Dumping shotgun log "
                          "and archiving dump directory: %s",
@@ -49,7 +51,7 @@ class Manager(object):
             self.action_single(self.conf.self_log_object, action='snapshot')
 
             utils.compress(self.conf.target, self.conf.compression_level,
-                           excludes)
+                           exclude=exclusions)
 
             with open(self.conf.lastdump, "w") as fo:
                 fo.write("{0}.tar.xz".format(self.conf.target))

shotgun/test/test_utils.py

@@ -49,6 +49,8 @@ class TestUtils(base.BaseTestCase):
         target = '/path/target'
         level = '-3'
 
+        mexecute.return_value = (None, None, None)
+
         utils.compress(target, level)
 
         compress_call = mexecute.call_args_list[0]
@@ -58,7 +60,7 @@ class TestUtils(base.BaseTestCase):
         self.assertEqual(compress_env['XZ_OPT'], level)
         self.assertEqual(
             compress_call[0][0],
-            'tar chJvf /path/target.tar.xz -C /path target')
+            'tar chJf /path/target.tar.xz -C /path target')
 
         self.assertEqual(rm_call[0][0], 'rm -r /path/target')
 
@@ -69,6 +71,8 @@ class TestUtils(base.BaseTestCase):
 
         exclusions = ['/path/to/exclude1', '/path/to/exclude2']
 
+        mexecute.return_value = (None, None, None)
+
         utils.compress(target, level, exclude=exclusions)
 
         compress_call = mexecute.call_args_list[0]
@@ -77,8 +81,8 @@ class TestUtils(base.BaseTestCase):
         self.assertEqual(compress_env['XZ_OPT'], level)
         self.assertEqual(
             compress_call[0][0],
-            'tar chJvf /path/target.tar.xz -C /path target '
-            '--exclude /path/to/exclude1 --exclude /path/to/exclude2')
+            'tar chJf /path/target.tar.xz -C /path target '
+            '--exclude=/path/to/exclude1 --exclude=/path/to/exclude2')
 
 
 class TestCCStringIO(base.BaseTestCase):

shotgun/utils.py

@@ -13,6 +13,7 @@
 #    under the License.
 
 import copy
+import errno
 import logging
 import os
 import re
@@ -58,6 +59,14 @@ def remove(full_dst_path, excludes):
         execute("shopt -s globstar; rm -rf {0}".format(path))
 
 
+def is_out_of_space(code, errdata):
+    if code:
+        # Since compression command is actually a pipeline
+        # we cannot use return code to detect out of space condition
+        # so, the only way is to parse stderr
+        return errdata.lower().find('no space left'.encode()) >= 0
+
+
 def compress(target, level, keep_target=False, exclude=None):
     """Runs compression of provided directory
 
@@ -70,14 +79,18 @@ def compress(target, level, keep_target=False, exclude=None):
 
     env = copy.deepcopy(os.environ)
     env['XZ_OPT'] = level
-    execute("tar chJvf {0}.tar.xz -C {1} {2}{3}"
-            "".format(target,
-                      os.path.dirname(target),
-                      os.path.basename(target),
-                      "".join(' --exclude {}'.format(e) for e in exclude)),
-            env=env)
+    env['LANG'] = 'C'  # We need non localized output
+    code, _, errdata = execute(
+        "tar chJf {0}.tar.xz -C {1} {2}{3}"
+        "".format(target,
+                  os.path.dirname(target),
+                  os.path.basename(target),
+                  "".join(' --exclude={}'.format(e) for e in exclude)),
+        env=env)
     if not keep_target:
         execute("rm -r {0}".format(target))
+    if is_out_of_space(code, errdata):
+        raise IOError(errno.ENOSPC)
 
 
 def execute(command, env=None):