Getting local files creates symlinks instead of copying

Dumping shotgun resources is now done through symlinks. All local
resources are symlinked in dump directory, after that dump is compressed
with tar using -h option (--dereference).

Excluding files from tarball is now done by passing --exclude option to
tar instead of removing files before taring to avoid deleting logs.

Symlinks are created by 'ln -s' command because of wildcards used in
shotgun settings.

Change-Id: Ie9a0ab51d5874cd46a3919179def0aef407e7340
Partial-bug: 1529182

shotgun/driver.py

@@ -145,6 +145,8 @@ class Driver(object):
         copied files or directories
         """
         try:
+            if not os.path.exists(target_path):
+                os.makedirs(target_path)
             if self.dest_host:
                 with fabric.api.settings(
                     host_string=self.dest_host,  # destination host
@@ -155,17 +157,19 @@ class Driver(object):
                 ):
                     logger.debug("Getting remote file: %s %s",
                                  path, target_path)
-                    utils.execute('mkdir -p "{0}"'.format(target_path))
                     try:
                         return fabric.api.get(path, target_path)
                     except SystemExit:
                         logger.error("Fabric aborted this iteration")
             else:
+                # NOTE(mkwiek): We need to use shell ln instead of os.symlink
+                # because of wildcards used in shotgun settings. ln utility
+                # will nicely handle wildcards and create all needed symlinks
+                # in target_path directory
+                symlink_command = 'ln -s "{}" "{}"'.format(path, target_path)
                 logger.debug(
-                    "Getting local file: cp -r %s %s", path, target_path)
-                utils.execute('mkdir -p "{0}"'.format(target_path))
-                return utils.execute(
-                    'cp -r "{0}" "{1}"'.format(path, target_path))
+                    "Symlinking to local file: {}".format(symlink_command))
+                return utils.execute(symlink_command)
         except fabric.exceptions.NetworkError as e:
             logger.error("NetworkError occured: %s", str(e))
             raise
@@ -199,8 +203,6 @@ class File(Driver):
         """
         self.get(self.path, self.target_path)
 
-        if self.exclude:
-            utils.remove(self.full_dst_path, self.exclude)
 
 Dir = File
 

shotgun/manager.py

@@ -32,15 +32,19 @@ class Manager(object):
     def snapshot(self):
         logger.debug("Making snapshot")
         utils.execute("rm -rf {0}".format(os.path.dirname(self.conf.target)))
+        excludes = []
         for obj_data in self.conf.objects:
             logger.debug("Dumping: %s", obj_data)
             self.action_single(obj_data, action='snapshot')
+            if 'exclude' in obj_data:
+                excludes.extend(os.path.join(obj_data['path'], ex)
+                                for ex in obj_data['exclude'])
 
         logger.debug("Dumping shotgun log and archiving dump directory: %s",
                      self.conf.target)
         self.action_single(self.conf.self_log_object, action='snapshot')
 
-        utils.compress(self.conf.target, self.conf.compression_level)
+        utils.compress(self.conf.target, self.conf.compression_level, excludes)
 
         with open(self.conf.lastdump, "w") as fo:
             fo.write("{0}.tar.xz".format(self.conf.target))

shotgun/test/test_driver.py

@@ -148,10 +148,12 @@ class TestDriver(base.BaseTestCase):
         mfabrun.assert_called_with(command, stdout=mstdout)
         self.assertEqual(result.stdout, 'FULL STDOUT')
 
-    @mock.patch('shotgun.driver.utils.execute')
+    @mock.patch('os.path.exists', return_value=False)
+    @mock.patch('os.makedirs')
+    @mock.patch('shotgun.utils.execute')
     @mock.patch('shotgun.driver.fabric.api.settings')
     @mock.patch('shotgun.driver.fabric.api.get')
-    def test_driver_get(self, mfabget, mfabset, mexecute):
+    def test_driver_get(self, mfabget, mfabset, mexecute, mmakedirs, _):
         mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR")
         remote_path = "/remote_dir/remote_file"
         target_path = "/target_dir"
@@ -165,7 +167,7 @@ class TestDriver(base.BaseTestCase):
             },
         }, conf)
         driver.get(remote_path, target_path)
-        mexecute.assert_called_with('mkdir -p "{0}"'.format(target_path))
+        mmakedirs.assert_called_once_with(target_path)
         mfabget.assert_called_with(remote_path, target_path)
 
         mfabset.assert_called_with(
@@ -173,11 +175,12 @@ class TestDriver(base.BaseTestCase):
             timeout=2, warn_only=True, abort_on_prompts=True)
 
         mexecute.reset_mock()
+        mmakedirs.reset_mock()
         driver = shotgun.driver.Driver({}, conf)
         driver.get(remote_path, target_path)
-        self.assertEqual(mexecute.mock_calls, [
-            mock.call('mkdir -p "{0}"'.format(target_path)),
-            mock.call('cp -r "{0}" "{1}"'.format(remote_path, target_path))])
+        mmakedirs.assert_called_once_with(target_path)
+        mexecute.assert_called_with('ln -s "{}" "{}"'.format(remote_path,
+                                                             target_path))
 
     def test_use_timeout_from_global_conf(self):
         data = {}
@@ -252,28 +255,6 @@ class TestFile(base.BaseTestCase):
 
         mget.assert_called_with(data["path"], target_path)
 
-    @mock.patch('shotgun.driver.utils.remove')
-    @mock.patch('shotgun.driver.Driver.get')
-    def test_dir_exclude_called(self, mget, mremove):
-        data = {
-            "type": "dir",
-            "path": "/remote_dir/",
-            "exclude": ["*test"],
-            "host": {
-                "hostname": "remote_host",
-                "address": "10.109.0.2",
-            },
-        }
-        conf = mock.MagicMock()
-        conf.target = "/target"
-        dir_driver = shotgun.driver.Dir(data, conf)
-
-        target_path = "/target/remote_host/remote_dir"
-        dir_driver.snapshot()
-
-        mget.assert_called_with(data["path"], target_path)
-        mremove.assert_called_with(dir_driver.full_dst_path, data['exclude'])
-
 
 class TestCommand(base.BaseTestCase):
     def setUp(self):

shotgun/test/test_utils.py

@@ -58,10 +58,28 @@ class TestUtils(base.BaseTestCase):
         self.assertEqual(compress_env['XZ_OPT'], level)
         self.assertEqual(
             compress_call[0][0],
-            'tar cJvf /path/target.tar.xz -C /path target')
+            'tar chJvf /path/target.tar.xz -C /path target')
 
         self.assertEqual(rm_call[0][0], 'rm -r /path/target')
 
+    @mock.patch('shotgun.utils.execute')
+    def test_compress_exclude(self, mexecute):
+        target = '/path/target'
+        level = '-3'
+
+        exclusions = ['/path/to/exclude1', '/path/to/exclude2']
+
+        utils.compress(target, level, exclude=exclusions)
+
+        compress_call = mexecute.call_args_list[0]
+
+        compress_env = compress_call[1]['env']
+        self.assertEqual(compress_env['XZ_OPT'], level)
+        self.assertEqual(
+            compress_call[0][0],
+            'tar chJvf /path/target.tar.xz -C /path target '
+            '--exclude /path/to/exclude1 --exclude /path/to/exclude2')
+
 
 class TestCCStringIO(base.BaseTestCase):
 

shotgun/utils.py

@@ -58,19 +58,23 @@ def remove(full_dst_path, excludes):
         execute("shopt -s globstar; rm -rf {0}".format(path))
 
 
-def compress(target, level, keep_target=False):
+def compress(target, level, keep_target=False, exclude=None):
     """Runs compression of provided directory
 
     :param target: directory to compress
     :param level: level of compression
     :param keep_target: bool, if True target directory wont be removed
     """
+    if exclude is None:
+        exclude = []
+
     env = copy.deepcopy(os.environ)
     env['XZ_OPT'] = level
-    execute("tar cJvf {0}.tar.xz -C {1} {2}"
+    execute("tar chJvf {0}.tar.xz -C {1} {2}{3}"
             "".format(target,
                       os.path.dirname(target),
-                      os.path.basename(target)),
+                      os.path.basename(target),
+                      "".join(' --exclude {}'.format(e) for e in exclude)),
             env=env)
     if not keep_target:
         execute("rm -r {0}".format(target))