From 6930543c708cc3df6327c1a109dad396d8147ad8 Mon Sep 17 00:00:00 2001 From: Maciej Kwiek Date: Wed, 13 Jan 2016 16:03:10 +0100 Subject: [PATCH] Getting local files creates symlinks instead of copying Dumping shotgun resources is now done through symlinks. All local resources are symlinked in dump directory, after that dump is compressed with tar using -h option (--dereference). Excluding files from tarball is now done by passing --exclude option to tar instead of removing files before taring to avoid deleting logs. Symlinks are created by 'ln -s' command because of wildcards used in shotgun settings. Change-Id: Ie9a0ab51d5874cd46a3919179def0aef407e7340 Partial-bug: 1529182 --- shotgun/driver.py | 16 +++++++++------- shotgun/manager.py | 6 +++++- shotgun/test/test_driver.py | 37 +++++++++---------------------------- shotgun/test/test_utils.py | 20 +++++++++++++++++++- shotgun/utils.py | 10 +++++++--- 5 files changed, 49 insertions(+), 40 deletions(-) diff --git a/shotgun/driver.py b/shotgun/driver.py index b8273ef..ee75571 100644 --- a/shotgun/driver.py +++ b/shotgun/driver.py @@ -145,6 +145,8 @@ class Driver(object): copied files or directories """ try: + if not os.path.exists(target_path): + os.makedirs(target_path) if self.dest_host: with fabric.api.settings( host_string=self.dest_host, # destination host @@ -155,17 +157,19 @@ class Driver(object): ): logger.debug("Getting remote file: %s %s", path, target_path) - utils.execute('mkdir -p "{0}"'.format(target_path)) try: return fabric.api.get(path, target_path) except SystemExit: logger.error("Fabric aborted this iteration") else: + # NOTE(mkwiek): We need to use shell ln instead of os.symlink + # because of wildcards used in shotgun settings. ln utility + # will nicely handle wildcards and create all needed symlinks + # in target_path directory + symlink_command = 'ln -s "{}" "{}"'.format(path, target_path) logger.debug( - "Getting local file: cp -r %s %s", path, target_path) - utils.execute('mkdir -p "{0}"'.format(target_path)) - return utils.execute( - 'cp -r "{0}" "{1}"'.format(path, target_path)) + "Symlinking to local file: {}".format(symlink_command)) + return utils.execute(symlink_command) except fabric.exceptions.NetworkError as e: logger.error("NetworkError occured: %s", str(e)) raise @@ -199,8 +203,6 @@ class File(Driver): """ self.get(self.path, self.target_path) - if self.exclude: - utils.remove(self.full_dst_path, self.exclude) Dir = File diff --git a/shotgun/manager.py b/shotgun/manager.py index 116e39a..c9d192f 100644 --- a/shotgun/manager.py +++ b/shotgun/manager.py @@ -32,15 +32,19 @@ class Manager(object): def snapshot(self): logger.debug("Making snapshot") utils.execute("rm -rf {0}".format(os.path.dirname(self.conf.target))) + excludes = [] for obj_data in self.conf.objects: logger.debug("Dumping: %s", obj_data) self.action_single(obj_data, action='snapshot') + if 'exclude' in obj_data: + excludes.extend(os.path.join(obj_data['path'], ex) + for ex in obj_data['exclude']) logger.debug("Dumping shotgun log and archiving dump directory: %s", self.conf.target) self.action_single(self.conf.self_log_object, action='snapshot') - utils.compress(self.conf.target, self.conf.compression_level) + utils.compress(self.conf.target, self.conf.compression_level, excludes) with open(self.conf.lastdump, "w") as fo: fo.write("{0}.tar.xz".format(self.conf.target)) diff --git a/shotgun/test/test_driver.py b/shotgun/test/test_driver.py index 71442e5..6115d9d 100644 --- a/shotgun/test/test_driver.py +++ b/shotgun/test/test_driver.py @@ -148,10 +148,12 @@ class TestDriver(base.BaseTestCase): mfabrun.assert_called_with(command, stdout=mstdout) self.assertEqual(result.stdout, 'FULL STDOUT') - @mock.patch('shotgun.driver.utils.execute') + @mock.patch('os.path.exists', return_value=False) + @mock.patch('os.makedirs') + @mock.patch('shotgun.utils.execute') @mock.patch('shotgun.driver.fabric.api.settings') @mock.patch('shotgun.driver.fabric.api.get') - def test_driver_get(self, mfabget, mfabset, mexecute): + def test_driver_get(self, mfabget, mfabset, mexecute, mmakedirs, _): mexecute.return_value = ("RETURN_CODE", "STDOUT", "STDERR") remote_path = "/remote_dir/remote_file" target_path = "/target_dir" @@ -165,7 +167,7 @@ class TestDriver(base.BaseTestCase): }, }, conf) driver.get(remote_path, target_path) - mexecute.assert_called_with('mkdir -p "{0}"'.format(target_path)) + mmakedirs.assert_called_once_with(target_path) mfabget.assert_called_with(remote_path, target_path) mfabset.assert_called_with( @@ -173,11 +175,12 @@ class TestDriver(base.BaseTestCase): timeout=2, warn_only=True, abort_on_prompts=True) mexecute.reset_mock() + mmakedirs.reset_mock() driver = shotgun.driver.Driver({}, conf) driver.get(remote_path, target_path) - self.assertEqual(mexecute.mock_calls, [ - mock.call('mkdir -p "{0}"'.format(target_path)), - mock.call('cp -r "{0}" "{1}"'.format(remote_path, target_path))]) + mmakedirs.assert_called_once_with(target_path) + mexecute.assert_called_with('ln -s "{}" "{}"'.format(remote_path, + target_path)) def test_use_timeout_from_global_conf(self): data = {} @@ -252,28 +255,6 @@ class TestFile(base.BaseTestCase): mget.assert_called_with(data["path"], target_path) - @mock.patch('shotgun.driver.utils.remove') - @mock.patch('shotgun.driver.Driver.get') - def test_dir_exclude_called(self, mget, mremove): - data = { - "type": "dir", - "path": "/remote_dir/", - "exclude": ["*test"], - "host": { - "hostname": "remote_host", - "address": "10.109.0.2", - }, - } - conf = mock.MagicMock() - conf.target = "/target" - dir_driver = shotgun.driver.Dir(data, conf) - - target_path = "/target/remote_host/remote_dir" - dir_driver.snapshot() - - mget.assert_called_with(data["path"], target_path) - mremove.assert_called_with(dir_driver.full_dst_path, data['exclude']) - class TestCommand(base.BaseTestCase): def setUp(self): diff --git a/shotgun/test/test_utils.py b/shotgun/test/test_utils.py index ec79547..ad9a718 100644 --- a/shotgun/test/test_utils.py +++ b/shotgun/test/test_utils.py @@ -58,10 +58,28 @@ class TestUtils(base.BaseTestCase): self.assertEqual(compress_env['XZ_OPT'], level) self.assertEqual( compress_call[0][0], - 'tar cJvf /path/target.tar.xz -C /path target') + 'tar chJvf /path/target.tar.xz -C /path target') self.assertEqual(rm_call[0][0], 'rm -r /path/target') + @mock.patch('shotgun.utils.execute') + def test_compress_exclude(self, mexecute): + target = '/path/target' + level = '-3' + + exclusions = ['/path/to/exclude1', '/path/to/exclude2'] + + utils.compress(target, level, exclude=exclusions) + + compress_call = mexecute.call_args_list[0] + + compress_env = compress_call[1]['env'] + self.assertEqual(compress_env['XZ_OPT'], level) + self.assertEqual( + compress_call[0][0], + 'tar chJvf /path/target.tar.xz -C /path target ' + '--exclude /path/to/exclude1 --exclude /path/to/exclude2') + class TestCCStringIO(base.BaseTestCase): diff --git a/shotgun/utils.py b/shotgun/utils.py index a00cc50..8531984 100644 --- a/shotgun/utils.py +++ b/shotgun/utils.py @@ -58,19 +58,23 @@ def remove(full_dst_path, excludes): execute("shopt -s globstar; rm -rf {0}".format(path)) -def compress(target, level, keep_target=False): +def compress(target, level, keep_target=False, exclude=None): """Runs compression of provided directory :param target: directory to compress :param level: level of compression :param keep_target: bool, if True target directory wont be removed """ + if exclude is None: + exclude = [] + env = copy.deepcopy(os.environ) env['XZ_OPT'] = level - execute("tar cJvf {0}.tar.xz -C {1} {2}" + execute("tar chJvf {0}.tar.xz -C {1} {2}{3}" "".format(target, os.path.dirname(target), - os.path.basename(target)), + os.path.basename(target), + "".join(' --exclude {}'.format(e) for e in exclude)), env=env) if not keep_target: execute("rm -r {0}".format(target))