openstack
/
nova
Code Issues Proposed changes

libvirt:Rsync compression removed 

During migration libvirt driver copies qcow image between nodes.
Compression ratio of qcow disk image is small because such type
of image doesn't  allocate the whole image space to a file. It
grows as data is added. But compression procedure takes many CPU
time.

Closes-Bug: #1478800
Change-Id: Iabcbbb576f7e9411310c540badb805eb1bf21bf5
This commit is contained in:
Marian Horban 2015-08-26 09:41:02 -04:00
parent bffdec0f0f
commit b7ad5a312a
6 changed files with 41 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
nova/tests/unit/virt/libvirt/test_driver.py
View File

@ -12396,7 +12396,8 @@ class LibvirtDriverTestCase(test.NoDBTestCase):
pass
def fake_copy_image(src, dest, host=None, receive=False,
on_execute=None, on_completion=None):
on_execute=None, on_completion=None,
compression=True):
self.assertIsNotNone(on_execute)
self.assertIsNotNone(on_completion)

4
nova/tests/unit/virt/libvirt/test_utils.py
View File

@ -67,14 +67,14 @@ blah BLAH: bb
self.flags(remote_filesystem_transport='ssh', group='libvirt')
libvirt_utils.copy_image('src', 'dest', host='host')
mock_rem_fs_remove.assert_called_once_with('src', 'host:dest',
on_completion=None, on_execute=None)
on_completion=None, on_execute=None, compression=True)
@mock.patch('nova.virt.libvirt.volume.remotefs.RsyncDriver.copy_file')
def test_copy_image_remote_rsync(self, mock_rem_fs_remove):
self.flags(remote_filesystem_transport='rsync', group='libvirt')
libvirt_utils.copy_image('src', 'dest', host='host')
mock_rem_fs_remove.assert_called_once_with('src', 'host:dest',
on_completion=None, on_execute=None)
on_completion=None, on_execute=None, compression=True)
@mock.patch('os.path.exists', return_value=True)
def test_disk_type(self, mock_exists):

19
nova/tests/unit/virt/libvirt/volume/test_remotefs.py
View File

@ -164,8 +164,21 @@ class RemoteFSTestCase(test.NoDBTestCase):
@mock.patch('nova.utils.execute')
def test_remote_copy_file_rsync(self, mock_execute):
remotefs.RsyncDriver().copy_file('1.2.3.4:/home/star_wars',
'/home/favourite', None, None)
mock_execute.assert_called_once_with('rsync', '--sparse', '--compress',
'/home/favourite', None, None,
compression=True)
mock_execute.assert_called_once_with('rsync', '--sparse',
'1.2.3.4:/home/star_wars',
'/home/favourite',
'--compress',
on_completion=None,
on_execute=None)
@mock.patch('nova.utils.execute')
def test_remote_copy_file_rsync_without_compression(self, mock_execute):
remotefs.RsyncDriver().copy_file('1.2.3.4:/home/star_wars',
'/home/favourite', None, None,
compression=False)
mock_execute.assert_called_once_with('rsync', '--sparse',
'1.2.3.4:/home/star_wars',
'/home/favourite',
on_completion=None,
@ -174,7 +187,7 @@ class RemoteFSTestCase(test.NoDBTestCase):
@mock.patch('nova.utils.execute')
def test_remote_copy_file_ssh(self, mock_execute):
remotefs.SshDriver().copy_file('1.2.3.4:/home/SpaceOdyssey',
'/home/favourite', None, None)
'/home/favourite', None, None, True)
mock_execute.assert_called_once_with('scp',
'1.2.3.4:/home/SpaceOdyssey',
'/home/favourite',

7
nova/virt/libvirt/driver.py
View File

@ -423,6 +423,9 @@ MIN_LIBVIRT_SET_ADMIN_PASSWD = (1, 2, 16)
MIN_LIBVIRT_KVM_S390_VERSION = (1, 2, 13)
MIN_QEMU_S390_VERSION = (2, 3, 0)
# Names of the types that do not get compressed during migration
NO_COMPRESSION_TYPES = ('qcow2',)
class LibvirtDriver(driver.ComputeDriver):
capabilities = {
@ -6728,9 +6731,11 @@ class LibvirtDriver(driver.ComputeDriver):
instance, process.pid)
on_completion = lambda process: self.job_tracker.\
remove_job(instance, process.pid)
compression = info['type'] not in NO_COMPRESSION_TYPES
libvirt_utils.copy_image(from_path, img_path, host=dest,
on_execute=on_execute,
on_completion=on_completion)
on_completion=on_completion,
compression=compression)
except Exception:
with excutils.save_and_reraise_exception():
self._cleanup_remote_migration(dest, inst_base,

8
nova/virt/libvirt/utils.py
View File

@ -185,7 +185,8 @@ def get_disk_backing_file(path, basename=True):
def copy_image(src, dest, host=None, receive=False,
on_execute=None, on_completion=None):
on_execute=None, on_completion=None,
compression=True):
"""Copy a disk image to an existing directory
:param src: Source image
@ -194,6 +195,8 @@ def copy_image(src, dest, host=None, receive=False,
:param receive: Reverse the rsync direction
:param on_execute: Callback method to store pid of process in cache
:param on_completion: Callback method to remove pid of process from cache
:param compression: Allows to use rsync operation with or without
compression
"""
if not host:
@ -210,7 +213,8 @@ def copy_image(src, dest, host=None, receive=False,
remote_filesystem_driver = remotefs.RemoteFilesystem()
remote_filesystem_driver.copy_file(src, dest,
on_execute=on_execute, on_completion=on_completion)
on_execute=on_execute, on_completion=on_completion,
compression=compression)
def write_to_file(path, contents, umask=None):

14
nova/virt/libvirt/volume/remotefs.py
View File

@ -117,10 +117,11 @@ class RemoteFilesystem(object):
on_completion=on_completion)
def copy_file(self, src, dst, on_execute=None,
on_completion=None):
on_completion=None, compression=True):
LOG.debug("Copying file %s to %s", src, dst)
self.driver.copy_file(src, dst, on_execute=on_execute,
on_completion=on_completion)
on_completion=on_completion,
compression=compression)
@six.add_metaclass(abc.ABCMeta)
@ -203,7 +204,7 @@ class SshDriver(RemoteFilesystemDriver):
utils.execute('ssh', host, 'rm', '-rf', dst,
on_execute=on_execute, on_completion=on_completion)
def copy_file(self, src, dst, on_execute, on_completion):
def copy_file(self, src, dst, on_execute, on_completion, compression):
utils.execute('scp', src, dst,
on_execute=on_execute, on_completion=on_completion)
@ -331,6 +332,9 @@ class RsyncDriver(RemoteFilesystemDriver):
relative_tmp_file_path, '%s:%s' % (host, os.path.sep),
on_execute=on_execute, on_completion=on_completion)
def copy_file(self, src, dst, on_execute, on_completion):
utils.execute('rsync', '--sparse', '--compress', src, dst,
def copy_file(self, src, dst, on_execute, on_completion, compression):
args = ['rsync', '--sparse', src, dst]
if compression:
args.append('--compress')
utils.execute(*args,
on_execute=on_execute, on_completion=on_completion)