From 192d408d700d39df3a74edb99da853ede01b5b82 Mon Sep 17 00:00:00 2001 From: lkuchlan Date: Thu, 7 Sep 2023 13:44:31 +0300 Subject: [PATCH] Optimize the dd command to improve performance Use if=/dev/zero: Instead of reading from /dev/urandom, which is slow for generating random data, use /dev/zero to write zeros to the disk. Zeros are generated much faster. Omit conv=fsync: The conv=fsync option forces dd to synchronize the data after each write, which can be slow for large writes. Change-Id: Id845c60c39072a03171fa5906461eb254c2736a7 --- manila_tempest_tests/config.py | 13 +++++++++++++ .../tests/scenario/manager_share.py | 10 ++++------ .../tests/scenario/test_share_extend.py | 12 ++++-------- .../tests/scenario/test_share_manage_unmanage.py | 2 +- .../tests/scenario/test_share_shrink.py | 4 ++-- zuul.d/manila-tempest-jobs.yaml | 1 + 6 files changed, 25 insertions(+), 17 deletions(-) diff --git a/manila_tempest_tests/config.py b/manila_tempest_tests/config.py index 0cd07d31..4d36944c 100644 --- a/manila_tempest_tests/config.py +++ b/manila_tempest_tests/config.py @@ -336,4 +336,17 @@ ShareGroup = [ "attempt to create an IPv6 subnet on the project network " "they create for ping and SSH to the client test VM " "where data path testing is performed."), + cfg.StrOpt("dd_input_file", + default="/dev/zero", + help="The input file (if) in the dd command specifies the " + "source of data that dd will read and process, which can " + "be a device, a regular file, or even standard input " + "(stdin). dd copies, transforms, or performs actions on " + "this data based on provided options and then writes it " + "to an output file or device (of). When using /dev/zero " + "in storage systems with default compression, although " + "it generates highly compressible null bytes (zeros), " + "writing data from /dev/zero might not yield significant " + "space savings as these systems are already optimized for " + "efficient compression."), ] diff --git a/manila_tempest_tests/tests/scenario/manager_share.py b/manila_tempest_tests/tests/scenario/manager_share.py index 6d6efc05..3e06323b 100644 --- a/manila_tempest_tests/tests/scenario/manager_share.py +++ b/manila_tempest_tests/tests/scenario/manager_share.py @@ -229,8 +229,7 @@ class ShareScenarioTest(manager.NetworkScenarioTest): def write_data_to_mounted_share_using_dd(self, remote_client, output_file, block_size, - block_count, - input_file='/dev/zero'): + block_count): """Writes data to mounted share using dd command Example Usage for writing 512Mb to a file on /mnt/ @@ -243,13 +242,12 @@ class ShareScenarioTest(manager.NetworkScenarioTest): :param block_size: The size of an individual block in bytes :param block_count: The number of blocks to write :param output_file: Path to the file to be written - :param input_file: Path to the file to read from """ block_count = int(block_count) remote_client.exec_command( - "sudo sh -c \"dd bs={} count={} if={} of={} conv=fsync" - " iflag=fullblock\"" - .format(block_size, block_count, input_file, output_file)) + "sudo sh -c \"dd bs={} count={} if={} of={} iflag=fullblock\"" + .format(block_size, block_count, CONF.share.dd_input_file, + output_file)) def read_data_from_mounted_share(self, remote_client, diff --git a/manila_tempest_tests/tests/scenario/test_share_extend.py b/manila_tempest_tests/tests/scenario/test_share_extend.py index 595ddd49..450d2d4c 100644 --- a/manila_tempest_tests/tests/scenario/test_share_extend.py +++ b/manila_tempest_tests/tests/scenario/test_share_extend.py @@ -76,8 +76,7 @@ class ShareExtendBase(manager.ShareScenarioTest): .format(three_quarter_blocks)) self.write_data_to_mounted_share_using_dd(remote_client, '/mnt/t1', '64M', - three_quarter_blocks, - '/dev/urandom') + three_quarter_blocks) ls_result = remote_client.exec_command("sudo ls -lAh /mnt/") LOG.debug(ls_result) @@ -86,8 +85,7 @@ class ShareExtendBase(manager.ShareScenarioTest): self.assertRaises( exceptions.SSHExecCommandFailed, self.write_data_to_mounted_share_using_dd, - remote_client, '/mnt/t2', '64M', over_one_quarter_blocks, - '/dev/urandom') + remote_client, '/mnt/t2', '64M', over_one_quarter_blocks) ls_result = remote_client.exec_command("sudo ls -lAh /mnt/") LOG.debug(ls_result) @@ -129,8 +127,7 @@ class ShareExtendBase(manager.ShareScenarioTest): self.write_data_to_mounted_share_using_dd(remote_client, output_file, block_size, - block_count, - '/dev/urandom') + block_count) except exceptions.SSHExecCommandFailed as e: if 'stale file handle' in str(e).lower(): LOG.warning("Client was disconnected during extend process") @@ -139,8 +136,7 @@ class ShareExtendBase(manager.ShareScenarioTest): self.write_data_to_mounted_share_using_dd(remote_client, output_file, block_size, - block_count, - '/dev/urandom') + block_count) else: raise diff --git a/manila_tempest_tests/tests/scenario/test_share_manage_unmanage.py b/manila_tempest_tests/tests/scenario/test_share_manage_unmanage.py index ef2a6afa..bbf5ba65 100644 --- a/manila_tempest_tests/tests/scenario/test_share_manage_unmanage.py +++ b/manila_tempest_tests/tests/scenario/test_share_manage_unmanage.py @@ -105,7 +105,7 @@ class ShareManageUnmanageBase(manager.ShareScenarioTest): LOG.debug('Step 6b - writing 640mb') self.write_data_to_mounted_share_using_dd(remote_client, '/mnt/t1', 1024, - 2048, '/dev/zero') + 2048) ls_result = remote_client.exec_command("sudo ls -lA /mnt/") LOG.debug(ls_result) diff --git a/manila_tempest_tests/tests/scenario/test_share_shrink.py b/manila_tempest_tests/tests/scenario/test_share_shrink.py index a4e59e8d..fab7e0a3 100644 --- a/manila_tempest_tests/tests/scenario/test_share_shrink.py +++ b/manila_tempest_tests/tests/scenario/test_share_shrink.py @@ -79,7 +79,7 @@ class ShareShrinkBase(manager.ShareScenarioTest): LOG.debug('Step 6 - writing {} * 64MB blocks'.format(blocks)) self.write_data_to_mounted_share_using_dd(remote_client, '/mnt/t1', '64M', - blocks, '/dev/urandom') + blocks) ls_result = remote_client.exec_command("sudo ls -lAh /mnt/") LOG.debug(ls_result) @@ -120,7 +120,7 @@ class ShareShrinkBase(manager.ShareScenarioTest): self.assertRaises( exceptions.SSHExecCommandFailed, self.write_data_to_mounted_share_using_dd, - remote_client, '/mnt/t1', '64M', blocks, '/dev/urandom') + remote_client, '/mnt/t1', '64M', blocks) LOG.debug('Step 12 - unmount') self.unmount_share(remote_client) diff --git a/zuul.d/manila-tempest-jobs.yaml b/zuul.d/manila-tempest-jobs.yaml index f2829bcb..6214867b 100644 --- a/zuul.d/manila-tempest-jobs.yaml +++ b/zuul.d/manila-tempest-jobs.yaml @@ -157,6 +157,7 @@ backend_names: LONDON,PARIS multi_backend: true image_password: manila + dd_input_file: /dev/urandom - job: name: manila-tempest-plugin-zfsonlinux