From 9073f7591ec398c8825fc82f8dda630c4d8ad3f3 Mon Sep 17 00:00:00 2001 From: Liang Fang Date: Wed, 31 Jul 2019 09:28:27 +0000 Subject: [PATCH] Leverage hw accelerator in image compression When trying to upload volume to glance as image, currently all the format transformation is done by software and the performance is not good. Leverage hardware accelerator to do the image compression, so as to offload CPU, release CPU to do more common and complex thing. Professional hardware accelerator will get better performance and shorter conversion time than software solution. Currently hardware accelerator is getting more popular and some are integrated in server chipset by default. This patch includes: 1. Uses the new image container_format 'compressed' introduced by Glance in the Train release 2. Implemented a simple framework: if there's an accelerator detected in system, then try to use it 3. Supported Intel QAT as one of the accelerator 4. Add command filter for command 'qzip' and 'gzip' in rootwrap 5. New configuration option 'allow_compression_on_image_upload' and 'compression_format' added 6. Releasenote added Change-Id: I8460f58d2ad95a6654cf4d6a6bb367f3c537536b Implements: blueprint leverage-compression-accelerator Signed-off-by: Liang Fang --- cinder/common/config.py | 12 + cinder/exception.py | 13 ++ cinder/image/accelerator.py | 105 +++++++++ cinder/image/accelerators/__init__.py | 0 cinder/image/accelerators/gzip.py | 98 ++++++++ cinder/image/accelerators/qat.py | 99 ++++++++ cinder/image/image_utils.py | 59 ++++- cinder/opts.py | 1 + .../tests/unit/image/accelerators/__init__.py | 0 .../unit/image/accelerators/test_qat_gzip.py | 217 ++++++++++++++++++ cinder/tests/unit/image/test_accelerator.py | 100 ++++++++ cinder/tests/unit/test_image_utils.py | 204 +++++++++++++++- cinder/volume/api.py | 7 + etc/cinder/rootwrap.d/volume.filters | 2 + ...pression-accelerator-579c7032290cd1e9.yaml | 49 ++++ 15 files changed, 948 insertions(+), 18 deletions(-) create mode 100644 cinder/image/accelerator.py create mode 100644 cinder/image/accelerators/__init__.py create mode 100644 cinder/image/accelerators/gzip.py create mode 100644 cinder/image/accelerators/qat.py create mode 100644 cinder/tests/unit/image/accelerators/__init__.py create mode 100644 cinder/tests/unit/image/accelerators/test_qat_gzip.py create mode 100644 cinder/tests/unit/image/test_accelerator.py create mode 100644 releasenotes/notes/leverage-compression-accelerator-579c7032290cd1e9.yaml diff --git a/cinder/common/config.py b/cinder/common/config.py index 79790c909f3..767f23df7d6 100644 --- a/cinder/common/config.py +++ b/cinder/common/config.py @@ -195,12 +195,24 @@ image_opts = [ 'value is used.'), ] +compression_opts = [ + cfg.StrOpt('compression_format', + default='gzip', + choices=[('gzip', 'GNUzip format')], + help='Image compression format on image upload'), + cfg.BoolOpt('allow_compression_on_image_upload', + default=False, + help='The strategy to use for image compression on upload. ' + 'Default is disallow compression.'), +] + CONF.register_opts(api_opts) CONF.register_opts(core_opts) CONF.register_opts(auth_opts) CONF.register_opts(backup_opts) CONF.register_opts(image_opts) CONF.register_opts(global_opts) +CONF.register_opts(compression_opts) def set_middleware_defaults(): diff --git a/cinder/exception.py b/cinder/exception.py index 1638ca499ca..2396b8ca722 100644 --- a/cinder/exception.py +++ b/cinder/exception.py @@ -1099,3 +1099,16 @@ class ServiceUserTokenNoAuth(CinderException): class RekeyNotSupported(CinderException): message = _("Rekey not supported.") + + +class ImageCompressionNotAllowed(CinderException): + message = _("Image compression upload disallowed, but container_format " + "is compressed") + + +class CinderAcceleratorError(CinderException): + message = _("Cinder accelerator %(accelerator)s encountered an error " + "while compressing/decompressing image.\n" + "Command %(cmd)s execution failed.\n" + "%(description)s\n" + "Reason: %(reason)s") diff --git a/cinder/image/accelerator.py b/cinder/image/accelerator.py new file mode 100644 index 00000000000..a471fb508d1 --- /dev/null +++ b/cinder/image/accelerator.py @@ -0,0 +1,105 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc + +from oslo_config import cfg +from oslo_log import log as logging +from oslo_utils import importutils +import six + +from cinder import exception +from cinder.i18n import _ + +LOG = logging.getLogger(__name__) + +CONF = cfg.CONF + +# NOTE(ZhengMa): The order of the option is improtant, accelerators +# are looked by this list order +# Be careful to edit it +_ACCEL_PATH_PREFERENCE_ORDER_LIST = [ + 'cinder.image.accelerators.qat.AccelQAT', + 'cinder.image.accelerators.gzip.AccelGZIP', +] + + +@six.add_metaclass(abc.ABCMeta) +class AccelBase(object): + def __init__(self): + return + + @abc.abstractmethod + def is_accel_exist(self): + return + + @abc.abstractmethod + def compress_img(self, run_as_root): + return + + @abc.abstractmethod + def decompress_img(self, run_as_root): + return + + +class ImageAccel(object): + + def __init__(self, src, dest): + self.src = src + self.dest = dest + self.compression_format = CONF.compression_format + if(self.compression_format == 'gzip'): + self._accel_engine_path = _ACCEL_PATH_PREFERENCE_ORDER_LIST + else: + self._accel_engine_path = None + self.engine = self._get_engine() + + def _get_engine(self, *args, **kwargs): + if self._accel_engine_path: + for accel in self._accel_engine_path: + engine_cls = importutils.import_class(accel) + eng = engine_cls(*args, **kwargs) + if eng.is_accel_exist(): + return eng + + ex_msg = _("No valid accelerator") + raise exception.CinderException(ex_msg) + + def is_engine_ready(self): + + if not self.engine: + return False + if not self.engine.is_accel_exist(): + return False + return True + + def compress_img(self, run_as_root): + if not self.is_engine_ready(): + return + self.engine.compress_img(self.src, + self.dest, + run_as_root) + + def decompress_img(self, run_as_root): + if not self.is_engine_ready(): + return + self.engine.decompress_img(self.src, + self.dest, + run_as_root) + + +def is_gzip_compressed(image_file): + # The first two bytes of a gzip file are: 1f 8b + GZIP_MAGIC_BYTES = b'\x1f\x8b' + with open(image_file, 'rb') as f: + return f.read(2) == GZIP_MAGIC_BYTES diff --git a/cinder/image/accelerators/__init__.py b/cinder/image/accelerators/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cinder/image/accelerators/gzip.py b/cinder/image/accelerators/gzip.py new file mode 100644 index 00000000000..1b92cb5b22a --- /dev/null +++ b/cinder/image/accelerators/gzip.py @@ -0,0 +1,98 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from oslo_concurrency import processutils +from oslo_log import log as logging + +from cinder import exception +from cinder.i18n import _ +from cinder.image import accelerator +from cinder import utils + +LOG = logging.getLogger(__name__) + + +class AccelGZIP(accelerator.AccelBase): + def is_accel_exist(self): + cmd = ['which', 'gzip'] + try: + utils.execute(*cmd) + except processutils.ProcessExecutionError: + LOG.error("GZIP package is not installed.") + return False + + return True + + # NOTE(ZhengMa): Gzip compresses a file in-place and adds a .gz + # extension to the filename, so we rename the compressed file back + # to the name Cinder expects it to have. + # (Cinder expects to have A to upload) + # Follow these steps: + # 1. compress A to A.gz (gzip_out_file is A.gz) + # 2. mv A.gz to A (gzip_out_file to dest) + def compress_img(self, src, dest, run_as_root): + try: + gzip_compress_cmd = ['gzip', '-k', src] + utils.execute(*gzip_compress_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + raise exception.CinderAcceleratorError( + accelerator='GZIP', + description=_("Volume compression failed while " + "uploading to glance. GZIP compression " + "command failed."), + cmd=gzip_compress_cmd, + reason=ex.stderr) + try: + gzip_output_filename = src + '.gz' + mv_cmd = ['mv', gzip_output_filename, dest] + utils.execute(*mv_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + fnames = {'i_fname': gzip_output_filename, 'o_fname': dest} + raise exception.CinderAcceleratorError( + accelerator='GZIP', + description = _("Failed to rename %(i_fname)s " + "to %(o_fname)s") % fnames, + cmd=mv_cmd, + reason=ex.stderr) + + # NOTE(ZhengMa): Gzip can only decompresses a file with a .gz + # extension to the filename, so we rename the original file so + # that it can be accepted by Gzip. + # Follow these steps: + # 1. mv A to A.gz (gzip_in_file is A.gz) + # 2. decompress A.gz to A (gzip_in_file to dest) + def decompress_img(self, src, dest, run_as_root): + try: + gzip_input_filename = dest + '.gz' + mv_cmd = ['mv', src, gzip_input_filename] + utils.execute(*mv_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + fnames = {'i_fname': src, 'o_fname': gzip_input_filename} + raise exception.CinderAcceleratorError( + accelerator='GZIP', + description = _("Failed to rename %(i_fname)s " + "to %(o_fname)s") % fnames, + cmd=mv_cmd, + reason=ex.stderr) + try: + gzip_decompress_cmd = ['gzip', '-d', gzip_input_filename] + utils.execute(*gzip_decompress_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + raise exception.CinderAcceleratorError( + accelerator='GZIP', + description = _("Image decompression failed while " + "downloading from glance. GZIP " + "decompression command failed."), + cmd=gzip_decompress_cmd, + reason=ex.stderr) diff --git a/cinder/image/accelerators/qat.py b/cinder/image/accelerators/qat.py new file mode 100644 index 00000000000..b07c583fb03 --- /dev/null +++ b/cinder/image/accelerators/qat.py @@ -0,0 +1,99 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from oslo_concurrency import processutils +from oslo_log import log as logging + +from cinder import exception +from cinder.i18n import _ +from cinder.image import accelerator +from cinder import utils + + +LOG = logging.getLogger(__name__) + + +class AccelQAT(accelerator.AccelBase): + def is_accel_exist(self): + cmd = ['which', 'qzip'] + try: + utils.execute(*cmd) + except processutils.ProcessExecutionError: + LOG.error("QATzip package is not installed.") + return False + + return True + + # NOTE(ZhengMa): QATzip compresses a file in-place and adds a .gz + # extension to the filename, so we rename the compressed file back + # to the name Cinder expects it to have. + # (Cinder expects to have A to upload) + # Follow these steps: + # 1. compress A to A.gz (src to qat_out_file) + # 2. mv A.gz to A (qat_out_file to dest) + def compress_img(self, src, dest, run_as_root): + try: + qat_compress_cmd = ['qzip', '-k', src, '-o', dest] + utils.execute(*qat_compress_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + raise exception.CinderAcceleratorError( + accelerator='QAT', + description=_("Volume compression failed while " + "uploading to glance. QAT compression " + "command failed."), + cmd=qat_compress_cmd, + reason=ex.stderr) + try: + qat_output_filename = src + '.gz' + mv_cmd = ['mv', qat_output_filename, dest] + utils.execute(*mv_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + fnames = {'i_fname': qat_output_filename, 'o_fname': dest} + raise exception.CinderAcceleratorError( + accelerator='QAT', + description = _("Failed to rename %(i_fname)s " + "to %(o_fname)s") % fnames, + cmd=mv_cmd, + reason=ex.stderr) + + # NOTE(ZhengMa): QATzip can only decompresses a file with a .gz + # extension to the filename, so we rename the original file so + # that it can be accepted by QATzip. + # Follow these steps: + # 1. mv A to A.gz (qat_in_file is A.gz) + # 2. decompress A.gz to A (qat_in_file to dest) + def decompress_img(self, src, dest, run_as_root): + try: + qat_input_filename = dest + '.gz' + mv_cmd = ['mv', src, qat_input_filename] + utils.execute(*mv_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + fnames = {'i_fname': src, 'o_fname': qat_input_filename} + raise exception.CinderAcceleratorError( + accelerator='QAT', + description = _("Failed to rename %(i_fname)s " + "to %(o_fname)s") % fnames, + cmd=mv_cmd, + reason=ex.stderr) + try: + qat_decompress_cmd = ['qzip', '-d', qat_input_filename] + utils.execute(*qat_decompress_cmd, run_as_root=run_as_root) + except processutils.ProcessExecutionError as ex: + raise exception.CinderAcceleratorError( + accelerator='QAT', + description = _("Image decompression failed while " + "downloading from glance. QAT " + "decompression command failed."), + cmd=qat_decompress_cmd, + reason=ex.stderr) diff --git a/cinder/image/image_utils.py b/cinder/image/image_utils.py index 7d57b839ebf..93c66a4be26 100644 --- a/cinder/image/image_utils.py +++ b/cinder/image/image_utils.py @@ -47,6 +47,7 @@ import six from cinder import exception from cinder.i18n import _ +from cinder.image import accelerator from cinder import utils from cinder.volume import throttling from cinder.volume import volume_utils @@ -279,7 +280,6 @@ def _convert_image(prefix, source, dest, out_format, LOG.error(message) raise - duration = timeutils.delta_seconds(start_time, timeutils.utcnow()) # NOTE(jdg): use a default of 1, mostly for unit test, but in @@ -551,6 +551,17 @@ def fetch_to_volume_format(context, image_service, qemu_img = True image_meta = image_service.show(context, image_id) + allow_image_compression = CONF.allow_compression_on_image_upload + if image_meta and (image_meta.get('container_format') == 'compressed'): + if allow_image_compression is False: + compression_param = {'container_format': + image_meta.get('container_format')} + raise exception.ImageUnacceptable( + image_id=image_id, + reason=_("Image compression disallowed, " + "but container_format is " + "%(container_format)s.") % compression_param) + # NOTE(avishay): I'm not crazy about creating temp files which may be # large and cause disk full errors which would confuse users. # Unfortunately it seems that you can't pipe to 'qemu-img convert' because @@ -608,6 +619,21 @@ def fetch_to_volume_format(context, image_service, reason=_("fmt=%(fmt)s backed by:%(backing_file)s") % {'fmt': fmt, 'backing_file': backing_file, }) + # NOTE(ZhengMa): This is used to do image decompression on image + # downloading with 'compressed' container_format. It is a + # transparent level between original image downloaded from + # Glance and Cinder image service. So the source file path is + # the same with destination file path. + if image_meta.get('container_format') == 'compressed': + LOG.debug("Found image with compressed container format") + if not accelerator.is_gzip_compressed(tmp): + raise exception.ImageUnacceptable( + image_id=image_id, + reason=_("Unsupported compressed image format found. " + "Only gzip is supported currently")) + accel = accelerator.ImageAccel(tmp, tmp) + accel.decompress_img(run_as_root=run_as_root) + # NOTE(jdg): I'm using qemu-img convert to write # to the volume regardless if it *needs* conversion or not # TODO(avishay): We can speed this up by checking if the image is raw @@ -615,8 +641,8 @@ def fetch_to_volume_format(context, image_service, # check via 'qemu-img info' that what we copied was in fact a raw # image and not a different format with a backing file, which may be # malicious. - LOG.debug("%s was %s, converting to %s ", image_id, fmt, volume_format) disk_format = fixup_disk_format(image_meta['disk_format']) + LOG.debug("%s was %s, converting to %s", image_id, fmt, volume_format) convert_image(tmp, dest, volume_format, out_subformat=volume_subformat, @@ -636,19 +662,20 @@ def _validate_file_format(image_data, expected_format): def upload_volume(context, image_service, image_meta, volume_path, volume_format='raw', run_as_root=True, compress=True): image_id = image_meta['id'] - if (image_meta['disk_format'] == volume_format): - LOG.debug("%s was %s, no need to convert to %s", - image_id, volume_format, image_meta['disk_format']) - if os.name == 'nt' or os.access(volume_path, os.R_OK): - with open(volume_path, 'rb') as image_file: - image_service.update(context, image_id, {}, - tpool.Proxy(image_file)) - else: - with utils.temporary_chown(volume_path): + if image_meta.get('container_format') != 'compressed': + if (image_meta['disk_format'] == volume_format): + LOG.debug("%s was %s, no need to convert to %s", + image_id, volume_format, image_meta['disk_format']) + if os.name == 'nt' or os.access(volume_path, os.R_OK): with open(volume_path, 'rb') as image_file: image_service.update(context, image_id, {}, tpool.Proxy(image_file)) - return + else: + with utils.temporary_chown(volume_path): + with open(volume_path, 'rb') as image_file: + image_service.update(context, image_id, {}, + tpool.Proxy(image_file)) + return with temporary_file() as tmp: LOG.debug("%s was %s, converting to %s", @@ -679,6 +706,14 @@ def upload_volume(context, image_service, image_meta, volume_path, reason=_("Converted to %(f1)s, but format is now %(f2)s") % {'f1': out_format, 'f2': data.file_format}) + # NOTE(ZhengMa): This is used to do image compression on image + # uploading with 'compressed' container_format. + # Compress file 'tmp' in-place + if image_meta.get('container_format') == 'compressed': + LOG.debug("Container_format set to 'compressed', compressing " + "image before uploading.") + accel = accelerator.ImageAccel(tmp, tmp) + accel.compress_img(run_as_root=run_as_root) with open(tmp, 'rb') as image_file: image_service.update(context, image_id, {}, tpool.Proxy(image_file)) diff --git a/cinder/opts.py b/cinder/opts.py index 6fbc39cf9a4..a6872c4ba85 100644 --- a/cinder/opts.py +++ b/cinder/opts.py @@ -226,6 +226,7 @@ def list_opts(): cinder_common_config.backup_opts, cinder_common_config.image_opts, cinder_common_config.global_opts, + cinder_common_config.compression_opts, cinder.compute.compute_opts, cinder_context.context_opts, cinder_db_api.db_opts, diff --git a/cinder/tests/unit/image/accelerators/__init__.py b/cinder/tests/unit/image/accelerators/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cinder/tests/unit/image/accelerators/test_qat_gzip.py b/cinder/tests/unit/image/accelerators/test_qat_gzip.py new file mode 100644 index 00000000000..3fa87a94e85 --- /dev/null +++ b/cinder/tests/unit/image/accelerators/test_qat_gzip.py @@ -0,0 +1,217 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from cinder import exception +from cinder.image import accelerator +from cinder import test + + +class TestAccelerators(test.TestCase): + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = True) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = True) + # Compress test, QAT and GZIP available + def test_compress_img_prefer_qat_when_available(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + + source = 'fake_path' + dest = 'fake_path' + + accel = accelerator.ImageAccel(source, dest) + accel.compress_img(run_as_root=True) + + expected = [ + mock.call('qzip', '-k', dest, '-o', dest, + run_as_root=True), + mock.call('mv', dest + '.gz', dest, + run_as_root=True) + ] + + mock_exec.assert_has_calls(expected) + + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = False) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = True) + # Compress test, QAT not available but GZIP available + def test_compress_img_qat_accel_not_exist_gzip_exist(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + source = 'fake_path' + dest = 'fake_path' + + accel = accelerator.ImageAccel(source, dest) + accel.compress_img(run_as_root=True) + + not_called = mock.call('qzip', '-k', dest, '-o', dest, + run_as_root=True) + + self.assertNotIn(not_called, mock_exec.call_args_list) + + expected = [ + mock.call('gzip', '-k', dest, + run_as_root=True), + mock.call('mv', dest + '.gz', dest, + run_as_root=True) + ] + mock_exec.assert_has_calls(expected) + + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = True) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = False) + # Compress test, QAT available but GZIP not available + def test_compress_img_prefer_qat_without_gzip(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + + source = 'fake_path' + dest = 'fake_path' + + accel = accelerator.ImageAccel(source, dest) + accel.compress_img(run_as_root=True) + + expected = [ + mock.call('qzip', '-k', dest, '-o', dest, + run_as_root=True), + mock.call('mv', dest + '.gz', dest, + run_as_root=True) + ] + + mock_exec.assert_has_calls(expected) + + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = False) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = False) + # Compress test, no accelerator available + def test_compress_img_no_accel_exist(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + source = 'fake_path' + dest = 'fake_path' + + self.assertRaises(exception.CinderException, + accelerator.ImageAccel, + source, + dest) + + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = True) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = True) + # Decompress test, QAT and GZIP available + def test_decompress_img_prefer_qat_when_available(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + source = 'fake_path' + dest = 'fake_path' + + accel = accelerator.ImageAccel(source, dest) + accel.decompress_img(run_as_root=True) + + expected = [ + mock.call('mv', source, source + '.gz', + run_as_root=True), + mock.call('qzip', '-d', source + '.gz', + run_as_root=True) + ] + + mock_exec.assert_has_calls(expected) + + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = False) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = True) + # Decompress test, QAT not available but GZIP available + def test_decompress_img_qat_accel_not_exist_gzip_exist(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + source = 'fake_path' + dest = 'fake_path' + + accel = accelerator.ImageAccel(source, dest) + accel.decompress_img(run_as_root=True) + + not_called = mock.call('qzip', '-d', source + '.gz', + run_as_root=True) + + self.assertNotIn(not_called, mock_exec.call_args_list) + + expected = [ + mock.call('mv', source, source + '.gz', + run_as_root=True), + mock.call('gzip', '-d', source + '.gz', + run_as_root=True) + ] + + mock_exec.assert_has_calls(expected) + + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = True) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = False) + # Decompress test, QAT available but GZIP not available + def test_decompress_img_prefer_qat_without_gzip(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + source = 'fake_path' + dest = 'fake_path' + + accel = accelerator.ImageAccel(source, dest) + accel.decompress_img(run_as_root=True) + + expected = [ + mock.call('mv', source, source + '.gz', + run_as_root=True), + mock.call('qzip', '-d', source + '.gz', + run_as_root=True) + ] + + mock_exec.assert_has_calls(expected) + + @mock.patch('cinder.utils.execute') + @mock.patch('cinder.image.accelerators.qat.AccelQAT.is_accel_exist', + return_value = False) + @mock.patch('cinder.image.accelerators.gzip.AccelGZIP.is_accel_exist', + return_value = False) + # Decompress test, no accelerator available + def test_decompress_img_no_accel_exist(self, + mock_gzip_exist, + mock_qat_exist, + mock_exec): + source = 'fake_path' + dest = 'fake_path' + + self.assertRaises(exception.CinderException, + accelerator.ImageAccel, + source, + dest) diff --git a/cinder/tests/unit/image/test_accelerator.py b/cinder/tests/unit/image/test_accelerator.py new file mode 100644 index 00000000000..4f7466c7b76 --- /dev/null +++ b/cinder/tests/unit/image/test_accelerator.py @@ -0,0 +1,100 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import mock + +from cinder.image import accelerator +from cinder import test + + +class fakeEngine(object): + + def __init__(self): + pass + + def compress_img(self, src, dest, run_as_root): + pass + + def decompress_img(self, src, dest, run_as_root): + pass + + +class TestAccelerator(test.TestCase): + + @mock.patch('cinder.image.accelerator.ImageAccel._get_engine') + @mock.patch('cinder.image.accelerator.ImageAccel.is_engine_ready', + return_value = True) + def test_compress_img_engine_ready(self, mock_accel_engine_ready, + mock_get_engine): + source = mock.sentinel.source + dest = mock.sentinel.dest + run_as_root = mock.sentinel.run_as_root + + mock_engine = mock.Mock(spec=fakeEngine) + mock_get_engine.return_value = mock_engine + accel = accelerator.ImageAccel(source, dest) + + accel.compress_img(run_as_root=run_as_root) + mock_engine.compress_img.assert_called() + + @mock.patch('cinder.image.accelerator.ImageAccel._get_engine') + @mock.patch('cinder.image.accelerator.ImageAccel.is_engine_ready', + return_value = False) + def test_compress_img_engine_not_ready(self, mock_accel_engine_ready, + mock_get_engine): + + source = mock.sentinel.source + dest = mock.sentinel.dest + run_as_root = mock.sentinel.run_as_root + + mock_engine = mock.Mock(spec=fakeEngine) + mock_get_engine.return_value = mock_engine + accel = accelerator.ImageAccel(source, dest) + + accel.compress_img(run_as_root=run_as_root) + mock_engine.compress_img.assert_not_called() + + @mock.patch('cinder.image.accelerator.ImageAccel._get_engine') + @mock.patch('cinder.image.accelerator.ImageAccel.is_engine_ready', + return_value = True) + def test_decompress_img_engine_ready(self, mock_accel_engine_ready, + mock_get_engine): + + source = mock.sentinel.source + dest = mock.sentinel.dest + run_as_root = mock.sentinel.run_as_root + + mock_engine = mock.Mock(spec=fakeEngine) + mock_get_engine.return_value = mock_engine + accel = accelerator.ImageAccel(source, dest) + + accel.decompress_img(run_as_root=run_as_root) + mock_engine.decompress_img.assert_called() + + @mock.patch('cinder.image.accelerator.ImageAccel._get_engine') + @mock.patch('cinder.image.accelerator.ImageAccel.is_engine_ready', + return_value = False) + def test_decompress_img_engine_not_ready(self, mock_accel_engine_ready, + mock_get_engine): + + source = mock.sentinel.source + dest = mock.sentinel.dest + run_as_root = mock.sentinel.run_as_root + + mock_engine = mock.Mock(spec=fakeEngine) + mock_get_engine.return_value = mock_engine + accel = accelerator.ImageAccel(source, dest) + + accel.decompress_img(run_as_root=run_as_root) + mock_engine.decompress_img.assert_not_called() diff --git a/cinder/tests/unit/test_image_utils.py b/cinder/tests/unit/test_image_utils.py index cb13dab5ae4..f7040ee3a06 100644 --- a/cinder/tests/unit/test_image_utils.py +++ b/cinder/tests/unit/test_image_utils.py @@ -1,5 +1,3 @@ -# Copyright (c) 2013 eNovance , Inc. -# All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -740,7 +738,8 @@ class TestUploadVolume(test.TestCase): ctxt = mock.sentinel.context image_service = mock.Mock() image_meta = {'id': 'test_id', - 'disk_format': input_format} + 'disk_format': input_format, + 'container_format': mock.sentinel.container_format} volume_path = mock.sentinel.volume_path mock_os.name = 'posix' data = mock_info.return_value @@ -778,7 +777,8 @@ class TestUploadVolume(test.TestCase): ctxt = mock.sentinel.context image_service = mock.Mock() image_meta = {'id': 'test_id', - 'disk_format': 'raw'} + 'disk_format': 'raw', + 'container_format': mock.sentinel.container_format} volume_path = mock.sentinel.volume_path mock_os.name = 'posix' mock_os.access.return_value = False @@ -796,6 +796,62 @@ class TestUploadVolume(test.TestCase): image_service.update.assert_called_once_with( ctxt, image_meta['id'], {}, mock_proxy.return_value) + @mock.patch('cinder.image.accelerator.ImageAccel._get_engine') + @mock.patch('cinder.image.accelerator.ImageAccel.is_engine_ready', + return_value = True) + @mock.patch('eventlet.tpool.Proxy') + @mock.patch('cinder.image.image_utils.utils.temporary_chown') + @mock.patch('cinder.image.image_utils.CONF') + @mock.patch('six.moves.builtins.open') + @mock.patch('cinder.image.image_utils.qemu_img_info') + @mock.patch('cinder.image.image_utils.convert_image') + @mock.patch('cinder.image.image_utils.temporary_file') + @mock.patch('cinder.image.image_utils.os') + def test_same_format_compressed(self, mock_os, mock_temp, mock_convert, + mock_info, mock_open, mock_conf, + mock_chown, mock_proxy, + mock_engine_ready, mock_get_engine): + class fakeEngine(object): + + def __init__(self): + pass + + def compress_img(self, src, dest, run_as_root): + pass + + ctxt = mock.sentinel.context + image_service = mock.Mock() + image_meta = {'id': 'test_id', + 'disk_format': 'raw', + 'container_format': 'compressed'} + mock_conf.allow_compression_on_image_upload = True + volume_path = mock.sentinel.volume_path + mock_os.name = 'posix' + data = mock_info.return_value + data.file_format = 'raw' + data.backing_file = None + temp_file = mock_temp.return_value.__enter__.return_value + mock_engine = mock.Mock(spec=fakeEngine) + mock_get_engine.return_value = mock_engine + + output = image_utils.upload_volume(ctxt, image_service, image_meta, + volume_path) + + self.assertIsNone(output) + mock_convert.assert_called_once_with(volume_path, + temp_file, + 'raw', + compress=True, + run_as_root=True) + mock_info.assert_called_with(temp_file, run_as_root=True) + self.assertEqual(2, mock_info.call_count) + mock_open.assert_called_once_with(temp_file, 'rb') + mock_proxy.assert_called_once_with( + mock_open.return_value.__enter__.return_value) + image_service.update.assert_called_once_with( + ctxt, image_meta['id'], {}, mock_proxy.return_value) + mock_engine.compress_img.assert_called() + @mock.patch('eventlet.tpool.Proxy') @mock.patch('cinder.image.image_utils.utils.temporary_chown') @mock.patch('cinder.image.image_utils.CONF') @@ -810,7 +866,8 @@ class TestUploadVolume(test.TestCase): ctxt = mock.sentinel.context image_service = mock.Mock() image_meta = {'id': 'test_id', - 'disk_format': 'raw'} + 'disk_format': 'raw', + 'container_format': 'bare'} volume_path = mock.sentinel.volume_path mock_os.name = 'nt' mock_os.access.return_value = False @@ -827,6 +884,63 @@ class TestUploadVolume(test.TestCase): image_service.update.assert_called_once_with( ctxt, image_meta['id'], {}, mock_proxy.return_value) + @mock.patch('cinder.image.accelerator.ImageAccel._get_engine') + @mock.patch('cinder.image.accelerator.ImageAccel.is_engine_ready', + return_value = True) + @mock.patch('eventlet.tpool.Proxy') + @mock.patch('cinder.image.image_utils.utils.temporary_chown') + @mock.patch('cinder.image.image_utils.CONF') + @mock.patch('six.moves.builtins.open') + @mock.patch('cinder.image.image_utils.qemu_img_info') + @mock.patch('cinder.image.image_utils.convert_image') + @mock.patch('cinder.image.image_utils.temporary_file') + @mock.patch('cinder.image.image_utils.os') + def test_same_format_on_nt_compressed(self, mock_os, mock_temp, + mock_convert, mock_info, + mock_open, mock_conf, + mock_chown, mock_proxy, + mock_engine_ready, mock_get_engine): + class fakeEngine(object): + + def __init__(self): + pass + + def compress_img(self, src, dest, run_as_root): + pass + + ctxt = mock.sentinel.context + image_service = mock.Mock() + image_meta = {'id': 'test_id', + 'disk_format': 'raw', + 'container_format': 'compressed'} + mock_conf.allow_compression_on_image_upload = True + volume_path = mock.sentinel.volume_path + mock_os.name = 'posix' + data = mock_info.return_value + data.file_format = 'raw' + data.backing_file = None + temp_file = mock_temp.return_value.__enter__.return_value + mock_engine = mock.Mock(spec=fakeEngine) + mock_get_engine.return_value = mock_engine + + output = image_utils.upload_volume(ctxt, image_service, image_meta, + volume_path) + + self.assertIsNone(output) + mock_convert.assert_called_once_with(volume_path, + temp_file, + 'raw', + compress=True, + run_as_root=True) + mock_info.assert_called_with(temp_file, run_as_root=True) + self.assertEqual(2, mock_info.call_count) + mock_open.assert_called_once_with(temp_file, 'rb') + mock_proxy.assert_called_once_with( + mock_open.return_value.__enter__.return_value) + image_service.update.assert_called_once_with( + ctxt, image_meta['id'], {}, mock_proxy.return_value) + mock_engine.compress_img.assert_called() + @mock.patch('cinder.image.image_utils.CONF') @mock.patch('six.moves.builtins.open') @mock.patch('cinder.image.image_utils.qemu_img_info') @@ -838,7 +952,8 @@ class TestUploadVolume(test.TestCase): ctxt = mock.sentinel.context image_service = mock.Mock() image_meta = {'id': 'test_id', - 'disk_format': mock.sentinel.disk_format} + 'disk_format': mock.sentinel.disk_format, + 'container_format': mock.sentinel.container_format} volume_path = mock.sentinel.volume_path mock_os.name = 'posix' data = mock_info.return_value @@ -1643,6 +1758,83 @@ class TestFetchToVolumeFormat(test.TestCase): image_utils.get_qemu_data, image_id, has_meta, disk_format_raw, dest, run_as_root=run_as_root) + @mock.patch('cinder.image.accelerator.is_gzip_compressed', + return_value = True) + @mock.patch('cinder.image.accelerator.ImageAccel._get_engine') + @mock.patch('cinder.image.accelerator.ImageAccel.is_engine_ready', + return_value = True) + @mock.patch('cinder.image.image_utils.check_available_space') + @mock.patch('cinder.image.image_utils.convert_image') + @mock.patch('cinder.image.image_utils.volume_utils.copy_volume') + @mock.patch( + 'cinder.image.image_utils.replace_xenserver_image_with_coalesced_vhd') + @mock.patch('cinder.image.image_utils.is_xenserver_format', + return_value=False) + @mock.patch('cinder.image.image_utils.fetch') + @mock.patch('cinder.image.image_utils.qemu_img_info') + @mock.patch('cinder.image.image_utils.temporary_file') + @mock.patch('cinder.image.image_utils.CONF') + def test_defaults_compressed(self, mock_conf, mock_temp, mock_info, + mock_fetch, mock_is_xen, mock_repl_xen, + mock_copy, mock_convert, mock_check_space, + mock_engine_ready, mock_get_engine, + mock_gzip_compressed): + class fakeEngine(object): + def __init__(self): + pass + + def decompress_img(self, src, dest, run_as_root): + pass + + class FakeImageService(object): + def __init__(self, db_driver=None, + image_service=None, disk_format='raw'): + self.temp_images = None + self.disk_format = disk_format + + def show(self, context, image_id): + return {'size': 2 * units.Gi, + 'disk_format': self.disk_format, + 'container_format': 'compressed', + 'status': 'active'} + + ctxt = mock.sentinel.context + ctxt.user_id = mock.sentinel.user_id + image_service = FakeImageService() + image_id = mock.sentinel.image_id + dest = mock.sentinel.dest + volume_format = mock.sentinel.volume_format + out_subformat = None + blocksize = mock.sentinel.blocksize + + data = mock_info.return_value + data.file_format = volume_format + data.backing_file = None + data.virtual_size = 1234 + tmp = mock_temp.return_value.__enter__.return_value + + mock_engine = mock.Mock(spec=fakeEngine) + mock_get_engine.return_value = mock_engine + + output = image_utils.fetch_to_volume_format(ctxt, image_service, + image_id, dest, + volume_format, blocksize) + + self.assertIsNone(output) + mock_temp.assert_called_once_with() + mock_info.assert_has_calls([ + mock.call(tmp, force_share=False, run_as_root=True), + mock.call(tmp, run_as_root=True)]) + mock_fetch.assert_called_once_with(ctxt, image_service, image_id, + tmp, None, None) + self.assertFalse(mock_repl_xen.called) + self.assertFalse(mock_copy.called) + mock_convert.assert_called_once_with(tmp, dest, volume_format, + out_subformat=out_subformat, + run_as_root=True, + src_format='raw') + mock_engine.decompress_img.assert_called() + class TestXenserverUtils(test.TestCase): def test_is_xenserver_format(self): diff --git a/cinder/volume/api.py b/cinder/volume/api.py index 185590c0d41..0cef6a84359 100644 --- a/cinder/volume/api.py +++ b/cinder/volume/api.py @@ -1316,6 +1316,13 @@ class API(base.Base): pass recv_metadata = self.image_service.create(context, metadata) + + # NOTE(ZhengMa): Check if allow image compression before image + # uploading + if recv_metadata.get('container_format') == 'compressed': + allow_compression = CONF.allow_compression_on_image_upload + if allow_compression is False: + raise exception.ImageCompressionNotAllowed() except Exception: # NOTE(geguileo): To mimic behavior before conditional_update we # will rollback status if image create fails diff --git a/etc/cinder/rootwrap.d/volume.filters b/etc/cinder/rootwrap.d/volume.filters index 4df1d66fb11..0c51152f6a4 100644 --- a/etc/cinder/rootwrap.d/volume.filters +++ b/etc/cinder/rootwrap.d/volume.filters @@ -80,6 +80,8 @@ cgexec: ChainingRegExpFilter, cgexec, root, cgexec, -g, blkio:\S+ # cinder/image/image_utils.py qemu-img: EnvFilter, env, root, LC_ALL=C, qemu-img qemu-img_convert: CommandFilter, qemu-img, root +qzip: CommandFilter, qzip, root +gzip: CommandFilter, gzip, root # cinder/volume/nfs.py stat: CommandFilter, stat, root diff --git a/releasenotes/notes/leverage-compression-accelerator-579c7032290cd1e9.yaml b/releasenotes/notes/leverage-compression-accelerator-579c7032290cd1e9.yaml new file mode 100644 index 00000000000..f870aaf631b --- /dev/null +++ b/releasenotes/notes/leverage-compression-accelerator-579c7032290cd1e9.yaml @@ -0,0 +1,49 @@ +--- +features: + - | + A general framework to accommodate hardware compression accelerators for + compression of volumes uploaded to the Image service (Glance) as images + and decompression of compressed images used to create volumes is + introduced. + + The only accelerator supported in this release is Intel QuickAssist + Technology (QAT), which produces a compressed file in gzip format. + Refer to this `Cinder documentation + `_ + for more information about using this feature. + + Additionally, the framework provides software-based compression using + GUNzip tool if a suitable hardware accelerator is not available. + Because this software fallback could cause performance problems if the + Cinder services are not deployed on sufficiently powerful nodes, the + default setting is *not* to enable compression on image upload or + download. + + The compressed image of a volume will be stored in the + Image service (Glance) with the ``container_format`` image property of + ``compressed``. See the `Image service documentation + `_ for more information about + this image container format. +issues: + - | + In the Image service (Glance), the ``compressed`` container format + identifier does not indicate a particular compression technology; it is up + to the image consumer to determine what compression has been used, and + there is no requirement that OpenStack services must support arbitrary + compression technologies. For the upload and download of compressed + images, Cinder supports *only* the gzip format. + + While you may expect that Cinder will be able to consume any image in + ``compressed`` container format *that Cinder has created*, you should not + expect Cinder to be able to successfully use an image in ``compressed`` + format that it has not created itself. +upgrade: + - | + Added string config option ``compression_format`` in [default] section of + cinder.conf to specify image compression format. Currently the only legal + value for this option is ``gzip``. + - | + Added boolean config option ``allow_compression_on_image_upload`` in + [default] section of cinder.conf to enable/disable image compression on + image upload. The default value of this option is ``false``, which means + image compression is disabled.