Change the force_raw_image config usage

The force_raw_image comes from nova side as a
fix for https://bugs.launchpad.net/nova/+bug/1383465.

The reason of the flag is, after all images are forced
to be raw because of security reason
(https://bugs.launchpad.net/nova/+bug/853330), this flag
is added later so that deployer can still select using raw images or
compressed images.

Currently this flag is used in the deeper of functions like image_to_raw() or
converted_size(). For example, image_to_raw() will convert the image
to be raw format only if the force_raw_image flag is set, that means
the caller totally have no idea if the image returned is a raw image or not.

This is possibly ok for libvirt since libvirt does not care for qcow2
or raw, but it's different for ironic, since we will take different action
for qcow2/raw format. A flag in such deeper layer is not so good.

This patch changes these functions, so that the caller in image cache level
accept parameter from deploy level like iscsi_deploy or pxe, which make
decision based on the config.

Change-Id: I118c9d8edcc13d15762593c254eaf27792d6c55f
Related-Bug: #1382164
This commit is contained in:
yunhong jiang 2014-10-23 04:49:57 -07:00
parent 16b9fff7c6
commit 37f1528d9f
10 changed files with 128 additions and 75 deletions

View File

@ -231,7 +231,7 @@ def convert_image(source, dest, out_format, run_as_root=False):
utils.execute(*cmd, run_as_root=run_as_root)
def fetch(context, image_href, path, image_service=None):
def fetch(context, image_href, path, image_service=None, force_raw=False):
# TODO(vish): Improve context handling and add owner and auth data
# when it is added to glance. Right now there is no
# auth checking in glance, so we assume that access was
@ -243,11 +243,8 @@ def fetch(context, image_href, path, image_service=None):
with open(path, "wb") as image_file:
image_service.download(image_href, image_file)
def fetch_to_raw(context, image_href, path, image_service=None):
path_tmp = "%s.part" % path
fetch(context, image_href, path_tmp, image_service)
image_to_raw(image_href, path, path_tmp)
if force_raw:
image_to_raw(image_href, path, "%s.part" % path)
def image_to_raw(image_href, path, path_tmp):
@ -267,7 +264,7 @@ def image_to_raw(image_href, path, path_tmp):
{'fmt': fmt,
'backing_file': backing_file})
if fmt != "raw" and CONF.force_raw_images:
if fmt != "raw":
staged = "%s.converted" % path
LOG.debug("%(image)s was %(format)s, converting to raw" %
{'image': image_href, 'format': fmt})
@ -303,8 +300,6 @@ def converted_size(path):
"""
data = qemu_img_info(path)
if data.file_format == "raw" or not CONF.force_raw_images:
return 0
return data.virtual_size
@ -357,8 +352,8 @@ def create_boot_iso(context, output_filename, kernel_uuid,
with utils.tempdir() as tmpdir:
kernel_path = os.path.join(tmpdir, kernel_uuid)
ramdisk_path = os.path.join(tmpdir, ramdisk_uuid)
fetch_to_raw(context, kernel_uuid, kernel_path)
fetch_to_raw(context, ramdisk_uuid, ramdisk_path)
fetch(context, kernel_uuid, kernel_path, CONF.force_raw_images)
fetch(context, ramdisk_uuid, ramdisk_path, CONF.force_raw_images)
params = []
if root_uuid:

View File

@ -28,6 +28,7 @@ from ironic.common import disk_partitioner
from ironic.common import exception
from ironic.common.i18n import _
from ironic.common.i18n import _LE
from ironic.common import images
from ironic.common import utils
from ironic.drivers.modules import image_cache
from ironic.openstack.common import log as logging
@ -200,10 +201,13 @@ def get_dev(address, port, iqn, lun):
return dev
def get_image_mb(image_path):
def get_image_mb(image_path, virtual_size=True):
"""Get size of an image in Megabyte."""
mb = 1024 * 1024
image_byte = os.path.getsize(image_path)
if not virtual_size:
image_byte = os.path.getsize(image_path)
else:
image_byte = images.converted_size(image_path)
# round up size to MB
image_mb = int((image_byte + mb - 1) / mb)
return image_mb
@ -407,12 +411,14 @@ def check_for_missing_params(info_dict, error_msg, param_prefix=''):
{'error_msg': error_msg, 'missing_info': missing_info})
def fetch_images(ctx, cache, images_info):
def fetch_images(ctx, cache, images_info, force_raw=True):
"""Check for available disk space and fetch images using ImageCache.
:param ctx: context
:param cache: ImageCache instance to use for fetching
:param images_info: list of tuples (image uuid, destination path)
:param force_raw: boolean value, whether to convert the image to raw
format
:raises: InstanceDeployFailure if unable to find enough disk space
"""
@ -426,4 +432,4 @@ def fetch_images(ctx, cache, images_info):
# This is probably unavoidable, as we can't control other
# (probably unrelated) processes
for uuid, path in images_info:
cache.fetch_image(uuid, path, ctx=ctx)
cache.fetch_image(uuid, path, ctx=ctx, force_raw=force_raw)

View File

@ -72,7 +72,7 @@ class ImageCache(object):
if master_dir is not None:
fileutils.ensure_tree(master_dir)
def fetch_image(self, uuid, dest_path, ctx=None):
def fetch_image(self, uuid, dest_path, ctx=None, force_raw=True):
"""Fetch image with given uuid to the destination path.
Does nothing if destination path exists.
@ -82,15 +82,18 @@ class ImageCache(object):
:param uuid: image UUID or href to fetch
:param dest_path: destination file path
:param ctx: context
:param force_raw: boolean value, whether to convert the image to raw
format
"""
img_download_lock_name = 'download-image'
if self.master_dir is None:
#NOTE(ghe): We don't share images between instances/hosts
if not CONF.parallel_image_downloads:
with lockutils.lock(img_download_lock_name, 'ironic-'):
_fetch_to_raw(ctx, uuid, dest_path, self._image_service)
_fetch(ctx, uuid, dest_path, self._image_service,
force_raw)
else:
_fetch_to_raw(ctx, uuid, dest_path, self._image_service)
_fetch(ctx, uuid, dest_path, self._image_service, force_raw)
return
#TODO(ghe): have hard links and counts the same behaviour in all fs
@ -123,12 +126,14 @@ class ImageCache(object):
{'uuid': uuid})
return
self._download_image(uuid, master_path, dest_path, ctx=ctx)
self._download_image(
uuid, master_path, dest_path, ctx=ctx, force_raw=force_raw)
# NOTE(dtantsur): we increased cache size - time to clean up
self.clean_up()
def _download_image(self, uuid, master_path, dest_path, ctx=None):
def _download_image(self, uuid, master_path, dest_path, ctx=None,
force_raw=True):
"""Download image from Glance and store at a given path.
This method should be called with uuid-specific lock taken.
@ -137,13 +142,15 @@ class ImageCache(object):
:param master_path: destination master path
:param dest_path: destination file path
:param ctx: context
:param force_raw: boolean value, whether to convert the image to raw
format
"""
#TODO(ghe): timeout and retry for downloads
#TODO(ghe): logging when image cannot be created
tmp_dir = tempfile.mkdtemp(dir=self.master_dir)
tmp_path = os.path.join(tmp_dir, uuid)
try:
_fetch_to_raw(ctx, uuid, tmp_path, self._image_service)
_fetch(ctx, uuid, tmp_path, self._image_service, force_raw)
# NOTE(dtantsur): no need for global lock here - master_path
# will have link count >1 at any moment, so won't be cleaned up
os.link(tmp_path, master_path)
@ -283,14 +290,20 @@ def _free_disk_space_for(path):
return stat.f_frsize * stat.f_bavail
def _fetch_to_raw(context, image_href, path, image_service=None):
def _fetch(context, image_href, path, image_service=None, force_raw=False):
"""Fetch image and convert to raw format if needed."""
path_tmp = "%s.part" % path
images.fetch(context, image_href, path_tmp, image_service)
required_space = images.converted_size(path_tmp)
directory = os.path.dirname(path_tmp)
_clean_up_caches(directory, required_space)
images.image_to_raw(image_href, path, path_tmp)
images.fetch(context, image_href, path_tmp, image_service,
force_raw=False)
# Notes(yjiang5): If glance can provide the virtual size information,
# then we can firstly clean cach and then invoke images.fetch().
if force_raw:
required_space = images.converted_size(path_tmp)
directory = os.path.dirname(path_tmp)
_clean_up_caches(directory, required_space)
images.image_to_raw(image_href, path, path_tmp)
else:
os.rename(path_tmp, path)
def _clean_up_caches(directory, amount):

View File

@ -187,7 +187,8 @@ def cache_instance_image(ctx, node):
LOG.debug("Fetching image %(ami)s for node %(uuid)s",
{'ami': uuid, 'uuid': node.uuid})
deploy_utils.fetch_images(ctx, InstanceImageCache(), [(uuid, image_path)])
deploy_utils.fetch_images(ctx, InstanceImageCache(), [(uuid, image_path)],
CONF.force_raw_images)
return (uuid, image_path)

View File

@ -210,7 +210,8 @@ def _cache_ramdisk_kernel(ctx, node, pxe_info):
os.path.join(pxe_utils.get_root_dir(), node.uuid))
LOG.debug("Fetching kernel and ramdisk for node %s",
node.uuid)
deploy_utils.fetch_images(ctx, TFTPImageCache(), pxe_info.values())
deploy_utils.fetch_images(ctx, TFTPImageCache(), pxe_info.values(),
CONF.force_raw_images)
def _get_image_info(node, ctx):

View File

@ -24,6 +24,7 @@ from oslo.config import cfg
from ironic.common import disk_partitioner
from ironic.common import exception
from ironic.common import images
from ironic.common import utils as common_utils
from ironic.drivers.modules import deploy_utils as utils
from ironic.drivers.modules import image_cache
@ -469,22 +470,27 @@ class OtherFunctionTestCase(tests_base.TestCase):
actual = utils.get_dev('1.2.3.4', 5678, 'iqn.fake', 9)
self.assertEqual(expected, actual)
def test_get_image_mb(self):
@mock.patch.object(os.path, 'getsize')
@mock.patch.object(images, 'converted_size')
def test_get_image_mb(self, mock_csize, mock_getsize):
mb = 1024 * 1024
size = None
def fake_getsize(path):
return size
self.useFixture(fixtures.MonkeyPatch('os.path.getsize', fake_getsize))
size = 0
self.assertEqual(0, utils.get_image_mb('x'))
size = 1
self.assertEqual(1, utils.get_image_mb('x'))
size = mb
self.assertEqual(1, utils.get_image_mb('x'))
size = mb + 1
self.assertEqual(2, utils.get_image_mb('x'))
mock_getsize.return_value = 0
mock_csize.return_value = 0
self.assertEqual(0, utils.get_image_mb('x', False))
self.assertEqual(0, utils.get_image_mb('x', True))
mock_getsize.return_value = 1
mock_csize.return_value = 1
self.assertEqual(1, utils.get_image_mb('x', False))
self.assertEqual(1, utils.get_image_mb('x', True))
mock_getsize.return_value = mb
mock_csize.return_value = mb
self.assertEqual(1, utils.get_image_mb('x', False))
self.assertEqual(1, utils.get_image_mb('x', True))
mock_getsize.return_value = mb + 1
mock_csize.return_value = mb + 1
self.assertEqual(2, utils.get_image_mb('x', False))
self.assertEqual(2, utils.get_image_mb('x', True))
@mock.patch.object(disk_partitioner.DiskPartitioner, 'commit', lambda _: None)
@ -760,7 +766,8 @@ class RealFilePartitioningTestCase(tests_base.TestCase):
mock_clean_up_caches.assert_called_once_with(None, 'master_dir',
[('uuid', 'path')])
mock_cache.fetch_image.assert_called_once_with('uuid', 'path',
ctx=None)
ctx=None,
force_raw=True)
@mock.patch.object(image_cache, 'clean_up_caches')
def test_fetch_images_fail(self, mock_clean_up_caches):

View File

@ -34,7 +34,7 @@ def touch(filename):
open(filename, 'w').close()
@mock.patch.object(image_cache, '_fetch_to_raw')
@mock.patch.object(image_cache, '_fetch')
class TestImageCacheFetch(base.TestCase):
def setUp(self):
@ -49,32 +49,32 @@ class TestImageCacheFetch(base.TestCase):
@mock.patch.object(image_cache.ImageCache, 'clean_up')
@mock.patch.object(image_cache.ImageCache, '_download_image')
def test_fetch_image_no_master_dir(self, mock_download, mock_clean_up,
mock_fetch_to_raw):
mock_fetch):
self.cache.master_dir = None
self.cache.fetch_image('uuid', self.dest_path)
self.assertFalse(mock_download.called)
mock_fetch_to_raw.assert_called_once_with(
None, 'uuid', self.dest_path, None)
mock_fetch.assert_called_once_with(
None, 'uuid', self.dest_path, None, True)
self.assertFalse(mock_clean_up.called)
@mock.patch.object(image_cache.ImageCache, 'clean_up')
@mock.patch.object(image_cache.ImageCache, '_download_image')
def test_fetch_image_dest_exists(self, mock_download, mock_clean_up,
mock_fetch_to_raw):
mock_fetch):
touch(self.dest_path)
self.cache.fetch_image(self.uuid, self.dest_path)
self.assertFalse(mock_download.called)
self.assertFalse(mock_fetch_to_raw.called)
self.assertFalse(mock_fetch.called)
self.assertFalse(mock_clean_up.called)
@mock.patch.object(image_cache.ImageCache, 'clean_up')
@mock.patch.object(image_cache.ImageCache, '_download_image')
def test_fetch_image_master_exists(self, mock_download, mock_clean_up,
mock_fetch_to_raw):
mock_fetch):
touch(self.master_path)
self.cache.fetch_image(self.uuid, self.dest_path)
self.assertFalse(mock_download.called)
self.assertFalse(mock_fetch_to_raw.called)
self.assertFalse(mock_fetch.called)
self.assertTrue(os.path.isfile(self.dest_path))
self.assertEqual(os.stat(self.dest_path).st_ino,
os.stat(self.master_path).st_ino)
@ -83,22 +83,23 @@ class TestImageCacheFetch(base.TestCase):
@mock.patch.object(image_cache.ImageCache, 'clean_up')
@mock.patch.object(image_cache.ImageCache, '_download_image')
def test_fetch_image(self, mock_download, mock_clean_up,
mock_fetch_to_raw):
mock_fetch):
self.cache.fetch_image(self.uuid, self.dest_path)
self.assertFalse(mock_fetch_to_raw.called)
self.assertFalse(mock_fetch.called)
mock_download.assert_called_once_with(
self.uuid, self.master_path, self.dest_path, ctx=None)
self.uuid, self.master_path, self.dest_path,
ctx=None, force_raw=True)
self.assertTrue(mock_clean_up.called)
def test__download_image(self, mock_fetch_to_raw):
def _fake_fetch_to_raw(ctx, uuid, tmp_path, *args):
def test__download_image(self, mock_fetch):
def _fake_fetch(ctx, uuid, tmp_path, *args):
self.assertEqual(self.uuid, uuid)
self.assertNotEqual(self.dest_path, tmp_path)
self.assertNotEqual(os.path.dirname(tmp_path), self.master_dir)
with open(tmp_path, 'w') as fp:
fp.write("TEST")
mock_fetch_to_raw.side_effect = _fake_fetch_to_raw
mock_fetch.side_effect = _fake_fetch
self.cache._download_image(self.uuid, self.master_path, self.dest_path)
self.assertTrue(os.path.isfile(self.dest_path))
self.assertTrue(os.path.isfile(self.master_path))
@ -242,9 +243,9 @@ class TestImageCacheCleanUp(base.TestCase):
mock_clean_ttl.assert_called_once_with(mock.ANY, None)
@mock.patch.object(utils, 'rmtree_without_raise')
@mock.patch.object(image_cache, '_fetch_to_raw')
def test_temp_images_not_cleaned(self, mock_fetch_to_raw, mock_rmtree):
def _fake_fetch_to_raw(ctx, uuid, tmp_path, *args):
@mock.patch.object(image_cache, '_fetch')
def test_temp_images_not_cleaned(self, mock_fetch, mock_rmtree):
def _fake_fetch(ctx, uuid, tmp_path, *args):
with open(tmp_path, 'w') as fp:
fp.write("TEST" * 10)
@ -252,16 +253,16 @@ class TestImageCacheCleanUp(base.TestCase):
self.cache.clean_up()
self.assertTrue(os.path.exists(tmp_path))
mock_fetch_to_raw.side_effect = _fake_fetch_to_raw
mock_fetch.side_effect = _fake_fetch
master_path = os.path.join(self.master_dir, 'uuid')
dest_path = os.path.join(tempfile.mkdtemp(), 'dest')
self.cache._download_image('uuid', master_path, dest_path)
self.assertTrue(mock_rmtree.called)
@mock.patch.object(utils, 'rmtree_without_raise')
@mock.patch.object(image_cache, '_fetch_to_raw')
def test_temp_dir_exception(self, mock_fetch_to_raw, mock_rmtree):
mock_fetch_to_raw.side_effect = exception.IronicException
@mock.patch.object(image_cache, '_fetch')
def test_temp_dir_exception(self, mock_fetch, mock_rmtree):
mock_fetch.side_effect = exception.IronicException
self.assertRaises(exception.IronicException,
self.cache._download_image,
'uuid', 'fake', 'fake')
@ -477,11 +478,12 @@ class TestFetchCleanup(base.TestCase):
@mock.patch.object(images, 'fetch')
@mock.patch.object(images, 'image_to_raw')
@mock.patch.object(image_cache, '_clean_up_caches')
def test__fetch_to_raw(self, mock_clean, mock_raw, mock_fetch, mock_size):
def test__fetch(self, mock_clean, mock_raw, mock_fetch, mock_size):
mock_size.return_value = 100
image_cache._fetch_to_raw('fake', 'fake-uuid', '/foo/bar')
image_cache._fetch('fake', 'fake-uuid', '/foo/bar', force_raw=True)
mock_fetch.assert_called_once_with('fake', 'fake-uuid',
'/foo/bar.part', None)
'/foo/bar.part', None,
force_raw=False)
mock_clean.assert_called_once_with('/foo', 100)
mock_raw.assert_called_once_with('fake-uuid', '/foo/bar',
'/foo/bar.part')

View File

@ -185,7 +185,7 @@ class IscsiDeployMethodsTestCase(db_base.DbTestCase):
(uuid, image_path) = iscsi_deploy.cache_instance_image(None, self.node)
mock_fetch_image.assert_called_once_with(None,
mock.ANY,
[(uuid, image_path)])
[(uuid, image_path)], True)
self.assertEqual('glance://image_uuid', uuid)
self.assertEqual(os.path.join(temp_dir,
self.node.uuid,

View File

@ -253,7 +253,8 @@ class PXEPrivateMethodsTestCase(db_base.DbTestCase):
mock_fetch_image.assert_called_once_with(None,
mock.ANY,
[('deploy_kernel',
image_path)])
image_path)],
True)
@mock.patch.object(pxe, 'TFTPImageCache', lambda: None)
@mock.patch.object(fileutils, 'ensure_tree')
@ -266,7 +267,7 @@ class PXEPrivateMethodsTestCase(db_base.DbTestCase):
pxe._cache_ramdisk_kernel(self.context, self.node, fake_pxe_info)
mock_ensure_tree.assert_called_with(expected_path)
mock_fetch_image.assert_called_once_with(self.context, mock.ANY,
fake_pxe_info.values())
fake_pxe_info.values(), True)
@mock.patch.object(pxe, 'TFTPImageCache', lambda: None)
@mock.patch.object(fileutils, 'ensure_tree')
@ -280,7 +281,8 @@ class PXEPrivateMethodsTestCase(db_base.DbTestCase):
pxe._cache_ramdisk_kernel(self.context, self.node, fake_pxe_info)
mock_ensure_tree.assert_called_with(expected_path)
mock_fetch_image.assert_called_once_with(self.context, mock.ANY,
fake_pxe_info.values())
fake_pxe_info.values(),
True)
class PXEDriverTestCase(db_base.DbTestCase):

View File

@ -93,6 +93,23 @@ class IronicImagesTestCase(base.TestCase):
image_service_mock.download.assert_called_once_with(
'image_href', 'file')
@mock.patch.object(images, 'image_to_raw')
@mock.patch.object(__builtin__, 'open')
def test_fetch_image_service_force_raw(self, open_mock, image_to_raw_mock):
mock_file_handle = mock.MagicMock(spec=file)
mock_file_handle.__enter__.return_value = 'file'
open_mock.return_value = mock_file_handle
image_service_mock = mock.Mock()
images.fetch('context', 'image_href', 'path', image_service_mock,
force_raw=True)
open_mock.assert_called_once_with('path', 'wb')
image_service_mock.download.assert_called_once_with(
'image_href', 'file')
image_to_raw_mock.assert_called_once_with(
'image_href', 'path', 'path.part')
@mock.patch.object(images, 'qemu_img_info')
def test_image_to_raw_no_file_format(self, qemu_img_info_mock):
info = self.FakeImgInfo()
@ -188,6 +205,15 @@ class IronicImagesTestCase(base.TestCase):
images.download_size('context', 'image_href', image_service_mock)
image_service_mock.show.assert_called_once_with('image_href')
@mock.patch.object(images, 'qemu_img_info')
def test_converted_size(self, qemu_img_info_mock):
info = self.FakeImgInfo()
info.virtual_size = 1
qemu_img_info_mock.return_value = info
size = images.converted_size('path')
qemu_img_info_mock.assert_called_once_with('path')
self.assertEqual(1, size)
class FsImageTestCase(base.TestCase):
@ -394,7 +420,7 @@ class FsImageTestCase(base.TestCase):
'path/to/ramdisk')
@mock.patch.object(images, 'create_isolinux_image')
@mock.patch.object(images, 'fetch_to_raw')
@mock.patch.object(images, 'fetch')
@mock.patch.object(utils, 'tempdir')
def test_create_boot_iso(self, tempdir_mock, fetch_images_mock,
create_isolinux_mock):
@ -406,9 +432,9 @@ class FsImageTestCase(base.TestCase):
'ramdisk-uuid', 'root-uuid', 'kernel-params')
fetch_images_mock.assert_any_call('ctx', 'kernel-uuid',
'tmpdir/kernel-uuid')
'tmpdir/kernel-uuid', True)
fetch_images_mock.assert_any_call('ctx', 'ramdisk-uuid',
'tmpdir/ramdisk-uuid')
'tmpdir/ramdisk-uuid', True)
params = ['root=UUID=root-uuid', 'kernel-params']
create_isolinux_mock.assert_called_once_with('output_file',
'tmpdir/kernel-uuid', 'tmpdir/ramdisk-uuid', params)