openstack
/
nova
Code Issues Proposed changes

image cache clean-up to clean swap disk 

Currently swap backing files are never removed from the system.
The image clean-up code should recognise at least obsolete unused files
need to be removed.

Change-Id: I09519a73c7233f2dc6f46f6cd094a3e1cc3dc395
Partial-Bug: 1253991a
This commit is contained in:
jichenjc 2014-10-29 09:36:58 +08:00
parent d9d04933a4
commit 7252839c9e
6 changed files with 243 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
nova/tests/unit/virt/libvirt/test_imagecache.py
View File

@ -20,13 +20,16 @@ import hashlib
import os
import time
import mock
from oslo.concurrency import processutils
from oslo.config import cfg
from oslo.serialization import jsonutils
from oslo.utils import importutils
from nova import conductor
from nova import context
from nova import db
from nova import objects
from nova.openstack.common import log as logging
from nova import test
from nova.tests.unit import fake_instance
@ -66,6 +69,22 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
csum = imagecache.read_stored_checksum('/tmp/foo', timestamped=False)
self.assertIsNone(csum)
@mock.patch.object(os.path, 'exists', return_value=True)
@mock.patch.object(time, 'time', return_value=2000000)
@mock.patch.object(os.path, 'getmtime', return_value=1000000)
def test_get_age_of_file(self, mock_getmtime, mock_time, mock_exists):
image_cache_manager = imagecache.ImageCacheManager()
exists, age = image_cache_manager._get_age_of_file('/tmp')
self.assertTrue(exists)
self.assertEqual(1000000, age)
@mock.patch.object(os.path, 'exists', return_value=False)
def test_get_age_of_file_not_exists(self, mock_exists):
image_cache_manager = imagecache.ImageCacheManager()
exists, age = image_cache_manager._get_age_of_file('/tmp')
self.assertFalse(exists)
self.assertEqual(0, age)
def test_read_stored_checksum(self):
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
@ -109,7 +128,8 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
listing = ['00000001',
'ephemeral_0_20_None',
'17d1b00b81642842e514494a78e804e9a511637c_5368709120.info',
'00000004']
'00000004',
'swap_1000']
images = ['e97222e91fc4241f49a7f520d1dcf446751129b3_sm',
'e09c675c2d1cfac32dae3c2d83689c8c94bc693b_sm',
'e97222e91fc4241f49a7f520d1dcf446751129b3',
@ -159,6 +179,9 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
'10737418240')
self.assertNotIn(unexpected, image_cache_manager.originals)
self.assertEqual(1, len(image_cache_manager.back_swap_images))
self.assertTrue('swap_1000' in image_cache_manager.back_swap_images)
def test_list_backing_images_small(self):
self.stubs.Set(os, 'listdir',
lambda x: ['_base', 'instance-00000001',
@ -664,9 +687,19 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
self.stubs.Set(os, 'remove', lambda x: remove(x))
self.mox.StubOutWithMock(objects.block_device.BlockDeviceMappingList,
'get_by_instance_uuid')
ctxt = context.get_admin_context()
objects.block_device.BlockDeviceMappingList.get_by_instance_uuid(
ctxt, '123').AndReturn(None)
objects.block_device.BlockDeviceMappingList.get_by_instance_uuid(
ctxt, '456').AndReturn(None)
self.mox.ReplayAll()
# And finally we can make the call we're actually testing...
# The argument here should be a context, but it is mocked out
image_cache_manager.update(None, all_instances)
image_cache_manager.update(ctxt, all_instances)
# Verify
active = [fq_path(hashed_1), fq_path('%s_5368709120' % hashed_1),
@ -750,8 +783,21 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
touch(base_filename + '.info')
os.utime(base_filename + '.info', (old, old))
self.mox.StubOutWithMock(
objects.block_device.BlockDeviceMappingList,
'get_by_instance_uuid')
ctxt = context.get_admin_context()
objects.block_device.BlockDeviceMappingList.get_by_instance_uuid(
ctxt, '123').AndReturn(None)
objects.block_device.BlockDeviceMappingList.get_by_instance_uuid(
ctxt, '456').AndReturn(None)
self.mox.ReplayAll()
image_cache_manager = imagecache.ImageCacheManager()
image_cache_manager.update(None, all_instances)
image_cache_manager.update(ctxt,
all_instances)
self.assertTrue(os.path.exists(base_filename))
self.assertTrue(os.path.exists(base_filename + '.info'))
@ -782,6 +828,56 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
compute._run_image_cache_manager_pass(None)
self.assertTrue(was['called'])
def test_store_swap_image(self):
image_cache_manager = imagecache.ImageCacheManager()
image_cache_manager._store_swap_image('swap_')
image_cache_manager._store_swap_image('swap_123')
image_cache_manager._store_swap_image('swap_456')
image_cache_manager._store_swap_image('swap_abc')
image_cache_manager._store_swap_image('123_swap')
image_cache_manager._store_swap_image('swap_129_')
self.assertEqual(len(image_cache_manager.back_swap_images), 2)
expect_set = set(['swap_123', 'swap_456'])
self.assertEqual(image_cache_manager.back_swap_images, expect_set)
@mock.patch.object(libvirt_utils, 'chown')
@mock.patch('os.path.exists', return_value=True)
@mock.patch('os.utime')
@mock.patch('os.path.getmtime')
@mock.patch('os.remove')
def test_age_and_verify_swap_images(self, mock_remove, mock_getmtime,
mock_utime, mock_exist, mock_chown):
image_cache_manager = imagecache.ImageCacheManager()
expected_remove = set()
expected_exist = set(['swap_128', 'swap_256'])
image_cache_manager.back_swap_images.add('swap_128')
image_cache_manager.back_swap_images.add('swap_256')
image_cache_manager.used_swap_images.add('swap_128')
def getmtime(path):
return time.time() - 1000000
mock_getmtime.side_effect = getmtime
def removefile(path):
if not path.startswith('/tmp_age_test'):
return os.remove(path)
fn = os.path.split(path)[-1]
expected_remove.add(fn)
expected_exist.remove(fn)
mock_remove.side_effect = removefile
image_cache_manager._age_and_verify_swap_images(None, '/tmp_age_test')
self.assertEqual(1, len(expected_exist))
self.assertEqual(1, len(expected_remove))
self.assertTrue('swap_128' in expected_exist)
self.assertTrue('swap_256' in expected_remove)
class VerifyChecksumTestCase(test.NoDBTestCase):

58
nova/tests/unit/virt/test_imagecache.py
View File

@ -14,13 +14,38 @@
from oslo.config import cfg
from nova import block_device
from nova.compute import vm_states
from nova import context
from nova import objects
from nova import test
from nova.tests.unit import fake_instance
from nova.virt import imagecache
CONF = cfg.CONF
swap_bdm_128 = [block_device.BlockDeviceDict(
{'id': 1, 'instance_uuid': 'fake-instance',
'device_name': '/dev/sdb1',
'source_type': 'blank',
'destination_type': 'local',
'delete_on_termination': True,
'guest_format': 'swap',
'disk_bus': 'scsi',
'volume_size': 128,
'boot_index': -1})]
swap_bdm_256 = [block_device.BlockDeviceDict(
{'id': 1, 'instance_uuid': 'fake-instance',
'device_name': '/dev/sdb1',
'source_type': 'blank',
'destination_type': 'local',
'delete_on_termination': True,
'guest_format': 'swap',
'disk_bus': 'scsi',
'volume_size': 256,
'boot_index': -1})]
class ImageCacheManagerTests(test.NoDBTestCase):
@ -72,9 +97,22 @@ class ImageCacheManagerTests(test.NoDBTestCase):
image_cache_manager = imagecache.ImageCacheManager()
self.mox.StubOutWithMock(objects.block_device.BlockDeviceMappingList,
'get_by_instance_uuid')
ctxt = context.get_admin_context()
objects.block_device.BlockDeviceMappingList.get_by_instance_uuid(
ctxt, '123').AndReturn(swap_bdm_256)
objects.block_device.BlockDeviceMappingList.get_by_instance_uuid(
ctxt, '456').AndReturn(swap_bdm_128)
objects.block_device.BlockDeviceMappingList.get_by_instance_uuid(
ctxt, '789').AndReturn(swap_bdm_128)
self.mox.ReplayAll()
# The argument here should be a context, but it's mocked out
running = image_cache_manager._list_running_instances(None,
all_instances)
running = image_cache_manager._list_running_instances(ctxt,
all_instances)
self.assertEqual(4, len(running['used_images']))
self.assertEqual((1, 0, ['instance-00000001']),
@ -96,6 +134,10 @@ class ImageCacheManagerTests(test.NoDBTestCase):
self.assertEqual(1, running['image_popularity']['21'])
self.assertEqual(1, running['image_popularity']['22'])
self.assertEqual(len(running['used_swap_images']), 2)
self.assertIn('swap_128', running['used_swap_images'])
self.assertIn('swap_256', running['used_swap_images'])
def test_list_resizing_instances(self):
instances = [{'image_ref': '1',
'host': CONF.host,
@ -108,8 +150,16 @@ class ImageCacheManagerTests(test.NoDBTestCase):
for instance in instances]
image_cache_manager = imagecache.ImageCacheManager()
running = image_cache_manager._list_running_instances(None,
all_instances)
self.mox.StubOutWithMock(objects.block_device.BlockDeviceMappingList,
'get_by_instance_uuid')
ctxt = context.get_admin_context()
objects.block_device.BlockDeviceMappingList.get_by_instance_uuid(
ctxt, '123').AndReturn(swap_bdm_256)
self.mox.ReplayAll()
running = image_cache_manager._list_running_instances(ctxt,
all_instances)
self.assertEqual(1, len(running['used_images']))
self.assertEqual((1, 0, ['instance-00000001']),

13
nova/tests/unit/virt/vmwareapi/test_driver_api.py
View File

@ -46,6 +46,7 @@ from nova import context
from nova import exception
from nova.image import glance
from nova.network import model as network_model
from nova import objects
from nova.openstack.common import uuidutils
from nova import test
from nova.tests.unit import fake_instance
@ -1948,7 +1949,9 @@ class VMwareAPIVMTestCase(test.NoDBTestCase):
self.conn.refresh_instance_security_rules,
instance=None)
def test_image_aging_image_used(self):
@mock.patch.object(objects.block_device.BlockDeviceMappingList,
'get_by_instance_uuid')
def test_image_aging_image_used(self, mock_get_by_inst):
self._create_vm()
all_instances = [self.instance]
self.conn.manage_image_cache(self.context, all_instances)
@ -1999,7 +2002,9 @@ class VMwareAPIVMTestCase(test.NoDBTestCase):
def test_timestamp_file_removed_spawn(self):
self._timestamp_file_removed()
def test_timestamp_file_removed_aging(self):
@mock.patch.object(objects.block_device.BlockDeviceMappingList,
'get_by_instance_uuid')
def test_timestamp_file_removed_aging(self, mock_get_by_inst):
self._timestamp_file_removed()
ts = self._get_timestamp_filename()
ts_path = ds_util.DatastorePath(self.ds, 'vmware_base',
@ -2010,7 +2015,9 @@ class VMwareAPIVMTestCase(test.NoDBTestCase):
self.conn.manage_image_cache(self.context, all_instances)
self._timestamp_file_exists(exists=False)
def test_image_aging_disabled(self):
@mock.patch.object(objects.block_device.BlockDeviceMappingList,
'get_by_instance_uuid')
def test_image_aging_disabled(self, mock_get_by_inst):
self._override_time()
self.flags(remove_unused_base_images=False)
self._create_vm()

5
nova/tests/unit/virt/vmwareapi/test_imagecache.py
View File

@ -19,6 +19,7 @@ import mock
from oslo.config import cfg
from oslo.utils import timeutils
from nova import objects
from nova import test
from nova.tests.unit import fake_instance
from nova.tests.unit.virt.vmwareapi import fake
@ -235,7 +236,9 @@ class ImageCacheManagerTestCase(test.NoDBTestCase):
ds_util.DatastorePath('fake-ds', 'fake-path'))
self.assertEqual(3, self._get_timestamp_called)
def test_update(self):
@mock.patch.object(objects.block_device.BlockDeviceMappingList,
'get_by_instance_uuid')
def test_update(self, mock_get_by_inst):
def fake_list_datastore_images(ds_path, datastore):
return {'unexplained_images': [],
'originals': self.images}

14
nova/virt/imagecache.py
View File

@ -17,6 +17,8 @@ from oslo.config import cfg
from nova.compute import task_states
from nova.compute import vm_states
from nova import objects
from nova.virt import block_device as driver_block_device
imagecache_opts = [
cfg.IntOpt('image_cache_manager_interval',
@ -71,6 +73,7 @@ class ImageCacheManager(object):
used_images = {}
image_popularity = {}
instance_names = set()
used_swap_images = set()
for instance in all_instances:
# NOTE(mikal): "instance name" here means "the name of a directory
@ -99,9 +102,18 @@ class ImageCacheManager(object):
image_popularity.setdefault(image_ref_str, 0)
image_popularity[image_ref_str] += 1
gb = objects.BlockDeviceMappingList.get_by_instance_uuid
bdms = gb(context, instance.uuid)
if bdms:
swap = driver_block_device.convert_swap(bdms)
if swap:
swap_image = 'swap_' + str(swap[0]['swap_size'])
used_swap_images.add(swap_image)
return {'used_images': used_images,
'image_popularity': image_popularity,
'instance_names': instance_names}
'instance_names': instance_names,
'used_swap_images': used_swap_images}
def _list_base_images(self, base_dir):
"""Return a list of the images present in _base.

80
nova/virt/libvirt/imagecache.py
View File

@ -245,6 +245,9 @@ class ImageCacheManager(imagecache.ImageCacheManager):
self.image_popularity = {}
self.instance_names = set()
self.back_swap_images = set()
self.used_swap_images = set()
self.active_base_files = []
self.corrupt_base_files = []
self.originals = []
@ -259,6 +262,14 @@ class ImageCacheManager(imagecache.ImageCacheManager):
if original:
self.originals.append(entpath)
def _store_swap_image(self, ent):
"""Store base swap images for later examination."""
names = ent.split('_')
if len(names) == 2 and names[0] == 'swap':
if len(names[1]) > 0 and names[1].isdigit():
LOG.debug('Adding %s into backend swap images', ent)
self.back_swap_images.add(ent)
def _list_base_images(self, base_dir):
"""Return a list of the images present in _base.
@ -276,6 +287,8 @@ class ImageCacheManager(imagecache.ImageCacheManager):
ent[digest_size] == '_' and
not is_valid_info_file(os.path.join(base_dir, ent))):
self._store_image(base_dir, ent, original=False)
else:
self._store_swap_image(ent)
return {'unexplained_images': self.unexplained_images,
'originals': self.originals}
@ -416,39 +429,54 @@ class ImageCacheManager(imagecache.ImageCacheManager):
return inner_verify_checksum()
def _remove_base_file(self, base_file):
"""Remove a single base file if it is old enough.
Returns nothing.
"""
@staticmethod
def _get_age_of_file(base_file):
if not os.path.exists(base_file):
LOG.debug('Cannot remove %s, it does not exist',
base_file)
return
LOG.debug('Cannot remove %s, it does not exist', base_file)
return (False, 0)
mtime = os.path.getmtime(base_file)
age = time.time() - mtime
maxage = CONF.libvirt.remove_unused_resized_minimum_age_seconds
if base_file in self.originals:
maxage = CONF.remove_unused_original_minimum_age_seconds
return (True, age)
def _remove_old_enough_file(self, base_file, maxage, remove_sig=True):
"""Remove a single swap or base file if it is old enough."""
exists, age = self._get_age_of_file(base_file)
if not exists:
return
if age < maxage:
LOG.info(_LI('Base file too young to remove: %s'),
base_file)
LOG.info(_LI('Base or swap file too young to remove: %s'),
base_file)
else:
LOG.info(_LI('Removing base file: %s'), base_file)
LOG.info(_LI('Removing base or swap file: %s'), base_file)
try:
os.remove(base_file)
signature = get_info_filename(base_file)
if os.path.exists(signature):
os.remove(signature)
if remove_sig:
signature = get_info_filename(base_file)
if os.path.exists(signature):
os.remove(signature)
except OSError as e:
LOG.error(_LE('Failed to remove %(base_file)s, '
'error was %(error)s'),
{'base_file': base_file,
'error': e})
def _remove_swap_file(self, base_file):
"""Remove a single swap base file if it is old enough."""
maxage = CONF.remove_unused_original_minimum_age_seconds
self._remove_old_enough_file(base_file, maxage, remove_sig=False)
def _remove_base_file(self, base_file):
"""Remove a single base file if it is old enough."""
maxage = CONF.libvirt.remove_unused_resized_minimum_age_seconds
if base_file in self.originals:
maxage = CONF.remove_unused_original_minimum_age_seconds
self._remove_old_enough_file(base_file, maxage)
def _handle_base_image(self, img_id, base_file):
"""Handle the checks for a single base image."""
@ -518,6 +546,22 @@ class ImageCacheManager(imagecache.ImageCacheManager):
libvirt_utils.chown(base_file, os.getuid())
os.utime(base_file, None)
def _age_and_verify_swap_images(self, context, base_dir):
LOG.debug('Verify swap images')
for ent in self.back_swap_images:
base_file = os.path.join(base_dir, ent)
if ent in self.used_swap_images and os.path.exists(base_file):
libvirt_utils.chown(base_file, os.getuid())
os.utime(base_file, None)
elif self.remove_unused_base_images:
self._remove_swap_file(base_file)
error_images = self.used_swap_images - self.back_swap_images
for error_image in error_images:
LOG.warn(_LW('%s swap image was used by instance'
' but no back files existing!'), error_image)
def _age_and_verify_cached_images(self, context, all_instances, base_dir):
LOG.debug('Verify base images')
# Determine what images are on disk because they're in use
@ -596,5 +640,7 @@ class ImageCacheManager(imagecache.ImageCacheManager):
self.used_images = running['used_images']
self.image_popularity = running['image_popularity']
self.instance_names = running['instance_names']
self.used_swap_images = running['used_swap_images']
# perform the aging and image verification
self._age_and_verify_cached_images(context, all_instances, base_dir)
self._age_and_verify_swap_images(context, base_dir)