openstack
/
nova
Code Issues Proposed changes

libvirt: set caps on maximum live migration time 

Currently Nova launches live migration and then just leaves it
to run, assuming it'll eventually finish. If the guest is
dirtying memory quicker than the network can transfer it, it
is entirely possible that a migration will never complete. In
such a case it is highly undesirable to leave it running
forever since it wastes valuable CPU and network resources
Rather than trying to come up with a new policy for aborting
migration in OpenStack, copy the existing logic that is
successfully battle tested by the oVirt project in VDSM.

This introduces two new host level configuration parameters
that cloud administrators can use to ensure migrations which
don't appear likely to complete will be aborted. First is
an overall cap on the total running time of migration. The
configured value is scaled by the number of GB of guest RAM,
since larger guests will obviously take proportionally longer
to migrate. The second is a timeout that is applied when Nova
detects that memory is being dirtied faster than it can be
transferred. It tracks a low watermark of data remaining and
if that low watermark doesn't decrease in the given time,
it assumes the VM is stuck and aborts migration

NB with the default values for the config parameters, the code
for detecting stuck migrations is only going to kick in for
guests which have more than 2 GB of RAM allocated, as for
smaller guests the overall time limit will abort migration
before this happens. This is reasonable as small guests are
less likely to get stuck during migration, as the network
will generally be able to keep up with dirtying data well
enough to get to a point where the final switchover can be
performed.

Related-bug: #1429220
DocImpact: two new libvirt configuration parameters in
           nova.conf allow the administrator to control
           length of migration before aborting it
Change-Id: I461affe6c85aaf2a6bf6e8749586bfbfe0ebc146
This commit is contained in:
Daniel P. Berrange 2015-03-06 17:34:16 +00:00
parent 07c7e5caf2
commit da33ab4f7b
3 changed files with 146 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
nova/tests/unit/virt/libvirt/fakelibvirt.py
View File

@ -768,6 +768,9 @@ class Domain(object):
def injectNMI(self, flags=0):
return 0
def abortJob(self):
pass
class DomainSnapshot(object):
def __init__(self, name, domain):

100
nova/tests/unit/virt/libvirt/test_driver.py
View File

@ -6225,16 +6225,22 @@ class LibvirtConnTestCase(test.NoDBTestCase):
self.assertFalse(mock_exist.called)
self.assertFalse(mock_shutil.called)
EXPECT_SUCCESS = 1
EXPECT_FAILURE = 2
EXPECT_ABORT = 3
@mock.patch.object(time, "time")
@mock.patch.object(time, "sleep",
side_effect=lambda x: eventlet.sleep(0))
@mock.patch.object(host.DomainJobInfo, "for_domain")
@mock.patch.object(objects.Instance, "save")
@mock.patch.object(fakelibvirt.Connection, "_mark_running")
@mock.patch.object(fakelibvirt.virDomain, "abortJob")
def _test_live_migration_monitoring(self,
job_info_records,
time_records,
expect_success,
expect_result,
mock_abort,
mock_running,
mock_save,
mock_job_info,
@ -6286,12 +6292,20 @@ class LibvirtConnTestCase(test.NoDBTestCase):
dom,
finish_event)
if expect_success:
if expect_result == self.EXPECT_SUCCESS:
self.assertFalse(fake_recover_method.called,
'Recover method called when success expected')
self.assertFalse(mock_abort.called,
'abortJob not called when success expected')
fake_post_method.assert_called_once_with(
self.context, instance, dest, False, migrate_data)
else:
if expect_result == self.EXPECT_ABORT:
self.assertTrue(mock_abort.called,
'abortJob called when abort expected')
else:
self.assertFalse(mock_abort.called,
'abortJob not called when failure expected')
self.assertFalse(fake_post_method.called,
'Post method called when success not expected')
fake_recover_method.assert_called_once_with(
@ -6314,7 +6328,8 @@ class LibvirtConnTestCase(test.NoDBTestCase):
type=fakelibvirt.VIR_DOMAIN_JOB_COMPLETED),
]
self._test_live_migration_monitoring(domain_info_records, [], True)
self._test_live_migration_monitoring(domain_info_records, [],
self.EXPECT_SUCCESS)
def test_live_migration_monitor_success_race(self):
# A normalish sequence but we're too slow to see the
@ -6334,7 +6349,8 @@ class LibvirtConnTestCase(test.NoDBTestCase):
type=fakelibvirt.VIR_DOMAIN_JOB_NONE),
]
self._test_live_migration_monitoring(domain_info_records, [], True)
self._test_live_migration_monitoring(domain_info_records, [],
self.EXPECT_SUCCESS)
def test_live_migration_monitor_failed(self):
# A failed sequence where we see all the expected events
@ -6352,7 +6368,8 @@ class LibvirtConnTestCase(test.NoDBTestCase):
type=fakelibvirt.VIR_DOMAIN_JOB_FAILED),
]
self._test_live_migration_monitoring(domain_info_records, [], False)
self._test_live_migration_monitoring(domain_info_records, [],
self.EXPECT_FAILURE)
def test_live_migration_monitor_failed_race(self):
# A failed sequence where we are too slow to see the
@ -6371,7 +6388,8 @@ class LibvirtConnTestCase(test.NoDBTestCase):
type=fakelibvirt.VIR_DOMAIN_JOB_NONE),
]
self._test_live_migration_monitoring(domain_info_records, [], False)
self._test_live_migration_monitoring(domain_info_records, [],
self.EXPECT_FAILURE)
def test_live_migration_monitor_cancelled(self):
# A cancelled sequence where we see all the events
@ -6390,13 +6408,17 @@ class LibvirtConnTestCase(test.NoDBTestCase):
type=fakelibvirt.VIR_DOMAIN_JOB_CANCELLED),
]
self._test_live_migration_monitoring(domain_info_records, [], False)
self._test_live_migration_monitoring(domain_info_records, [],
self.EXPECT_FAILURE)
@mock.patch.object(fakelibvirt.virDomain, "migrateSetMaxDowntime")
@mock.patch.object(libvirt_driver.LibvirtDriver,
"_migration_downtime_steps")
def test_live_migration_monitor_downtime(self, mock_downtime_steps,
mock_set_downtime):
self.flags(live_migration_completion_timeout=1000000,
live_migration_progress_timeout=1000000,
group='libvirt')
# We've setup 4 fake downtime steps - first value is the
# time delay, second is the downtime value
downtime_steps = [
@ -6434,12 +6456,74 @@ class LibvirtConnTestCase(test.NoDBTestCase):
]
self._test_live_migration_monitoring(domain_info_records,
fake_times, True)
fake_times, self.EXPECT_SUCCESS)
mock_set_downtime.assert_has_calls([mock.call(10),
mock.call(50),
mock.call(200)])
def test_live_migration_monitor_completion(self):
self.flags(live_migration_completion_timeout=100,
live_migration_progress_timeout=1000000,
group='libvirt')
# Each one of these fake times is used for time.time()
# when a new domain_info_records entry is consumed.
fake_times = [0, 40, 80, 120, 160, 200, 240, 280, 320]
# A normal sequence where see all the normal job states
domain_info_records = [
host.DomainJobInfo(
type=fakelibvirt.VIR_DOMAIN_JOB_NONE),
host.DomainJobInfo(
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
host.DomainJobInfo(
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
host.DomainJobInfo(
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
host.DomainJobInfo(
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
host.DomainJobInfo(
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
"thread-finish",
"domain-stop",
host.DomainJobInfo(
type=fakelibvirt.VIR_DOMAIN_JOB_CANCELLED),
]
self._test_live_migration_monitoring(domain_info_records,
fake_times, self.EXPECT_ABORT)
def test_live_migration_monitor_progress(self):
self.flags(live_migration_completion_timeout=1000000,
live_migration_progress_timeout=150,
group='libvirt')
# Each one of these fake times is used for time.time()
# when a new domain_info_records entry is consumed.
fake_times = [0, 40, 80, 120, 160, 200, 240, 280, 320]
# A normal sequence where see all the normal job states
domain_info_records = [
host.DomainJobInfo(
type=fakelibvirt.VIR_DOMAIN_JOB_NONE),
host.DomainJobInfo(
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
host.DomainJobInfo(
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
host.DomainJobInfo(
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
host.DomainJobInfo(
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
host.DomainJobInfo(
type=fakelibvirt.VIR_DOMAIN_JOB_UNBOUNDED),
"thread-finish",
"domain-stop",
host.DomainJobInfo(
type=fakelibvirt.VIR_DOMAIN_JOB_CANCELLED),
]
self._test_live_migration_monitoring(domain_info_records,
fake_times, self.EXPECT_ABORT)
def test_live_migration_downtime_steps(self):
self.flags(live_migration_downtime=400, group='libvirt')
self.flags(live_migration_downtime_steps=10, group='libvirt')

51
nova/virt/libvirt/driver.py
View File

@ -184,6 +184,19 @@ libvirt_opts = [
'of the migration downtime. Minimum delay is %d seconds. '
'Value is per GiB of guest RAM, with lower bound of a '
'minimum of 2 GiB' % LIVE_MIGRATION_DOWNTIME_DELAY_MIN),
cfg.IntOpt('live_migration_completion_timeout',
default=800,
help='Time to wait, in seconds, for migration to successfully '
'complete transferring data before aborting the '
'operation. Value is per GiB of guest RAM, with lower '
'bound of a minimum of 2 GiB. Should usually be larger '
'than downtime delay * downtime steps. Set to 0 to '
'disable timeouts.'),
cfg.IntOpt('live_migration_progress_timeout',
default=150,
help='Time to wait, in seconds, for migration to make forward '
'progress in transferring data before aborting the '
'operation. Set to 0 to disable timeouts.'),
cfg.StrOpt('snapshot_image_format',
choices=('raw', 'qcow2', 'vmdk', 'vdi'),
help='Snapshot image format. Defaults to same as source image'),
@ -5683,9 +5696,14 @@ class LibvirtDriver(driver.ComputeDriver):
migrate_data, dom, finish_event):
data_gb = self._live_migration_data_gb(instance)
downtime_steps = list(self._migration_downtime_steps(data_gb))
completion_timeout = int(
CONF.libvirt.live_migration_completion_timeout * data_gb)
progress_timeout = CONF.libvirt.live_migration_progress_timeout
n = 0
start = time.time()
progress_time = start
progress_watermark = None
while True:
info = host.DomainJobInfo.for_domain(dom)
@ -5743,6 +5761,32 @@ class LibvirtDriver(driver.ComputeDriver):
# the operation, change max bandwidth
now = time.time()
elapsed = now - start
abort = False
if ((progress_watermark is None) or
(progress_watermark > info.data_remaining)):
progress_watermark = info.data_remaining
progress_time = now
if (progress_timeout != 0 and
(now - progress_time) > progress_timeout):
LOG.warn(_LW("Live migration stuck for %d sec"),
(now - progress_time), instance=instance)
abort = True
if (completion_timeout != 0 and
elapsed > completion_timeout):
LOG.warn(_LW("Live migration not completed after %d sec"),
completion_timeout, instance=instance)
abort = True
if abort:
try:
dom.abortJob()
except libvirt.libvirtError as e:
LOG.warn(_LW("Failed to abort migration %s"),
e, instance=instance)
raise
# See if we need to increase the max downtime. We
# ignore failures, since we'd rather continue trying
@ -5801,6 +5845,13 @@ class LibvirtDriver(driver.ComputeDriver):
"processed_memory": info.memory_processed,
"remaining_memory": info.memory_remaining,
"total_memory": info.memory_total}, instance=instance)
if info.data_remaining > progress_watermark:
lg(_LI("Data remaining %(remaining)d bytes, "
"low watermark %(watermark)d bytes "
"%(last)d seconds ago"),
{"remaining": info.data_remaining,
"watermark": progress_watermark,
"last": (now - progress_time)}, instance=instance)
n = n + 1
elif info.type == libvirt.VIR_DOMAIN_JOB_COMPLETED: