diff --git a/watcher/applier/actions/migration.py b/watcher/applier/actions/migration.py index 0e10af6b8..6e6c26a3a 100644 --- a/watcher/applier/actions/migration.py +++ b/watcher/applier/actions/migration.py @@ -66,8 +66,10 @@ class Migrate(base.BaseAction): 'type': 'object', 'properties': { 'destination_node': { - 'type': 'string', - "minLength": 1 + "anyof": [ + {'type': 'string', "minLength": 1}, + {'type': 'None'} + ] }, 'migration_type': { 'type': 'string', @@ -85,8 +87,7 @@ class Migrate(base.BaseAction): "minLength": 1 } }, - 'required': ['destination_node', 'migration_type', - 'resource_id', 'source_node'], + 'required': ['migration_type', 'resource_id', 'source_node'], 'additionalProperties': False, } @@ -163,10 +164,14 @@ class Migrate(base.BaseAction): return nova.abort_live_migrate(instance_id=self.instance_uuid, source=source, destination=destination) - def migrate(self, destination): + def migrate(self, destination=None): nova = nova_helper.NovaHelper(osc=self.osc) - LOG.debug("Migrate instance %s to %s", self.instance_uuid, - destination) + if destination is None: + LOG.debug("Migrating instance %s, destination node will be " + "determined by nova-scheduler", self.instance_uuid) + else: + LOG.debug("Migrate instance %s to %s", self.instance_uuid, + destination) instance = nova.find_instance(self.instance_uuid) if instance: if self.migration_type == self.LIVE_MIGRATION: diff --git a/watcher/common/nova_helper.py b/watcher/common/nova_helper.py index b62953b73..52994f456 100644 --- a/watcher/common/nova_helper.py +++ b/watcher/common/nova_helper.py @@ -85,6 +85,20 @@ class NovaHelper(object): def find_instance(self, instance_id): return self.nova.servers.get(instance_id) + def confirm_resize(self, instance, previous_status, retry=60): + instance.confirm_resize() + instance = self.nova.servers.get(instance.id) + while instance.status != previous_status and retry: + instance = self.nova.servers.get(instance.id) + retry -= 1 + time.sleep(1) + if instance.status == previous_status: + return True + else: + LOG.debug("confirm resize failed for the " + "instance %s" % instance.id) + return False + def wait_for_volume_status(self, volume, status, timeout=60, poll_interval=1): """Wait until volume reaches given status. @@ -106,7 +120,8 @@ class NovaHelper(object): return volume.status == status def watcher_non_live_migrate_instance(self, instance_id, dest_hostname, - keep_original_image_name=True): + keep_original_image_name=True, + retry=120): """This method migrates a given instance using an image of this instance and creating a new instance @@ -118,6 +133,9 @@ class NovaHelper(object): It returns True if the migration was successful, False otherwise. + if destination hostname not given, this method calls nova api + to migrate the instance. + :param instance_id: the unique id of the instance to migrate. :param keep_original_image_name: flag indicating whether the image name from which the original instance was built must be @@ -125,10 +143,8 @@ class NovaHelper(object): If this flag is False, a temporary image name is built """ new_image_name = "" - LOG.debug( - "Trying a non-live migrate of instance '%s' " - "using a temporary image ..." % instance_id) + "Trying a non-live migrate of instance '%s' " % instance_id) # Looking for the instance to migrate instance = self.find_instance(instance_id) @@ -136,10 +152,38 @@ class NovaHelper(object): LOG.debug("Instance %s not found !" % instance_id) return False else: + # NOTE: If destination node is None call Nova API to migrate + # instance host_name = getattr(instance, "OS-EXT-SRV-ATTR:host") LOG.debug( "Instance %s found on host '%s'." % (instance_id, host_name)) + if dest_hostname is None: + previous_status = getattr(instance, 'status') + + instance.migrate() + instance = self.nova.servers.get(instance_id) + while (getattr(instance, 'status') not in + ["VERIFY_RESIZE", "ERROR"] and retry): + instance = self.nova.servers.get(instance.id) + time.sleep(2) + retry -= 1 + new_hostname = getattr(instance, 'OS-EXT-SRV-ATTR:host') + + if (host_name != new_hostname and + instance.status == 'VERIFY_RESIZE'): + if not self.confirm_resize(instance, previous_status): + return False + LOG.debug( + "cold migration succeeded : " + "instance %s is now on host '%s'." % ( + instance_id, new_hostname)) + return True + else: + LOG.debug( + "cold migration for instance %s failed" % instance_id) + return False + if not keep_original_image_name: # randrange gives you an integral value irand = random.randint(0, 1000) @@ -389,15 +433,15 @@ class NovaHelper(object): False otherwise. :param instance_id: the unique id of the instance to migrate. - :param dest_hostname: the name of the destination compute node. + :param dest_hostname: the name of the destination compute node, if + destination_node is None, nova scheduler choose + the destination host :param block_migration: No shared storage is required. """ - LOG.debug("Trying a live migrate of instance %s to host '%s'" % ( - instance_id, dest_hostname)) + LOG.debug("Trying to live migrate instance %s " % (instance_id)) # Looking for the instance to migrate instance = self.find_instance(instance_id) - if not instance: LOG.debug("Instance not found: %s" % instance_id) return False @@ -409,6 +453,29 @@ class NovaHelper(object): instance.live_migrate(host=dest_hostname, block_migration=block_migration, disk_over_commit=True) + + instance = self.nova.servers.get(instance_id) + + # NOTE: If destination host is not specified for live migration + # let nova scheduler choose the destination host. + if dest_hostname is None: + while (instance.status not in ['ACTIVE', 'ERROR'] and retry): + instance = self.nova.servers.get(instance.id) + LOG.debug( + 'Waiting the migration of {0}'.format(instance.id)) + time.sleep(1) + retry -= 1 + new_hostname = getattr(instance, 'OS-EXT-SRV-ATTR:host') + + if host_name != new_hostname and instance.status == 'ACTIVE': + LOG.debug( + "Live migration succeeded : " + "instance %s is now on host '%s'." % ( + instance_id, new_hostname)) + return True + else: + return False + while getattr(instance, 'OS-EXT-SRV-ATTR:host') != dest_hostname \ and retry: diff --git a/watcher/tests/applier/actions/test_migration.py b/watcher/tests/applier/actions/test_migration.py index 54b04b21b..7d85a0064 100644 --- a/watcher/tests/applier/actions/test_migration.py +++ b/watcher/tests/applier/actions/test_migration.py @@ -117,15 +117,14 @@ class TestMigration(base.TestCase): self.assertRaises(jsonschema.ValidationError, self.action.validate_parameters) - def test_parameters_exception_destination_node(self): + def test_parameters_destination_node_none(self): parameters = {baction.BaseAction.RESOURCE_ID: self.INSTANCE_UUID, 'migration_type': 'live', 'source_node': 'compute-1', 'destination_node': None} self.action.input_parameters = parameters - self.assertRaises(jsonschema.ValidationError, - self.action.validate_parameters) + self.assertTrue(self.action.validate_parameters) def test_parameters_exception_resource_id(self): parameters = {baction.BaseAction.RESOURCE_ID: "EFEF", diff --git a/watcher/tests/common/test_nova_helper.py b/watcher/tests/common/test_nova_helper.py index f037c5bcb..06daf6fa5 100644 --- a/watcher/tests/common/test_nova_helper.py +++ b/watcher/tests/common/test_nova_helper.py @@ -69,6 +69,31 @@ class TestNovaHelper(base.TestCase): else: nova_util.nova.server_migration.list.return_value = [list] + @staticmethod + def fake_live_migrate(server, *args, **kwargs): + + def side_effect(*args, **kwargs): + setattr(server, 'OS-EXT-SRV-ATTR:host', "compute-2") + + server.live_migrate.side_effect = side_effect + + @staticmethod + def fake_confirm_resize(server, *args, **kwargs): + + def side_effect(*args, **kwargs): + setattr(server, 'status', 'ACTIVE') + + server.confirm_resize.side_effect = side_effect + + @staticmethod + def fake_cold_migrate(server, *args, **kwargs): + + def side_effect(*args, **kwargs): + setattr(server, 'OS-EXT-SRV-ATTR:host', "compute-2") + setattr(server, 'status', 'VERIFY_RESIZE') + + server.migrate.side_effect = side_effect + @mock.patch.object(time, 'sleep', mock.Mock()) def test_stop_instance(self, mock_glance, mock_cinder, mock_neutron, mock_nova): @@ -140,6 +165,19 @@ class TestNovaHelper(base.TestCase): ) self.assertFalse(is_success) + @mock.patch.object(time, 'sleep', mock.Mock()) + def test_live_migrate_instance_no_destination_node( + self, mock_glance, mock_cinder, mock_neutron, mock_nova): + nova_util = nova_helper.NovaHelper() + server = self.fake_server(self.instance_uuid) + self.destination_node = None + self.fake_nova_find_list(nova_util, find=server, list=server) + self.fake_live_migrate(server) + is_success = nova_util.live_migrate_instance( + self.instance_uuid, self.destination_node + ) + self.assertTrue(is_success) + def test_watcher_non_live_migrate_instance_not_found( self, mock_glance, mock_cinder, mock_neutron, mock_nova): nova_util = nova_helper.NovaHelper() @@ -228,6 +266,21 @@ class TestNovaHelper(base.TestCase): (self.instance_uuid, self.source_node, self.destination_node)) + def test_non_live_migrate_instance_no_destination_node( + self, mock_glance, mock_cinder, mock_neutron, mock_nova): + nova_util = nova_helper.NovaHelper() + server = self.fake_server(self.instance_uuid) + setattr(server, 'OS-EXT-SRV-ATTR:host', + self.source_node) + self.destination_node = None + self.fake_nova_find_list(nova_util, find=server, list=server) + self.fake_cold_migrate(server) + self.fake_confirm_resize(server) + is_success = nova_util.watcher_non_live_migrate_instance( + self.instance_uuid, self.destination_node + ) + self.assertTrue(is_success) + @mock.patch.object(time, 'sleep', mock.Mock()) def test_create_image_from_instance(self, mock_glance, mock_cinder, mock_neutron, mock_nova):