From 8c46b5c73e9e92c714c051e8cb165c97fcc8645b Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Wed, 8 Aug 2018 18:34:50 +0200 Subject: [PATCH] Continue running on_end() when instance deletion fails A user can delete their instances while the on_end function is executing. This can result in novaclient raising a NotFound exception, preventing on_end() from completing successfully. The host allocations are deleted at this point, but the hosts are left in the reservation's aggregate, preventing future leases from starting. This patch catches all exceptions and continues running on_end(). Change-Id: Ib495a26f9ec2e7eae64e1eaf69a6cf635c3aadcd Closes-Bug: #1786073 --- blazar/plugins/oshosts/host_plugin.py | 9 ++++++++- .../tests/plugins/oshosts/test_physical_host_plugin.py | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/blazar/plugins/oshosts/host_plugin.py b/blazar/plugins/oshosts/host_plugin.py index fa5d1294..15d0033a 100644 --- a/blazar/plugins/oshosts/host_plugin.py +++ b/blazar/plugins/oshosts/host_plugin.py @@ -16,6 +16,7 @@ import datetime +from novaclient import exceptions as nova_exceptions from oslo_config import cfg from oslo_log import log as logging from oslo_utils import strutils @@ -198,7 +199,13 @@ class PhysicalHostPlugin(base.BasePlugin, nova.NovaClientWrapper): for host in pool.get_computehosts(host_reservation['aggregate_id']): for server in self.nova.servers.list( search_opts={"host": host, "all_tenants": 1}): - self.nova.servers.delete(server=server) + try: + self.nova.servers.delete(server=server) + except nova_exceptions.NotFound: + LOG.info('Could not find server %s, may have been deleted ' + 'concurrently.', server) + except Exception as e: + LOG.exception('Failed to delete %s: %s.', server, str(e)) try: pool.delete(host_reservation['aggregate_id']) except manager_ex.AggregateNotFound: diff --git a/blazar/tests/plugins/oshosts/test_physical_host_plugin.py b/blazar/tests/plugins/oshosts/test_physical_host_plugin.py index 10b30ce5..d92bdcf3 100644 --- a/blazar/tests/plugins/oshosts/test_physical_host_plugin.py +++ b/blazar/tests/plugins/oshosts/test_physical_host_plugin.py @@ -17,6 +17,7 @@ import datetime import mock from novaclient import client as nova_client +from novaclient import exceptions as nova_exceptions from oslo_config import cfg from oslo_config import fixture as conf_fixture import testtools @@ -1404,6 +1405,11 @@ class PhysicalHostPluginTestCase(tests.TestCase): list_servers = self.patch(self.ServerManager, 'list') list_servers.return_value = ['server1', 'server2'] delete_server = self.patch(self.ServerManager, 'delete') + # Mock delete_server so the first call fails to find the instance. + # This can happen when the user is deleting instances concurrently. + delete_server.side_effect = mock.Mock( + side_effect=[nova_exceptions.NotFound( + 404, 'Instance server1 could not be found.'), None]) delete_pool = self.patch(self.nova.ReservationPool, 'delete') self.fake_phys_plugin.on_end(u'04de74e8-193a-49d2-9ab8-cba7b49e45e8') host_reservation_update.assert_called_with(