diff --git a/heat/engine/service.py b/heat/engine/service.py index bd3c807c6f..bcd7801b5a 100644 --- a/heat/engine/service.py +++ b/heat/engine/service.py @@ -79,6 +79,10 @@ cfg.CONF.import_opt('enable_stack_abandon', 'heat.common.config') cfg.CONF.import_opt('enable_stack_adopt', 'heat.common.config') cfg.CONF.import_opt('convergence_engine', 'heat.common.config') +# Time to wait for a stack to stop when cancelling running threads, before +# giving up on being able to start a delete. +STOP_STACK_TIMEOUT = 30 + LOG = logging.getLogger(__name__) @@ -1147,7 +1151,8 @@ class EngineService(service.Service): # Another active engine has the lock elif service_utils.engine_alive(cnxt, engine_id): cancel_result = self._remote_call( - cnxt, engine_id, self.listener.SEND, + cnxt, engine_id, cfg.CONF.engine_life_check_timeout, + self.listener.SEND, stack_identity=stack_identity, message=cancel_message) if cancel_result is None: LOG.debug("Successfully sent %(msg)s message " @@ -1337,8 +1342,7 @@ class EngineService(service.Service): return api.format_stack_output(outputs[output_key]) - def _remote_call(self, cnxt, lock_engine_id, call, **kwargs): - timeout = cfg.CONF.engine_life_check_timeout + def _remote_call(self, cnxt, lock_engine_id, timeout, call, **kwargs): self.cctxt = self._client.prepare( version='1.0', timeout=timeout, @@ -1396,7 +1400,8 @@ class EngineService(service.Service): # Another active engine has the lock elif service_utils.engine_alive(cnxt, acquire_result): cancel_result = self._remote_call( - cnxt, acquire_result, self.listener.SEND, + cnxt, acquire_result, cfg.CONF.engine_life_check_timeout, + self.listener.SEND, stack_identity=stack_identity, message=rpc_api.THREAD_CANCEL) if cancel_result is None: LOG.debug("Successfully sent %(msg)s message " @@ -1436,7 +1441,8 @@ class EngineService(service.Service): elif service_utils.engine_alive(cnxt, acquire_result): # Another active engine has the lock stop_result = self._remote_call( - cnxt, acquire_result, self.listener.STOP_STACK, + cnxt, acquire_result, STOP_STACK_TIMEOUT, + self.listener.STOP_STACK, stack_identity=stack_identity) if stop_result is None: LOG.debug("Successfully stopped remote task " diff --git a/heat/tests/engine/service/test_stack_delete.py b/heat/tests/engine/service/test_stack_delete.py index ab0dd282d8..418c9018e6 100644 --- a/heat/tests/engine/service/test_stack_delete.py +++ b/heat/tests/engine/service/test_stack_delete.py @@ -165,7 +165,8 @@ class StackDeleteTest(common.HeatTestCase): mock_load.assert_called_once_with(self.ctx, stack=st) mock_try.assert_called_once_with() mock_alive.assert_called_once_with(self.ctx, OTHER_ENGINE) - mock_call.assert_called_once_with(self.ctx, OTHER_ENGINE, "send", + mock_call.assert_called_once_with(self.ctx, OTHER_ENGINE, mock.ANY, + "send", message='cancel', stack_identity=mock.ANY) @@ -203,10 +204,10 @@ class StackDeleteTest(common.HeatTestCase): mock_try.assert_called_with() mock_alive.assert_called_with(self.ctx, OTHER_ENGINE) mock_call.assert_has_calls([ - mock.call(self.ctx, OTHER_ENGINE, "send", + mock.call(self.ctx, OTHER_ENGINE, mock.ANY, "send", message='cancel', stack_identity=mock.ANY), - mock.call(self.ctx, OTHER_ENGINE, "stop_stack", + mock.call(self.ctx, OTHER_ENGINE, mock.ANY, "stop_stack", stack_identity=mock.ANY) ]) mock_acquire.assert_called_once_with(True) diff --git a/heat_integrationtests/functional/test_delete.py b/heat_integrationtests/functional/test_delete.py new file mode 100644 index 0000000000..92b1c74512 --- /dev/null +++ b/heat_integrationtests/functional/test_delete.py @@ -0,0 +1,42 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time + +from heat_integrationtests.functional import functional_base + + +class DeleteInProgressTest(functional_base.FunctionalTestsBase): + + root_template = ''' +heat_template_version: 2013-05-23 +resources: + rg: + type: OS::Heat::ResourceGroup + properties: + count: 125 + resource_def: + type: empty.yaml +''' + + empty_template = ''' +heat_template_version: 2013-05-23 +resources: +''' + + def test_delete_nested_stacks_create_in_progress(self): + files = {'empty.yaml': self.empty_template} + identifier = self.stack_create(template=self.root_template, + files=files, + expected_status='CREATE_IN_PROGRESS') + time.sleep(20) + self._stack_delete(identifier)