Catch exceptions when cancelling tasks in DependencyTaskGroup

TaskRunner.cancel() can (and, in the case of a running Resource action with
grace_period=None, will) raise an exception. To ensure that all of the
tasks in the DependencyTaskGroup get cancelled, catch and ignore any
exceptions raised when cancelling each task.

Change-Id: I612e76d55d6f427156f6d74991e7c8f717c32200

heat/engine/scheduler.py

@@ -427,7 +427,10 @@ class DependencyTaskGroup(object):
 
     def cancel_all(self, grace_period=None):
         for r in six.itervalues(self._runners):
-            r.cancel(grace_period=grace_period)
+            try:
+                r.cancel(grace_period=grace_period)
+            except Exception as ex:
+                LOG.debug('Exception cancelling task: %s' % six.text_type(ex))
 
     def _cancel_recursively(self, key, runner):
         runner.cancel()

heat/tests/engine/test_scheduler.py

@@ -280,6 +280,37 @@ class DependencyTaskGroupTest(common.HeatTestCase):
                                 run_tasks_with_exceptions, e1)
         self.assertEqual([e1], exc.exceptions)
 
+    def test_exceptions_on_cancel(self):
+        class TestException(Exception):
+            pass
+
+        class ExceptionOnExit(Exception):
+            pass
+
+        cancelled = []
+
+        def task_func(arg):
+            for i in range(4):
+                if i > 1:
+                    raise TestException
+
+                try:
+                    yield
+                except GeneratorExit:
+                    cancelled.append(arg)
+                    raise ExceptionOnExit
+
+        tasks = (('A', None), ('B', None), ('C', None))
+        deps = dependencies.Dependencies(tasks)
+
+        tg = scheduler.DependencyTaskGroup(deps, task_func)
+        task = tg()
+
+        next(task)
+        next(task)
+        self.assertRaises(TestException, next, task)
+        self.assertEqual(len(tasks) - 1, len(cancelled))
+
     def test_exception_grace_period(self):
         e1 = Exception('e1')