InstanceFailure: Ignore notifications for certain events

For instance failure recovery Masakari doesn't check for the event from
payload received in notification, as a result it executes workflow for
all the events received.

When a instance is stopped (normal operation), masakari-instancemonitor
sends notifications related to following two events one after another to
masakari-api service.
    A) event: LIFECYCLE detail: STOPPED_SHUTDOWN
    B) event: LIFECYCLE detail: SHUTDOWN_FINISHED

When masakari-engine processes the first notification, it checks for vmstate
of the instance, if it’s 'stopped', it won't stop the vm. But if it's not
stopped it will attempt to stop it. Then again, libvirt will send the above
two events to masakari-instancemonitor and masakari-instancemonitor will send
related notifications to masakari-api and this way it will go into the recursion.

To solve this issue we can only process the below mentioned events at masakari
side which actually requires actions to be taken:
    A) 'event': 'LIFECYCLE', 'vir_domain_event': 'STOPPED_FAILED'
    B) 'event': 'IO_ERROR', 'vir_domain_event': 'IO_ERROR_REPORT'

So removed handling of all other events. For all other events masakari will
simply log the message at INFO level and mark the notification status as
'ignored'.

Closes-Bug: #1662994
Change-Id: If1113a599894cc7bab32b9656b6c42de5e5d7a54

masakari/engine/instance_events.py

@@ -0,0 +1,34 @@
+# Copyright 2017 NTT DATA
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+""" VM libvirt events
+
+These are the events which needs to be processed by masakari in case of
+instance recovery failure.
+"""
+
+INSTANCE_EVENTS = {
+    # Add more events and vir_domain_events here.
+    'LIFECYCLE': ['STOPPED_FAILED'],
+    'IO_ERROR': ['IO_ERROR_REPORT']
+}
+
+
+def is_valid_event(payload):
+    vir_domain_event_list = INSTANCE_EVENTS.get(payload.get('event'))
+    if vir_domain_event_list and payload.get(
+            'vir_domain_event') in vir_domain_event_list:
+        return True
+    return False

masakari/engine/manager.py

@@ -29,6 +29,7 @@ from oslo_utils import timeutils
 
 import masakari.conf
 from masakari.engine import driver
+from masakari.engine import instance_events as virt_events
 from masakari import exception
 from masakari.i18n import _LE, _LI, _LW
 from masakari import manager
@@ -100,6 +101,13 @@ class MasakariManager(manager.Manager):
         return notification_status
 
     def _handle_notification_type_instance(self, context, notification):
+        if not virt_events.is_valid_event(notification.payload):
+            LOG.info(_LI("Notification '%(uuid)s' received with payload "
+                         "%(payload)s is ignored."), {
+                "uuid": notification.notification_uuid,
+                "payload": notification.payload})
+            return fields.NotificationStatus.IGNORED
+
         notification_status = fields.NotificationStatus.FINISHED
         try:
             self.driver.execute_instance_failure(

masakari/tests/unit/engine/test_engine_mgr.py

@@ -45,8 +45,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
     def _get_vm_type_notification(self):
         return fakes.create_fake_notification(
             type="VM", id=1, payload={
-                'event': 'stopped', 'instance_uuid': uuidsentinel.fake_ins,
-                'vir_domain_event': 'fake_event'
+                'event': 'LIFECYCLE', 'instance_uuid': uuidsentinel.fake_ins,
+                'vir_domain_event': 'STOPPED_FAILED'
             },
             source_host_uuid=uuidsentinel.fake_host,
             generated_time=NOW, status="new",
@@ -100,6 +100,22 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
             self.context, notification.payload.get('instance_uuid'),
             notification.notification_uuid)
 
+    @mock.patch.object(notification_obj.Notification, "save")
+    def test_process_notification_type_vm_error_event_unmatched(
+            self, mock_save):
+        notification = fakes.create_fake_notification(
+            type="VM", id=1, payload={
+                'event': 'fake_event', 'instance_uuid': uuidsentinel.fake_ins,
+                'vir_domain_event': 'fake_vir_domain_event'
+            },
+            source_host_uuid=uuidsentinel.fake_host,
+            generated_time=NOW, status="new",
+            notification_uuid=uuidsentinel.fake_notification)
+
+        self.engine.process_notification(self.context,
+                                         notification=notification)
+        self.assertEqual("ignored", notification.status)
+
     @mock.patch("masakari.engine.drivers.taskflow."
                 "TaskFlowDriver.execute_instance_failure")
     @mock.patch.object(notification_obj.Notification, "save")