Merge "Add testcases of ha and processmonitor"

masakarimonitors/tests/unit/ha/test_masakari.py

@@ -0,0 +1,178 @@
+# Copyright(c) 2017 Nippon Telegraph and Telephone Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+import testtools
+
+import eventlet
+from openstack import connection
+from openstack import exceptions
+from openstack import profile
+from oslo_utils import timeutils
+
+from masakariclient.sdk.ha import ha_service
+from masakarimonitors.ha import masakari
+from masakarimonitors.objects import event_constants as ec
+
+PROFILE_TYPE = "ha"
+PROFILE_NAME = "masakari"
+
+
+class TestSendNotification(testtools.TestCase):
+
+    def setUp(self):
+        super(TestSendNotification, self).setUp()
+        self.api_retry_max = 3
+        self.api_retry_interval = 1
+        self.event = {
+            'notification': {
+                'type': ec.EventConstants.TYPE_COMPUTE_HOST,
+                'hostname': 'compute-node1',
+                'generated_time': timeutils.utcnow(),
+                'payload': {
+                    'event': ec.EventConstants.EVENT_STOPPED,
+                    'cluster_status': 'OFFLINE',
+                    'host_status': ec.EventConstants.HOST_STATUS_NORMAL
+                }
+            }
+        }
+
+    @mock.patch.object(connection, 'Connection')
+    @mock.patch.object(profile, 'Profile')
+    def test_send_notification(self,
+                               mock_Profile,
+                               mock_Connection):
+
+        mock_prof = mock.Mock()
+        mock_Profile.return_value = mock_prof
+        mock_conn = mock.Mock()
+        mock_Connection.return_value = mock_conn
+
+        notifier = masakari.SendNotification()
+        notifier.send_notification(
+            self.api_retry_max, self.api_retry_interval, self.event)
+
+        mock_prof._add_service.assert_called_once_with(
+            ha_service.HAService(version='v1'))
+        mock_prof.set_name.assert_called_once_with(
+            PROFILE_TYPE, PROFILE_NAME)
+        mock_prof.set_region.assert_called_once_with(
+            PROFILE_TYPE, 'RegionOne')
+        mock_prof.set_version.assert_called_once_with(
+            PROFILE_TYPE, 'v1')
+        mock_prof.set_interface.assert_called_once_with(
+            PROFILE_TYPE, 'public')
+
+        mock_Connection.assert_called_once_with(
+            auth_url=None,
+            project_name=None,
+            username=None,
+            password=None,
+            project_domain_id=None,
+            user_domain_id=None,
+            profile=mock_prof)
+        mock_conn.ha.create_notification.assert_called_once_with(
+            type=self.event['notification']['type'],
+            hostname=self.event['notification']['hostname'],
+            generated_time=self.event['notification']['generated_time'],
+            payload=self.event['notification']['payload'])
+
+    @mock.patch.object(connection, 'Connection')
+    @mock.patch.object(profile, 'Profile')
+    def test_send_notification_409_error(self,
+                                         mock_Profile,
+                                         mock_Connection):
+
+        mock_prof = mock.Mock()
+        mock_Profile.return_value = mock_prof
+        mock_conn = mock.Mock()
+        mock_Connection.return_value = mock_conn
+        mock_conn.ha.create_notification.side_effect = \
+            exceptions.HttpException(http_status=409)
+
+        notifier = masakari.SendNotification()
+        notifier.send_notification(
+            self.api_retry_max, self.api_retry_interval, self.event)
+
+        mock_prof._add_service.assert_called_once_with(
+            ha_service.HAService(version='v1'))
+        mock_prof.set_name.assert_called_once_with(
+            PROFILE_TYPE, PROFILE_NAME)
+        mock_prof.set_region.assert_called_once_with(
+            PROFILE_TYPE, 'RegionOne')
+        mock_prof.set_version.assert_called_once_with(
+            PROFILE_TYPE, 'v1')
+        mock_prof.set_interface.assert_called_once_with(
+            PROFILE_TYPE, 'public')
+
+        mock_Connection.assert_called_once_with(
+            auth_url=None,
+            project_name=None,
+            username=None,
+            password=None,
+            project_domain_id=None,
+            user_domain_id=None,
+            profile=mock_prof)
+        mock_conn.ha.create_notification.assert_called_once_with(
+            type=self.event['notification']['type'],
+            hostname=self.event['notification']['hostname'],
+            generated_time=self.event['notification']['generated_time'],
+            payload=self.event['notification']['payload'])
+
+    @mock.patch.object(eventlet.greenthread, 'sleep')
+    @mock.patch.object(connection, 'Connection')
+    @mock.patch.object(profile, 'Profile')
+    def test_send_notification_500_error(self,
+                                         mock_Profile,
+                                         mock_Connection,
+                                         mock_sleep):
+
+        mock_prof = mock.Mock()
+        mock_Profile.return_value = mock_prof
+        mock_conn = mock.Mock()
+        mock_Connection.return_value = mock_conn
+        mock_conn.ha.create_notification.side_effect = \
+            exceptions.HttpException(http_status=500)
+        mock_sleep.return_value = None
+
+        notifier = masakari.SendNotification()
+        notifier.send_notification(
+            self.api_retry_max, self.api_retry_interval, self.event)
+
+        mock_prof._add_service.assert_called_once_with(
+            ha_service.HAService(version='v1'))
+        mock_prof.set_name.assert_called_once_with(
+            PROFILE_TYPE, PROFILE_NAME)
+        mock_prof.set_region.assert_called_once_with(
+            PROFILE_TYPE, 'RegionOne')
+        mock_prof.set_version.assert_called_once_with(
+            PROFILE_TYPE, 'v1')
+        mock_prof.set_interface.assert_called_once_with(
+            PROFILE_TYPE, 'public')
+
+        mock_Connection.assert_called_once_with(
+            auth_url=None,
+            project_name=None,
+            username=None,
+            password=None,
+            project_domain_id=None,
+            user_domain_id=None,
+            profile=mock_prof)
+        mock_conn.ha.create_notification.assert_called_with(
+            type=self.event['notification']['type'],
+            hostname=self.event['notification']['hostname'],
+            generated_time=self.event['notification']['generated_time'],
+            payload=self.event['notification']['payload'])
+        self.assertEqual(self.api_retry_max + 1,
+                         mock_conn.ha.create_notification.call_count)

masakarimonitors/tests/unit/processmonitor/process_handler/test_handle_process.py

@@ -13,13 +13,19 @@
 # limitations under the License.
 
 import mock
+import socket
 import testtools
 
 import eventlet
+from oslo_utils import timeutils
 
+import masakarimonitors.conf
+from masakarimonitors.ha import masakari
+from masakarimonitors.objects import event_constants as ec
 from masakarimonitors.processmonitor.process_handler import handle_process
 from masakarimonitors import utils
 
+CONF = masakarimonitors.conf.CONF
 eventlet.monkey_patch(os=False)
 
 MOCK_PROCESS_LIST = [
@@ -65,6 +71,8 @@ class TestHandleProcess(testtools.TestCase):
         obj = handle_process.HandleProcess()
         obj.set_process_list(process_list)
 
+        self.assertEqual(process_list, obj.process_list)
+
     @mock.patch.object(utils, 'execute')
     def test_start_processes(self,
                              mock_execute):
@@ -88,6 +96,20 @@ class TestHandleProcess(testtools.TestCase):
             MOCK_PROCESS_LIST[0].get('post_start_command'),
             run_as_root=MOCK_PROCESS_LIST[0].get('run_as_root'))
 
+    @mock.patch.object(utils, 'execute')
+    def test_start_processes_pre_cmd_fail(self, mock_execute):
+        process_list = MOCK_PROCESS_LIST
+        obj = handle_process.HandleProcess()
+        obj.set_process_list(process_list)
+
+        mock_execute.return_value = ('test_stdout', 'test_stderr')
+
+        obj.start_processes()
+
+        mock_execute.assert_called_once_with(
+            MOCK_PROCESS_LIST[0].get('pre_start_command'),
+            run_as_root=MOCK_PROCESS_LIST[0].get('run_as_root'))
+
     @mock.patch.object(utils, 'execute')
     def test_monitor_processes(self,
                                mock_execute):
@@ -99,6 +121,32 @@ class TestHandleProcess(testtools.TestCase):
 
         down_process_list = obj.monitor_processes()
         self.assertEqual([], down_process_list)
+        mock_execute.assert_called_once_with(
+            'ps', '-ef', run_as_root=False)
+
+    @mock.patch.object(utils, 'execute')
+    def test_monitor_processes_not_found(self, mock_execute):
+        process_list = MOCK_PROCESS_LIST
+        obj = handle_process.HandleProcess()
+        obj.set_process_list(process_list)
+
+        mock_execute.return_value = ('', '')
+
+        down_process_list = obj.monitor_processes()
+        self.assertEqual(MOCK_PROCESS_LIST, down_process_list)
+        mock_execute.assert_called_once_with(
+            'ps', '-ef', run_as_root=False)
+
+    @mock.patch.object(utils, 'execute')
+    def test_monitor_processes_exception(self, mock_execute):
+        process_list = MOCK_PROCESS_LIST
+        obj = handle_process.HandleProcess()
+        obj.set_process_list(process_list)
+
+        mock_execute.side_effect = Exception("Test exception.")
+
+        down_process_list = obj.monitor_processes()
+        self.assertEqual([], down_process_list)
 
     @mock.patch.object(utils, 'execute')
     def test_restart_processes(self,
@@ -115,11 +163,217 @@ class TestHandleProcess(testtools.TestCase):
         obj.restart_processes(down_process_list)
 
         mock_execute.assert_any_call(
-            MOCK_DOWN_PROCESS_LIST[0].get('pre_restart_command'),
-            run_as_root=MOCK_DOWN_PROCESS_LIST[0].get('run_as_root'))
+            down_process_list[0].get('pre_restart_command'),
+            run_as_root=down_process_list[0].get('run_as_root'))
         mock_execute.assert_any_call(
-            MOCK_DOWN_PROCESS_LIST[0].get('restart_command'),
-            run_as_root=MOCK_DOWN_PROCESS_LIST[0].get('run_as_root'))
+            down_process_list[0].get('restart_command'),
+            run_as_root=down_process_list[0].get('run_as_root'))
         mock_execute.assert_any_call(
-            MOCK_DOWN_PROCESS_LIST[0].get('post_restart_command'),
-            run_as_root=MOCK_DOWN_PROCESS_LIST[0].get('run_as_root'))
+            down_process_list[0].get('post_restart_command'),
+            run_as_root=down_process_list[0].get('run_as_root'))
+        self.assertEqual([], obj.restart_failure_list)
+
+    @mock.patch.object(utils, 'execute')
+    def test_restart_processes_failed_to_restart_previously(
+        self, mock_execute):
+        process_list = MOCK_PROCESS_LIST
+        obj = handle_process.HandleProcess()
+        obj.set_process_list(process_list)
+        restart_failure_list = [MOCK_DOWN_PROCESS_LIST[0].get('process_name')]
+        obj.restart_failure_list = restart_failure_list
+        down_process_list = MOCK_DOWN_PROCESS_LIST
+
+        obj.restart_processes(down_process_list)
+
+        self.assertEqual(restart_failure_list, obj.restart_failure_list)
+        mock_execute.assert_not_called()
+
+    @mock.patch.object(masakari.SendNotification, 'send_notification')
+    @mock.patch.object(timeutils, 'utcnow')
+    @mock.patch.object(eventlet.greenthread, 'sleep')
+    @mock.patch.object(utils, 'execute')
+    def test_restart_processes_pre_restart_command_retry_over(
+        self, mock_execute, mock_sleep, mock_utcnow, mock_send_notification):
+        process_list = MOCK_PROCESS_LIST
+        obj = handle_process.HandleProcess()
+        obj.set_process_list(process_list)
+        down_process_list = MOCK_DOWN_PROCESS_LIST
+
+        mock_execute.side_effect = [('test_stdout', 'test_stderr'),
+                                    ('test_stdout', 'test_stderr'),
+                                    ('test_stdout', 'test_stderr'),
+                                    ('test_stdout', 'test_stderr')]
+        mock_sleep.return_value = None
+        current_time = timeutils.utcnow()
+        mock_utcnow.return_value = current_time
+        mock_send_notification.return_value = None
+
+        obj.restart_processes(down_process_list)
+
+        pre_execute_count = CONF.process.restart_retries + 1
+        self.assertEqual(pre_execute_count, mock_execute.call_count)
+
+        for var in range(0, mock_execute.call_count):
+            args, kwargs = mock_execute.call_args_list[var]
+            self.assertEqual(
+                (down_process_list[0].get('pre_restart_command'),),
+                args)
+            self.assertEqual({'run_as_root': True}, kwargs)
+
+        event = {
+            'notification': {
+                'type': ec.EventConstants.TYPE_PROCESS,
+                'hostname': socket.gethostname(),
+                'generated_time': current_time,
+                'payload': {
+                    'event': ec.EventConstants.EVENT_STOPPED,
+                    'process_name': down_process_list[0].get('process_name')
+                }
+            }
+        }
+        mock_send_notification.assert_called_once_with(
+            CONF.process.api_retry_max,
+            CONF.process.api_retry_interval,
+            event)
+
+        self.assertEqual(
+            [down_process_list[0].get('process_name')],
+            obj.restart_failure_list)
+
+    @mock.patch.object(masakari.SendNotification, 'send_notification')
+    @mock.patch.object(timeutils, 'utcnow')
+    @mock.patch.object(eventlet.greenthread, 'sleep')
+    @mock.patch.object(utils, 'execute')
+    def test_restart_processes_restart_command_retry_over(
+        self, mock_execute, mock_sleep, mock_utcnow, mock_send_notification):
+        process_list = MOCK_PROCESS_LIST
+        obj = handle_process.HandleProcess()
+        obj.set_process_list(process_list)
+        down_process_list = MOCK_DOWN_PROCESS_LIST
+
+        mock_execute.side_effect = [('test_stdout', ''),
+                                    ('test_stdout', 'test_stderr'),
+                                    ('test_stdout', ''),
+                                    ('test_stdout', 'test_stderr'),
+                                    ('test_stdout', ''),
+                                    ('test_stdout', 'test_stderr'),
+                                    ('test_stdout', ''),
+                                    ('test_stdout', 'test_stderr')]
+        mock_sleep.return_value = None
+        current_time = timeutils.utcnow()
+        mock_utcnow.return_value = current_time
+        mock_send_notification.return_value = None
+
+        obj.restart_processes(down_process_list)
+
+        pre_execute_count = CONF.process.restart_retries + 1
+        execute_count = CONF.process.restart_retries + 1
+        total_execute_count = pre_execute_count + execute_count
+        self.assertEqual(total_execute_count, mock_execute.call_count)
+
+        for var in range(0, mock_execute.call_count):
+            # Execute order of restart_command is the second.
+            execute_order = 2
+
+            if (var + 1) % execute_order == 0:
+                args, kwargs = mock_execute.call_args_list[var]
+                self.assertEqual(
+                    (down_process_list[0].get('restart_command'),),
+                    args)
+                self.assertEqual({'run_as_root': True}, kwargs)
+
+        event = {
+            'notification': {
+                'type': ec.EventConstants.TYPE_PROCESS,
+                'hostname': socket.gethostname(),
+                'generated_time': current_time,
+                'payload': {
+                    'event': ec.EventConstants.EVENT_STOPPED,
+                    'process_name': down_process_list[0].get('process_name')
+                }
+            }
+        }
+        mock_send_notification.assert_called_once_with(
+            CONF.process.api_retry_max,
+            CONF.process.api_retry_interval,
+            event)
+        self.assertEqual(
+            [down_process_list[0].get('process_name')],
+            obj.restart_failure_list)
+
+    @mock.patch.object(masakari.SendNotification, 'send_notification')
+    @mock.patch.object(timeutils, 'utcnow')
+    @mock.patch.object(eventlet.greenthread, 'sleep')
+    @mock.patch.object(utils, 'execute')
+    def test_restart_processes_post_restart_command_retry_over(
+        self, mock_execute, mock_sleep, mock_utcnow, mock_send_notification):
+        process_list = MOCK_PROCESS_LIST
+        obj = handle_process.HandleProcess()
+        obj.set_process_list(process_list)
+        down_process_list = MOCK_DOWN_PROCESS_LIST
+
+        mock_execute.side_effect = [('test_stdout', ''),
+                                    ('test_stdout', ''),
+                                    ('test_stdout', 'test_stderr'),
+                                    ('test_stdout', ''),
+                                    ('test_stdout', ''),
+                                    ('test_stdout', 'test_stderr'),
+                                    ('test_stdout', ''),
+                                    ('test_stdout', ''),
+                                    ('test_stdout', 'test_stderr'),
+                                    ('test_stdout', ''),
+                                    ('test_stdout', ''),
+                                    ('test_stdout', 'test_stderr')]
+        mock_sleep.return_value = None
+        current_time = timeutils.utcnow()
+        mock_utcnow.return_value = current_time
+        mock_send_notification.return_value = None
+
+        obj.restart_processes(down_process_list)
+
+        pre_execute_count = CONF.process.restart_retries + 1
+        execute_count = CONF.process.restart_retries + 1
+        post_execute_count = CONF.process.restart_retries + 1
+        total_execute_count = \
+            pre_execute_count + execute_count + post_execute_count
+        self.assertEqual(total_execute_count, mock_execute.call_count)
+
+        for var in range(0, mock_execute.call_count):
+            # Execute order of restart_command is the third.
+            execute_order = 3
+
+            if (var + 1) % execute_order == 0:
+                args, kwargs = mock_execute.call_args_list[var]
+                self.assertEqual(
+                    (down_process_list[0].get('post_restart_command'),),
+                    args)
+                self.assertEqual({'run_as_root': True}, kwargs)
+
+        event = {
+            'notification': {
+                'type': ec.EventConstants.TYPE_PROCESS,
+                'hostname': socket.gethostname(),
+                'generated_time': current_time,
+                'payload': {
+                    'event': ec.EventConstants.EVENT_STOPPED,
+                    'process_name': down_process_list[0].get('process_name')
+                }
+            }
+        }
+        mock_send_notification.assert_called_once_with(
+            CONF.process.api_retry_max,
+            CONF.process.api_retry_interval,
+            event)
+        self.assertEqual(
+            [down_process_list[0].get('process_name')],
+            obj.restart_failure_list)
+
+    @mock.patch.object(utils, 'execute')
+    def test_execute_cmd_exception(self, mock_execute):
+        mock_execute.side_effect = Exception("Test exception.")
+
+        obj = handle_process.HandleProcess()
+        ret = obj._execute_cmd(MOCK_PROCESS_LIST[0].get('start_command'),
+                               MOCK_PROCESS_LIST[0].get('run_as_root'))
+
+        self.assertEqual(ret, 1)

masakarimonitors/tests/unit/processmonitor/test_process.py

@@ -47,6 +47,19 @@ MOCK_PROCESS_LIST = [
     },
 ]
 
+MOCK_DOWN_PROCESS_LIST = [
+    {
+        'process_name': 'mock_process_name_A',
+        'start_command': 'mock_start_command',
+        'pre_start_command': 'mock_pre_start_command',
+        'post_start_command': 'mock_post_start_command',
+        'restart_command': 'mock_restart_command',
+        'pre_restart_command': 'mock_pre_restart_command',
+        'post_restart_command': 'mock_post_restart_command',
+        'run_as_root': True
+    },
+]
+
 
 class TestProcessmonitorManager(testtools.TestCase):
 
@@ -59,21 +72,102 @@ class TestProcessmonitorManager(testtools.TestCase):
         else:
             return
 
+    @mock.patch.object(eventlet.greenthread, 'sleep')
+    @mock.patch.object(handle_process.HandleProcess, 'restart_processes')
     @mock.patch.object(handle_process.HandleProcess, 'monitor_processes')
     @mock.patch.object(handle_process.HandleProcess, 'start_processes')
     @mock.patch.object(handle_process.HandleProcess, 'set_process_list')
     @mock.patch.object(yaml, 'load')
+    @mock.patch('__builtin__.file')
     def test_main(self,
+                  mock_file,
                   mock_load,
                   mock_set_process_list,
                   mock_start_processes,
-                  mock_monitor_processes):
+                  mock_monitor_processes,
+                  mock_restart_processes,
+                  mock_sleep):
 
+        mock_file.return_value = None
         mock_load.side_effect = [self._get_mock_process_list(0),
+                                 self._get_mock_process_list(0),
                                  self._get_mock_process_list(1)]
         mock_set_process_list.return_value = None
         mock_start_processes.return_value = None
-        mock_monitor_processes.return_value = []
+        mock_monitor_processes.side_effect = [MOCK_DOWN_PROCESS_LIST, []]
+        mock_restart_processes.return_value = None
+        mock_sleep.return_value = None
 
         obj = processmonitor_manager.ProcessmonitorManager()
         obj.main()
+
+        mock_set_process_list.assert_called_with(MOCK_PROCESS_LIST)
+        mock_start_processes.assert_called_once_with()
+        self.assertEqual(2, mock_monitor_processes.call_count)
+        mock_restart_processes.assert_called_once_with(MOCK_DOWN_PROCESS_LIST)
+
+    @mock.patch.object(handle_process.HandleProcess, 'restart_processes')
+    @mock.patch.object(handle_process.HandleProcess, 'monitor_processes')
+    @mock.patch.object(handle_process.HandleProcess, 'start_processes')
+    @mock.patch.object(handle_process.HandleProcess, 'set_process_list')
+    @mock.patch.object(yaml, 'load')
+    @mock.patch('__builtin__.file')
+    def test_main_exception(self,
+                            mock_file,
+                            mock_load,
+                            mock_set_process_list,
+                            mock_start_processes,
+                            mock_monitor_processes,
+                            mock_restart_processes):
+
+        mock_file.return_value = None
+        mock_load.return_value = self._get_mock_process_list(0)
+        mock_set_process_list.return_value = None
+        mock_start_processes.side_effect = Exception("Test exception.")
+
+        obj = processmonitor_manager.ProcessmonitorManager()
+        obj.main()
+
+        mock_set_process_list.assert_called_once_with(MOCK_PROCESS_LIST)
+        mock_start_processes.assert_called_once_with()
+        mock_monitor_processes.assert_not_called()
+        mock_restart_processes.assert_not_called()
+
+    @mock.patch.object(handle_process.HandleProcess, 'set_process_list')
+    @mock.patch.object(yaml, 'load')
+    @mock.patch('__builtin__.file')
+    def test_load_process_list_yaml_error(self,
+                                         mock_file,
+                                         mock_load,
+                                         mock_set_process_list):
+
+        mock_file.return_value = None
+        mock_load.side_effect = yaml.YAMLError
+
+        obj = processmonitor_manager.ProcessmonitorManager()
+        obj.main()
+
+        mock_set_process_list.assert_not_called()
+
+    @mock.patch.object(handle_process.HandleProcess, 'set_process_list')
+    @mock.patch.object(yaml, 'load')
+    @mock.patch('__builtin__.file')
+    def test_load_process_list_exception(self,
+                                         mock_file,
+                                         mock_load,
+                                         mock_set_process_list):
+
+        mock_file.return_value = None
+        mock_load.side_effect = Exception("Test exception.")
+
+        obj = processmonitor_manager.ProcessmonitorManager()
+        obj.main()
+
+        mock_set_process_list.assert_not_called()
+
+    def test_stop(self):
+
+        obj = processmonitor_manager.ProcessmonitorManager()
+        obj.stop()
+
+        self.assertFalse(obj.running)