Move db query to fetch down bindings under try/except

In case of intermittent DB failures router and network auto-rescheduling
tasks may fail due to error on fetching down bindings from db.
Need to put this queries under try/except to prevent unexpected exit.

Conflicts:
	neutron/db/agentschedulers_db.py
	neutron/db/l3_agentschedulers_db.py
	neutron/tests/unit/scheduler/test_dhcp_agent_scheduler.py

Closes-Bug: #1546110
Change-Id: Id48e899a5b3d906c6d1da4d03923bdda2681cd92
(cherry picked from commit b6ec40cbf7)
(cherry picked from commit 48a6196718)

neutron/db/agentschedulers_db.py

@@ -259,14 +259,14 @@ class DhcpAgentSchedulerDbMixin(dhcpagentscheduler
         cutoff = self.get_cutoff_time(agent_dead_limit)
 
         context = ncontext.get_admin_context()
-        down_bindings = (
-            context.session.query(NetworkDhcpAgentBinding).
-            join(agents_db.Agent).
-            filter(agents_db.Agent.heartbeat_timestamp < cutoff,
-                   agents_db.Agent.admin_state_up))
-        dhcp_notifier = self.agent_notifiers.get(constants.AGENT_TYPE_DHCP)
-
         try:
+            down_bindings = (
+                context.session.query(NetworkDhcpAgentBinding).
+                join(agents_db.Agent).
+                filter(agents_db.Agent.heartbeat_timestamp < cutoff,
+                       agents_db.Agent.admin_state_up))
+            dhcp_notifier = self.agent_notifiers.get(constants.AGENT_TYPE_DHCP)
+
             for binding in self._filter_bindings(context, down_bindings):
                 LOG.warn(_LW("Removing network %(network)s from agent "
                              "%(agent)s because the agent did not report "

neutron/db/l3_agentschedulers_db.py

@@ -89,17 +89,19 @@ class L3AgentSchedulerDbMixin(l3agentscheduler.L3AgentSchedulerPluginBase,
         cutoff = self.get_cutoff_time(agent_dead_limit)
 
         context = n_ctx.get_admin_context()
-        down_bindings = (
-            context.session.query(RouterL3AgentBinding).
-            join(agents_db.Agent).
-            filter(agents_db.Agent.heartbeat_timestamp < cutoff,
-                   agents_db.Agent.admin_state_up).
-            outerjoin(l3_attrs_db.RouterExtraAttributes,
-                      l3_attrs_db.RouterExtraAttributes.router_id ==
-                      RouterL3AgentBinding.router_id).
-            filter(sa.or_(l3_attrs_db.RouterExtraAttributes.ha == sql.false(),
-                          l3_attrs_db.RouterExtraAttributes.ha == sql.null())))
         try:
+            down_bindings = (
+                context.session.query(RouterL3AgentBinding).
+                join(agents_db.Agent).
+                filter(agents_db.Agent.heartbeat_timestamp < cutoff,
+                       agents_db.Agent.admin_state_up).
+                outerjoin(l3_attrs_db.RouterExtraAttributes,
+                          l3_attrs_db.RouterExtraAttributes.router_id ==
+                          RouterL3AgentBinding.router_id).
+                filter(sa.or_(
+                    l3_attrs_db.RouterExtraAttributes.ha == sql.false(),
+                    l3_attrs_db.RouterExtraAttributes.ha == sql.null())))
+
             for binding in down_bindings:
                 LOG.warn(_LW(
                     "Rescheduling router %(router)s from agent %(agent)s "

neutron/tests/unit/plugins/openvswitch/test_agent_scheduler.py

@@ -741,6 +741,16 @@ class OvsAgentSchedulerTestCase(OvsAgentSchedulerTestCaseBase):
             self._take_down_agent_and_run_reschedule(L3_HOSTA)  # Value error
             self._take_down_agent_and_run_reschedule(L3_HOSTA)  # Exception
 
+    def test_router_rescheduler_catches_exceptions_on_fetching_bindings(self):
+        with mock.patch('neutron.context.get_admin_context') as get_ctx:
+            mock_ctx = mock.Mock()
+            get_ctx.return_value = mock_ctx
+            mock_ctx.session.query.side_effect = db_exc.DBError()
+            plugin = manager.NeutronManager.get_service_plugins().get(
+                service_constants.L3_ROUTER_NAT)
+            # check that no exception is raised
+            plugin.reschedule_routers_from_down_agents()
+
     def test_router_rescheduler_iterates_after_reschedule_failure(self):
         plugin = manager.NeutronManager.get_service_plugins().get(
             service_constants.L3_ROUTER_NAT)

neutron/tests/unit/scheduler/test_dhcp_agent_scheduler.py

@@ -351,6 +351,14 @@ class TestNetworksFailover(TestDhcpSchedulerBaseTestCase,
             # just make sure that no exception is raised
             self.remove_networks_from_down_agents()
 
+    def test_reschedule_network_catches_exceptions_on_fetching_bindings(self):
+        with mock.patch('neutron.context.get_admin_context') as get_ctx:
+            mock_ctx = mock.Mock()
+            get_ctx.return_value = mock_ctx
+            mock_ctx.session.query.side_effect = Exception()
+            # just make sure that no exception is raised
+            self.remove_networks_from_down_agents()
+
 
 class DHCPAgentWeightSchedulerTestCase(TestDhcpSchedulerBaseTestCase):
     """Unit test scenarios for WeightScheduler.schedule."""