From dd26b010a6191f8870779fb0ba654b6ed5b094e7 Mon Sep 17 00:00:00 2001 From: Artur Korzeniewski Date: Fri, 27 Jan 2017 11:19:16 +0100 Subject: [PATCH] Addressing L3 HA keepalived failures in functional tests Current testing of Keepalived was not configuring the connectivity between 2 agent namespaces. Added setting up the veth pair. Also, bridges external qg- and internal qr- were removed from agent1 namespace and moved to agent2 namespace, because they had the same name. Added patching the qg and qr bridges name creation to be different for functional tests. Change-Id: I82b3218091da4feb39a9e820d0e54639ae27c97d Closes-Bug: #1580648 (cherry picked from commit 8d3f216e2421a01b54a4049c639bdb803df72510) --- .../tests/functional/agent/l3/framework.py | 85 ++++++++++++++++++- .../functional/agent/l3/test_ha_router.py | 36 +++----- 2 files changed, 95 insertions(+), 26 deletions(-) diff --git a/neutron/tests/functional/agent/l3/framework.py b/neutron/tests/functional/agent/l3/framework.py index 66541ab68df..7da9daf44b0 100644 --- a/neutron/tests/functional/agent/l3/framework.py +++ b/neutron/tests/functional/agent/l3/framework.py @@ -28,6 +28,8 @@ import textwrap from neutron.agent.common import config as agent_config from neutron.agent.common import ovs_lib from neutron.agent.l3 import agent as neutron_l3_agent +from neutron.agent.l3 import namespaces +from neutron.agent.l3 import router_info as l3_router_info from neutron.agent import l3_agent as l3_agent_main from neutron.agent.linux import external_process from neutron.agent.linux import ip_lib @@ -48,6 +50,7 @@ def get_ovs_bridge(br_name): class L3AgentTestFramework(base.BaseSudoTestCase): INTERFACE_DRIVER = 'neutron.agent.linux.interface.OVSInterfaceDriver' + NESTED_NAMESPACE_SEPARATOR = '@' def setUp(self): super(L3AgentTestFramework, self).setUp() @@ -205,12 +208,15 @@ class L3AgentTestFramework(base.BaseSudoTestCase): self._gateway_check(self.agent.conf.ipv6_gateway, external_device) - def _assert_external_device(self, router): + def _check_external_device(self, router): external_port = router.get_ex_gw_port() - self.assertTrue(self.device_exists_with_ips_and_mac( + return (self.device_exists_with_ips_and_mac( external_port, router.get_external_device_name, router.ns_name)) + def _assert_external_device(self, router): + self.assertTrue(self._check_external_device(router)) + def _assert_ipv6_accept_ra(self, router): external_port = router.get_ex_gw_port() external_device_name = router.get_external_device_name( @@ -501,6 +507,81 @@ class L3AgentTestFramework(base.BaseSudoTestCase): external_port['mac_address'], namespace=router.ns_name) for fip in floating_ips) + def _create_router(self, router_info, agent): + + ns_name = "%s%s%s" % ( + 'qrouter-' + router_info['id'], + self.NESTED_NAMESPACE_SEPARATOR, agent.host) + ext_name = "qg-%s-%s" % (agent.host, _uuid()[-4:]) + int_name = "qr-%s-%s" % (agent.host, _uuid()[-4:]) + + get_ns_name = mock.patch.object( + namespaces.RouterNamespace, '_get_ns_name').start() + get_ns_name.return_value = ns_name + get_ext_name = mock.patch.object(l3_router_info.RouterInfo, + 'get_external_device_name').start() + get_ext_name.return_value = ext_name + get_int_name = mock.patch.object(l3_router_info.RouterInfo, + 'get_internal_device_name').start() + get_int_name.return_value = int_name + + router = self.manage_router(agent, router_info) + + router_ext_name = mock.patch.object(router, + 'get_external_device_name').start() + router_ext_name.return_value = get_ext_name.return_value + router_int_name = mock.patch.object(router, + 'get_internal_device_name').start() + router_int_name.return_value = get_int_name.return_value + + return router + + def create_ha_routers(self): + router_info = self.generate_router_info(enable_ha=True) + router1 = self._create_router(router_info, self.agent) + self._add_fip(router1, '192.168.111.12') + + r1_br = ip_lib.IPDevice(router1.driver.conf.external_network_bridge) + r1_br.addr.add('19.4.4.1/24') + r1_br.link.set_up() + + router_info_2 = copy.deepcopy(router_info) + router_info_2[constants.HA_INTERFACE_KEY] = ( + l3_test_common.get_ha_interface(ip='169.254.192.2', + mac='22:22:22:22:22:22')) + router2 = self._create_router(router_info_2, self.failover_agent) + + r2_br = ip_lib.IPDevice(router2.driver.conf.external_network_bridge) + r2_br.addr.add('19.4.4.1/24') + r2_br.link.set_up() + + return (router1, router2) + + def _get_master_and_slave_routers(self, router1, router2): + + try: + common_utils.wait_until_true( + lambda: router1.ha_state == 'master') + common_utils.wait_until_true( + lambda: self._check_external_device(router1)) + master_router = router1 + slave_router = router2 + except common_utils.WaitTimeout: + common_utils.wait_until_true( + lambda: router2.ha_state == 'master') + common_utils.wait_until_true( + lambda: self._check_external_device(router2)) + master_router = router2 + slave_router = router1 + + common_utils.wait_until_true( + lambda: master_router.ha_state == 'master') + common_utils.wait_until_true( + lambda: self._check_external_device(master_router)) + common_utils.wait_until_true( + lambda: slave_router.ha_state == 'backup') + return master_router, slave_router + def fail_ha_router(self, router): device_name = router.get_ha_device_name() ha_device = ip_lib.IPDevice(device_name, router.ha_namespace) diff --git a/neutron/tests/functional/agent/l3/test_ha_router.py b/neutron/tests/functional/agent/l3/test_ha_router.py index 1725f9968ec..d10bfc08ff9 100644 --- a/neutron/tests/functional/agent/l3/test_ha_router.py +++ b/neutron/tests/functional/agent/l3/test_ha_router.py @@ -21,7 +21,6 @@ import six import testtools from neutron.agent.l3 import agent as neutron_l3_agent -from neutron.agent.l3 import namespaces from neutron.agent.linux import ip_lib from neutron.common import ipv6_utils from neutron.common import utils as common_utils @@ -305,8 +304,6 @@ class L3HATestCase(framework.L3AgentTestFramework): class L3HATestFailover(framework.L3AgentTestFramework): - NESTED_NAMESPACE_SEPARATOR = '@' - def setUp(self): super(L3HATestFailover, self).setUp() conf = self._configure_agent('agent2') @@ -317,35 +314,26 @@ class L3HATestFailover(framework.L3AgentTestFramework): br_int_2 = self._get_agent_ovs_integration_bridge(self.failover_agent) veth1, veth2 = self.useFixture(net_helpers.VethFixture()).ports + veth1.link.set_up() + veth2.link.set_up() br_int_1.add_port(veth1.name) br_int_2.add_port(veth2.name) def test_ha_router_failover(self): - router_info = self.generate_router_info(enable_ha=True) - get_ns_name = mock.patch.object( - namespaces.RouterNamespace, '_get_ns_name').start() - get_ns_name.return_value = "%s%s%s" % ( - 'qrouter-' + router_info['id'], - self.NESTED_NAMESPACE_SEPARATOR, self.agent.host) - router1 = self.manage_router(self.agent, router_info) + router1, router2 = self.create_ha_routers() - router_info_2 = copy.deepcopy(router_info) - router_info_2[constants.HA_INTERFACE_KEY] = ( - l3_test_common.get_ha_interface(ip='169.254.192.2', - mac='22:22:22:22:22:22')) - - get_ns_name.return_value = "%s%s%s" % ( - namespaces.RouterNamespace._get_ns_name(router_info_2['id']), - self.NESTED_NAMESPACE_SEPARATOR, self.failover_agent.host) - router2 = self.manage_router(self.failover_agent, router_info_2) - - common_utils.wait_until_true(lambda: router1.ha_state == 'master') - common_utils.wait_until_true(lambda: router2.ha_state == 'backup') + master_router, slave_router = self._get_master_and_slave_routers( + router1, router2) self.fail_ha_router(router1) - common_utils.wait_until_true(lambda: router2.ha_state == 'master') - common_utils.wait_until_true(lambda: router1.ha_state == 'backup') + # NOTE: passing slave_router as first argument, because we expect + # that this router should be the master + new_master, new_slave = self._get_master_and_slave_routers( + slave_router, master_router) + + self.assertEqual(master_router, new_slave) + self.assertEqual(slave_router, new_master) class LinuxBridgeL3HATestCase(L3HATestCase):