diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_healthmonitor.py b/octavia_tempest_plugin/tests/scenario/v2/test_healthmonitor.py index 7b769e1e..b2b1d18c 100644 --- a/octavia_tempest_plugin/tests/scenario/v2/test_healthmonitor.py +++ b/octavia_tempest_plugin/tests/scenario/v2/test_healthmonitor.py @@ -103,7 +103,7 @@ class HealthMonitorScenarioTest(test_base.LoadBalancerBaseTest): } hm = self.mem_healthmonitor_client.create_healthmonitor(**hm_kwargs) - self.addClassResourceCleanup( + self.addCleanup( self.mem_healthmonitor_client.cleanup_healthmonitor, hm[const.ID], lb_client=self.mem_lb_client, lb_id=self.lb_id) diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_l7policy.py b/octavia_tempest_plugin/tests/scenario/v2/test_l7policy.py index 98d3bc62..9e09f353 100644 --- a/octavia_tempest_plugin/tests/scenario/v2/test_l7policy.py +++ b/octavia_tempest_plugin/tests/scenario/v2/test_l7policy.py @@ -115,7 +115,7 @@ class L7PolicyScenarioTest(test_base.LoadBalancerBaseTest): } l7policy = self.mem_l7policy_client.create_l7policy(**l7policy_kwargs) - self.addClassResourceCleanup( + self.addCleanup( self.mem_l7policy_client.cleanup_l7policy, l7policy[const.ID], lb_client=self.mem_lb_client, lb_id=self.lb_id) diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_l7rule.py b/octavia_tempest_plugin/tests/scenario/v2/test_l7rule.py index 3e14a74a..114ea3e3 100644 --- a/octavia_tempest_plugin/tests/scenario/v2/test_l7rule.py +++ b/octavia_tempest_plugin/tests/scenario/v2/test_l7rule.py @@ -112,7 +112,7 @@ class L7RuleScenarioTest(test_base.LoadBalancerBaseTest): } l7rule = self.mem_l7rule_client.create_l7rule(**l7rule_kwargs) - self.addClassResourceCleanup( + self.addCleanup( self.mem_l7rule_client.cleanup_l7rule, l7rule[const.ID], l7policy_id=self.l7policy_id, lb_client=self.mem_lb_client, lb_id=self.lb_id) diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_listener.py b/octavia_tempest_plugin/tests/scenario/v2/test_listener.py index 685c2004..7720d278 100644 --- a/octavia_tempest_plugin/tests/scenario/v2/test_listener.py +++ b/octavia_tempest_plugin/tests/scenario/v2/test_listener.py @@ -132,7 +132,7 @@ class ListenerScenarioTest(test_base.LoadBalancerBaseTest): }) listener = self.mem_listener_client.create_listener(**listener_kwargs) - self.addClassResourceCleanup( + self.addCleanup( self.mem_listener_client.cleanup_listener, listener[const.ID], lb_client=self.mem_lb_client, lb_id=self.lb_id) diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_load_balancer.py b/octavia_tempest_plugin/tests/scenario/v2/test_load_balancer.py index cfc3ee51..79f1876b 100644 --- a/octavia_tempest_plugin/tests/scenario/v2/test_load_balancer.py +++ b/octavia_tempest_plugin/tests/scenario/v2/test_load_balancer.py @@ -59,7 +59,7 @@ class LoadBalancerScenarioTest(test_base.LoadBalancerBaseTest): self._setup_lb_network_kwargs(lb_kwargs, ip_version) lb = self.mem_lb_client.create_loadbalancer(**lb_kwargs) - self.addClassResourceCleanup( + self.addCleanup( self.mem_lb_client.cleanup_loadbalancer, lb[const.ID]) diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_member.py b/octavia_tempest_plugin/tests/scenario/v2/test_member.py index b30d651d..6bd18dee 100644 --- a/octavia_tempest_plugin/tests/scenario/v2/test_member.py +++ b/octavia_tempest_plugin/tests/scenario/v2/test_member.py @@ -123,7 +123,7 @@ class MemberScenarioTest(test_base.LoadBalancerBaseTest): const.ID] member = self.mem_member_client.create_member(**member_kwargs) - self.addClassResourceCleanup( + self.addCleanup( self.mem_member_client.cleanup_member, member[const.ID], pool_id=self.pool_id, lb_client=self.mem_lb_client, lb_id=self.lb_id) diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_pool.py b/octavia_tempest_plugin/tests/scenario/v2/test_pool.py index 1ddc8b2f..cd266870 100644 --- a/octavia_tempest_plugin/tests/scenario/v2/test_pool.py +++ b/octavia_tempest_plugin/tests/scenario/v2/test_pool.py @@ -108,7 +108,7 @@ class PoolScenarioTest(test_base.LoadBalancerBaseTest): pool_kwargs[const.LOADBALANCER_ID] = self.lb_id pool = self.mem_pool_client.create_pool(**pool_kwargs) - self.addClassResourceCleanup( + self.addCleanup( self.mem_pool_client.cleanup_pool, pool[const.ID], lb_client=self.mem_lb_client, lb_id=self.lb_id) diff --git a/octavia_tempest_plugin/tests/test_base.py b/octavia_tempest_plugin/tests/test_base.py index a48ba0b9..b9354097 100644 --- a/octavia_tempest_plugin/tests/test_base.py +++ b/octavia_tempest_plugin/tests/test_base.py @@ -30,6 +30,7 @@ from tempest.lib.common.utils import data_utils from tempest.lib.common.utils.linux import remote_client from tempest.lib import exceptions from tempest import test +import tenacity from octavia_tempest_plugin import clients from octavia_tempest_plugin.common import constants as const @@ -39,6 +40,11 @@ from octavia_tempest_plugin.tests import waiters CONF = config.CONF LOG = logging.getLogger(__name__) +RETRY_ATTEMPTS = 15 +RETRY_INITIAL_DELAY = 1 +RETRY_BACKOFF = 1 +RETRY_MAX = 5 + class LoadBalancerBaseTest(test.BaseTestCase): """Base class for load balancer tests.""" @@ -203,6 +209,42 @@ class LoadBalancerBaseTest(test.BaseTestCase): LOG.debug('Octavia Setup: lb_member_2_ipv6_subnet = {}'.format( cls.lb_member_2_ipv6_subnet[const.ID])) + @classmethod + # Neutron can be slow to clean up ports from the subnets/networks. + # Retry this delete a few times if we get a "Conflict" error to give + # neutron time to fully cleanup the ports. + @tenacity.retry( + retry=tenacity.retry_if_exception_type(exceptions.Conflict), + wait=tenacity.wait_incrementing( + RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX), + stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS)) + def _logging_delete_network(cls, net_id): + try: + cls.lb_mem_net_client.delete_network(net_id) + except Exception: + LOG.error('Unable to delete network {}. Active ports:'.format( + net_id)) + LOG.error(cls.lb_mem_ports_client.list_ports()) + raise + + @classmethod + # Neutron can be slow to clean up ports from the subnets/networks. + # Retry this delete a few times if we get a "Conflict" error to give + # neutron time to fully cleanup the ports. + @tenacity.retry( + retry=tenacity.retry_if_exception_type(exceptions.Conflict), + wait=tenacity.wait_incrementing( + RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX), + stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS)) + def _logging_delete_subnet(cls, subnet_id): + try: + cls.lb_mem_subnet_client.delete_subnet(subnet_id) + except Exception: + LOG.error('Unable to delete subnet {}. Active ports:'.format( + subnet_id)) + LOG.error(cls.lb_mem_ports_client.list_ports()) + raise + @classmethod def _create_networks(cls): """Creates networks, subnets, and routers used in tests. @@ -230,7 +272,7 @@ class LoadBalancerBaseTest(test.BaseTestCase): LOG.info('lb_member_vip_net: {}'.format(cls.lb_member_vip_net)) cls.addClassResourceCleanup( waiters.wait_for_not_found, - cls.lb_mem_net_client.delete_network, + cls._logging_delete_network, cls.lb_mem_net_client.show_network, cls.lb_member_vip_net['id']) @@ -245,7 +287,7 @@ class LoadBalancerBaseTest(test.BaseTestCase): LOG.info('lb_member_vip_subnet: {}'.format(cls.lb_member_vip_subnet)) cls.addClassResourceCleanup( waiters.wait_for_not_found, - cls.lb_mem_subnet_client.delete_subnet, + cls._logging_delete_subnet, cls.lb_mem_subnet_client.show_subnet, cls.lb_member_vip_subnet['id']) @@ -270,7 +312,7 @@ class LoadBalancerBaseTest(test.BaseTestCase): cls.lb_member_vip_ipv6_subnet = result['subnet'] cls.addClassResourceCleanup( waiters.wait_for_not_found, - cls.lb_mem_subnet_client.delete_subnet, + cls._logging_delete_subnet, cls.lb_mem_subnet_client.show_subnet, cls.lb_member_vip_ipv6_subnet['id']) LOG.info('lb_member_vip_ipv6_subnet: {}'.format( @@ -289,7 +331,7 @@ class LoadBalancerBaseTest(test.BaseTestCase): LOG.info('lb_member_1_net: {}'.format(cls.lb_member_1_net)) cls.addClassResourceCleanup( waiters.wait_for_not_found, - cls.lb_mem_net_client.delete_network, + cls._logging_delete_network, cls.lb_mem_net_client.show_network, cls.lb_member_1_net['id']) @@ -304,7 +346,7 @@ class LoadBalancerBaseTest(test.BaseTestCase): LOG.info('lb_member_1_subnet: {}'.format(cls.lb_member_1_subnet)) cls.addClassResourceCleanup( waiters.wait_for_not_found, - cls.lb_mem_subnet_client.delete_subnet, + cls._logging_delete_subnet, cls.lb_mem_subnet_client.show_subnet, cls.lb_member_1_subnet['id']) @@ -325,7 +367,7 @@ class LoadBalancerBaseTest(test.BaseTestCase): cls.lb_member_1_ipv6_subnet)) cls.addClassResourceCleanup( waiters.wait_for_not_found, - cls.lb_mem_subnet_client.delete_subnet, + cls._logging_delete_subnet, cls.lb_mem_subnet_client.show_subnet, cls.lb_member_1_ipv6_subnet['id']) @@ -342,7 +384,7 @@ class LoadBalancerBaseTest(test.BaseTestCase): LOG.info('lb_member_2_net: {}'.format(cls.lb_member_2_net)) cls.addClassResourceCleanup( waiters.wait_for_not_found, - cls.lb_mem_net_client.delete_network, + cls._logging_delete_network, cls.lb_mem_net_client.show_network, cls.lb_member_2_net['id']) @@ -357,7 +399,7 @@ class LoadBalancerBaseTest(test.BaseTestCase): LOG.info('lb_member_2_subnet: {}'.format(cls.lb_member_2_subnet)) cls.addClassResourceCleanup( waiters.wait_for_not_found, - cls.lb_mem_subnet_client.delete_subnet, + cls._logging_delete_subnet, cls.lb_mem_subnet_client.show_subnet, cls.lb_member_2_subnet['id']) @@ -378,7 +420,7 @@ class LoadBalancerBaseTest(test.BaseTestCase): cls.lb_member_2_ipv6_subnet)) cls.addClassResourceCleanup( waiters.wait_for_not_found, - cls.lb_mem_subnet_client.delete_subnet, + cls._logging_delete_subnet, cls.lb_mem_subnet_client.show_subnet, cls.lb_member_2_ipv6_subnet['id'])