Fix bugs in functional tests

Pulls in some fixes from master that address new issues that recently
started affecting our gate jobs.

Includes:

Fix bugs in functional tests

- Dont cache management address of appliances
- Wait for nova instance status==active before asserting router status
- Wait a full config_timeout (600s) for rebuild
- Also look for ha router interfaces when cleaning up

Change-Id: I69341c6abee4e36b10f8e9b676587babe861f0c0

And:

devstack: Bump alive_timeout

Bump alive timeout to 60s to deal with slow qemu instances.

Also, give astara more time to recover from a degraded cluster. This
can be slow to recover when a slow qemu node is waiting on a long-running
config update.

Change-Id: I491b96c43d898f7a961725e4f5720af7b9547038

astara/test/functional/base.py

@@ -412,7 +412,11 @@ class TestTenant(object):
         LOG.debug('Cleaning up created neutron resources')
         router_interface_ports = [
             p for p in self.clients.neutronclient.list_ports()['ports']
-            if 'router_interface' in p['device_owner']]
+            if (
+                'router_interface' in p['device_owner'] or
+                'ha_router_replicated_interface' in p['device_owner']
+            )]
+
         for rip in router_interface_ports:
             LOG.debug('Deleting router interface port: %s', rip)
             self.clients.neutronclient.remove_interface_router(
@@ -521,7 +525,6 @@ class AstaraFunctionalBase(testtools.TestCase):
         parse_config()
         self.ak_client = astara_client
         self.admin_clients = AdminClientManager()
-        self._management_address = None
 
     @classmethod
     def setUpClass(cls):
@@ -553,19 +556,17 @@ class AstaraFunctionalBase(testtools.TestCase):
 
     def get_management_address(self, router_uuid):
         LOG.debug('Getting management address for resource %s', router_uuid)
-        if self._management_address:
-            return self._management_address['addr']
 
         service_instance = self.get_router_appliance_server(router_uuid)
 
         try:
-            self._management_address = service_instance.addresses['mgt'][0]
+            management_address = service_instance.addresses['mgt'][0]
         except KeyError:
             raise Exception(
                 '"mgt" port not found on service instance %s (%s)' %
                 (service_instance.id, service_instance.name))
         LOG.debug('Got management address for resource %s', router_uuid)
-        return self._management_address['addr']
+        return management_address['addr']
 
     def assert_router_is_active(self, router_uuid, ha_router=False):
         LOG.debug('Waiting for resource %s to become ACTIVE', router_uuid)
@@ -577,7 +578,7 @@ class AstaraFunctionalBase(testtools.TestCase):
                 return
 
             service_instances = self.get_router_appliance_server(
-                router_uuid, ha_router=ha_router)
+                router_uuid, wait_for_active=True, ha_router=ha_router)
             if not ha_router:
                 service_instances = [service_instances]
 

astara/test/functional/test_tenant_router.py

@@ -43,8 +43,6 @@ class AstaraRouterTestBase(base.AstaraFunctionalBase):
         router = self.neutronclient.show_router(self.router['id'])
         self.router = router['router']
 
-    HA_ROUTER = False
-
     @property
     def router_ha(self):
         router = self.admin_clients.neutronclient.show_router(
@@ -69,7 +67,7 @@ class TestAstaraHARouter(AstaraRouterTestBase):
         time.sleep(CONF.health_check_period)
 
         service_instances = self.get_router_appliance_server(
-            self.router['id'], ha_router=self.HA_ROUTER)
+            self.router['id'], retries=600, ha_router=self.HA_ROUTER)
         self.assertEqual(len(service_instances), 2)
         self.assertEqual(service_instances[0], backup)
 
@@ -122,7 +120,7 @@ class TestAstaraRouter(AstaraRouterTestBase):
 
         # look for the new server, retry giving rug time to do its thing.
         new_server = self.get_router_appliance_server(
-            self.router['id'], retries=60, wait_for_active=True)
+            self.router['id'], retries=600, wait_for_active=True)
         LOG.debug('Rebuilt new server found: %s', new_server)
         self.assertNotEqual(old_server.id, new_server.id)
 

devstack/plugin.sh

@@ -51,7 +51,8 @@ function configure_astara() {
     # NOTE(adam_g) When running in the gate on slow VMs, gunicorn workers in the appliance
     # sometimes hang during config update and eventually timeout after 60s.  Update
     # config_timeout in the RUG to reflect that timeout.
-    iniset $ASTARA_CONF DEFAULT config_timeout 60
+    iniset $ASTARA_CONF DEFAULT alive_timeout 60
+    iniset $ASTARA_CONF DEFAULT config_timeout 600
 
     iniset $ASTARA_CONF DEFAULT enabled_drivers $ASTARA_ENABLED_DRIVERS