Add new test decorator skip_if_timeout

In some cases our db migration tests which run on MySQL are
failing with timeout and it happens due to slow VMs on which
job is running.
Sometimes it may also happen that timeout exception is raised
in the middle of some sqlalchemy operations and
sqlalchemy.InterfaceError is raised as last one.
Details about this exception can be found in [1].

To avoid many rechecks because of this reason this patch
introduces new decorator which is very similar to "unstable_test"
but will skip test only if one of exceptions mentioned above will
be raised.
In all other cases it will fail test.

That should be a bit more safe for us because we will not miss
some other failures raised in those tests and will avoid rechecks
because of this "well-known" reason described in related bug.

[1] http://sqlalche.me/e/rvf5

Conflicts:
    neutron/tests/functional/db/test_migrations.py

Change-Id: Ie291fda7d23a696aaa1160d126a3cf72b08c522f
Related-Bug: #1687027
(cherry picked from commit c0fec67672)

neutron/tests/base.py

@@ -37,6 +37,7 @@ from oslo_utils import fileutils
 from oslo_utils import strutils
 from oslotest import base
 import six
+from sqlalchemy import exc as sqlalchemy_exc
 import testtools
 
 from neutron._i18n import _
@@ -112,6 +113,28 @@ def unstable_test(reason):
     return decor
 
 
+def skip_if_timeout(reason):
+    def decor(f):
+        @functools.wraps(f)
+        def inner(self, *args, **kwargs):
+            try:
+                return f(self, *args, **kwargs)
+            except fixtures.TimeoutException:
+                msg = ("Timeout raised for test %s, skipping it "
+                       "because of: %s") % (self.id(), reason)
+                raise self.skipTest(msg)
+            except sqlalchemy_exc.InterfaceError:
+                # In case of db tests very often TimeoutException is reason of
+                # some sqlalchemy InterfaceError exception and that is final
+                # raised exception which needs to be handled
+                msg = ("DB connection broken in test %s. It is very likely "
+                       "that this happend because of test timeout. "
+                       "Skipping test because of: %s") % (self.id(), reason)
+                raise self.skipTest(msg)
+        return inner
+    return decor
+
+
 def set_timeout(timeout):
     """Timeout decorator for test methods.
 

neutron/tests/functional/db/test_migrations.py

@@ -350,6 +350,7 @@ class TestModelsMigrationsMysql(testlib_api.MySQLTestCaseMixin,
                                 _TestModelsMigrations,
                                 testlib_api.SqlTestCaseLight):
 
+    @test_base.skip_if_timeout("bug 1687027")
     def test_check_mysql_engine(self):
         engine = self.get_engine()
         cfg.CONF.set_override('connection', engine.url, group='database')
@@ -367,6 +368,32 @@ class TestModelsMigrationsMysql(testlib_api.MySQLTestCaseMixin,
                    and table != 'alembic_version']
             self.assertEqual(0, len(res), "%s non InnoDB tables created" % res)
 
+    @test_base.skip_if_timeout("bug 1687027")
+    def test_upgrade_expand_branch(self):
+        super(TestModelsMigrationsMysql, self).test_upgrade_expand_branch()
+
+    @test_base.skip_if_timeout("bug 1687027")
+    def test_upgrade_contract_branch(self):
+        super(TestModelsMigrationsMysql, self).test_upgrade_contract_branch()
+
+    @test_base.skip_if_timeout("bug 1687027")
+    def test_branches(self):
+        super(TestModelsMigrationsMysql, self).test_branches()
+
+    @test_base.skip_if_timeout("bug 1687027")
+    def test_has_offline_migrations_pending_contract_scripts(self):
+        super(TestModelsMigrationsMysql,
+              self).test_has_offline_migrations_pending_contract_scripts()
+
+    @test_base.skip_if_timeout("bug 1687027")
+    def test_has_offline_migrations_all_heads_upgraded(self):
+        super(TestModelsMigrationsMysql,
+              self).test_has_offline_migrations_all_heads_upgraded()
+
+    @test_base.skip_if_timeout("bug 1687027")
+    def test_models_sync(self):
+        super(TestModelsMigrationsMysql, self).test_models_sync()
+
 
 class TestModelsMigrationsPsql(testlib_api.PostgreSQLTestCaseMixin,
                                _TestModelsMigrations,
@@ -574,6 +601,7 @@ class TestWalkMigrationsMysql(testlib_api.MySQLTestCaseMixin,
     # on slow nodes than 'psycopg2' and because of that this increased
     # timeout is required only when for testing with 'mysql' backend.
     @test_base.set_timeout(600)
+    @test_base.skip_if_timeout("bug 1687027")
     def test_walk_versions(self):
         super(TestWalkMigrationsMysql, self).test_walk_versions()