Add DEFAULT_TIMEOUT and TIMEOUT_SCALING_FACTOR

The openstacksdk test suite sets a default value for OS_TEST_TIMEOUT to
enforce that no test in the suite should ever run long. It has to do
that by using the EnvironmentVariable fixture before running the super
setUp to get the value set before oslotest reads the environment
variable. Add a class variable that can be overridden to allow doing
that more cleanly.

Additionally, in the openstacksdk functional test suite, there is a
variable called "TIMEOUT_SCALING_FACTOR" that lets us mark that some of
the tests (I'm looking at you volume functional tests) naturally run a
bit longer than other tests in the suite. It's been invaluable for us
for a couple of ugly cases, so it seemed like we should upstream it into
oslotest.

Change-Id: I2ef5d0194185bf58c0945efb0725202e6d177e3f

oslotest/base.py

@@ -40,6 +40,15 @@ class BaseTestCase(testtools.TestCase):
     individual test cases can run. This lets tests fail for taking too
     long, and prevents deadlocks from completely hanging test runs.
 
+    The class variable ``DEFAULT_TIMEOUT`` can be set to configure
+    a test suite default test value for cases in which ``OS_TEST_TIMEOUT``
+    is not set. It defaults to ``0``, which means no timeout.
+
+    The class variable ``TIMEOUT_SCALING_FACTOR`` can be set on an
+    individual test class for tests that reasonably take longer than
+    the rest of the test suite so that the overall timeout can be
+    kept small. It defaults to ``1``.
+
     If the environment variable ``OS_STDOUT_CAPTURE`` is set, a fake
     stream replaces ``sys.stdout`` so the test can look at the output
     it produces.
@@ -75,6 +84,8 @@ class BaseTestCase(testtools.TestCase):
     .. _fixtures: https://pypi.org/project/fixtures
 
     """
+    DEFAULT_TIMEOUT = 0
+    TIMEOUT_SCALING_FACTOR = 1
 
     def __init__(self, *args, **kwds):
         super(BaseTestCase, self).__init__(*args, **kwds)
@@ -112,7 +123,10 @@ class BaseTestCase(testtools.TestCase):
         self.useFixture(fixtures.TempHomeDir())
 
     def _set_timeout(self):
-        self.useFixture(timeout.Timeout())
+        self.useFixture(timeout.Timeout(
+            default_timeout=self.DEFAULT_TIMEOUT,
+            scaling_factor=self.TIMEOUT_SCALING_FACTOR,
+        ))
 
     def _fake_output(self):
         self.output_fixture = self.useFixture(output.CaptureOutput())

oslotest/tests/unit/test_timeout.py

@@ -43,3 +43,51 @@ class TimeoutTestCase(testtools.TestCase):
         env_get_mock.assert_called_once_with('OS_TEST_TIMEOUT', 0)
         self.assertEqual(0, fixture_timeout_mock.call_count)
         self.assertEqual(0, fixture_mock.call_count)
+
+    @mock.patch('os.environ.get')
+    @mock.patch.object(timeout.Timeout, 'useFixture')
+    @mock.patch('fixtures.Timeout')
+    def test_timeout_default(
+            self, fixture_timeout_mock, fixture_mock, env_get_mock):
+        env_get_mock.return_value = 5
+        tc = timeout.Timeout(default_timeout=5)
+        tc.setUp()
+        env_get_mock.assert_called_once_with('OS_TEST_TIMEOUT', 5)
+        fixture_timeout_mock.assert_called_once_with(5, gentle=True)
+        self.assertEqual(1, fixture_mock.call_count)
+
+    @mock.patch('os.environ.get')
+    @mock.patch.object(timeout.Timeout, 'useFixture')
+    @mock.patch('fixtures.Timeout')
+    def test_timeout_bad_default(
+            self, fixture_timeout_mock, fixture_mock, env_get_mock):
+        env_get_mock.return_value = 'invalid'
+        tc = timeout.Timeout(default_timeout='invalid')
+        tc.setUp()
+        env_get_mock.assert_called_once_with('OS_TEST_TIMEOUT', 0)
+        self.assertEqual(0, fixture_timeout_mock.call_count)
+        self.assertEqual(0, fixture_mock.call_count)
+
+    @mock.patch('os.environ.get')
+    @mock.patch.object(timeout.Timeout, 'useFixture')
+    @mock.patch('fixtures.Timeout')
+    def test_timeout_scaling(
+            self, fixture_timeout_mock, fixture_mock, env_get_mock):
+        env_get_mock.return_value = 2
+        tc = timeout.Timeout(scaling_factor=1.5)
+        tc.setUp()
+        env_get_mock.assert_called_once_with('OS_TEST_TIMEOUT', 0)
+        fixture_timeout_mock.assert_called_once_with(3, gentle=True)
+        self.assertEqual(1, fixture_mock.call_count)
+
+    @mock.patch('os.environ.get')
+    @mock.patch.object(timeout.Timeout, 'useFixture')
+    @mock.patch('fixtures.Timeout')
+    def test_timeout_bad_scaling(
+            self, fixture_timeout_mock, fixture_mock, env_get_mock):
+        env_get_mock.return_value = 2
+        tc = timeout.Timeout(scaling_factor='invalid')
+        tc.setUp()
+        env_get_mock.assert_called_once_with('OS_TEST_TIMEOUT', 0)
+        fixture_timeout_mock.assert_called_once_with(2, gentle=True)
+        self.assertEqual(1, fixture_mock.call_count)

oslotest/timeout.py

@@ -22,13 +22,27 @@ class Timeout(fixtures.Fixture):
 
     """
 
+    def __init__(self, default_timeout=0, scaling_factor=1):
+        super(Timeout, self).__init__()
+        try:
+            self._default_timeout = int(default_timeout)
+        except ValueError:
+            # If timeout value is invalid do not set a timeout.
+            self._default_timeout = 0
+        self._scaling_factor = scaling_factor
+
     def setUp(self):
         super(Timeout, self).setUp()
-        test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
+        test_timeout = os.environ.get('OS_TEST_TIMEOUT', self._default_timeout)
         try:
             test_timeout = int(test_timeout)
         except ValueError:
             # If timeout value is invalid do not set a timeout.
-            test_timeout = 0
-        if test_timeout > 0:
-            self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
+            test_timeout = self._default_timeout
+        try:
+            scaled_timeout = int(test_timeout * self._scaling_factor)
+        except ValueError:
+            # If scaling factor is invalid, use the basic test timeout.
+            scaled_timeout = test_timeout
+        if scaled_timeout > 0:
+            self.useFixture(fixtures.Timeout(scaled_timeout, gentle=True))

releasenotes/notes/timeout-scaling-52741beadde528e5.yaml

@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    New class variable, ``TIMEOUT_SCALING_FACTOR`` was added that allows
+    modifying a test class to have a longer timeout than other tests in the
+    suite without having to raise the default timeout for all tests.
+  - |
+    New class varable, ``DEFAULT_TIMEOUT`` was added that lets test suite
+    authors override the default value of ``OS_TEST_TIMEOUT`` at the
+    test suite level.