Harden placement init under wsgi

- This change tries to address an edge case discovered
  when running placement under mod_wsgi where
  if the placement wsgi application is re-initialize the db_api
  configure method attempts to reconfigure a started transaction
  factory.

- This since oslo.db transaction factories do not support
  reconfiguration at runtime this result in an exception being
  raised preventing reloading of the Placement API without
  restarting apache to force mod_wsgi to recreate the
  python interpreter.

- This change introduces a run once decorator to allow annotating
  functions that should only be executed once for the lifetime fo
  an interpreter.

- This change applies the run_once decorator to the db_api configure
  method, to suppress the attempt to reconfigure the current
  TransactionFactory on application reload.

Co-Authored-By: Balazs Gibizer <balazs.gibizer@ericsson.com>
Closes-Bug: #1799246
Related-Bug: #1784155
Change-Id: I704196711d30c1124e713ac31111a8ea6fa2f1ba

nova/api/openstack/placement/db_api.py

@@ -13,17 +13,21 @@
 own file so the nova db_api (which has cascading imports) is not imported.
 """
 
-
 from oslo_db.sqlalchemy import enginefacade
+from oslo_log import log as logging
 
+from nova.utils import run_once
 
 placement_context_manager = enginefacade.transaction_context()
+LOG = logging.getLogger(__name__)
 
 
 def _get_db_conf(conf_group):
     return dict(conf_group.items())
 
 
+@run_once("TransactionFactory already started, not reconfiguring.",
+          LOG.warning)
 def configure(conf):
     # If [placement_database]/connection is not set in conf, then placement
     # data will be stored in the nova_api database.

nova/tests/unit/api/openstack/placement/test_db_api.py

@@ -0,0 +1,52 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import mock
+import testtools
+
+from oslo_config import cfg
+from oslo_config import fixture as config_fixture
+
+from nova.api.openstack.placement import db_api
+
+
+CONF = cfg.CONF
+
+
+class DbApiTests(testtools.TestCase):
+
+    def setUp(self):
+        super(DbApiTests, self).setUp()
+        self.conf_fixture = self.useFixture(config_fixture.Config(CONF))
+        db_api.configure.reset()
+
+    @mock.patch.object(db_api.placement_context_manager, "configure")
+    def test_can_call_configure_twice(self, configure_mock):
+        """This test asserts that configure can be safely called twice
+        which may happen if placement is run under mod_wsgi and the
+        wsgi application is reloaded.
+        """
+        db_api.configure(self.conf_fixture.conf)
+        configure_mock.assert_called_once()
+
+        # a second invocation of configure on a transaction context
+        # should raise an exception so mock this and assert its not
+        # called on a second invocation of db_api's configure function
+        configure_mock.side_effect = TypeError()
+
+        db_api.configure(self.conf_fixture.conf)
+        # Note we have not reset the mock so it should
+        # have been called once from the first invocation of
+        # db_api.configure and the second invocation should not
+        # have called it again
+        configure_mock.assert_called_once()

nova/tests/unit/test_utils.py

@@ -1321,3 +1321,101 @@ class GetEndpointTestCase(test.NoDBTestCase):
         self.adap.get_endpoint_data.assert_not_called()
         self.assertEqual(3, self.adap.get_endpoint.call_count)
         self.assertEqual('public', self.adap.interface)
+
+
+class RunOnceTests(test.NoDBTestCase):
+
+    fake_logger = mock.MagicMock()
+
+    @utils.run_once("already ran once", fake_logger)
+    def dummy_test_func(self, fail=False):
+        if fail:
+            raise ValueError()
+        return True
+
+    def setUp(self):
+        super(RunOnceTests, self).setUp()
+        self.dummy_test_func.reset()
+        RunOnceTests.fake_logger.reset_mock()
+
+    def test_wrapped_funtions_called_once(self):
+        self.assertFalse(self.dummy_test_func.called)
+        result = self.dummy_test_func()
+        self.assertTrue(result)
+        self.assertTrue(self.dummy_test_func.called)
+
+        # assert that on second invocation no result
+        # is returned and that the logger is invoked.
+        result = self.dummy_test_func()
+        RunOnceTests.fake_logger.assert_called_once()
+        self.assertIsNone(result)
+
+    def test_wrapped_funtions_called_once_raises(self):
+        self.assertFalse(self.dummy_test_func.called)
+        self.assertRaises(ValueError, self.dummy_test_func, fail=True)
+        self.assertTrue(self.dummy_test_func.called)
+
+        # assert that on second invocation no result
+        # is returned and that the logger is invoked.
+        result = self.dummy_test_func()
+        RunOnceTests.fake_logger.assert_called_once()
+        self.assertIsNone(result)
+
+    def test_wrapped_funtions_can_be_reset(self):
+        # assert we start with a clean state
+        self.assertFalse(self.dummy_test_func.called)
+        result = self.dummy_test_func()
+        self.assertTrue(result)
+
+        self.dummy_test_func.reset()
+        # assert we restored a clean state
+        self.assertFalse(self.dummy_test_func.called)
+        result = self.dummy_test_func()
+        self.assertTrue(result)
+
+        # assert that we never called the logger
+        RunOnceTests.fake_logger.assert_not_called()
+
+    def test_reset_calls_cleanup(self):
+        mock_clean = mock.Mock()
+
+        @utils.run_once("already ran once", self.fake_logger,
+                        cleanup=mock_clean)
+        def f():
+            pass
+
+        f()
+        self.assertTrue(f.called)
+
+        f.reset()
+        self.assertFalse(f.called)
+        mock_clean.assert_called_once_with()
+
+    def test_clean_is_not_called_at_reset_if_wrapped_not_called(self):
+        mock_clean = mock.Mock()
+
+        @utils.run_once("already ran once", self.fake_logger,
+                        cleanup=mock_clean)
+        def f():
+            pass
+
+        self.assertFalse(f.called)
+
+        f.reset()
+        self.assertFalse(f.called)
+        self.assertFalse(mock_clean.called)
+
+    def test_reset_works_even_if_cleanup_raises(self):
+        mock_clean = mock.Mock(side_effect=ValueError())
+
+        @utils.run_once("already ran once", self.fake_logger,
+                        cleanup=mock_clean)
+        def f():
+            pass
+
+        f()
+        self.assertTrue(f.called)
+
+        self.assertRaises(ValueError, f.reset)
+        self.assertFalse(f.called)
+        mock_clean.assert_called_once_with()

nova/utils.py

@@ -1312,3 +1312,46 @@ else:
     def nested_contexts(*contexts):
         with contextlib.ExitStack() as stack:
             yield [stack.enter_context(c) for c in contexts]
+
+
+def run_once(message, logger, cleanup=None):
+    """This is a utility function decorator to ensure a function
+    is run once and only once in an interpreter instance.
+    The decorated function object can be reset by calling its
+    reset function. All exceptions raised by the wrapped function,
+    logger and cleanup function will be propagated to the caller.
+    """
+    def outer_wrapper(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            if not wrapper.called:
+                # Note(sean-k-mooney): the called state is always
+                # updated even if the wrapped function completes
+                # by raising an exception. If the caller catches
+                # the exception it is their responsibility to call
+                # reset if they want to re-execute the wrapped function.
+                try:
+                    return func(*args, **kwargs)
+                finally:
+                    wrapper.called = True
+            else:
+                logger(message)
+
+        wrapper.called = False
+
+        def reset(wrapper, *args, **kwargs):
+            # Note(sean-k-mooney): we conditionally call the
+            # cleanup function if one is provided only when the
+            # wrapped function has been called previously. We catch
+            # and reraise any exception that may be raised and update
+            # the called state in a finally block to ensure its
+            # always updated if reset is called.
+            try:
+                if cleanup and wrapper.called:
+                    return cleanup(*args, **kwargs)
+            finally:
+                wrapper.called = False
+
+        wrapper.reset = functools.partial(reset, wrapper)
+        return wrapper
+    return outer_wrapper