Fix Heartbeats stop when time is changed

eventlet's hub mechanism defaults to using time.time for figuring
out how long to sleep, when to wake up etc:
https://github.com/eventlet/eventlet/blob/master/eventlet/hubs/epolls.py#L37
https://github.com/eventlet/eventlet/blob/master/eventlet/hubs/hub.py#L344

So if you mess with the system time, anything using loopingcall
will stop working. Best example as described in the bug is
`nova-manage service list` which fails for some services when
the clock is advanced.

If we use a monotonic timer, then we can get past this problem.
In this review we use the EVENTLET_HUB hook to specify
a custom Hub. In our custom Hub, we just delegate to the
existing hub mechanism (get_default_hub()) to create an
instance of the Hub and override the clock with one based
on monotonic.

Note that use_hub() mechanism will not work because that sets
the Hub just for the current thread and will not work for other
threads that may be started later. In order to set this
environment variable early, we set it in the __init__.py so
anyone who uses oslo service will enable this support just
by a simple import referencing anything in oslo.service

Co-Authored-By: Victor Stinner <vstinner@redhat.com>
Closes-Bug: 1510234
Change-Id: I4c1cf223e9f6faa1e6dcaf432be1aa709072a343

oslo_service/__init__.py

@@ -0,0 +1,41 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import eventlet.patcher
+import monotonic
+from oslo_log import log as logging
+
+time = eventlet.patcher.original('time')
+
+
+LOG = logging.getLogger(__name__)
+
+if hasattr(time, 'monotonic'):
+    # Use builtin monotonic clock, Python 3.3+
+    _monotonic = time.monotonic
+else:
+    _monotonic = monotonic.monotonic
+
+
+def service_hub():
+    # NOTE(dims): Add a custom impl for EVENTLET_HUB, so we can
+    # override the clock used in the eventlet hubs. The default
+    # uses time.time() and we need to use a monotonic timer
+    # to ensure that things like loopingcall work properly.
+    hub = eventlet.hubs.get_default_hub().Hub()
+    hub.clock = _monotonic
+    return hub
+
+
+os.environ['EVENTLET_HUB'] = 'oslo_service:service_hub'

oslo_service/tests/test_loopingcall.py

@@ -12,12 +12,17 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import eventlet
 from eventlet.green import threading as greenthreading
 import mock
 from oslotest import base as test_base
 
+import oslo_service
 from oslo_service import loopingcall
 
+threading = eventlet.patcher.original('threading')
+time = eventlet.patcher.original('time')
+
 
 class LoopingCallTestCase(test_base.BaseTestCase):
 
@@ -32,6 +37,61 @@ class LoopingCallTestCase(test_base.BaseTestCase):
         timer = loopingcall.FixedIntervalLoopingCall(_raise_it)
         self.assertTrue(timer.start(interval=0.5).wait())
 
+    def test_monotonic_timer(self):
+        def _raise_it():
+            clock = eventlet.hubs.get_hub().clock
+            ok = (clock == oslo_service._monotonic)
+            raise loopingcall.LoopingCallDone(ok)
+
+        timer = loopingcall.FixedIntervalLoopingCall(_raise_it)
+        self.assertTrue(timer.start(interval=0.5).wait())
+
+    def test_eventlet_clock(self):
+        # Make sure that by default the oslo_service.service_hub() kicks in,
+        # test in the main thread
+        hub = eventlet.hubs.get_hub()
+        self.assertEqual(hub.clock,
+                         oslo_service._monotonic)
+
+    def test_eventlet_use_hub_override(self):
+        ns = {}
+
+        def task():
+            try:
+                self._test_eventlet_use_hub_override()
+            except Exception as exc:
+                ns['result'] = exc
+            else:
+                ns['result'] = 'ok'
+
+        # test overriding the hub of in a new thread to not modify the hub
+        # of the main thread
+        thread = threading.Thread(target=task)
+        thread.start()
+        thread.join()
+        self.assertEqual(ns['result'], 'ok')
+
+    def _test_eventlet_use_hub_override(self):
+        # Make sure that by default the
+        # oslo_service.service_hub() kicks in
+        old_clock = eventlet.hubs.get_hub().clock
+        self.assertEqual(old_clock,
+                         oslo_service._monotonic)
+
+        # eventlet will use time.monotonic() by default, same clock than
+        # oslo.service_hub():
+        # https://github.com/eventlet/eventlet/pull/303
+        if not hasattr(time, 'monotonic'):
+            # If anyone wants to override it
+            try:
+                eventlet.hubs.use_hub('poll')
+            except Exception:
+                eventlet.hubs.use_hub('selects')
+
+            # then we get a new clock and the override works fine too!
+            clock = eventlet.hubs.get_hub().clock
+            self.assertNotEqual(old_clock, clock)
+
     def test_return_false(self):
         def _raise_it():
             raise loopingcall.LoopingCallDone(False)
@@ -133,6 +193,15 @@ class DynamicLoopingCallTestCase(test_base.BaseTestCase):
         timer = loopingcall.DynamicLoopingCall(_raise_it)
         self.assertTrue(timer.start().wait())
 
+    def test_monotonic_timer(self):
+        def _raise_it():
+            clock = eventlet.hubs.get_hub().clock
+            ok = (clock == oslo_service._monotonic)
+            raise loopingcall.LoopingCallDone(ok)
+
+        timer = loopingcall.DynamicLoopingCall(_raise_it)
+        self.assertTrue(timer.start().wait())
+
     def test_no_double_start(self):
         wait_ev = greenthreading.Event()