From 2c4574bfdcf9bd08efcb3c82becb26787a635338 Mon Sep 17 00:00:00 2001 From: Dinesh Bhor Date: Wed, 13 Sep 2017 16:12:33 +0530 Subject: [PATCH] Make eventlet hub use a monotonic clock If system time is adjusted first forward and then backward while a masakari-engine service is running, then the periodic tasks stops for the duration of time the system clock was adjusted backward. This was supposed to be fixed by the following patch to oslo.service https://review.openstack.org/#/c/286838/ , but the order of imports in unit tests and production code is different, so masakari services end up starting with the default eventlet hub, that does not use a monotonic clock and, thus, is affected by changes of system time. Testing the change done in the patch is problematic, as it's a subject of imports order and is not reproduced in functional or unit tests (oslo_service is always imported earlier than eventlet hub is initialized, so it just does "the right thing"). The alternative is to make an assertion when services start. Co-Authored-By: Roman Podoliaka Closes-Bug: #1510234 Change-Id: I9d917b3151d9cdf7340a173b5baf98def63c76cd --- masakari/__init__.py | 5 +++++ masakari/service.py | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/masakari/__init__.py b/masakari/__init__.py index 28f10424..4073dc93 100644 --- a/masakari/__init__.py +++ b/masakari/__init__.py @@ -26,4 +26,9 @@ import os os.environ['EVENTLET_NO_GREENDNS'] = 'yes' +# NOTE(rpodolyaka): import oslo_service first, so that it makes eventlet hub +# use a monotonic clock to avoid issues with drifts of system time (see +# LP 1510234 for details) +import oslo_service # noqa + import eventlet # noqa diff --git a/masakari/service.py b/masakari/service.py index ba5168f1..949a105f 100644 --- a/masakari/service.py +++ b/masakari/service.py @@ -41,6 +41,17 @@ LOG = logging.getLogger(__name__) CONF = masakari.conf.CONF +def assert_eventlet_uses_monotonic_clock(): + import eventlet.hubs as hubs + import monotonic + + hub = hubs.get_hub() + if hub.clock is not monotonic.monotonic: + raise RuntimeError( + 'eventlet hub is not using a monotonic clock - ' + 'periodic tasks will be affected by drifts of system time.') + + class Service(service.Service): """Service object for binaries running on hosts. @@ -78,6 +89,7 @@ class Service(service.Service): } def start(self): + assert_eventlet_uses_monotonic_clock() verstr = version.version_string_with_package() LOG.info('Starting %(topic)s (version %(version)s)', { 'topic': self.topic,