summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDinesh Bhor <dinesh.bhor@nttdata.com>2017-09-13 16:12:33 +0530
committerSampath Priyankara (samP) <sam47priya@gmail.com>2017-11-14 07:15:45 +0000
commit2c4574bfdcf9bd08efcb3c82becb26787a635338 (patch)
treee890a720b612d9027c681d15e8f473dc11b4ff4a
parent57bbe7deada89ad35da885451315f216d2f53919 (diff)
Make eventlet hub use a monotonic clock
If system time is adjusted first forward and then backward while a masakari-engine service is running, then the periodic tasks stops for the duration of time the system clock was adjusted backward. This was supposed to be fixed by the following patch to oslo.service https://review.openstack.org/#/c/286838/ , but the order of imports in unit tests and production code is different, so masakari services end up starting with the default eventlet hub, that does not use a monotonic clock and, thus, is affected by changes of system time. Testing the change done in the patch is problematic, as it's a subject of imports order and is not reproduced in functional or unit tests (oslo_service is always imported earlier than eventlet hub is initialized, so it just does "the right thing"). The alternative is to make an assertion when services start. Co-Authored-By: Roman Podoliaka <rpodolyaka@mirantis.com> Closes-Bug: #1510234 Change-Id: I9d917b3151d9cdf7340a173b5baf98def63c76cd
Notes
Notes (review): Code-Review+1: Jon Schlueter <jschluet@redhat.com> Code-Review+2: Rikimaru Honjo <honjo.rikimaru@po.ntt-tx.co.jp> Code-Review+2: Sampath Priyankara (samP) <sam47priya@gmail.com> Code-Review+1: Nam Nguyen Hoai <namnh@vn.fujitsu.com> Workflow+1: Sampath Priyankara (samP) <sam47priya@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Tue, 14 Nov 2017 07:49:18 +0000 Reviewed-on: https://review.openstack.org/503939 Project: openstack/masakari Branch: refs/heads/master
-rw-r--r--masakari/__init__.py5
-rw-r--r--masakari/service.py12
2 files changed, 17 insertions, 0 deletions
diff --git a/masakari/__init__.py b/masakari/__init__.py
index 28f1042..4073dc9 100644
--- a/masakari/__init__.py
+++ b/masakari/__init__.py
@@ -26,4 +26,9 @@ import os
26 26
27os.environ['EVENTLET_NO_GREENDNS'] = 'yes' 27os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
28 28
29# NOTE(rpodolyaka): import oslo_service first, so that it makes eventlet hub
30# use a monotonic clock to avoid issues with drifts of system time (see
31# LP 1510234 for details)
32import oslo_service # noqa
33
29import eventlet # noqa 34import eventlet # noqa
diff --git a/masakari/service.py b/masakari/service.py
index ba5168f..949a105 100644
--- a/masakari/service.py
+++ b/masakari/service.py
@@ -41,6 +41,17 @@ LOG = logging.getLogger(__name__)
41CONF = masakari.conf.CONF 41CONF = masakari.conf.CONF
42 42
43 43
44def assert_eventlet_uses_monotonic_clock():
45 import eventlet.hubs as hubs
46 import monotonic
47
48 hub = hubs.get_hub()
49 if hub.clock is not monotonic.monotonic:
50 raise RuntimeError(
51 'eventlet hub is not using a monotonic clock - '
52 'periodic tasks will be affected by drifts of system time.')
53
54
44class Service(service.Service): 55class Service(service.Service):
45 """Service object for binaries running on hosts. 56 """Service object for binaries running on hosts.
46 57
@@ -78,6 +89,7 @@ class Service(service.Service):
78 } 89 }
79 90
80 def start(self): 91 def start(self):
92 assert_eventlet_uses_monotonic_clock()
81 verstr = version.version_string_with_package() 93 verstr = version.version_string_with_package()
82 LOG.info('Starting %(topic)s (version %(version)s)', { 94 LOG.info('Starting %(topic)s (version %(version)s)', {
83 'topic': self.topic, 95 'topic': self.topic,