openstack
/
oslo.log
Code Issues Proposed changes

Allow oslo.log to work on non-linux platforms 

Especially needed for all the HyperV work on windows.

Depends-On: I1a0a2dacbefa548296a6cfef1663ea9d48253c0c
Change-Id: Ifcc854e6eb6f4d0e575d1489f0d00f3339705e0b
This commit is contained in:
Davanum Srinivas 2015-10-05 10:51:36 -07:00
parent 97614133a8
commit 8ad1403d8f
6 changed files with 128 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
doc/source/api/watchers.rst Normal file
View File

@ -0,0 +1,12 @@
==================
oslo_log.watchers
==================
.. automodule:: oslo_log.watchers
:members:
:undoc-members:
:show-inheritance:
.. seealso::
:doc:`../usage`

89
oslo_log/handlers.py
View File

@ -10,15 +10,11 @@
# License for the specific language governing permissions and limitations
# under the License.
import errno
import inspect
import logging
import logging.config
import logging.handlers
import os
import pyinotify
import stat
import time
try:
import syslog
except ImportError:
@ -119,88 +115,3 @@ class ColorHandler(logging.StreamHandler):
def format(self, record):
record.color = self.LEVEL_COLORS[record.levelno]
return logging.StreamHandler.format(self, record)
class FileKeeper(pyinotify.ProcessEvent):
def my_init(self, watched_handler, watched_file):
self._watched_handler = watched_handler
self._watched_file = watched_file
def process_default(self, event):
if event.name == self._watched_file:
self._watched_handler.reopen_file()
class EventletThreadedNotifier(pyinotify.ThreadedNotifier):
def loop(self):
"""Eventlet friendly ThreadedNotifier
EventletFriendlyThreadedNotifier contains additional time.sleep()
call insude loop to allow switching to other thread when eventlet
is used.
It can be used with eventlet and native threads as well.
"""
while not self._stop_event.is_set():
self.process_events()
time.sleep(0)
ref_time = time.time()
if self.check_events():
self._sleep(ref_time)
self.read_events()
class FastWatchedFileHandler(logging.handlers.WatchedFileHandler, object):
"""Frequency of reading events.
Watching thread sleeps max(0, READ_FREQ - (TIMEOUT / 1000)) seconds.
"""
READ_FREQ = 1
"""Poll timeout in milliseconds.
See https://docs.python.org/2/library/select.html#select.poll.poll"""
TIMEOUT = 500
def __init__(self, logpath, *args, **kwargs):
self._log_file = os.path.basename(logpath)
self._log_dir = os.path.dirname(logpath)
super(FastWatchedFileHandler, self).__init__(logpath, *args, **kwargs)
self._watch_file()
def _watch_file(self):
mask = pyinotify.IN_MOVED_FROM | pyinotify.IN_DELETE
watch_manager = pyinotify.WatchManager()
handler = FileKeeper(watched_handler=self,
watched_file=self._log_file)
notifier = EventletThreadedNotifier(
watch_manager,
default_proc_fun=handler,
read_freq=FastWatchedFileHandler.READ_FREQ,
timeout=FastWatchedFileHandler.TIMEOUT)
notifier.daemon = True
watch_manager.add_watch(self._log_dir, mask)
notifier.start()
def reopen_file(self):
try:
# stat the file by path, checking for existence
sres = os.stat(self.baseFilename)
except OSError as err:
if err.errno == errno.ENOENT:
sres = None
else:
raise
# compare file system stat with that of our stream file handle
if (not sres or
sres[stat.ST_DEV] != self.dev or
sres[stat.ST_INO] != self.ino):
if self.stream is not None:
# we have an open file handle, clean it up
self.stream.flush()
self.stream.close()
self.stream = None
# open a new file handle and get new stat info from that fd
self.stream = self._open()
self._statstream()

3
oslo_log/log.py
View File

@ -313,7 +313,8 @@ def _setup_logging_from_conf(conf, project, version):
logpath = _get_log_file_path(conf)
if logpath:
if conf.watch_log_file and platform.system() == 'Linux':
file_handler = handlers.FastWatchedFileHandler
from oslo_log import watchers
file_handler = watchers.FastWatchedFileHandler
else:
file_handler = logging.handlers.WatchedFileHandler

3
oslo_log/tests/unit/test_log.py
View File

@ -684,8 +684,9 @@ class FastWatchedFileHandlerTestCase(BaseTestCase):
self._config()
logger = log._loggers[None].logger
self.assertEqual(1, len(logger.handlers))
from oslo_log import watchers
self.assertIsInstance(logger.handlers[0],
handlers.FastWatchedFileHandler)
watchers.FastWatchedFileHandler)
def test_log(self):
log_path = self._config()

111
oslo_log/watchers.py Normal file
View File

@ -0,0 +1,111 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import errno
import logging
import logging.config
import logging.handlers
import os
import pyinotify
import stat
import time
try:
import syslog
except ImportError:
syslog = None
"""Linux specific pyinotify based logging handlers"""
class _FileKeeper(pyinotify.ProcessEvent):
def my_init(self, watched_handler, watched_file):
self._watched_handler = watched_handler
self._watched_file = watched_file
def process_default(self, event):
if event.name == self._watched_file:
self._watched_handler.reopen_file()
class _EventletThreadedNotifier(pyinotify.ThreadedNotifier):
def loop(self):
"""Eventlet friendly ThreadedNotifier
EventletFriendlyThreadedNotifier contains additional time.sleep()
call insude loop to allow switching to other thread when eventlet
is used.
It can be used with eventlet and native threads as well.
"""
while not self._stop_event.is_set():
self.process_events()
time.sleep(0)
ref_time = time.time()
if self.check_events():
self._sleep(ref_time)
self.read_events()
class FastWatchedFileHandler(logging.handlers.WatchedFileHandler, object):
"""Frequency of reading events.
Watching thread sleeps max(0, READ_FREQ - (TIMEOUT / 1000)) seconds.
"""
READ_FREQ = 1
"""Poll timeout in milliseconds.
See https://docs.python.org/2/library/select.html#select.poll.poll"""
TIMEOUT = 500
def __init__(self, logpath, *args, **kwargs):
self._log_file = os.path.basename(logpath)
self._log_dir = os.path.dirname(logpath)
super(FastWatchedFileHandler, self).__init__(logpath, *args, **kwargs)
self._watch_file()
def _watch_file(self):
mask = pyinotify.IN_MOVED_FROM | pyinotify.IN_DELETE
watch_manager = pyinotify.WatchManager()
handler = _FileKeeper(watched_handler=self,
watched_file=self._log_file)
notifier = _EventletThreadedNotifier(
watch_manager,
default_proc_fun=handler,
read_freq=FastWatchedFileHandler.READ_FREQ,
timeout=FastWatchedFileHandler.TIMEOUT)
notifier.daemon = True
watch_manager.add_watch(self._log_dir, mask)
notifier.start()
def reopen_file(self):
try:
# stat the file by path, checking for existence
sres = os.stat(self.baseFilename)
except OSError as err:
if err.errno == errno.ENOENT:
sres = None
else:
raise
# compare file system stat with that of our stream file handle
if (not sres or
sres[stat.ST_DEV] != self.dev or
sres[stat.ST_INO] != self.ino):
if self.stream is not None:
# we have an open file handle, clean it up
self.stream.flush()
self.stream.close()
self.stream = None
# open a new file handle and get new stat info from that fd
self.stream = self._open()
self._statstream()

2
requirements.txt
View File

@ -12,4 +12,4 @@ oslo.i18n>=1.5.0 # Apache-2.0
oslo.utils>=2.0.0 # Apache-2.0
oslo.serialization>=1.4.0 # Apache-2.0
debtcollector>=0.3.0 # Apache-2.0
pyinotify>=0.9.6 # MIT
pyinotify>=0.9.6;sys_platform!='win32' and sys_platform!='darwin' # MIT