Fix #7: Add support for timestamped logs
This patch adds support for timestamped log files with the format "[ 0.003036]". Since timestamp many times will not take epoc time as the source of the timestamp but the time the system started, the initial datetime will be calculated by substracting from the file modified datetime the last timestamp in the file. Option added is --timestamp-logs or -tl, and it also supports ALIAS and globs, and is also affected by base directory and postfix options.
This commit is contained in:
parent
8c3ae25c17
commit
ef9b9bd54f
|
@ -11,6 +11,7 @@ Changelog
|
||||||
- Auto alias generation: `-a` `--alias-level`.
|
- Auto alias generation: `-a` `--alias-level`.
|
||||||
- Add support for default /var/log/messages datetime format files with
|
- Add support for default /var/log/messages datetime format files with
|
||||||
`-ml [FILE [FILE]]`
|
`-ml [FILE [FILE]]`
|
||||||
|
- Add support for timestamped log files with `-tl [FILE [FILE]]`
|
||||||
|
|
||||||
**Bugfixes:**
|
**Bugfixes:**
|
||||||
|
|
||||||
|
|
23
README.rst
23
README.rst
|
@ -114,6 +114,29 @@ Example for Cinder:
|
||||||
$ os-log-merger -b /var/log/ cinder/api.log:API -ml messages:MSG *.log
|
$ os-log-merger -b /var/log/ cinder/api.log:API -ml messages:MSG *.log
|
||||||
|
|
||||||
|
|
||||||
|
Timestamped logs
|
||||||
|
~~~~~~~~~~~~~~~~
|
||||||
|
|
||||||
|
os-log-merger also supports timestamped -[ 0.003036]- with options `-tl`
|
||||||
|
and `--timestamp-logs` options.
|
||||||
|
|
||||||
|
Since timestamp many times will not take epoc time as the source of the
|
||||||
|
timestamp but the time the system started, the initial datetime will be
|
||||||
|
calculated by substracting from the file modified datetime the last timestamp
|
||||||
|
in the file.
|
||||||
|
|
||||||
|
These files can also be specified with globs and they support alias definition
|
||||||
|
as well.
|
||||||
|
|
||||||
|
Beware that openstack files should be listed before `-tl` option files.
|
||||||
|
|
||||||
|
Example for Cinder:
|
||||||
|
|
||||||
|
.. code:: bash
|
||||||
|
|
||||||
|
$ os-log-merger -b /var/log/ cinder/api.log:API -tl dmesg:DMSG
|
||||||
|
|
||||||
|
|
||||||
Auto Alias
|
Auto Alias
|
||||||
~~~~~~~~~~
|
~~~~~~~~~~
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
from __future__ import print_function
|
from __future__ import print_function
|
||||||
import argparse
|
import argparse
|
||||||
from datetime import datetime
|
from datetime import datetime, timedelta
|
||||||
import hashlib
|
import hashlib
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
@ -80,10 +80,15 @@ class LogEntry(object):
|
||||||
date_format = None
|
date_format = None
|
||||||
_date_parse_msg = 'unconverted data remains: '
|
_date_parse_msg = 'unconverted data remains: '
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self, **kwargs):
|
||||||
self._date_length = None
|
self._date_length = None
|
||||||
|
self.__dict__.update(**kwargs)
|
||||||
|
|
||||||
def prepare_line(self, line, file_datetime):
|
@classmethod
|
||||||
|
def get_init_args(cls, filename):
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def prepare_line(self, line):
|
||||||
return line
|
return line
|
||||||
|
|
||||||
def parse_date(self, line):
|
def parse_date(self, line):
|
||||||
|
@ -108,15 +113,15 @@ class LogEntry(object):
|
||||||
return self._date_length
|
return self._date_length
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def factory(cls, filename, line, file_datetime):
|
def factory(cls, filename, line, **kwargs):
|
||||||
self = cls()
|
self = cls(**kwargs)
|
||||||
|
|
||||||
self.filename = filename
|
self.filename = filename
|
||||||
if not line:
|
if not line:
|
||||||
raise ValueError
|
raise ValueError
|
||||||
|
|
||||||
# Prepare the line for date parsing
|
# Prepare the line for date parsing
|
||||||
prepared_line = self.prepare_line(line, file_datetime)
|
prepared_line = self.prepare_line(line)
|
||||||
|
|
||||||
# Extract the datetime
|
# Extract the datetime
|
||||||
self.date = self.parse_date(prepared_line)
|
self.date = self.parse_date(prepared_line)
|
||||||
|
@ -144,6 +149,7 @@ class LogFile(object):
|
||||||
def factory(cls, filename):
|
def factory(cls, filename):
|
||||||
instance = LogFile(filename)
|
instance = LogFile(filename)
|
||||||
instance.log_entry_class = cls
|
instance.log_entry_class = cls
|
||||||
|
instance.entry_kwargs = cls.get_init_args(filename)
|
||||||
return instance
|
return instance
|
||||||
|
|
||||||
def __init__(self, filename):
|
def __init__(self, filename):
|
||||||
|
@ -202,7 +208,7 @@ class LogFile(object):
|
||||||
try:
|
try:
|
||||||
new_entry = self.log_entry_class.factory(self._filename,
|
new_entry = self.log_entry_class.factory(self._filename,
|
||||||
line,
|
line,
|
||||||
self.mtime)
|
**self.entry_kwargs)
|
||||||
if new_entry is None:
|
if new_entry is None:
|
||||||
continue
|
continue
|
||||||
if entry:
|
if entry:
|
||||||
|
@ -242,11 +248,18 @@ class MsgLogEntry(LogEntry):
|
||||||
"""Message format: Oct 15 14:11:19"""
|
"""Message format: Oct 15 14:11:19"""
|
||||||
date_format = '%Y%b %d %H:%M:%S'
|
date_format = '%Y%b %d %H:%M:%S'
|
||||||
|
|
||||||
def prepare_line(self, line, file_datetime):
|
@classmethod
|
||||||
|
def get_init_args(cls, filename):
|
||||||
|
kwargs = super(MsgLogEntry, cls).get_init_args(filename)
|
||||||
|
stat = os.stat(filename)
|
||||||
|
kwargs['file_year'] = datetime.fromtimestamp(stat.st_mtime).year
|
||||||
|
return kwargs
|
||||||
|
|
||||||
|
def prepare_line(self, line):
|
||||||
# TODO: If year of file creation and file last modification are
|
# TODO: If year of file creation and file last modification are
|
||||||
# different we should start with the cration year and then change to
|
# different we should start with the cration year and then change to
|
||||||
# the next year once the months go back.
|
# the next year once the months go back.
|
||||||
return '%s%s' % (file_datetime.year, line)
|
return '%s%s' % (self.file_year, line)
|
||||||
|
|
||||||
def _calculate_date_length(self):
|
def _calculate_date_length(self):
|
||||||
return super(MsgLogEntry, self)._calculate_date_length() - 4
|
return super(MsgLogEntry, self)._calculate_date_length() - 4
|
||||||
|
@ -260,6 +273,50 @@ class OSLogEntry(LogEntry):
|
||||||
return super(OSLogEntry, self)._calculate_date_length() - 3
|
return super(OSLogEntry, self)._calculate_date_length() - 3
|
||||||
|
|
||||||
|
|
||||||
|
class TSLogEntry(LogEntry):
|
||||||
|
"""Timestamped log: [275514.814982]"""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_init_args(cls, filename):
|
||||||
|
kwargs = super(TSLogEntry, cls).get_init_args(filename)
|
||||||
|
stat = os.stat(filename)
|
||||||
|
mtime = datetime.fromtimestamp(stat.st_mtime)
|
||||||
|
timestamp = cls._get_last_timestamp(filename)
|
||||||
|
kwargs['start_date'] = mtime - timedelta(seconds=timestamp)
|
||||||
|
return kwargs
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _get_last_timestamp(cls, filename):
|
||||||
|
result = None
|
||||||
|
with open(filename, 'r') as f:
|
||||||
|
file_size = os.fstat(f.fileno()).st_size
|
||||||
|
# We will jump to the last KB so we don't have to read all file
|
||||||
|
offset = max(0, file_size - 1024)
|
||||||
|
f.seek(offset)
|
||||||
|
for line in f:
|
||||||
|
try:
|
||||||
|
__, result = cls._read_timestamp(line)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _read_timestamp(line):
|
||||||
|
start = line.index('[') + 1
|
||||||
|
end = line.index(']')
|
||||||
|
|
||||||
|
if end < start:
|
||||||
|
raise ValueError
|
||||||
|
|
||||||
|
return end, float(line[start:end])
|
||||||
|
|
||||||
|
def parse_date(self, date_str):
|
||||||
|
end, timestamp = self._read_timestamp(date_str)
|
||||||
|
self._date_length = end + 1
|
||||||
|
return self.start_date + timedelta(seconds=timestamp)
|
||||||
|
|
||||||
|
|
||||||
def process_logs_limit_memory_usage(logs):
|
def process_logs_limit_memory_usage(logs):
|
||||||
oslogs = [iter(log) for log in logs]
|
oslogs = [iter(log) for log in logs]
|
||||||
|
|
||||||
|
@ -298,6 +355,7 @@ def process_logs_memory_hog(logs):
|
||||||
LOG_TYPES = [
|
LOG_TYPES = [
|
||||||
('logfiles', OSLogEntry),
|
('logfiles', OSLogEntry),
|
||||||
('logfiles_m', MsgLogEntry),
|
('logfiles_m', MsgLogEntry),
|
||||||
|
('logfiles_t', TSLogEntry),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@ -497,6 +555,12 @@ Y-m-d H:M:S.mmm ............
|
||||||
can optionally be merged as well using "--msg-logs" or "-ml"
|
can optionally be merged as well using "--msg-logs" or "-ml"
|
||||||
options. Year will be taken from the last modified time of the file.
|
options. Year will be taken from the last modified time of the file.
|
||||||
|
|
||||||
|
Logs with timestamp format -[ 0.003036]- are also supported with
|
||||||
|
options "--timestamp-logs" or "-tl". Since timestamp many times will
|
||||||
|
not take epoc time as the source of the timestamp but the time the
|
||||||
|
system started, the initial datetime will be calculated by substracting
|
||||||
|
from the file modified datetime the last timestamp in the file.
|
||||||
|
|
||||||
These log files will aso be affected by log base directory and log
|
These log files will aso be affected by log base directory and log
|
||||||
postfix.
|
postfix.
|
||||||
"""
|
"""
|
||||||
|
@ -556,6 +620,9 @@ one has not been provided:'
|
||||||
parser.add_argument('--msg-logs', '-ml', default=[], nargs='+',
|
parser.add_argument('--msg-logs', '-ml', default=[], nargs='+',
|
||||||
dest='logfiles_m', metavar='file[:ALIAS]',
|
dest='logfiles_m', metavar='file[:ALIAS]',
|
||||||
help='Message log files with format: Oct 15 14:11:19')
|
help='Message log files with format: Oct 15 14:11:19')
|
||||||
|
parser.add_argument('--timestamp-logs', '-tl', default=[], nargs='+',
|
||||||
|
dest='logfiles_t', metavar='file[:ALIAS]',
|
||||||
|
help='Message log files with timestamp: [ 0.003036]')
|
||||||
|
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue