Implement format auto-detection

Add a new option --os-logs/-ol to explicitly specify OpenStack format
logs. Logs whose format is not given explicitly are auto-detected.

Change-Id: Ie97127f24f34ab386138db28b626abf4c259ac24
This commit is contained in:
Matthew Booth 2017-09-20 17:31:49 +01:00
parent 27599915cb
commit 06de0ba054
1 changed files with 65 additions and 12 deletions

View File

@ -152,6 +152,8 @@ class TSLogParser(LogParser):
stat = os.stat(filename)
mtime = datetime.fromtimestamp(stat.st_mtime)
timestamp = self._get_last_timestamp(filename)
if timestamp is None:
raise ValueError("Didn't find timestamp")
self.start_date = mtime - timedelta(seconds=timestamp)
@classmethod
@ -187,10 +189,45 @@ class TSLogParser(LogParser):
class LogFile(object):
def __init__(self, filename, alias, parser_cls):
def _detect_format(self, filename):
self.open(filename)
parsers = []
for cls in LOG_TYPES.values():
if cls is None:
continue
try:
parsers.append(cls(filename))
except ValueError:
# Don't consider the parser if we can't instantiate it for this
# file
pass
# Try to parse the first few lines with each parser in turn, returning
# the first to successfully parse a line
for i in range(0, 5):
line = self._readline()
for parser in parsers:
try:
parser.parse_line(line)
# It worked!
return parser
except ValueError:
pass
raise ValueError("Failed to detect format of %s" % self.alias)
def __init__(self, filename, alias, parser_cls=None):
self.alias = alias
self.parser = parser_cls(filename)
if parser_cls is None:
self.parser = self._detect_format(filename)
else:
self.parser = parser_cls(filename)
self.open(filename)
def open(self, filename):
self._filename = filename
@ -234,12 +271,18 @@ class LogFile(object):
self.next_entry = None
return self
def _readline(self, entry):
def _readline(self):
line = self._file.readline()
if line == "":
return None
line.replace('\0', ' ')
return line
def _next_entry(self, entry):
while True:
line = self._file.readline()
if line == "":
line = self._readline()
if line is None:
return entry, None
line.replace('\0', ' ')
try:
dt, dt_str, data = self.parser.parse_line(line)
@ -258,7 +301,7 @@ class LogFile(object):
return self.next()
def next(self):
self.entry, self.next_entry = self._readline(self.next_entry)
self.entry, self.next_entry = self._next_entry(self.next_entry)
if self.entry is None:
raise StopIteration()
return self.entry
@ -278,7 +321,8 @@ class LogFile(object):
LOG_TYPES = {
'logfiles': OSLogParser,
'logfiles_detect': None,
'logfiles_o': OSLogParser,
'logfiles_m': MsgLogParser,
'logfiles_t': TSLogParser,
}
@ -302,8 +346,13 @@ def process_logs(cfg):
# now, though.
aliases = generate_aliases(paths_aliases, cfg)
logs = [LogFile(path, aliases[path], parser_cls)
for path, parser_cls in paths_parsers.items()]
logs = []
for path, parser_cls in paths_parsers.items():
try:
logs.append(LogFile(path, aliases[path], parser_cls))
except ValueError:
print('WARNING: %s unable to determine format, ignoring' % path,
file=sys.stderr)
entry_iters = [iter(log) for log in logs]
for entry in heapq.merge(*entry_iters):
@ -551,14 +600,18 @@ one has not been provided:'
help='Base path for all the log files')
parser.add_argument('--log-postfix ', '-p', dest='log_postfix',
help='Append to all the log files path')
parser.add_argument('logfiles', nargs='+', metavar='log_file[:ALIAS]',
help='OpenStack log file.')
parser.add_argument('logfiles_detect', nargs='+',
metavar='log_file[:ALIAS]',
help='Log file (auto-detect format)')
parser.add_argument('--alias-level', '-a', type=int, default=0,
dest='alias_level',
help='Level of smart alias naming (0-3)')
parser.add_argument('--min-memory', '-m', default=False,
action='store_true', dest='limit_memory',
help='This option is deprecated and has no effect')
parser.add_argument('--os-logs', '-ol', default=[], nargs='+',
dest='logfiles_o', metavar='file[:ALIAS]',
help='Openstack log files')
parser.add_argument('--msg-logs', '-ml', default=[], nargs='+',
dest='logfiles_m', metavar='file[:ALIAS]',
help='Message log files with format: Oct 15 14:11:19')