From cc2907689ffd9865f18003c7486f9060031259d8 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Fri, 24 Apr 2015 12:41:05 +0000 Subject: [PATCH] Revert "Tidy up generators into contained objects" This reverts commit b62ed8abc362fc8c6a377a3d0a3678284650baa6. Change-Id: I56d947d913010b733a3b5c906a9403313b6ea350 --- os_loganalyze/filter.py | 12 +-- os_loganalyze/generator.py | 156 +++++++++++++----------------- os_loganalyze/tests/test_views.py | 5 +- os_loganalyze/view.py | 9 +- os_loganalyze/wsgi.py | 19 ++-- 5 files changed, 90 insertions(+), 111 deletions(-) diff --git a/os_loganalyze/filter.py b/os_loganalyze/filter.py index 8c94b9a..a397763 100644 --- a/os_loganalyze/filter.py +++ b/os_loganalyze/filter.py @@ -96,11 +96,11 @@ class LogLine(object): class Filter(object): - def __init__(self, file_generator, minsev="NONE", limit=None): + def __init__(self, fname, generator, minsev="NONE", limit=None): self.minsev = minsev - self.file_generator = file_generator - self.supports_sev = \ - SUPPORTS_SEV.search(file_generator.logname) is not None + self.gen = generator + self.supports_sev = SUPPORTS_SEV.search(fname) is not None + self.fname = fname self.limit = limit self.strip_control = False @@ -110,9 +110,9 @@ class Filter(object): def __iter__(self): old_sev = "NONE" lineno = 1 - for line in self.file_generator: + for line in self.gen: # bail early for limits - if self.limit and lineno > int(self.limit): + if self.limit and lineno >= int(self.limit): raise StopIteration() # strip control chars in case the console is ascii colored if self.strip_control: diff --git a/os_loganalyze/generator.py b/os_loganalyze/generator.py index 37ff3f7..f238ead 100644 --- a/os_loganalyze/generator.py +++ b/os_loganalyze/generator.py @@ -16,7 +16,6 @@ # License for the specific language governing permissions and limitations # under the License. -import collections import fileinput import os.path import re @@ -105,99 +104,73 @@ def _get_swift_connection(swift_config): _get_swift_connection.con = None -class SwiftIterableBuffer(collections.Iterable): - file_headers = {} +def get_swift_line_generator(logname, config): + resp_headers = {} + if not config.has_section('swift'): + sys.stderr.write('Not configured to use swift..\n') + sys.stderr.write('logname: %s\n' % logname) + return resp_headers, None - def __init__(self, logname, config): - self.logname = logname - self.resp_headers = {} - self.obj = None - self.file_headers['filename'] = logname + try: + swift_config = dict(config.items('swift')) + con = _get_swift_connection(swift_config) - if not config.has_section('swift'): - sys.stderr.write('Not configured to use swift..\n') - sys.stderr.write('logname: %s\n' % logname) - else: - try: - swift_config = dict(config.items('swift')) - # NOTE(jhesketh): While _get_siwft_connection seems like it - # should be part of this class we actually still need it - # outside to maintain the connection across multiple objects. - # Each SwiftIterableBuffer is a new object request, not - # necessarily a new swift connection (hopefully we can reuse - # connections). I think the place to put the get connection - # in the future would be in the server.py (todo). - con = _get_swift_connection(swift_config) + chunk_size = int(swift_config.get('chunk_size', 64)) + if chunk_size < 1: + chunk_size = None - chunk_size = int(swift_config.get('chunk_size', 64)) - if chunk_size < 1: - chunk_size = None + resp_headers, obj = con.get_object( + swift_config['container'], logname, + resp_chunk_size=chunk_size) - self.resp_headers, self.obj = con.get_object( - swift_config['container'], logname, - resp_chunk_size=chunk_size) - self.file_headers.update(self.resp_headers) - except Exception: - import traceback - sys.stderr.write("Error fetching from swift.\n") - sys.stderr.write('logname: %s\n' % logname) - traceback.print_exc() + def line_generator(): + ext = os.path.splitext(logname)[1] + if ext == '.gz': + # Set up a decompression object assuming the deflate + # compression algorithm was used + d = zlib.decompressobj(16 + zlib.MAX_WBITS) - def __iter__(self): - ext = os.path.splitext(self.logname)[1] - if ext == '.gz': - # Set up a decompression object assuming the deflate - # compression algorithm was used - d = zlib.decompressobj(16 + zlib.MAX_WBITS) - - if isinstance(self.obj, types.GeneratorType): - buf = next(self.obj) - partial = '' - while buf: + if isinstance(obj, types.GeneratorType): + buf = next(obj) + partial = '' + while buf: + if ext == '.gz': + string = partial + d.decompress(buf) + else: + string = partial + buf + split = string.split('\n') + for line in split[:-1]: + yield line + '\n' + partial = split[-1] + try: + buf = next(obj) + except StopIteration: + break + if partial != '': + yield partial + else: + output = obj if ext == '.gz': - string = partial + d.decompress(buf) - else: - string = partial + buf - split = string.split('\n') + output = d.decompress(output) + + split = output.split('\n') for line in split[:-1]: yield line + '\n' partial = split[-1] - try: - buf = next(self.obj) - except StopIteration: - break - if partial != '': - yield partial - else: - output = self.obj - if ext == '.gz': - output = d.decompress(output) + if partial != '': + yield partial - split = output.split('\n') - for line in split[:-1]: - yield line + '\n' - partial = split[-1] - if partial != '': - yield partial + return resp_headers, line_generator() + + except Exception: + import traceback + sys.stderr.write("Error fetching from swift.\n") + sys.stderr.write('logname: %s\n' % logname) + traceback.print_exc() + return resp_headers, None -class DiskIterableBuffer(collections.Iterable): - file_headers = {} - - def __init__(self, logname, logpath, config): - self.logname = logname - self.logpath = logpath - self.resp_headers = {} - self.obj = fileinput.FileInput(self.logpath, - openhook=fileinput.hook_compressed) - self.file_headers['filename'] = logname - self.file_headers.update(util.get_headers_for_file(logpath)) - - def __iter__(self): - return self.obj - - -def get_file_generator(environ, root_path, config=None): +def get(environ, root_path, config=None): logname = log_name(environ) logpath = safe_path(root_path, logname) file_headers = {} @@ -205,19 +178,24 @@ def get_file_generator(environ, root_path, config=None): raise UnsafePath() file_headers['filename'] = os.path.basename(logpath) - file_generator = None + flines_generator = None # if we want swift only, we'll skip processing files use_files = (util.parse_param(environ, 'source', default='all') != 'swift') if use_files and does_file_exist(logpath): - file_generator = DiskIterableBuffer(logname, logpath, config) + flines_generator = fileinput.FileInput( + logpath, openhook=fileinput.hook_compressed) + file_headers.update(util.get_headers_for_file(logpath)) else: - file_generator = SwiftIterableBuffer(logname, config) - if not file_generator.obj: + resp_headers, flines_generator = get_swift_line_generator(logname, + config) + if not flines_generator: logname = os.path.join(logname, 'index.html') - file_generator = SwiftIterableBuffer(logname, config) + resp_headers, flines_generator = get_swift_line_generator(logname, + config) + file_headers.update(resp_headers) - if not file_generator.obj: + if not flines_generator: raise NoSuchFile() - return file_generator + return logname, flines_generator, file_headers diff --git a/os_loganalyze/tests/test_views.py b/os_loganalyze/tests/test_views.py index 96fa9af..a682683 100644 --- a/os_loganalyze/tests/test_views.py +++ b/os_loganalyze/tests/test_views.py @@ -28,9 +28,8 @@ class TestViews(base.TestCase): # wsgi application. We just need the generator to give to Views. root_path = base.samples_path(self.samples_directory) kwargs = {'PATH_INFO': '/htmlify/%s' % fname} - file_generator = osgen.get_file_generator(self.fake_env(**kwargs), - root_path) - flines_generator = osfilter.Filter(file_generator) + logname, gen, headers = osgen.get(self.fake_env(**kwargs), root_path) + flines_generator = osfilter.Filter(logname, gen) return flines_generator def test_html_detection(self): diff --git a/os_loganalyze/view.py b/os_loganalyze/view.py index af24757..fddcff4 100644 --- a/os_loganalyze/view.py +++ b/os_loganalyze/view.py @@ -161,8 +161,7 @@ class HTMLView(collections.Iterable): return newline def __iter__(self): - igen = (x for x in self.gen) - first_line = next(igen) + first_line = next(x for x in self.gen) self._discover_html(first_line.line) if not self.is_html: @@ -176,7 +175,7 @@ class HTMLView(collections.Iterable): if first: yield first - for line in igen: + for line in self.gen: newline = self._process_line(line) if newline: yield newline @@ -199,9 +198,9 @@ class TextView(collections.Iterable): class PassthroughView(collections.Iterable): headers = [] - def __init__(self, gen): + def __init__(self, gen, file_headers): self.gen = gen - for hn, hv in self.gen.file_headers.items(): + for hn, hv in file_headers.items(): self.headers.append((hn, hv)) def __iter__(self): diff --git a/os_loganalyze/wsgi.py b/os_loganalyze/wsgi.py index 3bd2d5d..b77f1a8 100755 --- a/os_loganalyze/wsgi.py +++ b/os_loganalyze/wsgi.py @@ -99,7 +99,8 @@ def application(environ, start_response, root_path=None, status = '200 OK' try: - file_generator = osgen.get_file_generator(environ, root_path, config) + logname, flines_generator, file_headers = osgen.get(environ, root_path, + config) except osgen.UnsafePath: status = '400 Bad Request' response_headers = [('Content-type', 'text/plain')] @@ -111,21 +112,23 @@ def application(environ, start_response, root_path=None, start_response(status, response_headers) return ['File Not Found'] - if use_passthrough_view(file_generator.file_headers): - view_generator = osview.PassthroughView(file_generator) + if use_passthrough_view(file_headers): + generator = osview.PassthroughView(flines_generator, + file_headers) else: minsev = util.parse_param(environ, 'level', default="NONE") limit = util.parse_param(environ, 'limit') - flines_generator = osfilter.Filter(file_generator, minsev, limit) + flines_generator = osfilter.Filter( + logname, flines_generator, minsev, limit) if environ.get('OS_LOGANALYZE_STRIP', None): flines_generator.strip_control = True if should_be_html(environ): - view_generator = osview.HTMLView(flines_generator) + generator = osview.HTMLView(flines_generator) else: - view_generator = osview.TextView(flines_generator) + generator = osview.TextView(flines_generator) - start_response(status, view_generator.headers) - return view_generator + start_response(status, generator.headers) + return generator # for development purposes, makes it easy to test the filter output