introduce limit parameter

limit parameter allows you to specify a max number of lines that
you want, and will not go above that. It's added to deal with
run away failure conditions in logs which make their use in things
like Elastic Search really problematic, as a run away log can
kill logstash workers.

Change-Id: I25cfa10012060214046da8787eba8832e9eb802a

README.rst

@@ -27,6 +27,8 @@ Features
 * filtering based on severity using the level=XXXX parameter (works in
   either text/html or text/plain responses
 * linking and highlighting of lines based on timestamp
+* control of max number of lines that will be returned using the
+  limit=XXXX parameter
 * Provides a script named htmlify_server.py that serves htmlified logs
   over HTTP. To view devstack logs: set
   SCREEN_LOGDIR=$DEST/logs/screen and LOG_COLOR=false in localrc

os_loganalyze/filter.py

@@ -84,20 +84,28 @@ class LogLine(object):
 
 class Filter(object):
 
-    def __init__(self, fname, generator, minsev="NONE"):
+    def __init__(self, fname, generator, minsev="NONE", limit=None):
         self.minsev = minsev
         self.gen = generator
         self.supports_sev = SUPPORTS_SEV.search(fname) is not None
         self.fname = fname
+        self.limit = limit
 
     def __iter__(self):
         old_sev = "NONE"
+        lineno = 0
         for line in self.gen:
+            # bail early for limits
+            if self.limit and lineno >= int(self.limit):
+                raise StopIteration()
+
             logline = LogLine(line, old_sev)
+
             if self.supports_sev and self.skip_by_sev(logline.status):
                 old_sev = logline.status
                 continue
 
+            lineno += 1
             old_sev = logline.status
             yield logline.date + logline.line
 

os_loganalyze/tests/base.py

@@ -17,6 +17,7 @@
 
 import os
 import os.path
+import urllib
 from wsgiref.util import setup_testing_defaults
 
 import fixtures
@@ -80,11 +81,15 @@ class TestCase(testtools.TestCase):
         setup_testing_defaults(environ)
         return environ
 
-    def get_generator(self, fname, level=None, html=True):
+    def get_generator(self, fname, level=None, html=True, limit=None):
         kwargs = {'PATH_INFO': '/htmlify/%s' % fname}
-
+        qs = {}
         if level:
-            kwargs['QUERY_STRING'] = 'level=%s' % level
+            qs['level'] = level
+        if limit:
+            qs['limit'] = limit
+        if qs:
+            kwargs['QUERY_STRING'] = urllib.urlencode(qs)
 
         if html:
             kwargs['HTTP_ACCEPT'] = 'text/html'

os_loganalyze/tests/test_filters.py

@@ -118,3 +118,16 @@ class TestFilters(base.TestCase):
         line = gen.next()
         self.assertIn("<span class='INFO", line)
         self.assertIn('object-server: SIGTERM received', line)
+
+    def test_limit_filters(self):
+        gen = self.get_generator('devstacklog.txt.gz', limit=10)
+        # dump the header
+        gen.next()
+
+        # first line
+        lines = 0
+        for line in gen:
+            lines += 1
+
+        # this is an html file, so 1 extra line for the footers
+        self.assertEqual(11, lines)

os_loganalyze/wsgi.py

@@ -271,6 +271,14 @@ def get_min_sev(environ):
         return "NONE"
 
 
+def get_limit(environ):
+    parameters = cgi.parse_qs(environ.get('QUERY_STRING', ''))
+    if 'limit' in parameters:
+        return cgi.escape(parameters['limit'][0])
+    else:
+        return None
+
+
 def get_config(wsgi_config):
     config = ConfigParser.ConfigParser()
     config.read(os.path.expanduser(wsgi_config))
@@ -304,7 +312,9 @@ def application(environ, start_response, root_path=None,
         return ['File Not Found']
 
     minsev = get_min_sev(environ)
-    flines_generator = osfilter.Filter(logname, flines_generator, minsev)
+    limit = get_limit(environ)
+    flines_generator = osfilter.Filter(
+        logname, flines_generator, minsev, limit)
     if should_be_html(environ):
         response_headers = [('Content-type', 'text/html')]
         generator = html_filter(logname, flines_generator, minsev)