Add support for non-text files

A first pass at adding support for non-text files.

This passes the content-type headers from swift to determine
how to serve up the file.

Change-Id: I905a31b68308c6a79ef863b94cebdaa14914b221

README.rst

@@ -42,7 +42,6 @@ Todo
 ------------
 Next steps, roughly in order
 
-* support swift logs (timestamp linking only, no sevs in swift logs)
 * provide links to logstash for request streams (link well know
   request ids to logstash queries for them)
 

os_loganalyze/generator.py

@@ -2,6 +2,7 @@
 #
 # Copyright (c) 2013 IBM Corp.
 # Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+# Copyright (c) 2014 Rackspace Australia
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -104,10 +105,11 @@ _get_swift_connection.con = None
 
 
 def get_swift_line_generator(logname, config):
+    resp_headers = {}
     if not config.has_section('swift'):
         sys.stderr.write('Not configured to use swift..\n')
         sys.stderr.write('logname: %s\n' % logname)
-        return None
+        return resp_headers, None
 
     try:
         swift_config = dict(config.items('swift'))
@@ -158,21 +160,23 @@ def get_swift_line_generator(logname, config):
                 if partial != '':
                     yield partial
 
-        return line_generator()
+        return resp_headers, line_generator()
 
     except Exception:
         import traceback
         sys.stderr.write("Error fetching from swift.\n")
         sys.stderr.write('logname: %s\n' % logname)
         traceback.print_exc()
-        return None
+        return resp_headers, None
 
 
 def get(environ, root_path, config=None):
     logname = log_name(environ)
     logpath = safe_path(root_path, logname)
+    file_headers = {}
     if not logpath:
         raise UnsafePath()
+    file_headers['filename'] = os.path.basename(logpath)
 
     flines_generator = None
     # if we want swift only, we'll skip processing files
@@ -181,13 +185,17 @@ def get(environ, root_path, config=None):
     if use_files and does_file_exist(logpath):
         flines_generator = fileinput.FileInput(
             logpath, openhook=fileinput.hook_compressed)
+        file_headers.update(util.get_headers_for_file(logpath))
     else:
-        flines_generator = get_swift_line_generator(logname, config)
+        resp_headers, flines_generator = get_swift_line_generator(logname,
+                                                                  config)
         if not flines_generator:
             logname = os.path.join(logname, 'index.html')
-            flines_generator = get_swift_line_generator(logname, config)
+            resp_headers, flines_generator = get_swift_line_generator(logname,
+                                                                      config)
+        file_headers.update(resp_headers)
 
     if not flines_generator:
         raise NoSuchFile()
 
-    return logname, flines_generator
+    return logname, flines_generator, file_headers

os_loganalyze/tests/base.py

@@ -25,7 +25,6 @@ import testtools
 
 import os_loganalyze.wsgi as log_wsgi
 
-
 _TRUE_VALUES = ('true', '1', 'yes')
 
 

os_loganalyze/tests/test_views.py

@@ -28,7 +28,7 @@ class TestViews(base.TestCase):
         # wsgi application. We just need the generator to give to Views.
         root_path = base.samples_path(self.samples_directory)
         kwargs = {'PATH_INFO': '/htmlify/%s' % fname}
-        logname, gen = osgen.get(self.fake_env(**kwargs), root_path)
+        logname, gen, headers = osgen.get(self.fake_env(**kwargs), root_path)
         flines_generator = osfilter.Filter(logname, gen)
         return flines_generator
 

os_loganalyze/tests/test_wsgi.py

@@ -24,6 +24,7 @@ import fixtures
 import swiftclient  # noqa needed for monkeypatching
 
 from os_loganalyze.tests import base
+import os_loganalyze.util
 import os_loganalyze.wsgi as log_wsgi
 
 
@@ -162,6 +163,14 @@ class TestWsgiDisk(base.TestCase):
             '+ ln -sf /opt/stack/new/screen-logs/screen-c-api.2013-09-27-1815',
             first)
 
+    def test_passthrough_filter(self):
+        # Test the passthrough filter returns an image stream
+        gen = self.get_generator('openstack_logo.png')
+        first = gen.next()
+        self.assertNotIn('html', first)
+        with open(base.samples_path('samples') + 'openstack_logo.png') as f:
+            self.assertEqual(first, f.readline())
+
 
 class TestWsgiSwift(TestWsgiDisk):
     """Test loading files from swift."""
@@ -186,7 +195,9 @@ class TestWsgiSwift(TestWsgiDisk):
                 else:
                     with open(base.samples_path('samples') + name) as f:
                         object_body = f.read()
-                return [], object_body
+                resp_headers = os_loganalyze.util.get_headers_for_file(
+                    base.samples_path('samples') + name)
+                return resp_headers, object_body
 
         self.useFixture(fixtures.MonkeyPatch(
             'swiftclient.client.Connection', fake_swiftclient))

os_loganalyze/util.py

@@ -15,6 +15,10 @@
 # under the License.
 
 import cgi
+import os
+import time
+
+import magic
 
 
 def parse_param(env, name, default=None):
@@ -23,3 +27,36 @@ def parse_param(env, name, default=None):
         return cgi.escape(parameters[name][0])
     else:
         return default
+
+
+def get_file_mime(file_path):
+    """Get the file mime using libmagic."""
+
+    if not os.path.isfile(file_path):
+        return None
+
+    if hasattr(magic, 'from_file'):
+        return magic.from_file(file_path, mime=True)
+    else:
+        # no magic.from_file, we might be using the libmagic bindings
+        m = magic.open(magic.MAGIC_MIME)
+        m.load()
+        return m.file(file_path).split(';')[0]
+
+
+def get_headers_for_file(file_path):
+    """Get some headers for a real (on-disk) file.
+
+    In a format similar to fetching from swift.
+    """
+
+    resp = {}
+    resp['content-length'] = str(os.stat(file_path).st_size)
+    resp['accept-ranges'] = 'bytes'
+    resp['last-modified'] = time.strftime(
+        "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(os.path.getmtime(file_path)))
+    resp['etag'] = ''
+    resp['x-trans-id'] = str(time.time())
+    resp['date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT")
+    resp['content-type'] = get_file_mime(file_path)
+    return resp

os_loganalyze/view.py

@@ -193,3 +193,16 @@ class TextView(collections.Iterable):
     def __iter__(self):
         for line in self.gen:
             yield line.date + line.line + "\n"
+
+
+class PassthroughView(collections.Iterable):
+    headers = []
+
+    def __init__(self, gen, file_headers):
+        self.gen = gen
+        for hn, hv in file_headers.items():
+            self.headers.append((hn, hv))
+
+    def __iter__(self):
+        for line in self.gen:
+            yield line

os_loganalyze/wsgi.py

@@ -63,6 +63,28 @@ def get_config(wsgi_config):
     return config
 
 
+def use_passthrough_view(file_headers):
+    """Determine if we need to use the passthrough filter."""
+
+    if 'content-type' not in file_headers:
+        # For legacy we'll try and format. This shouldn't occur though.
+        return False
+    else:
+        if file_headers['content-type'] in ['text/plain', 'text/html']:
+            # We want to format these files
+            return False
+        if file_headers['content-type'] in ['application/x-gzip',
+                                            'application/gzip']:
+            # We'll need to guess if we should render the output or offer a
+            # download.
+            filename = file_headers['filename']
+            filename = filename[:-3] if filename[-3:] == '.gz' else filename
+            if os.path.splitext(filename)[1] in ['.txt', '.html']:
+                return False
+
+    return True
+
+
 def application(environ, start_response, root_path=None,
                 wsgi_config='/etc/os_loganalyze/wsgi.conf'):
     if root_path is None:
@@ -77,7 +99,8 @@ def application(environ, start_response, root_path=None,
     status = '200 OK'
 
     try:
-        logname, flines_generator = osgen.get(environ, root_path, config)
+        logname, flines_generator, file_headers = osgen.get(environ, root_path,
+                                                            config)
     except osgen.UnsafePath:
         status = '400 Bad Request'
         response_headers = [('Content-type', 'text/plain')]
@@ -89,16 +112,20 @@ def application(environ, start_response, root_path=None,
         start_response(status, response_headers)
         return ['File Not Found']
 
-    minsev = util.parse_param(environ, 'level', default="NONE")
-    limit = util.parse_param(environ, 'limit')
-    flines_generator = osfilter.Filter(
-        logname, flines_generator, minsev, limit)
-    if environ.get('OS_LOGANALYZE_STRIP', None):
-        flines_generator.strip_control = True
-    if should_be_html(environ):
-        generator = osview.HTMLView(flines_generator)
+    if use_passthrough_view(file_headers):
+        generator = osview.PassthroughView(flines_generator,
+                                           file_headers)
     else:
-        generator = osview.TextView(flines_generator)
+        minsev = util.parse_param(environ, 'level', default="NONE")
+        limit = util.parse_param(environ, 'limit')
+        flines_generator = osfilter.Filter(
+            logname, flines_generator, minsev, limit)
+        if environ.get('OS_LOGANALYZE_STRIP', None):
+            flines_generator.strip_control = True
+        if should_be_html(environ):
+            generator = osview.HTMLView(flines_generator)
+        else:
+            generator = osview.TextView(flines_generator)
 
     start_response(status, generator.headers)
     return generator

requirements.txt

@@ -2,3 +2,4 @@ pbr>=0.5.21,<1.0
 Babel>=0.9.6
 python-swiftclient>=1.6
 python-keystoneclient>=0.4.2
+python-magic