openstack
/
os-loganalyze
Code Issues Proposed changes

Print file sizes in indexes for on disk files 

File sizes are an important piece of information in log file indexes as
they help readers pick out inappropriately sized files which may
indicate the bug can be found there. Add file sizes to the index so that
we don't lose this info when compared to mod_autoindex.

Change-Id: I248a4fc9bd5ff3fa536e30bcc981a2d5f640307c
This commit is contained in:
Clark Boylan 2015-10-07 13:04:27 -07:00
parent a6fa3bfd2d
commit a9b91f6a05
3 changed files with 62 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
os_loganalyze/generator.py
View File

@ -91,6 +91,16 @@ def safe_path(root, log_name):
return None
def sizeof_fmt(num, suffix='B'):
# From http://stackoverflow.com/questions/1094841/
# reusable-library-to-get-human-readable-version-of-file-size
for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, 'Y', suffix)
def _get_swift_connection(swift_config):
# TODO(jhesketh): refactor the generator into a class so we can keep a
# persistent connection. For now, emulate a static variable on this method
@ -213,18 +223,22 @@ class IndexIterableBuffer(collections.Iterable):
# Use sets here to dedup. We can have duplicates
# if disk and swift based paths have overlap.
file_set = self.disk_list() | self.swift_list()
# file_list is a list of tuples (relpath, name)
# file_list is a list of tuples (relpath, name, size)
self.file_list = sorted(file_set, key=lambda tup: tup[0])
def disk_list(self):
file_set = set()
if os.path.isdir(self.logpath):
for f in os.listdir(self.logpath):
if os.path.isdir(os.path.join(self.logpath, f)):
full_path = os.path.join(self.logpath, f)
stat_info = os.stat(full_path)
size = sizeof_fmt(stat_info.st_size)
if os.path.isdir(full_path):
f = f + '/' if f[-1] != '/' else f
file_set.add((
os.path.join('/', self.logname, f),
f
f,
size
))
return file_set
@ -248,9 +262,11 @@ class IndexIterableBuffer(collections.Iterable):
fname
else:
fname = os.path.relpath(f['name'], self.logname)
# TODO(clarkb) get size from swift and write it here.
file_set.add((
os.path.join('/', self.logname, fname),
fname
fname,
"NA"
))
except Exception:
import traceback

8
os_loganalyze/templates/file_index.html
View File

@ -5,10 +5,10 @@
</head>
<body>
<h1>Index of {{ logname }}</h1>
<ul>
{% for link, title in file_list %}
<li><a href="{{ link }}">{{ title }}</a></li>
<table><tr><th>Name</th><th>Size</th></tr>
{% for link, title, size in file_list %}
<tr><td><a href="{{ link }}">{{ title }}</a></td><td style="text-align: right">{{ size }}</td></tr>
{% endfor %}
</ul>
</table>
</body>
</html>

61
os_loganalyze/tests/test_wsgi.py
View File

@ -98,7 +98,7 @@ def fake_get_object(self, container, name, resp_chunk_size=None):
return resp_headers, object_body
def fake_get_container_factory(_swift_index_items=[]):
def fake_get_container_factory(_swift_index_items=None):
def fake_get_container(self, container, prefix=None, delimiter=None):
index_items = []
if _swift_index_items:
@ -107,7 +107,7 @@ def fake_get_container_factory(_swift_index_items=[]):
index_items.append({'subdir': os.path.join(prefix, i)})
else:
index_items.append({'name': os.path.join(prefix, i)})
else:
elif _swift_index_items == []:
name = prefix[len('non-existent/'):]
p = os.path.join(base.samples_path('samples'), name)
for i in os.listdir(p):
@ -116,7 +116,9 @@ def fake_get_container_factory(_swift_index_items=[]):
{'subdir': os.path.join(prefix, i + '/')})
else:
index_items.append({'name': os.path.join(prefix, i)})
else:
# No swift container data.
pass
return {}, index_items
return fake_get_container
@ -300,12 +302,14 @@ class TestWsgiDisk(base.TestCase):
self.assertEqual('<!DOCTYPE html>', full_lines[0])
self.assertIn('samples/</title>', full_lines[3])
self.assertEqual(
' <li><a href="/samples/console.html.gz">'
'console.html.gz</a></li>',
' <tr><td><a href="/samples/console.html.gz">'
'console.html.gz</a></td><td style="text-align: right">'
'277.4KB</td></tr>',
full_lines[9])
self.assertEqual(
' <li><a href="/samples/wsgi_plain.conf">'
'wsgi_plain.conf</a></li>',
' <tr><td><a href="/samples/wsgi_plain.conf">'
'wsgi_plain.conf</a></td><td style="text-align: right">'
'177.0B</td></tr>',
full_lines[-5])
self.assertEqual('</html>', full_lines[-1])
@ -397,7 +401,7 @@ class TestWsgiSwift(TestWsgiDisk):
@mock.patch.object(swiftclient.client.Connection, 'get_object',
fake_get_object)
@mock.patch.object(swiftclient.client.Connection, 'get_container',
fake_get_container_factory())
fake_get_container_factory([]))
def test_folder_index(self):
self.wsgi_config_file = (base.samples_path('samples') +
'wsgi_folder_index.conf')
@ -410,12 +414,14 @@ class TestWsgiSwift(TestWsgiDisk):
self.assertEqual('<!DOCTYPE html>', full_lines[0])
self.assertIn('non-existent/</title>', full_lines[3])
self.assertEqual(
' <li><a href="/non-existent/console.html.gz">'
'console.html.gz</a></li>',
' <tr><td><a href="/non-existent/console.html.gz">'
'console.html.gz</a></td><td style="text-align: right">'
'NA</td></tr>',
full_lines[9])
self.assertEqual(
' <li><a href="/non-existent/wsgi_plain.conf">'
'wsgi_plain.conf</a></li>',
' <tr><td><a href="/non-existent/wsgi_plain.conf">'
'wsgi_plain.conf</a></td><td style="text-align: right">'
'NA</td></tr>',
full_lines[-5])
self.assertEqual('</html>', full_lines[-1])
@ -436,21 +442,30 @@ class TestWsgiSwift(TestWsgiDisk):
full_lines = full.split('\n')
self.assertEqual('<!DOCTYPE html>', full_lines[0])
self.assertIn('samples/</title>', full_lines[3])
self.assertEqual(' <li><a href="/samples/a">a</a></li>',
full_lines[9])
self.assertEqual(' <li><a href="/samples/b">b</a></li>',
full_lines[11])
self.assertEqual(
' <li><a href="/samples/console.html.gz">'
'console.html.gz</a></li>',
' <tr><td><a href="/samples/a">a</a></td>'
'<td style="text-align: right">NA</td></tr>',
full_lines[9])
self.assertEqual(
' <tr><td><a href="/samples/b">b</a></td>'
'<td style="text-align: right">NA</td></tr>',
full_lines[11])
self.assertEqual(
' <tr><td><a href="/samples/console.html.gz">'
'console.html.gz</a></td><td style="text-align: right">'
'277.4KB</td></tr>',
full_lines[13])
self.assertEqual(
' <li><a href="/samples/dir/">dir/</a></li>',
' <tr><td><a href="/samples/dir/">dir/</a></td>'
'<td style="text-align: right">NA</td></tr>',
full_lines[17])
self.assertEqual(
' <li><a href="/samples/wsgi_plain.conf">'
'wsgi_plain.conf</a></li>',
' <tr><td><a href="/samples/wsgi_plain.conf">'
'wsgi_plain.conf</a></td><td style="text-align: right">'
'177.0B</td></tr>',
full_lines[-7])
self.assertEqual(' <li><a href="/samples/z">z</a></li>',
full_lines[-5])
self.assertEqual(
' <tr><td><a href="/samples/z">z</a></td>'
'<td style="text-align: right">NA</td></tr>',
full_lines[-5])
self.assertEqual('</html>', full_lines[-1])