Remove swift support
The swift support was never moved into production for infra because of an unsolved issue around index files, so the logs in swift approach was ended. This takes out all the swift supporting code to drop the complexity of the code base. As this is very intermittently maintained tool, less code makes any future work easier to do. Change-Id: Iabc4cffc35633fd808556ebb68a8bdd5a5d1fde0
This commit is contained in:
Sean Dague
parent
9916cb3a3d
commit
5b7ad194dd
|
|
@ -29,9 +29,6 @@ Features
|
|||
* linking and highlighting of lines based on timestamp
|
||||
* control of max number of lines that will be returned using the
|
||||
limit=XXXX parameter
|
||||
* Support for specifying which backend source will be used. As we
|
||||
support filesystems and swift containers simultaneously, you can
|
||||
specify source=swift to only use the swift container backend.
|
||||
* Provides a script named htmlify_server.py that serves htmlified logs
|
||||
over HTTP. To view devstack logs: set
|
||||
SCREEN_LOGDIR=$DEST/logs/screen and LOG_COLOR=false in localrc
|
||||
|
|
|
|||
|
|
@ -3,12 +3,3 @@ filter = SevFilter
|
|||
view = HTMLView
|
||||
file_conditions = /etc/os-loganalyze/file_conditions.yaml
|
||||
generate_folder_index = true
|
||||
|
||||
[swift]
|
||||
authurl=https://keystone.example.org/v2.0/
|
||||
user=example
|
||||
password=example
|
||||
container=logs
|
||||
region=EXP
|
||||
tenant=
|
||||
chunk_size=64
|
||||
|
|
|
|||
|
|
@ -21,19 +21,11 @@ import datetime
|
|||
import fileinput
|
||||
import os.path
|
||||
import re
|
||||
import sys
|
||||
import types
|
||||
import zlib
|
||||
|
||||
import jinja2
|
||||
|
||||
import os_loganalyze.util as util
|
||||
|
||||
try:
|
||||
import swiftclient
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
class UnsafePath(Exception):
|
||||
pass
|
||||
|
|
@ -102,101 +94,6 @@ def sizeof_fmt(num, suffix='B'):
|
|||
return "%.1f%s%s" % (num, 'Y', suffix)
|
||||
|
||||
|
||||
def _get_swift_connection(swift_config):
|
||||
# TODO(jhesketh): refactor the generator into a class so we can keep a
|
||||
# persistent connection. For now, emulate a static variable on this method
|
||||
# called 'con'.
|
||||
if not _get_swift_connection.con:
|
||||
_get_swift_connection.con = swiftclient.client.Connection(
|
||||
authurl=swift_config['authurl'],
|
||||
user=swift_config['user'],
|
||||
key=swift_config['password'],
|
||||
os_options={'region_name': swift_config['region']},
|
||||
tenant_name=swift_config['tenant'],
|
||||
auth_version=2.0
|
||||
)
|
||||
return _get_swift_connection.con
|
||||
_get_swift_connection.con = None
|
||||
|
||||
|
||||
class SwiftIterableBuffer(collections.Iterable):
|
||||
def __init__(self, logname, config):
|
||||
self.logname = logname
|
||||
self.resp_headers = {}
|
||||
self.obj = None
|
||||
self.file_headers = {}
|
||||
self.file_headers['filename'] = logname
|
||||
|
||||
if not config.has_section('swift'):
|
||||
sys.stderr.write('Not configured to use swift..\n')
|
||||
sys.stderr.write('logname: %s\n' % logname)
|
||||
else:
|
||||
try:
|
||||
swift_config = dict(config.items('swift'))
|
||||
# NOTE(jhesketh): While _get_siwft_connection seems like it
|
||||
# should be part of this class we actually still need it
|
||||
# outside to maintain the connection across multiple objects.
|
||||
# Each SwiftIterableBuffer is a new object request, not
|
||||
# necessarily a new swift connection (hopefully we can reuse
|
||||
# connections). I think the place to put the get connection
|
||||
# in the future would be in the server.py (todo).
|
||||
con = _get_swift_connection(swift_config)
|
||||
|
||||
chunk_size = int(swift_config.get('chunk_size', 64))
|
||||
if chunk_size < 1:
|
||||
chunk_size = None
|
||||
|
||||
self.resp_headers, self.obj = con.get_object(
|
||||
swift_config['container'], logname,
|
||||
resp_chunk_size=chunk_size)
|
||||
self.file_headers.update(self.resp_headers)
|
||||
except Exception as e:
|
||||
# Only print the traceback if the error was anything but a
|
||||
# 404. File not found errors are handled separately.
|
||||
if 'http_status' not in dir(e) or e.http_status != 404:
|
||||
import traceback
|
||||
sys.stderr.write("Error fetching from swift.\n")
|
||||
sys.stderr.write('logname: %s\n' % logname)
|
||||
traceback.print_exc()
|
||||
|
||||
def __iter__(self):
|
||||
ext = os.path.splitext(self.logname)[1]
|
||||
if ext == '.gz':
|
||||
# Set up a decompression object assuming the deflate
|
||||
# compression algorithm was used
|
||||
d = zlib.decompressobj(16 + zlib.MAX_WBITS)
|
||||
|
||||
if isinstance(self.obj, types.GeneratorType):
|
||||
buf = next(self.obj)
|
||||
partial = ''
|
||||
while buf:
|
||||
if ext == '.gz':
|
||||
string = partial + d.decompress(buf)
|
||||
else:
|
||||
string = partial + buf
|
||||
split = string.split('\n')
|
||||
for line in split[:-1]:
|
||||
yield line + '\n'
|
||||
partial = split[-1]
|
||||
try:
|
||||
buf = next(self.obj)
|
||||
except StopIteration:
|
||||
break
|
||||
if partial != '':
|
||||
yield partial
|
||||
else:
|
||||
output = self.obj
|
||||
if ext == '.gz':
|
||||
output = d.decompress(output)
|
||||
|
||||
split = output.split('\n')
|
||||
for line in split[:-1]:
|
||||
yield line + '\n'
|
||||
partial = split[-1]
|
||||
if partial != '':
|
||||
yield partial
|
||||
|
||||
|
||||
class DiskIterableBuffer(collections.Iterable):
|
||||
def __init__(self, logname, logpath, config):
|
||||
self.logname = logname
|
||||
|
|
@ -223,7 +120,7 @@ class IndexIterableBuffer(collections.Iterable):
|
|||
|
||||
# Use sets here to dedup. We can have duplicates
|
||||
# if disk and swift based paths have overlap.
|
||||
file_set = self.disk_list() | self.swift_list()
|
||||
file_set = self.disk_list()
|
||||
# file_list is a list of tuples (relpath, name, mtime, size)
|
||||
self.file_list = sorted(file_set, key=lambda tup: tup[0])
|
||||
|
||||
|
|
@ -246,42 +143,6 @@ class IndexIterableBuffer(collections.Iterable):
|
|||
))
|
||||
return file_set
|
||||
|
||||
def swift_list(self):
|
||||
file_set = set()
|
||||
if self.config.has_section('swift'):
|
||||
try:
|
||||
swift_config = dict(self.config.items('swift'))
|
||||
con = _get_swift_connection(swift_config)
|
||||
|
||||
prefix = self.logname + '/' if self.logname[-1] != '/' \
|
||||
else self.logname
|
||||
resp, files = con.get_container(swift_config['container'],
|
||||
prefix=prefix,
|
||||
delimiter='/')
|
||||
|
||||
for f in files:
|
||||
size = sizeof_fmt(f.get('bytes', 0))
|
||||
mtime = f.get('last_modified', 'unknown')
|
||||
if 'subdir' in f:
|
||||
fname = os.path.relpath(f['subdir'], self.logname)
|
||||
fname = fname + '/' if f['subdir'][-1] == '/' else \
|
||||
fname
|
||||
else:
|
||||
fname = os.path.relpath(f['name'], self.logname)
|
||||
file_set.add((
|
||||
os.path.join('/', self.logname, fname),
|
||||
fname,
|
||||
mtime,
|
||||
size
|
||||
))
|
||||
except Exception:
|
||||
import traceback
|
||||
sys.stderr.write("Error fetching index list from swift.\n")
|
||||
sys.stderr.write('logname: %s\n' % self.logname)
|
||||
traceback.print_exc()
|
||||
|
||||
return file_set
|
||||
|
||||
def __iter__(self):
|
||||
env = jinja2.Environment(
|
||||
loader=jinja2.PackageLoader('os_loganalyze', 'templates'))
|
||||
|
|
@ -299,28 +160,8 @@ def get_file_generator(environ, root_path, config=None):
|
|||
raise UnsafePath()
|
||||
|
||||
file_generator = None
|
||||
# if we want swift only, we'll skip processing files
|
||||
use_files = (util.parse_param(environ, 'source', default='all')
|
||||
!= 'swift')
|
||||
if use_files and does_file_exist(logpath):
|
||||
if does_file_exist(logpath):
|
||||
file_generator = DiskIterableBuffer(logname, logpath, config)
|
||||
else:
|
||||
# NOTE(jhesketh): If the requested URL ends in a trailing slash we
|
||||
# assume that this is meaning to load an index.html from our pseudo
|
||||
# filesystem and attempt that first.
|
||||
if logname and logname[-1] == '/':
|
||||
file_generator = SwiftIterableBuffer(
|
||||
os.path.join(logname, 'index.html'), config)
|
||||
if not file_generator.obj:
|
||||
# Maybe our assumption was wrong, lets go back to trying the
|
||||
# original object name.
|
||||
file_generator = SwiftIterableBuffer(logname, config)
|
||||
else:
|
||||
file_generator = SwiftIterableBuffer(logname, config)
|
||||
if not file_generator.obj:
|
||||
# The object doesn't exist. Try again appending index.html
|
||||
file_generator = SwiftIterableBuffer(
|
||||
os.path.join(logname, 'index.html'), config)
|
||||
|
||||
if not file_generator or not file_generator.obj:
|
||||
if (config.has_section('general') and
|
||||
|
|
|
|||
|
|
@ -1,11 +1,2 @@
|
|||
[general]
|
||||
# Don't override the filter or view default detection
|
||||
|
||||
[swift]
|
||||
authurl=https://keystone.example.org/v2.0/
|
||||
user=example
|
||||
password=example
|
||||
container=logs
|
||||
region=EXP
|
||||
tenant=
|
||||
chunk_size=64
|
||||
|
|
|
|||
|
|
@ -1,11 +1,2 @@
|
|||
[general]
|
||||
file_conditions = file_conditions.yaml
|
||||
|
||||
[swift]
|
||||
authurl=https://keystone.example.org/v2.0/
|
||||
user=example
|
||||
password=example
|
||||
container=logs
|
||||
region=EXP
|
||||
tenant=
|
||||
chunk_size=64
|
||||
|
|
|
|||
|
|
@ -1,11 +1,2 @@
|
|||
[general]
|
||||
generate_folder_index = true
|
||||
|
||||
[swift]
|
||||
authurl=https://keystone.example.org/v2.0/
|
||||
user=example
|
||||
password=example
|
||||
container=logs
|
||||
region=EXP
|
||||
tenant=
|
||||
chunk_size=64
|
||||
|
|
|
|||
|
|
@ -1,8 +0,0 @@
|
|||
[swift]
|
||||
authurl=https://keystone.example.org/v2.0/
|
||||
user=example
|
||||
password=example
|
||||
container=logs
|
||||
region=EXP
|
||||
tenant=
|
||||
chunk_size=0
|
||||
|
|
@ -1,12 +1,3 @@
|
|||
[general]
|
||||
filter = nofilter
|
||||
view = passthrough
|
||||
|
||||
[swift]
|
||||
authurl=https://keystone.example.org/v2.0/
|
||||
user=example
|
||||
password=example
|
||||
container=logs
|
||||
region=EXP
|
||||
tenant=
|
||||
chunk_size=64
|
||||
|
|
|
|||
|
|
@ -19,16 +19,12 @@ Test the ability to convert files into wsgi generators
|
|||
"""
|
||||
|
||||
import collections
|
||||
import os
|
||||
import re
|
||||
import types
|
||||
|
||||
import mock
|
||||
import swiftclient # noqa needed for monkeypatching
|
||||
import testtools
|
||||
|
||||
from os_loganalyze.tests import base
|
||||
import os_loganalyze.util
|
||||
import os_loganalyze.wsgi as log_wsgi
|
||||
|
||||
|
||||
|
|
@ -78,58 +74,6 @@ def compute_total(level, counts):
|
|||
return total
|
||||
|
||||
|
||||
def fake_get_object(self, container, name, resp_chunk_size=None):
|
||||
name = name[len('non-existent/'):]
|
||||
if not os.path.isfile(base.samples_path('samples') + name):
|
||||
return {}, None
|
||||
if resp_chunk_size:
|
||||
|
||||
def _object_body():
|
||||
with open(base.samples_path('samples') + name) as f:
|
||||
|
||||
buf = f.read(resp_chunk_size)
|
||||
while buf:
|
||||
yield buf
|
||||
buf = f.read(resp_chunk_size)
|
||||
|
||||
object_body = _object_body()
|
||||
else:
|
||||
with open(base.samples_path('samples') + name) as f:
|
||||
object_body = f.read()
|
||||
resp_headers = os_loganalyze.util.get_headers_for_file(
|
||||
base.samples_path('samples') + name)
|
||||
return resp_headers, object_body
|
||||
|
||||
|
||||
def fake_get_container_factory(_swift_index_items=None):
|
||||
def fake_get_container(self, container, prefix=None, delimiter=None):
|
||||
index_items = []
|
||||
if _swift_index_items:
|
||||
for i in _swift_index_items:
|
||||
if i[-1] == '/':
|
||||
index_items.append({'subdir': os.path.join(prefix, i)})
|
||||
else:
|
||||
index_items.append({'name': os.path.join(prefix, i),
|
||||
'last_modified': '2042-12-31T23:59:59',
|
||||
'bytes': 4200})
|
||||
elif _swift_index_items == []:
|
||||
name = prefix[len('non-existent/'):]
|
||||
p = os.path.join(base.samples_path('samples'), name)
|
||||
for i in os.listdir(p):
|
||||
if os.path.isdir(os.path.join(p, i)):
|
||||
index_items.append(
|
||||
{'subdir': os.path.join(prefix, i + '/')})
|
||||
else:
|
||||
index_items.append({'name': os.path.join(prefix, i),
|
||||
'last_modified': '2042-12-31T23:59:59',
|
||||
'bytes': 4200})
|
||||
else:
|
||||
# No swift container data.
|
||||
pass
|
||||
return {}, index_items
|
||||
return fake_get_container
|
||||
|
||||
|
||||
class TestWsgiDisk(base.TestCase):
|
||||
"""Test loading files from samples on disk."""
|
||||
|
||||
|
|
@ -169,8 +113,6 @@ class TestWsgiDisk(base.TestCase):
|
|||
},
|
||||
}
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_pass_through_all(self):
|
||||
for fname in self.files:
|
||||
gen = self.get_generator(fname, html=False)
|
||||
|
|
@ -178,8 +120,6 @@ class TestWsgiDisk(base.TestCase):
|
|||
compute_total('DEBUG', count_types(gen)),
|
||||
compute_total('DEBUG', self.files[fname]))
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_pass_through_at_levels(self):
|
||||
for fname in self.files:
|
||||
for level in self.files[fname]:
|
||||
|
|
@ -192,23 +132,17 @@ class TestWsgiDisk(base.TestCase):
|
|||
compute_total(level, counts),
|
||||
compute_total(level, self.files[fname]))
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_invalid_file(self):
|
||||
gen = log_wsgi.application(
|
||||
self.fake_env(PATH_INFO='../'), self._start_response)
|
||||
self.assertEqual(gen, ['Invalid file url'])
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_file_not_found(self):
|
||||
gen = log_wsgi.application(
|
||||
self.fake_env(PATH_INFO='/htmlify/foo.txt'),
|
||||
self._start_response)
|
||||
self.assertEqual(gen, ['File Not Found'])
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_plain_text(self):
|
||||
gen = self.get_generator('screen-c-api.txt.gz', html=False)
|
||||
self.assertEqual(type(gen), types.GeneratorType)
|
||||
|
|
@ -218,15 +152,11 @@ class TestWsgiDisk(base.TestCase):
|
|||
'+ ln -sf /opt/stack/new/screen-logs/screen-c-api.2013-09-27-1815',
|
||||
first)
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_html_gen(self):
|
||||
gen = self.get_generator('screen-c-api.txt.gz')
|
||||
first = gen.next()
|
||||
self.assertIn('<html>', first)
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_plain_non_compressed(self):
|
||||
gen = self.get_generator('screen-c-api.txt', html=False)
|
||||
self.assertEqual(type(gen), types.GeneratorType)
|
||||
|
|
@ -236,8 +166,6 @@ class TestWsgiDisk(base.TestCase):
|
|||
'+ ln -sf /opt/stack/new/screen-logs/screen-c-api.2013-09-27-1815',
|
||||
first)
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_passthrough_filter(self):
|
||||
# Test the passthrough filter returns an image stream
|
||||
gen = self.get_generator('openstack_logo.png')
|
||||
|
|
@ -246,8 +174,6 @@ class TestWsgiDisk(base.TestCase):
|
|||
with open(base.samples_path('samples') + 'openstack_logo.png') as f:
|
||||
self.assertEqual(first, f.readline())
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_config_no_filter(self):
|
||||
self.wsgi_config_file = (base.samples_path('samples') +
|
||||
'wsgi_plain.conf')
|
||||
|
|
@ -263,8 +189,6 @@ class TestWsgiDisk(base.TestCase):
|
|||
# given the header and footer, but we expect to get the full file
|
||||
self.assertNotEqual(12, lines)
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_config_passthrough_view(self):
|
||||
self.wsgi_config_file = (base.samples_path('samples') +
|
||||
'wsgi_plain.conf')
|
||||
|
|
@ -274,8 +198,6 @@ class TestWsgiDisk(base.TestCase):
|
|||
first = gen.next()
|
||||
self.assertNotIn('<html>', first)
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_file_conditions(self):
|
||||
self.wsgi_config_file = (base.samples_path('samples') +
|
||||
'wsgi_file_conditions.conf')
|
||||
|
|
@ -298,51 +220,37 @@ class TestWsgiDisk(base.TestCase):
|
|||
with open(base.samples_path('samples') + 'openstack_logo.png') as f:
|
||||
self.assertEqual(first, f.readline())
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_accept_range(self):
|
||||
gen = self.get_generator('screen-c-api.txt.gz', range_bytes='0-5')
|
||||
body = gen.next()
|
||||
self.assertEqual('<html>', body)
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_accept_range_seek(self):
|
||||
gen = self.get_generator('screen-c-api.txt.gz', range_bytes='7-12')
|
||||
body = gen.next()
|
||||
self.assertEqual('<head>', body)
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_accept_range_no_start(self):
|
||||
gen = self.get_generator('screen-c-api.txt.gz', range_bytes='-8')
|
||||
body = gen.next()
|
||||
self.assertEqual('</html>\n', body)
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_accept_range_no_end(self):
|
||||
gen = self.get_generator('screen-c-api.txt.gz', range_bytes='7-')
|
||||
body = gen.next()
|
||||
self.assertNotIn('<html>', body)
|
||||
self.assertIn('<head>', body)
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_accept_range_no_start_no_end(self):
|
||||
gen = self.get_generator('screen-c-api.txt.gz', range_bytes='-')
|
||||
body = gen.next()
|
||||
self.assertEqual('Invalid Range', body)
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_accept_range_no_hyphen(self):
|
||||
gen = self.get_generator('screen-c-api.txt.gz', range_bytes='7')
|
||||
body = gen.next()
|
||||
self.assertEqual('Invalid Range', body)
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_container',
|
||||
fake_get_container_factory())
|
||||
def test_folder_index(self):
|
||||
self.wsgi_config_file = (base.samples_path('samples') +
|
||||
'wsgi_folder_index.conf')
|
||||
|
|
@ -354,183 +262,20 @@ class TestWsgiDisk(base.TestCase):
|
|||
full_lines = full.split('\n')
|
||||
self.assertEqual('<!DOCTYPE html>', full_lines[0])
|
||||
self.assertIn('samples/</title>', full_lines[3])
|
||||
# self.assertEqual("foo", full_lines)
|
||||
self.assertThat(
|
||||
full_lines[9],
|
||||
testtools.matchers.MatchesRegex(
|
||||
r' <tr><td><a href="/samples/console.html.gz">'
|
||||
r'console.html.gz</a></td><td>' + ISO8601RE + r'</td>'
|
||||
r'<td style="text-align: right">277.4KB</td></tr>'))
|
||||
r'<td style="text-align: right">277.4KB</td></tr>'),
|
||||
"\nFull log: %s" % full_lines)
|
||||
self.assertThat(
|
||||
full_lines[-5],
|
||||
testtools.matchers.MatchesRegex(
|
||||
r' <tr><td><a href="/samples/wsgi_plain.conf">'
|
||||
r'wsgi_plain.conf</a></td><td>' + ISO8601RE + r'</td>'
|
||||
r'<td style="text-align: right">177.0B</td></tr>'))
|
||||
self.assertEqual('</html>', full_lines[-1])
|
||||
|
||||
|
||||
class TestWsgiSwift(TestWsgiDisk):
|
||||
"""Test loading files from swift."""
|
||||
def setUp(self):
|
||||
super(TestWsgiSwift, self).setUp()
|
||||
|
||||
# Set the samples directory to somewhere non-existent so that swift
|
||||
# is checked for files
|
||||
self.samples_directory = 'non-existent'
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_compare_disk_to_swift_html(self):
|
||||
"""Compare loading logs from disk vs swift."""
|
||||
# Load from disk
|
||||
self.samples_directory = 'samples'
|
||||
gen = self.get_generator('screen-c-api.txt.gz')
|
||||
result_disk = ''
|
||||
for line in gen:
|
||||
result_disk += line
|
||||
|
||||
self.samples_directory = 'non-existent'
|
||||
gen = self.get_generator('screen-c-api.txt.gz')
|
||||
result_swift = ''
|
||||
for line in gen:
|
||||
result_swift += line
|
||||
|
||||
self.assertEqual(result_disk, result_swift)
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_compare_disk_to_swift_plain(self):
|
||||
"""Compare loading logs from disk vs swift."""
|
||||
# Load from disk
|
||||
self.samples_directory = 'samples'
|
||||
gen = self.get_generator('screen-c-api.txt.gz', html=False)
|
||||
result_disk = ''
|
||||
for line in gen:
|
||||
result_disk += line
|
||||
|
||||
self.samples_directory = 'non-existent'
|
||||
gen = self.get_generator('screen-c-api.txt.gz', html=False)
|
||||
result_swift = ''
|
||||
for line in gen:
|
||||
result_swift += line
|
||||
|
||||
self.assertEqual(result_disk, result_swift)
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_skip_file(self):
|
||||
# this should generate a TypeError because we're telling it to
|
||||
# skip the filesystem, but we don't have a working swift here.
|
||||
self.assertRaises(
|
||||
TypeError,
|
||||
self.get_generator('screen-c-api.txt.gz', source='swift'))
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_compare_disk_to_swift_no_compression(self):
|
||||
"""Compare loading logs from disk vs swift."""
|
||||
# Load from disk
|
||||
self.samples_directory = 'samples'
|
||||
gen = self.get_generator('screen-c-api.txt')
|
||||
result_disk = ''
|
||||
for line in gen:
|
||||
result_disk += line
|
||||
|
||||
self.samples_directory = 'non-existent'
|
||||
gen = self.get_generator('screen-c-api.txt')
|
||||
result_swift = ''
|
||||
for line in gen:
|
||||
result_swift += line
|
||||
|
||||
self.assertEqual(result_disk, result_swift)
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
def test_compare_disk_to_swift_no_chunks(self):
|
||||
self.wsgi_config_file = (base.samples_path('samples') +
|
||||
'wsgi_no_chunks.conf')
|
||||
self.test_compare_disk_to_swift_no_compression()
|
||||
self.test_compare_disk_to_swift_plain()
|
||||
self.test_compare_disk_to_swift_html()
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_object',
|
||||
fake_get_object)
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_container',
|
||||
fake_get_container_factory([]))
|
||||
def test_folder_index(self):
|
||||
self.wsgi_config_file = (base.samples_path('samples') +
|
||||
'wsgi_folder_index.conf')
|
||||
gen = self.get_generator('')
|
||||
full = ''
|
||||
for line in gen:
|
||||
full += line
|
||||
|
||||
full_lines = full.split('\n')
|
||||
self.assertEqual('<!DOCTYPE html>', full_lines[0])
|
||||
self.assertIn('non-existent/</title>', full_lines[3])
|
||||
self.assertThat(
|
||||
full_lines[9],
|
||||
testtools.matchers.MatchesRegex(
|
||||
r' <tr><td><a href="/non-existent/console.html.gz">'
|
||||
r'console.html.gz</a></td><td>' + ISO8601RE + r'</td>'
|
||||
r'<td style="text-align: right">4.1KB</td></tr>'))
|
||||
self.assertThat(
|
||||
full_lines[-5],
|
||||
testtools.matchers.MatchesRegex(
|
||||
r' <tr><td><a href="/non-existent/wsgi_plain.conf">'
|
||||
r'wsgi_plain.conf</a></td><td>' + ISO8601RE + r'</td>'
|
||||
r'<td style="text-align: right">4.1KB</td></tr>'))
|
||||
self.assertEqual('</html>', full_lines[-1])
|
||||
|
||||
@mock.patch.object(swiftclient.client.Connection, 'get_container',
|
||||
fake_get_container_factory(['a', 'b', 'dir/', 'z']))
|
||||
def test_folder_index_dual(self):
|
||||
# Test an index is correctly generated where files may exist on disk as
|
||||
# well as in swift.
|
||||
self.samples_directory = 'samples'
|
||||
self.wsgi_config_file = (base.samples_path('samples') +
|
||||
'wsgi_folder_index.conf')
|
||||
|
||||
gen = self.get_generator('')
|
||||
full = ''
|
||||
for line in gen:
|
||||
full += line
|
||||
|
||||
full_lines = full.split('\n')
|
||||
self.assertEqual('<!DOCTYPE html>', full_lines[0])
|
||||
self.assertIn('samples/</title>', full_lines[3])
|
||||
self.assertThat(
|
||||
full_lines[9],
|
||||
testtools.matchers.MatchesRegex(
|
||||
r' <tr><td><a href="/samples/a">a</a></td>'
|
||||
r'<td>' + ISO8601RE + r'</td>'
|
||||
r'<td style="text-align: right">4.1KB</td></tr>'))
|
||||
self.assertThat(
|
||||
full_lines[11],
|
||||
testtools.matchers.MatchesRegex(
|
||||
r' <tr><td><a href="/samples/b">b</a></td>'
|
||||
r'<td>' + ISO8601RE + r'</td>'
|
||||
r'<td style="text-align: right">4.1KB</td></tr>'))
|
||||
self.assertThat(
|
||||
full_lines[13],
|
||||
testtools.matchers.MatchesRegex(
|
||||
r' <tr><td><a href="/samples/console.html.gz">'
|
||||
r'console.html.gz</a></td><td>' + ISO8601RE + r'</td>'
|
||||
r'<td style="text-align: right">277.4KB</td></tr>'))
|
||||
self.assertEqual(
|
||||
full_lines[17],
|
||||
' <tr><td><a href="/samples/dir/">dir/</a></td>'
|
||||
'<td>unknown</td><td style="text-align: right">0.0B</td></tr>')
|
||||
self.assertThat(
|
||||
full_lines[-7],
|
||||
testtools.matchers.MatchesRegex(
|
||||
r' <tr><td><a href="/samples/wsgi_plain.conf">'
|
||||
r'wsgi_plain.conf</a></td><td>' + ISO8601RE + r'</td>'
|
||||
r'<td style="text-align: right">177.0B</td></tr>'))
|
||||
self.assertThat(
|
||||
full_lines[-5],
|
||||
testtools.matchers.MatchesRegex(
|
||||
r' <tr><td><a href="/samples/z">z</a></td>'
|
||||
r'<td>' + ISO8601RE + r'</td>'
|
||||
r'<td style="text-align: right">4.1KB</td></tr>'))
|
||||
self.assertEqual('</html>', full_lines[-1])
|
||||
r'<td style="text-align: right">47.0B</td></tr>'),
|
||||
"\nFull log: %s" % full_lines)
|
||||
self.assertEqual('</html>', full_lines[-1],
|
||||
"\nFull log: %s" % full_lines)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,5 @@
|
|||
pbr>=1.6
|
||||
Babel>=1.3
|
||||
Jinja2>=2.6 # BSD License (3 clause)
|
||||
python-swiftclient>=2.2.0,<=2.3.1
|
||||
python-keystoneclient>=1.6.0
|
||||
PyYAML>=3.1.0
|
||||
python-magic
|
||||
|
|
|
|||
Loading…
Reference in New Issue