openstack
/
os-loganalyze
Code Issues Proposed changes

Add support for rfc7233 range headers. 

This is primarily needed by our log_processor gearman workers which currently
attempt to retry failed downloads with partial GETs.

Change-Id: Ic49579d253ea773651e4b9641223c2d2ae961c94
This commit is contained in:
K Jonathan Harker 2015-11-02 16:34:02 -08:00
parent ec9a338ddc
commit 5828d818ed
3 changed files with 141 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
os_loganalyze/tests/base.py
View File

@ -98,7 +98,7 @@ class TestCase(testtools.TestCase):
return filename
def get_generator(self, fname, level=None, html=True,
limit=None, source=None):
limit=None, source=None, range_bytes=None):
kwargs = {'PATH_INFO': '/htmlify/%s/%s' % (self.samples_directory,
fname)}
qs = {}
@ -114,6 +114,9 @@ class TestCase(testtools.TestCase):
if html:
kwargs['HTTP_ACCEPT'] = 'text/html'
if range_bytes:
kwargs['HTTP_RANGE'] = 'bytes=%s' % range_bytes
gen = log_wsgi.application(
self.fake_env(**kwargs),
self._start_response,

43
os_loganalyze/tests/test_wsgi.py
View File

@ -295,6 +295,49 @@ class TestWsgiDisk(base.TestCase):
with open(base.samples_path('samples') + 'openstack_logo.png') as f:
self.assertEqual(first, f.readline())
@mock.patch.object(swiftclient.client.Connection, 'get_object',
fake_get_object)
def test_accept_range(self):
gen = self.get_generator('screen-c-api.txt.gz', range_bytes='0-5')
body = gen.next()
self.assertEqual('<html>', body)
@mock.patch.object(swiftclient.client.Connection, 'get_object',
fake_get_object)
def test_accept_range_seek(self):
gen = self.get_generator('screen-c-api.txt.gz', range_bytes='7-12')
body = gen.next()
self.assertEqual('<head>', body)
@mock.patch.object(swiftclient.client.Connection, 'get_object',
fake_get_object)
def test_accept_range_no_start(self):
gen = self.get_generator('screen-c-api.txt.gz', range_bytes='-8')
body = gen.next()
self.assertEqual('</html>\n', body)
@mock.patch.object(swiftclient.client.Connection, 'get_object',
fake_get_object)
def test_accept_range_no_end(self):
gen = self.get_generator('screen-c-api.txt.gz', range_bytes='7-')
body = gen.next()
self.assertNotIn('<html>', body)
self.assertIn('<head>', body)
@mock.patch.object(swiftclient.client.Connection, 'get_object',
fake_get_object)
def test_accept_range_no_start_no_end(self):
gen = self.get_generator('screen-c-api.txt.gz', range_bytes='-')
body = gen.next()
self.assertEqual('Invalid Range', body)
@mock.patch.object(swiftclient.client.Connection, 'get_object',
fake_get_object)
def test_accept_range_no_hyphen(self):
gen = self.get_generator('screen-c-api.txt.gz', range_bytes='7')
body = gen.next()
self.assertEqual('Invalid Range', body)
@mock.patch.object(swiftclient.client.Connection, 'get_container',
fake_get_container_factory())
def test_folder_index(self):

96
os_loganalyze/wsgi.py
View File

@ -38,6 +38,94 @@ def get_config(wsgi_config):
return config
def get_range(environ, start_response, view_generator):
if '-' not in environ['HTTP_RANGE']:
status = '400 Bad Request'
response_headers = [('Content-type', 'text/plain')]
start_response(status, response_headers)
yield 'Invalid Range'
return
_range = environ['HTTP_RANGE'].split('=')[1].split('-')
try:
start = None
if _range[0] is not '':
start = int(_range[0])
end = None
if _range[1] is not '':
end = int(_range[1])
except ValueError:
status = '400 Bad Request'
response_headers = [('Content-type', 'text/plain')]
start_response(status, response_headers)
yield 'Invalid Range'
return
if start is None and end is None:
status = '400 Bad Request'
response_headers = [('Content-type', 'text/plain')]
start_response(status, response_headers)
yield 'Invalid Range'
return
# bytes=-5 means the last 5 bytes
remainder = None
if start is None:
remainder = -end # this will be a negative index
start = 0
end = None
position = 0
body = []
status = '206 Partial Content'
start_response(status, view_generator.headers)
stop = False
for chunk in view_generator:
new_chunk = None
if (position + len(chunk)) < start:
# ignore this chunk
position += len(chunk)
elif position < start:
# start is between position and position + len(chunk)
# we only want a portion of this chunk
offset = start - position
if end and end < (position + len(chunk)):
# the entire range is a subset of this chunk
cutoff = end - position + 1
new_chunk = chunk[offset:cutoff]
stop = True # don't break until we have yielded
else:
new_chunk = chunk[offset:]
position += len(chunk[offset:])
elif end and end <= (position + len(chunk)):
cutoff = end - position + 1
new_chunk = chunk[:cutoff]
stop = True # don't break until we have yielded
else:
new_chunk = chunk
position += len(chunk)
if remainder and new_chunk:
body.append(new_chunk)
elif new_chunk:
yield new_chunk
if stop:
break
if remainder:
_body = ''.join(body)
if len(_body) > -remainder: # remainder is negative
yield _body[remainder:]
else:
yield _body
def application(environ, start_response, root_path=None,
wsgi_config='/etc/os_loganalyze/wsgi.conf'):
if root_path is None:
@ -69,8 +157,12 @@ def application(environ, start_response, root_path=None,
view_generator = osview.get_view_generator(filter_generator, environ,
root_path, config)
start_response(status, view_generator.headers)
return view_generator
if 'HTTP_RANGE' in environ:
return get_range(environ, start_response, view_generator)
else:
start_response(status, view_generator.headers)
return view_generator
# for development purposes, makes it easy to test the filter output