Fix parsing request path with slash delimiters
This patch fixes access_processor to parse request path correctly when the request path includes query with slash delimiters. For example, existing access_processor's result of log line with a request path "/v1/acc/con?prefix=YYYY/MM/DD" is as follows: account = "acc" container_name = "con" object_name = "MM/DD" It is because, existing access_processor splits query from request paths *AFTER* splits them into account, container_name, object_name. This patch changes splitting query timing.
This commit is contained in:
parent
e3a4de56cf
commit
b973f522b8
|
@ -20,6 +20,7 @@ from tzlocal import get_localzone
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from slogging import common
|
from slogging import common
|
||||||
import pytz
|
import pytz
|
||||||
|
from urlparse import urlparse
|
||||||
|
|
||||||
# conditionalize the return_ips method based on whether or not iptools
|
# conditionalize the return_ips method based on whether or not iptools
|
||||||
# is present in the system. Without iptools, you will lack CIDR support.
|
# is present in the system. Without iptools, you will lack CIDR support.
|
||||||
|
@ -98,6 +99,9 @@ class AccessLogProcessor(object):
|
||||||
{'found': server, 'expected': self.server_name})
|
{'found': server, 'expected': self.server_name})
|
||||||
return {}
|
return {}
|
||||||
try:
|
try:
|
||||||
|
parsed_url = urlparse(request)
|
||||||
|
request = parsed_url.path
|
||||||
|
query = parsed_url.query
|
||||||
(version, account, container_name, object_name) = \
|
(version, account, container_name, object_name) = \
|
||||||
split_path(request, 2, 4, True)
|
split_path(request, 2, 4, True)
|
||||||
except ValueError, e:
|
except ValueError, e:
|
||||||
|
@ -112,15 +116,9 @@ class AccessLogProcessor(object):
|
||||||
self.logger.debug(_('Unexpected Swift version string: found ' \
|
self.logger.debug(_('Unexpected Swift version string: found ' \
|
||||||
'"%s" expected "v1"') % version)
|
'"%s" expected "v1"') % version)
|
||||||
return {}
|
return {}
|
||||||
if container_name is not None:
|
if query != "":
|
||||||
container_name = container_name.split('?', 1)[0]
|
|
||||||
if object_name is not None:
|
|
||||||
object_name = object_name.split('?', 1)[0]
|
|
||||||
account = account.split('?', 1)[0]
|
|
||||||
query = None
|
|
||||||
if '?' in request:
|
|
||||||
request, query = request.split('?', 1)
|
|
||||||
args = query.split('&')
|
args = query.split('&')
|
||||||
|
d['query'] = query
|
||||||
# Count each query argument. This is used later to aggregate
|
# Count each query argument. This is used later to aggregate
|
||||||
# the number of format, prefix, etc. queries.
|
# the number of format, prefix, etc. queries.
|
||||||
for q in args:
|
for q in args:
|
||||||
|
@ -138,8 +136,6 @@ class AccessLogProcessor(object):
|
||||||
d['lb_ip'] = lb_ip
|
d['lb_ip'] = lb_ip
|
||||||
d['method'] = method
|
d['method'] = method
|
||||||
d['request'] = request
|
d['request'] = request
|
||||||
if query:
|
|
||||||
d['query'] = query
|
|
||||||
d['http_version'] = http_version
|
d['http_version'] = http_version
|
||||||
d['code'] = code
|
d['code'] = code
|
||||||
d['referrer'] = referrer
|
d['referrer'] = referrer
|
||||||
|
|
|
@ -72,6 +72,38 @@ class TestAccessProcessor(unittest.TestCase):
|
||||||
expected['query'] = query
|
expected['query'] = query
|
||||||
self.assertEquals(res, expected)
|
self.assertEquals(res, expected)
|
||||||
|
|
||||||
|
def test_log_line_parser_query_args_with_slash_delimiter_to_container(self):
|
||||||
|
p = access_processor.AccessLogProcessor({})
|
||||||
|
log_line = [str(x) for x in range(18)]
|
||||||
|
log_line[1] = 'proxy-server'
|
||||||
|
log_line[4] = '1/Jan/3/4/5/6'
|
||||||
|
query = 'prefix=YYYY/MM/DD'
|
||||||
|
log_line[6] = '/v1/a/c?%s' % query
|
||||||
|
log_line = 'x' * 16 + ' '.join(log_line)
|
||||||
|
res = p.log_line_parser(log_line)
|
||||||
|
|
||||||
|
self.assertEquals(res['object_name'], None)
|
||||||
|
self.assertEquals(res['container_name'], 'c')
|
||||||
|
self.assertEquals(res['account'], 'a')
|
||||||
|
self.assertEquals(res['request'], '/v1/a/c')
|
||||||
|
self.assertEquals(res['query'], query)
|
||||||
|
|
||||||
|
def test_log_line_parser_query_args_with_slash_delimiter_to_account(self):
|
||||||
|
p = access_processor.AccessLogProcessor({})
|
||||||
|
log_line = [str(x) for x in range(18)]
|
||||||
|
log_line[1] = 'proxy-server'
|
||||||
|
log_line[4] = '1/Jan/3/4/5/6'
|
||||||
|
query = 'prefix=YYYY/MM/DD'
|
||||||
|
log_line[6] = '/v1/a?%s' % query
|
||||||
|
log_line = 'x' * 16 + ' '.join(log_line)
|
||||||
|
res = p.log_line_parser(log_line)
|
||||||
|
|
||||||
|
self.assertEquals(res['object_name'], None)
|
||||||
|
self.assertEquals(res['container_name'], None)
|
||||||
|
self.assertEquals(res['account'], 'a')
|
||||||
|
self.assertEquals(res['request'], '/v1/a')
|
||||||
|
self.assertEquals(res['query'], query)
|
||||||
|
|
||||||
def test_log_line_parser_field_count(self):
|
def test_log_line_parser_field_count(self):
|
||||||
p = access_processor.AccessLogProcessor({})
|
p = access_processor.AccessLogProcessor({})
|
||||||
# too few fields
|
# too few fields
|
||||||
|
|
Loading…
Reference in New Issue