openstack
/
deb-python-kafka
Code Issues Proposed changes

LZ4 support in kafka 0.8/0.9 does not accept a ContentSize header

This commit is contained in:
Dana Powers 2017-03-14 11:01:58 -07:00
parent fb023fe85d
commit fea10d9c16
1 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
kafka/codec.py
View File

@ -229,13 +229,21 @@ def lz4_encode_old_kafka(payload):
assert xxhash is not None assert xxhash is not None
data = lz4_encode(payload) data = lz4_encode(payload)
header_size = 7 header_size = 7
if isinstance(data[4], int): flg = data[4]
flg = data[4] if not isinstance(flg, int):
else: flg = ord(flg)
flg = ord(data[4])
content_size_bit = ((flg >> 3) & 1) content_size_bit = ((flg >> 3) & 1)
if content_size_bit: if content_size_bit:
header_size += 8 # Old kafka does not accept the content-size field
# so we need to discard it and reset the header flag
flg -= 8
data = bytearray(data)
data[4] = flg
data = bytes(data)
payload = data[header_size+8:]
else:
payload = data[header_size:]
# This is the incorrect hc # This is the incorrect hc
hc = xxhash.xxh32(data[0:header_size-1]).digest()[-2:-1] # pylint: disable-msg=no-member hc = xxhash.xxh32(data[0:header_size-1]).digest()[-2:-1] # pylint: disable-msg=no-member
@ -243,7 +251,7 @@ def lz4_encode_old_kafka(payload):
return b''.join([ return b''.join([
data[0:header_size-1], data[0:header_size-1],
hc, hc,
data[header_size:] payload
]) ])