Properly read py2 object metadata on py3
Replicated, unencrypted metadata is written down differently on py2 vs py3, and has been since we started supporting py3. Fortunately, we can inspect the raw xattr bytes to determine whether the pickle was written using py2 or py3, so we can properly read legacy py2 meta under py3 rather than hitting a unicode error. Closes-Bug: #2012531 Change-Id: I5876e3b88f0bb1224299b57541788f590f64ddd4
This commit is contained in:
Tim Burke
parent
f99a6e5762
commit
7807540962
|
|
@ -30,6 +30,7 @@ from swift.container.backend import ContainerBroker, DATADIR as CBDATADIR
|
|||
from swift.obj.diskfile import get_data_dir, read_metadata, DATADIR_BASE, \
|
||||
extract_policy
|
||||
from swift.common.storage_policy import POLICIES
|
||||
from swift.common.swob import wsgi_to_str
|
||||
from swift.common.middleware.crypto.crypto_utils import load_crypto_meta
|
||||
from swift.common.utils import md5
|
||||
|
||||
|
|
@ -537,6 +538,8 @@ def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
|
|||
except EOFError:
|
||||
print("Invalid metadata")
|
||||
raise InfoSystemExit()
|
||||
metadata = {wsgi_to_str(k): v if k == 'name' else wsgi_to_str(v)
|
||||
for k, v in metadata.items()}
|
||||
|
||||
etag = metadata.pop('ETag', '')
|
||||
length = metadata.pop('Content-Length', '')
|
||||
|
|
|
|||
|
|
@ -167,24 +167,36 @@ def _encode_metadata(metadata):
|
|||
return dict(((encode_str(k), encode_str(v)) for k, v in metadata.items()))
|
||||
|
||||
|
||||
def _decode_metadata(metadata):
|
||||
def _decode_metadata(metadata, metadata_written_by_py3):
|
||||
"""
|
||||
Given a metadata dict from disk, convert keys and values to native strings.
|
||||
|
||||
:param metadata: a dict
|
||||
:param metadata_written_by_py3:
|
||||
"""
|
||||
if six.PY2:
|
||||
def to_str(item):
|
||||
def to_str(item, is_name=False):
|
||||
# For years, py2 and py3 handled non-ascii metadata differently;
|
||||
# see https://bugs.launchpad.net/swift/+bug/2012531
|
||||
if metadata_written_by_py3 and not is_name:
|
||||
# do our best to read new-style data replicated from a py3 node
|
||||
item = item.decode('utf8').encode('latin1')
|
||||
if isinstance(item, six.text_type):
|
||||
return item.encode('utf8')
|
||||
return item
|
||||
else:
|
||||
def to_str(item):
|
||||
def to_str(item, is_name=False):
|
||||
# For years, py2 and py3 handled non-ascii metadata differently;
|
||||
# see https://bugs.launchpad.net/swift/+bug/2012531
|
||||
if not metadata_written_by_py3 and isinstance(item, bytes) \
|
||||
and not is_name:
|
||||
# do our best to read old py2 data
|
||||
item = item.decode('latin1')
|
||||
if isinstance(item, six.binary_type):
|
||||
return item.decode('utf8', 'surrogateescape')
|
||||
return item
|
||||
|
||||
return dict(((to_str(k), to_str(v)) for k, v in metadata.items()))
|
||||
return {to_str(k): to_str(v, k == b'name') for k, v in metadata.items()}
|
||||
|
||||
|
||||
def read_metadata(fd, add_missing_checksum=False):
|
||||
|
|
@ -238,6 +250,7 @@ def read_metadata(fd, add_missing_checksum=False):
|
|||
"stored checksum='%s', computed='%s'" % (
|
||||
fd, metadata_checksum, computed_checksum))
|
||||
|
||||
metadata_written_by_py3 = (b'_codecs\nencode' in metadata[:32])
|
||||
# strings are utf-8 encoded when written, but have not always been
|
||||
# (see https://bugs.launchpad.net/swift/+bug/1678018) so encode them again
|
||||
# when read
|
||||
|
|
@ -245,7 +258,7 @@ def read_metadata(fd, add_missing_checksum=False):
|
|||
metadata = pickle.loads(metadata)
|
||||
else:
|
||||
metadata = pickle.loads(metadata, encoding='bytes')
|
||||
return _decode_metadata(metadata)
|
||||
return _decode_metadata(metadata, metadata_written_by_py3)
|
||||
|
||||
|
||||
def write_metadata(fd, metadata, xattr_size=65536):
|
||||
|
|
|
|||
|
|
@ -1219,7 +1219,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
|
|||
md = {'name': '/AUTH_admin/c/obj',
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
|
||||
'Content-Length': 0}
|
||||
'Content-Length': '0'}
|
||||
write_metadata(fp, md)
|
||||
|
||||
def test_print_obj(self):
|
||||
|
|
@ -1256,7 +1256,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
|
|||
md = {'name': '/AUTH_admin/c/obj',
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
|
||||
'Content-Length': 0}
|
||||
'Content-Length': '0'}
|
||||
write_metadata(fp, md)
|
||||
|
||||
object_ring = ring.Ring(self.testdir, ring_name='object-2')
|
||||
|
|
@ -1297,7 +1297,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
|
|||
md = {'name': '/AUTH_admin/c/obj',
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
|
||||
'Content-Length': 0}
|
||||
'Content-Length': '0'}
|
||||
write_metadata(fp, md)
|
||||
|
||||
object_ring = ring.Ring(self.testdir, ring_name='object-3')
|
||||
|
|
@ -1368,7 +1368,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
|
|||
md = {'name': '/AUTH_admin/c/obj',
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'ETag': 'badetag',
|
||||
'Content-Length': 0}
|
||||
'Content-Length': '0'}
|
||||
write_metadata(fp, md)
|
||||
|
||||
out = StringIO()
|
||||
|
|
@ -1703,5 +1703,5 @@ class TestPrintObjWeirdPath(TestPrintObjFullMeta):
|
|||
md = {'name': '/AUTH_admin/c/obj',
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
|
||||
'Content-Length': 0}
|
||||
'Content-Length': '0'}
|
||||
write_metadata(fp, md)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@
|
|||
|
||||
"""Tests for swift.obj.diskfile"""
|
||||
|
||||
from __future__ import print_function
|
||||
import binascii
|
||||
import six.moves.cPickle as pickle
|
||||
import os
|
||||
import errno
|
||||
|
|
@ -28,6 +30,7 @@ import uuid
|
|||
import xattr
|
||||
import re
|
||||
import six
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from random import shuffle, randint
|
||||
from shutil import rmtree
|
||||
|
|
@ -554,6 +557,233 @@ class TestDiskFileModuleMethods(unittest.TestCase):
|
|||
exp_dir = '/srv/node/sdb5/objects-1/123'
|
||||
self.assertEqual(part_dir, exp_dir)
|
||||
|
||||
def test_can_read_old_meta(self):
|
||||
# outputs taken from `xattr -l <diskfile>`
|
||||
cases = {
|
||||
'python_2.7.18_swift_2.13_replicated': '''
|
||||
0000 80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D ..}q.(U.Content-
|
||||
0010 4C 65 6E 67 74 68 71 02 55 02 31 33 55 04 6E 61 Lengthq.U.13U.na
|
||||
0020 6D 65 71 03 55 12 2F 41 55 54 48 5F 74 65 73 74 meq.U./AUTH_test
|
||||
0030 2F E2 98 83 2F E2 98 83 71 04 55 13 58 2D 4F 62 /.../...q.U.X-Ob
|
||||
0040 6A 65 63 74 2D 4D 65 74 61 2D 4D 74 69 6D 65 55 ject-Meta-MtimeU
|
||||
0050 11 31 36 38 32 39 35 39 38 37 34 2E 37 35 36 32 .1682959874.7562
|
||||
0060 30 35 71 05 55 04 45 54 61 67 71 06 55 20 36 62 05q.U.ETagq.U 6b
|
||||
0070 37 64 39 61 31 63 35 64 31 36 37 63 63 35 30 30 7d9a1c5d167cc500
|
||||
0080 33 37 66 32 39 66 32 39 30 62 62 33 37 35 71 07 37f29f290bb375q.
|
||||
0090 55 0B 58 2D 54 69 6D 65 73 74 61 6D 70 71 08 55 U.X-Timestampq.U
|
||||
00A0 10 31 36 38 32 39 36 32 36 35 31 2E 39 37 34 39 .1682962651.9749
|
||||
00B0 34 55 11 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 61 4U.X-Object-Meta
|
||||
00C0 2D E2 98 83 55 03 E2 98 83 71 09 55 0C 43 6F 6E -...U....q.U.Con
|
||||
00D0 74 65 6E 74 2D 54 79 70 65 71 0A 55 18 61 70 70 tent-Typeq.U.app
|
||||
00E0 6C 69 63 61 74 69 6F 6E 2F 6F 63 74 65 74 2D 73 lication/octet-s
|
||||
00F0 74 72 65 61 6D 71 0B 75 2E treamq.u.
|
||||
''',
|
||||
'python_2.7.18_swift_2.13_ec': '''
|
||||
0000 80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D ..}q.(U.Content-
|
||||
0010 4C 65 6E 67 74 68 71 02 55 02 38 34 55 04 6E 61 Lengthq.U.84U.na
|
||||
0020 6D 65 71 03 55 12 2F 41 55 54 48 5F 74 65 73 74 meq.U./AUTH_test
|
||||
0030 2F E2 98 83 2F E2 98 83 71 04 58 1E 00 00 00 58 /.../...q.X....X
|
||||
0040 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 2D -Object-Sysmeta-
|
||||
0050 45 63 2D 46 72 61 67 2D 49 6E 64 65 78 71 05 55 Ec-Frag-Indexq.U
|
||||
0060 01 35 55 13 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 .5U.X-Object-Met
|
||||
0070 61 2D 4D 74 69 6D 65 55 11 31 36 38 32 39 35 39 a-MtimeU.1682959
|
||||
0080 38 37 34 2E 37 35 36 32 30 35 71 06 58 22 00 00 874.756205q.X"..
|
||||
0090 00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 .X-Object-Sysmet
|
||||
00A0 61 2D 45 63 2D 43 6F 6E 74 65 6E 74 2D 4C 65 6E a-Ec-Content-Len
|
||||
00B0 67 74 68 71 07 55 02 31 33 71 08 58 18 00 00 00 gthq.U.13q.X....
|
||||
00C0 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 X-Object-Sysmeta
|
||||
00D0 2D 45 63 2D 45 74 61 67 71 09 55 20 36 62 37 64 -Ec-Etagq.U 6b7d
|
||||
00E0 39 61 31 63 35 64 31 36 37 63 63 35 30 30 33 37 9a1c5d167cc50037
|
||||
00F0 66 32 39 66 32 39 30 62 62 33 37 35 71 0A 55 04 f29f290bb375q.U.
|
||||
0100 45 54 61 67 71 0B 55 20 65 32 66 64 34 33 30 65 ETagq.U e2fd430e
|
||||
0110 61 66 37 32 32 33 63 32 35 30 33 63 34 65 38 33 af7223c2503c4e83
|
||||
0120 30 31 63 66 66 33 37 63 71 0C 55 0B 58 2D 54 69 01cff37cq.U.X-Ti
|
||||
0130 6D 65 73 74 61 6D 70 71 0D 55 10 31 36 38 32 39 mestampq.U.16829
|
||||
0140 36 32 32 36 32 2E 31 36 31 39 39 55 11 58 2D 4F 62262.16199U.X-O
|
||||
0150 62 6A 65 63 74 2D 4D 65 74 61 2D E2 98 83 55 03 bject-Meta-...U.
|
||||
0160 E2 98 83 71 0E 58 1A 00 00 00 58 2D 4F 62 6A 65 ...q.X....X-Obje
|
||||
0170 63 74 2D 53 79 73 6D 65 74 61 2D 45 63 2D 53 63 ct-Sysmeta-Ec-Sc
|
||||
0180 68 65 6D 65 71 0F 55 1A 6C 69 62 65 72 61 73 75 hemeq.U.liberasu
|
||||
0190 72 65 63 6F 64 65 5F 72 73 5F 76 61 6E 64 20 34 recode_rs_vand 4
|
||||
01A0 2B 32 71 10 55 0C 43 6F 6E 74 65 6E 74 2D 54 79 +2q.U.Content-Ty
|
||||
01B0 70 65 71 11 55 18 61 70 70 6C 69 63 61 74 69 6F peq.U.applicatio
|
||||
01C0 6E 2F 6F 63 74 65 74 2D 73 74 72 65 61 6D 71 12 n/octet-streamq.
|
||||
01D0 58 20 00 00 00 58 2D 4F 62 6A 65 63 74 2D 53 79 X ...X-Object-Sy
|
||||
01E0 73 6D 65 74 61 2D 45 63 2D 53 65 67 6D 65 6E 74 smeta-Ec-Segment
|
||||
01F0 2D 53 69 7A 65 71 13 55 07 31 30 34 38 35 37 36 -Sizeq.U.1048576
|
||||
0200 71 14 75 2E q.u.
|
||||
''',
|
||||
'python_2.7.18_swift_2.23_replicated': '''
|
||||
0000 80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D ..}q.(U.Content-
|
||||
0010 4C 65 6E 67 74 68 71 02 55 02 31 33 71 03 55 04 Lengthq.U.13q.U.
|
||||
0020 6E 61 6D 65 71 04 55 12 2F 41 55 54 48 5F 74 65 nameq.U./AUTH_te
|
||||
0030 73 74 2F E2 98 83 2F E2 98 83 71 05 55 0C 43 6F st/.../...q.U.Co
|
||||
0040 6E 74 65 6E 74 2D 54 79 70 65 71 06 55 18 61 70 ntent-Typeq.U.ap
|
||||
0050 70 6C 69 63 61 74 69 6F 6E 2F 6F 63 74 65 74 2D plication/octet-
|
||||
0060 73 74 72 65 61 6D 71 07 55 04 45 54 61 67 71 08 streamq.U.ETagq.
|
||||
0070 55 20 36 62 37 64 39 61 31 63 35 64 31 36 37 63 U 6b7d9a1c5d167c
|
||||
0080 63 35 30 30 33 37 66 32 39 66 32 39 30 62 62 33 c50037f29f290bb3
|
||||
0090 37 35 71 09 55 0B 58 2D 54 69 6D 65 73 74 61 6D 75q.U.X-Timestam
|
||||
00A0 70 71 0A 55 10 31 36 38 32 39 36 33 32 30 39 2E pq.U.1682963209.
|
||||
00B0 38 32 32 37 32 71 0B 55 11 58 2D 4F 62 6A 65 63 82272q.U.X-Objec
|
||||
00C0 74 2D 4D 65 74 61 2D E2 98 83 71 0C 55 03 E2 98 t-Meta-...q.U...
|
||||
00D0 83 71 0D 55 13 58 2D 4F 62 6A 65 63 74 2D 4D 65 .q.U.X-Object-Me
|
||||
00E0 74 61 2D 4D 74 69 6D 65 71 0E 55 11 31 36 38 32 ta-Mtimeq.U.1682
|
||||
00F0 39 35 39 38 37 34 2E 37 35 36 32 30 35 71 0F 75 959874.756205q.u
|
||||
0100 2E .
|
||||
''',
|
||||
'python_3.10.6_swift_2.23_replicated': '''
|
||||
0000 80 02 7D 71 00 28 63 5F 63 6F 64 65 63 73 0A 65 ..}q.(c_codecs.e
|
||||
0010 6E 63 6F 64 65 0A 71 01 58 0B 00 00 00 58 2D 54 ncode.q.X....X-T
|
||||
0020 69 6D 65 73 74 61 6D 70 71 02 58 06 00 00 00 6C imestampq.X....l
|
||||
0030 61 74 69 6E 31 71 03 86 71 04 52 71 05 68 01 58 atin1q..q.Rq.h.X
|
||||
0040 10 00 00 00 31 36 38 32 39 36 33 30 31 37 2E 31 ....1682963017.1
|
||||
0050 30 34 37 32 71 06 68 03 86 71 07 52 71 08 68 01 0472q.h..q.Rq.h.
|
||||
0060 58 0C 00 00 00 43 6F 6E 74 65 6E 74 2D 54 79 70 X....Content-Typ
|
||||
0070 65 71 09 68 03 86 71 0A 52 71 0B 68 01 58 18 00 eq.h..q.Rq.h.X..
|
||||
0080 00 00 61 70 70 6C 69 63 61 74 69 6F 6E 2F 6F 63 ..application/oc
|
||||
0090 74 65 74 2D 73 74 72 65 61 6D 71 0C 68 03 86 71 tet-streamq.h..q
|
||||
00A0 0D 52 71 0E 68 01 58 0E 00 00 00 43 6F 6E 74 65 .Rq.h.X....Conte
|
||||
00B0 6E 74 2D 4C 65 6E 67 74 68 71 0F 68 03 86 71 10 nt-Lengthq.h..q.
|
||||
00C0 52 71 11 68 01 58 02 00 00 00 31 33 71 12 68 03 Rq.h.X....13q.h.
|
||||
00D0 86 71 13 52 71 14 68 01 58 04 00 00 00 45 54 61 .q.Rq.h.X....ETa
|
||||
00E0 67 71 15 68 03 86 71 16 52 71 17 68 01 58 20 00 gq.h..q.Rq.h.X .
|
||||
00F0 00 00 36 62 37 64 39 61 31 63 35 64 31 36 37 63 ..6b7d9a1c5d167c
|
||||
0100 63 35 30 30 33 37 66 32 39 66 32 39 30 62 62 33 c50037f29f290bb3
|
||||
0110 37 35 71 18 68 03 86 71 19 52 71 1A 68 01 58 13 75q.h..q.Rq.h.X.
|
||||
0120 00 00 00 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 61 ...X-Object-Meta
|
||||
0130 2D 4D 74 69 6D 65 71 1B 68 03 86 71 1C 52 71 1D -Mtimeq.h..q.Rq.
|
||||
0140 68 01 58 11 00 00 00 31 36 38 32 39 35 39 38 37 h.X....168295987
|
||||
0150 34 2E 37 35 36 32 30 35 71 1E 68 03 86 71 1F 52 4.756205q.h..q.R
|
||||
0160 71 20 68 01 58 1A 00 00 00 58 2D 4F 62 6A 65 63 q h.X....X-Objec
|
||||
0170 74 2D 4D 65 74 61 2D C3 83 C2 A2 C3 82 C2 98 C3 t-Meta-.........
|
||||
0180 82 C2 83 71 21 68 03 86 71 22 52 71 23 68 01 58 ...q!h..q"Rq#h.X
|
||||
0190 0C 00 00 00 C3 83 C2 A2 C3 82 C2 98 C3 82 C2 83 ................
|
||||
01A0 71 24 68 03 86 71 25 52 71 26 68 01 58 04 00 00 q$h..q%Rq&h.X...
|
||||
01B0 00 6E 61 6D 65 71 27 68 03 86 71 28 52 71 29 68 .nameq'h..q(Rq)h
|
||||
01C0 01 58 18 00 00 00 2F 41 55 54 48 5F 74 65 73 74 .X..../AUTH_test
|
||||
01D0 2F C3 A2 C2 98 C2 83 2F C3 A2 C2 98 C2 83 71 2A /....../......q*
|
||||
01E0 68 03 86 71 2B 52 71 2C 75 2E h..q+Rq,u.
|
||||
''',
|
||||
'python_2.7.18_swift_2.23_ec': '''
|
||||
0000 80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D ..}q.(U.Content-
|
||||
0010 4C 65 6E 67 74 68 71 02 55 02 38 34 71 03 55 04 Lengthq.U.84q.U.
|
||||
0020 6E 61 6D 65 71 04 55 12 2F 41 55 54 48 5F 74 65 nameq.U./AUTH_te
|
||||
0030 73 74 2F E2 98 83 2F E2 98 83 71 05 55 1E 58 2D st/.../...q.U.X-
|
||||
0040 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 2D 45 Object-Sysmeta-E
|
||||
0050 63 2D 46 72 61 67 2D 49 6E 64 65 78 55 01 35 55 c-Frag-IndexU.5U
|
||||
0060 0C 43 6F 6E 74 65 6E 74 2D 54 79 70 65 71 06 55 .Content-Typeq.U
|
||||
0070 18 61 70 70 6C 69 63 61 74 69 6F 6E 2F 6F 63 74 .application/oct
|
||||
0080 65 74 2D 73 74 72 65 61 6D 71 07 55 22 58 2D 4F et-streamq.U"X-O
|
||||
0090 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 2D 45 63 bject-Sysmeta-Ec
|
||||
00A0 2D 43 6F 6E 74 65 6E 74 2D 4C 65 6E 67 74 68 55 -Content-LengthU
|
||||
00B0 02 31 33 71 08 55 18 58 2D 4F 62 6A 65 63 74 2D .13q.U.X-Object-
|
||||
00C0 53 79 73 6D 65 74 61 2D 45 63 2D 45 74 61 67 55 Sysmeta-Ec-EtagU
|
||||
00D0 20 36 62 37 64 39 61 31 63 35 64 31 36 37 63 63 6b7d9a1c5d167cc
|
||||
00E0 35 30 30 33 37 66 32 39 66 32 39 30 62 62 33 37 50037f29f290bb37
|
||||
00F0 35 71 09 55 04 45 54 61 67 71 0A 55 20 65 32 66 5q.U.ETagq.U e2f
|
||||
0100 64 34 33 30 65 61 66 37 32 32 33 63 32 35 30 33 d430eaf7223c2503
|
||||
0110 63 34 65 38 33 30 31 63 66 66 33 37 63 71 0B 55 c4e8301cff37cq.U
|
||||
0120 0B 58 2D 54 69 6D 65 73 74 61 6D 70 71 0C 55 10 .X-Timestampq.U.
|
||||
0130 31 36 38 32 39 36 33 31 33 30 2E 33 35 39 38 36 1682963130.35986
|
||||
0140 71 0D 55 11 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 q.U.X-Object-Met
|
||||
0150 61 2D E2 98 83 71 0E 55 03 E2 98 83 71 0F 55 1A a-...q.U....q.U.
|
||||
0160 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 X-Object-Sysmeta
|
||||
0170 2D 45 63 2D 53 63 68 65 6D 65 55 1A 6C 69 62 65 -Ec-SchemeU.libe
|
||||
0180 72 61 73 75 72 65 63 6F 64 65 5F 72 73 5F 76 61 rasurecode_rs_va
|
||||
0190 6E 64 20 34 2B 32 71 10 55 13 58 2D 4F 62 6A 65 nd 4+2q.U.X-Obje
|
||||
01A0 63 74 2D 4D 65 74 61 2D 4D 74 69 6D 65 71 11 55 ct-Meta-Mtimeq.U
|
||||
01B0 11 31 36 38 32 39 35 39 38 37 34 2E 37 35 36 32 .1682959874.7562
|
||||
01C0 30 35 71 12 55 20 58 2D 4F 62 6A 65 63 74 2D 53 05q.U X-Object-S
|
||||
01D0 79 73 6D 65 74 61 2D 45 63 2D 53 65 67 6D 65 6E ysmeta-Ec-Segmen
|
||||
01E0 74 2D 53 69 7A 65 55 07 31 30 34 38 35 37 36 71 t-SizeU.1048576q
|
||||
01F0 13 75 2E .u.
|
||||
''',
|
||||
'python_3.10.6_swift_2.23_ec': '''
|
||||
0000 80 02 7D 71 00 28 63 5F 63 6F 64 65 63 73 0A 65 ..}q.(c_codecs.e
|
||||
0010 6E 63 6F 64 65 0A 71 01 58 0B 00 00 00 58 2D 54 ncode.q.X....X-T
|
||||
0020 69 6D 65 73 74 61 6D 70 71 02 58 06 00 00 00 6C imestampq.X....l
|
||||
0030 61 74 69 6E 31 71 03 86 71 04 52 71 05 68 01 58 atin1q..q.Rq.h.X
|
||||
0040 10 00 00 00 31 36 38 32 39 36 32 39 35 35 2E 33 ....1682962955.3
|
||||
0050 37 35 34 36 71 06 68 03 86 71 07 52 71 08 68 01 7546q.h..q.Rq.h.
|
||||
0060 58 0C 00 00 00 43 6F 6E 74 65 6E 74 2D 54 79 70 X....Content-Typ
|
||||
0070 65 71 09 68 03 86 71 0A 52 71 0B 68 01 58 18 00 eq.h..q.Rq.h.X..
|
||||
0080 00 00 61 70 70 6C 69 63 61 74 69 6F 6E 2F 6F 63 ..application/oc
|
||||
0090 74 65 74 2D 73 74 72 65 61 6D 71 0C 68 03 86 71 tet-streamq.h..q
|
||||
00A0 0D 52 71 0E 68 01 58 0E 00 00 00 43 6F 6E 74 65 .Rq.h.X....Conte
|
||||
00B0 6E 74 2D 4C 65 6E 67 74 68 71 0F 68 03 86 71 10 nt-Lengthq.h..q.
|
||||
00C0 52 71 11 68 01 58 02 00 00 00 38 34 71 12 68 03 Rq.h.X....84q.h.
|
||||
00D0 86 71 13 52 71 14 68 01 58 04 00 00 00 45 54 61 .q.Rq.h.X....ETa
|
||||
00E0 67 71 15 68 03 86 71 16 52 71 17 68 01 58 20 00 gq.h..q.Rq.h.X .
|
||||
00F0 00 00 65 32 66 64 34 33 30 65 61 66 37 32 32 33 ..e2fd430eaf7223
|
||||
0100 63 32 35 30 33 63 34 65 38 33 30 31 63 66 66 33 c2503c4e8301cff3
|
||||
0110 37 63 71 18 68 03 86 71 19 52 71 1A 68 01 58 13 7cq.h..q.Rq.h.X.
|
||||
0120 00 00 00 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 61 ...X-Object-Meta
|
||||
0130 2D 4D 74 69 6D 65 71 1B 68 03 86 71 1C 52 71 1D -Mtimeq.h..q.Rq.
|
||||
0140 68 01 58 11 00 00 00 31 36 38 32 39 35 39 38 37 h.X....168295987
|
||||
0150 34 2E 37 35 36 32 30 35 71 1E 68 03 86 71 1F 52 4.756205q.h..q.R
|
||||
0160 71 20 68 01 58 1A 00 00 00 58 2D 4F 62 6A 65 63 q h.X....X-Objec
|
||||
0170 74 2D 4D 65 74 61 2D C3 83 C2 A2 C3 82 C2 98 C3 t-Meta-.........
|
||||
0180 82 C2 83 71 21 68 03 86 71 22 52 71 23 68 01 58 ...q!h..q"Rq#h.X
|
||||
0190 0C 00 00 00 C3 83 C2 A2 C3 82 C2 98 C3 82 C2 83 ................
|
||||
01A0 71 24 68 03 86 71 25 52 71 26 68 01 58 18 00 00 q$h..q%Rq&h.X...
|
||||
01B0 00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 .X-Object-Sysmet
|
||||
01C0 61 2D 45 63 2D 45 74 61 67 71 27 68 03 86 71 28 a-Ec-Etagq'h..q(
|
||||
01D0 52 71 29 68 01 58 20 00 00 00 36 62 37 64 39 61 Rq)h.X ...6b7d9a
|
||||
01E0 31 63 35 64 31 36 37 63 63 35 30 30 33 37 66 32 1c5d167cc50037f2
|
||||
01F0 39 66 32 39 30 62 62 33 37 35 71 2A 68 03 86 71 9f290bb375q*h..q
|
||||
0200 2B 52 71 2C 68 01 58 22 00 00 00 58 2D 4F 62 6A +Rq,h.X"...X-Obj
|
||||
0210 65 63 74 2D 53 79 73 6D 65 74 61 2D 45 63 2D 43 ect-Sysmeta-Ec-C
|
||||
0220 6F 6E 74 65 6E 74 2D 4C 65 6E 67 74 68 71 2D 68 ontent-Lengthq-h
|
||||
0230 03 86 71 2E 52 71 2F 68 01 58 02 00 00 00 31 33 ..q.Rq/h.X....13
|
||||
0240 71 30 68 03 86 71 31 52 71 32 68 01 58 1E 00 00 q0h..q1Rq2h.X...
|
||||
0250 00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 .X-Object-Sysmet
|
||||
0260 61 2D 45 63 2D 46 72 61 67 2D 49 6E 64 65 78 71 a-Ec-Frag-Indexq
|
||||
0270 33 68 03 86 71 34 52 71 35 68 01 58 01 00 00 00 3h..q4Rq5h.X....
|
||||
0280 35 71 36 68 03 86 71 37 52 71 38 68 01 58 1A 00 5q6h..q7Rq8h.X..
|
||||
0290 00 00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 ..X-Object-Sysme
|
||||
02A0 74 61 2D 45 63 2D 53 63 68 65 6D 65 71 39 68 03 ta-Ec-Schemeq9h.
|
||||
02B0 86 71 3A 52 71 3B 68 01 58 1A 00 00 00 6C 69 62 .q:Rq;h.X....lib
|
||||
02C0 65 72 61 73 75 72 65 63 6F 64 65 5F 72 73 5F 76 erasurecode_rs_v
|
||||
02D0 61 6E 64 20 34 2B 32 71 3C 68 03 86 71 3D 52 71 and 4+2q<h..q=Rq
|
||||
02E0 3E 68 01 58 20 00 00 00 58 2D 4F 62 6A 65 63 74 >h.X ...X-Object
|
||||
02F0 2D 53 79 73 6D 65 74 61 2D 45 63 2D 53 65 67 6D -Sysmeta-Ec-Segm
|
||||
0300 65 6E 74 2D 53 69 7A 65 71 3F 68 03 86 71 40 52 ent-Sizeq?h..q@R
|
||||
0310 71 41 68 01 58 07 00 00 00 31 30 34 38 35 37 36 qAh.X....1048576
|
||||
0320 71 42 68 03 86 71 43 52 71 44 68 01 58 04 00 00 qBh..qCRqDh.X...
|
||||
0330 00 6E 61 6D 65 71 45 68 03 86 71 46 52 71 47 68 .nameqEh..qFRqGh
|
||||
0340 01 58 18 00 00 00 2F 41 55 54 48 5F 74 65 73 74 .X..../AUTH_test
|
||||
0350 2F C3 A2 C2 98 C2 83 2F C3 A2 C2 98 C2 83 71 48 /....../......qH
|
||||
0360 68 03 86 71 49 52 71 4A 75 2E h..qIRqJu.
|
||||
''',
|
||||
}
|
||||
|
||||
def raw_xattr(output):
|
||||
return binascii.unhexlify(''.join(
|
||||
line[7:55] for line in output.split('\n')
|
||||
).replace(' ', ''))
|
||||
|
||||
path = os.path.join(self.testdir, str(uuid.uuid4()))
|
||||
for case, xattr_output in cases.items():
|
||||
try:
|
||||
to_write = raw_xattr(xattr_output)
|
||||
with open(path, 'wb') as fp:
|
||||
xattr.setxattr(
|
||||
fp.fileno(), 'user.swift.metadata', to_write)
|
||||
with open(path, 'rb') as fd:
|
||||
actual = diskfile.read_metadata(fd)
|
||||
# name should come out as native strings
|
||||
expected_name = b'/AUTH_test/\xe2\x98\x83/\xe2\x98\x83'
|
||||
if not six.PY2:
|
||||
expected_name = expected_name.decode('utf8')
|
||||
self.assertEqual(actual['name'], expected_name)
|
||||
# other meta will be WSGI strings, though
|
||||
self.assertEqual(
|
||||
actual['X-Object-Meta-\xe2\x98\x83'], '\xe2\x98\x83')
|
||||
except Exception:
|
||||
print('Failure in %s' % case, file=sys.stderr)
|
||||
raise
|
||||
|
||||
def test_write_read_metadata(self):
|
||||
path = os.path.join(self.testdir, str(uuid.uuid4()))
|
||||
metadata = {'name': '/a/c/o',
|
||||
|
|
@ -597,29 +827,6 @@ class TestDiskFileModuleMethods(unittest.TestCase):
|
|||
diskfile.write_metadata(fd, metadata)
|
||||
check_metadata(as_native, str)
|
||||
|
||||
# mock the read path to check the write path encoded persisted metadata
|
||||
with mock.patch.object(diskfile, '_decode_metadata', lambda x: x):
|
||||
check_metadata(as_bytes, bytes)
|
||||
|
||||
# simulate a legacy diskfile that might have persisted
|
||||
# (some) unicode metadata
|
||||
with mock.patch.object(diskfile, '_encode_metadata', lambda x: x):
|
||||
with open(path, 'wb') as fd:
|
||||
diskfile.write_metadata(fd, metadata)
|
||||
# sanity check: mock read path again to see that we did persist unicode
|
||||
with mock.patch.object(diskfile, '_decode_metadata', lambda x: x):
|
||||
with open(path, 'rb') as fd:
|
||||
actual = diskfile.read_metadata(fd)
|
||||
for k, v in actual.items():
|
||||
if isinstance(k, six.text_type) and \
|
||||
k == u'X-Object-Meta-Strange':
|
||||
self.assertIsInstance(v, six.text_type)
|
||||
break
|
||||
else:
|
||||
self.fail('Did not find X-Object-Meta-Strange')
|
||||
# check that read_metadata converts binary_type
|
||||
check_metadata(as_native, str)
|
||||
|
||||
|
||||
@patch_policies
|
||||
class TestObjectAuditLocationGenerator(unittest.TestCase):
|
||||
|
|
|
|||
Loading…
Reference in New Issue