Merge master to feature/ec
Conflicts: test/unit/obj/test_server.py Change-Id: I4b4efa6fd8816fc9f059e488ab0e2a0454e0245e
This commit is contained in:
paul luse
commit
621089e7be
1
.mailmap
1
.mailmap
|
|
@ -68,3 +68,4 @@ Pawel Palucki <pawel.palucki@gmail.com> <pawel.palucki@gmail.com>
|
|||
Guang Yee <guang.yee@hp.com> <guang.yee@hp.com>
|
||||
Jing Liuqing <jing.liuqing@99cloud.net> <jing.liuqing@99cloud.net>
|
||||
Lorcan Browne <lorcan.browne@hp.com> <lorcan.browne@hp.com>
|
||||
Eohyung Lee <liquidnuker@gmail.com> <liquid@kt.com>
|
||||
|
|
|
|||
3
AUTHORS
3
AUTHORS
|
|
@ -46,6 +46,7 @@ Thiago da Silva (thiago@redhat.com)
|
|||
Julien Danjou (julien@danjou.info)
|
||||
Ksenia Demina (kdemina@mirantis.com)
|
||||
Dan Dillinger (dan.dillinger@sonian.net)
|
||||
Cedric Dos Santos (cedric.dos.sant@gmail.com)
|
||||
Gerry Drudy (gerry.drudy@hp.com)
|
||||
Morgan Fainberg (morgan.fainberg@gmail.com)
|
||||
ZhiQiang Fan (aji.zqfan@gmail.com)
|
||||
|
|
@ -134,6 +135,7 @@ Colin Nicholson (colin.nicholson@iomart.com)
|
|||
Zhenguo Niu (zhenguo@unitedstack.com)
|
||||
Timothy Okwii (tokwii@cisco.com)
|
||||
Matthew Oliver (matt@oliver.net.au)
|
||||
Hisashi Osanai (osanai.hisashi@jp.fujitsu.com)
|
||||
Eamonn O'Toole (eamonn.otoole@hp.com)
|
||||
James Page (james.page@ubuntu.com)
|
||||
Prashanth Pai (ppai@redhat.com)
|
||||
|
|
@ -151,6 +153,7 @@ Rafael Rivero (rafael@cloudscaling.com)
|
|||
Victor Rodionov (victor.rodionov@nexenta.com)
|
||||
Aaron Rosen (arosen@nicira.com)
|
||||
Brent Roskos (broskos@internap.com)
|
||||
Shilla Saebi (shilla.saebi@gmail.com)
|
||||
Cristian A Sanchez (cristian.a.sanchez@intel.com)
|
||||
saranjan (saranjan@cisco.com)
|
||||
Christian Schwede (info@cschwede.de)
|
||||
|
|
|
|||
47
CHANGELOG
47
CHANGELOG
|
|
@ -1,3 +1,50 @@
|
|||
swift (2.2.1)
|
||||
|
||||
* Swift now rejects object names with Unicode surrogates.
|
||||
|
||||
* Return 403 (instead of 413) on unauthorized upload when over account
|
||||
quota.
|
||||
|
||||
* Fix a rare condition when a rebalance could cause swift-ring-builder
|
||||
to crash. This would only happen on old ring files when "rebalance"
|
||||
was the first command run.
|
||||
|
||||
* Storage node error limits now survive a ring reload.
|
||||
|
||||
* Speed up reading and writing xattrs for object metadata by using larger
|
||||
xattr value sizes. The change is moving from 254 byte values to 64KiB
|
||||
values. There is no migration issue with this.
|
||||
|
||||
* Deleted containers beyond the reclaim age are now properly reclaimed.
|
||||
|
||||
* Full Simplified Chinese translation (zh_CN locale) for errors and logs.
|
||||
|
||||
* Container quota is now properly enforced during cross-account COPY.
|
||||
|
||||
* ssync replication now properly uses the configured replication_ip.
|
||||
|
||||
* Fixed issue were ssync did not replicate custom object headers.
|
||||
|
||||
* swift-drive-audit now has the 'unmount_failed_device' config option
|
||||
(default to True) that controls if the process will unmount failed
|
||||
drives or not.
|
||||
|
||||
* swift-drive-audit will now dump drive error rates to a recon file.
|
||||
The file location is controlled by the 'recon_cache_path' config value
|
||||
and it includes each drive and its associated number of errors.
|
||||
|
||||
* When a filesystem does't support xattr, the object server now returns
|
||||
a 507 Insufficient Storage error to the proxy server.
|
||||
|
||||
* Clean up empty account and container partitions directories if they
|
||||
are empty. This keeps the system healthy and prevents a large number
|
||||
of empty directories from slowing down the replication process.
|
||||
|
||||
* Show the sum of every policy's amount of async pendings in swift-recon.
|
||||
|
||||
* Various other minor bug fixes and improvements.
|
||||
|
||||
|
||||
swift (2.2.0)
|
||||
|
||||
* Added support for Keystone v3 auth.
|
||||
|
|
|
|||
|
|
@ -150,6 +150,7 @@ if __name__ == '__main__':
|
|||
recon_cache_path = conf.get('recon_cache_path', "/var/cache/swift")
|
||||
log_file_pattern = conf.get('log_file_pattern',
|
||||
'/var/log/kern.*[!.][!g][!z]')
|
||||
log_to_console = config_true_value(conf.get('log_to_console', False))
|
||||
error_re = []
|
||||
for conf_key in conf:
|
||||
if conf_key.startswith('regex_pattern_'):
|
||||
|
|
@ -166,7 +167,8 @@ if __name__ == '__main__':
|
|||
re.compile(r'\b(sd[a-z]{1,2}\d?)\b.*\berror\b'),
|
||||
]
|
||||
conf['log_name'] = conf.get('log_name', 'drive-audit')
|
||||
logger = get_logger(conf, log_route='drive-audit')
|
||||
logger = get_logger(conf, log_to_console=log_to_console,
|
||||
log_route='drive-audit')
|
||||
devices = get_devices(device_dir, logger)
|
||||
logger.debug("Devices found: %s" % str(devices))
|
||||
if not devices:
|
||||
|
|
|
|||
|
|
@ -92,9 +92,6 @@ Other
|
|||
* `Swiftsync <https://github.com/stackforge/swiftsync>`_ - A massive syncer between two swift clusters.
|
||||
* `Django Swiftbrowser <https://github.com/cschwede/django-swiftbrowser>`_ - Simple Django web app to access Openstack Swift.
|
||||
* `Swift-account-stats <https://github.com/enovance/swift-account-stats>`_ - Swift-account-stats is a tool to report statistics on Swift usage at tenant and global levels.
|
||||
<<<<<<< HEAD
|
||||
* `PyECLib <https://bitbucket.org/kmgreen2/pyeclib>`_ - Erasure Code library used by Swift
|
||||
* TODO: tsg add liberasurecode reference here
|
||||
=======
|
||||
* `PyECLib <https://bitbucket.org/kmgreen2/pyeclib>`_ - High Level Erasure Code library used by Swift
|
||||
* `liberasurecode <http://www.bytebucket.org/tsg-/liberasurecode>`_ - Low Level Erasure Code library used by PyECLib
|
||||
* `Swift Browser <https://github.com/zerovm/swift-browser>`_ - JavaScript interface for Swift
|
||||
>>>>>>> remotes/origin/master
|
||||
|
|
|
|||
|
|
@ -11,6 +11,10 @@
|
|||
# recon_cache_path = /var/cache/swift
|
||||
# unmount_failed_device = True
|
||||
#
|
||||
# By default, drive-audit logs only to syslog. Setting this option True
|
||||
# makes drive-audit log to console in addition to syslog.
|
||||
# log_to_console = False
|
||||
#
|
||||
# Location of the log file with globbing
|
||||
# pattern to check against device errors.
|
||||
# log_file_pattern = /var/log/kern.*[!.][!g][!z]
|
||||
|
|
|
|||
|
|
@ -115,8 +115,6 @@ class CNAMELookupMiddleware(object):
|
|||
port = ''
|
||||
if ':' in given_domain:
|
||||
given_domain, port = given_domain.rsplit(':', 1)
|
||||
if given_domain == self.storage_domain[1:]: # strip initial '.'
|
||||
return self.app(env, start_response)
|
||||
if is_ip(given_domain):
|
||||
return self.app(env, start_response)
|
||||
a_domain = given_domain
|
||||
|
|
|
|||
|
|
@ -0,0 +1,199 @@
|
|||
# Copyright (c) 2014 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
'''
|
||||
Bindings to the `tee` and `splice` system calls
|
||||
'''
|
||||
|
||||
import os
|
||||
import operator
|
||||
|
||||
import ctypes
|
||||
import ctypes.util
|
||||
|
||||
__all__ = ['tee', 'splice']
|
||||
|
||||
|
||||
c_loff_t = ctypes.c_long
|
||||
|
||||
|
||||
class Tee(object):
|
||||
'''Binding to `tee`'''
|
||||
|
||||
__slots__ = '_c_tee',
|
||||
|
||||
def __init__(self):
|
||||
libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True)
|
||||
|
||||
try:
|
||||
c_tee = libc.tee
|
||||
except AttributeError:
|
||||
self._c_tee = None
|
||||
return
|
||||
|
||||
c_tee.argtypes = [
|
||||
ctypes.c_int,
|
||||
ctypes.c_int,
|
||||
ctypes.c_size_t,
|
||||
ctypes.c_uint
|
||||
]
|
||||
|
||||
c_tee.restype = ctypes.c_ssize_t
|
||||
|
||||
def errcheck(result, func, arguments):
|
||||
if result == -1:
|
||||
errno = ctypes.set_errno(0)
|
||||
|
||||
raise IOError(errno, 'tee: %s' % os.strerror(errno))
|
||||
else:
|
||||
return result
|
||||
|
||||
c_tee.errcheck = errcheck
|
||||
|
||||
self._c_tee = c_tee
|
||||
|
||||
def __call__(self, fd_in, fd_out, len_, flags):
|
||||
'''See `man 2 tee`
|
||||
|
||||
File-descriptors can be file-like objects with a `fileno` method, or
|
||||
integers.
|
||||
|
||||
Flags can be an integer value, or a list of flags (exposed on
|
||||
`splice`).
|
||||
|
||||
This function returns the number of bytes transferred (i.e. the actual
|
||||
result of the call to `tee`).
|
||||
|
||||
Upon other errors, an `IOError` is raised with the proper `errno` set.
|
||||
'''
|
||||
|
||||
if not self.available:
|
||||
raise EnvironmentError('tee not available')
|
||||
|
||||
if not isinstance(flags, (int, long)):
|
||||
c_flags = reduce(operator.or_, flags, 0)
|
||||
else:
|
||||
c_flags = flags
|
||||
|
||||
c_fd_in = getattr(fd_in, 'fileno', lambda: fd_in)()
|
||||
c_fd_out = getattr(fd_out, 'fileno', lambda: fd_out)()
|
||||
|
||||
return self._c_tee(c_fd_in, c_fd_out, len_, c_flags)
|
||||
|
||||
@property
|
||||
def available(self):
|
||||
'''Availability of `tee`'''
|
||||
|
||||
return self._c_tee is not None
|
||||
|
||||
tee = Tee()
|
||||
del Tee
|
||||
|
||||
|
||||
class Splice(object):
|
||||
'''Binding to `splice`'''
|
||||
|
||||
# From `bits/fcntl-linux.h`
|
||||
SPLICE_F_MOVE = 1
|
||||
SPLICE_F_NONBLOCK = 2
|
||||
SPLICE_F_MORE = 4
|
||||
SPLICE_F_GIFT = 8
|
||||
|
||||
__slots__ = '_c_splice',
|
||||
|
||||
def __init__(self):
|
||||
libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True)
|
||||
|
||||
try:
|
||||
c_splice = libc.splice
|
||||
except AttributeError:
|
||||
self._c_splice = None
|
||||
return
|
||||
|
||||
c_loff_t_p = ctypes.POINTER(c_loff_t)
|
||||
|
||||
c_splice.argtypes = [
|
||||
ctypes.c_int, c_loff_t_p,
|
||||
ctypes.c_int, c_loff_t_p,
|
||||
ctypes.c_size_t,
|
||||
ctypes.c_uint
|
||||
]
|
||||
|
||||
c_splice.restype = ctypes.c_ssize_t
|
||||
|
||||
def errcheck(result, func, arguments):
|
||||
if result == -1:
|
||||
errno = ctypes.set_errno(0)
|
||||
|
||||
raise IOError(errno, 'splice: %s' % os.strerror(errno))
|
||||
else:
|
||||
off_in = arguments[1]
|
||||
off_out = arguments[3]
|
||||
|
||||
return (
|
||||
result,
|
||||
off_in.contents.value if off_in is not None else None,
|
||||
off_out.contents.value if off_out is not None else None)
|
||||
|
||||
c_splice.errcheck = errcheck
|
||||
|
||||
self._c_splice = c_splice
|
||||
|
||||
def __call__(self, fd_in, off_in, fd_out, off_out, len_, flags):
|
||||
'''See `man 2 splice`
|
||||
|
||||
File-descriptors can be file-like objects with a `fileno` method, or
|
||||
integers.
|
||||
|
||||
Flags can be an integer value, or a list of flags (exposed on this
|
||||
object).
|
||||
|
||||
Returns a tuple of the result of the `splice` call, the output value of
|
||||
`off_in` and the output value of `off_out` (or `None` for any of these
|
||||
output values, if applicable).
|
||||
|
||||
Upon other errors, an `IOError` is raised with the proper `errno` set.
|
||||
|
||||
Note: if you want to pass `NULL` as value for `off_in` or `off_out` to
|
||||
the system call, you must pass `None`, *not* 0!
|
||||
'''
|
||||
|
||||
if not self.available:
|
||||
raise EnvironmentError('splice not available')
|
||||
|
||||
if not isinstance(flags, (int, long)):
|
||||
c_flags = reduce(operator.or_, flags, 0)
|
||||
else:
|
||||
c_flags = flags
|
||||
|
||||
c_fd_in = getattr(fd_in, 'fileno', lambda: fd_in)()
|
||||
c_fd_out = getattr(fd_out, 'fileno', lambda: fd_out)()
|
||||
|
||||
c_off_in = \
|
||||
ctypes.pointer(c_loff_t(off_in)) if off_in is not None else None
|
||||
c_off_out = \
|
||||
ctypes.pointer(c_loff_t(off_out)) if off_out is not None else None
|
||||
|
||||
return self._c_splice(
|
||||
c_fd_in, c_off_in, c_fd_out, c_off_out, len_, c_flags)
|
||||
|
||||
@property
|
||||
def available(self):
|
||||
'''Availability of `splice`'''
|
||||
|
||||
return self._c_splice is not None
|
||||
|
||||
splice = Splice()
|
||||
del Splice
|
||||
|
|
@ -87,8 +87,6 @@ _posix_fadvise = None
|
|||
_libc_socket = None
|
||||
_libc_bind = None
|
||||
_libc_accept = None
|
||||
_libc_splice = None
|
||||
_libc_tee = None
|
||||
|
||||
# If set to non-zero, fallocate routines will fail based on free space
|
||||
# available being at or below this amount, in bytes.
|
||||
|
|
@ -3225,65 +3223,3 @@ def get_md5_socket():
|
|||
raise IOError(ctypes.get_errno(), "Failed to accept MD5 socket")
|
||||
|
||||
return md5_sockfd
|
||||
|
||||
|
||||
# Flags for splice() and tee()
|
||||
SPLICE_F_MOVE = 1
|
||||
SPLICE_F_NONBLOCK = 2
|
||||
SPLICE_F_MORE = 4
|
||||
SPLICE_F_GIFT = 8
|
||||
|
||||
|
||||
def splice(fd_in, off_in, fd_out, off_out, length, flags):
|
||||
"""
|
||||
Calls splice - a Linux-specific syscall for zero-copy data movement.
|
||||
|
||||
On success, returns the number of bytes moved.
|
||||
|
||||
On failure where errno is EWOULDBLOCK, returns None.
|
||||
|
||||
On all other failures, raises IOError.
|
||||
"""
|
||||
global _libc_splice
|
||||
if _libc_splice is None:
|
||||
_libc_splice = load_libc_function('splice', fail_if_missing=True)
|
||||
|
||||
ret = _libc_splice(ctypes.c_int(fd_in), ctypes.c_long(off_in),
|
||||
ctypes.c_int(fd_out), ctypes.c_long(off_out),
|
||||
ctypes.c_int(length), ctypes.c_int(flags))
|
||||
if ret < 0:
|
||||
err = ctypes.get_errno()
|
||||
if err == errno.EWOULDBLOCK:
|
||||
return None
|
||||
else:
|
||||
raise IOError(err, "splice() failed: %s" % os.strerror(err))
|
||||
return ret
|
||||
|
||||
|
||||
def tee(fd_in, fd_out, length, flags):
|
||||
"""
|
||||
Calls tee - a Linux-specific syscall to let pipes share data.
|
||||
|
||||
On success, returns the number of bytes "copied".
|
||||
|
||||
On failure, raises IOError.
|
||||
"""
|
||||
global _libc_tee
|
||||
if _libc_tee is None:
|
||||
_libc_tee = load_libc_function('tee', fail_if_missing=True)
|
||||
|
||||
ret = _libc_tee(ctypes.c_int(fd_in), ctypes.c_int(fd_out),
|
||||
ctypes.c_int(length), ctypes.c_int(flags))
|
||||
if ret < 0:
|
||||
err = ctypes.get_errno()
|
||||
raise IOError(err, "tee() failed: %s" % os.strerror(err))
|
||||
return ret
|
||||
|
||||
|
||||
def system_has_splice():
|
||||
global _libc_splice
|
||||
try:
|
||||
_libc_splice = load_libc_function('splice', fail_if_missing=True)
|
||||
return True
|
||||
except AttributeError:
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -56,8 +56,8 @@ from swift.common.utils import mkdirs, Timestamp, \
|
|||
storage_directory, hash_path, renamer, fallocate, fsync, \
|
||||
fdatasync, drop_buffer_cache, ThreadPool, lock_path, write_pickle, \
|
||||
config_true_value, listdir, split_path, ismount, remove_file, \
|
||||
get_md5_socket, system_has_splice, splice, tee, SPLICE_F_MORE, \
|
||||
F_SETPIPE_SZ
|
||||
get_md5_socket, F_SETPIPE_SZ
|
||||
from swift.common.splice import splice, tee
|
||||
from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \
|
||||
DiskFileCollision, DiskFileNoSpace, DiskFileDeviceUnavailable, \
|
||||
DiskFileDeleted, DiskFileError, DiskFileNotOpen, PathNotDir, \
|
||||
|
|
@ -544,17 +544,15 @@ class DiskFileManager(object):
|
|||
self.use_splice = False
|
||||
self.pipe_size = None
|
||||
|
||||
splice_available = system_has_splice()
|
||||
|
||||
conf_wants_splice = config_true_value(conf.get('splice', 'no'))
|
||||
# If the operator wants zero-copy with splice() but we don't have the
|
||||
# requisite kernel support, complain so they can go fix it.
|
||||
if conf_wants_splice and not splice_available:
|
||||
if conf_wants_splice and not splice.available:
|
||||
self.logger.warn(
|
||||
"Use of splice() requested (config says \"splice = %s\"), "
|
||||
"but the system does not support it. "
|
||||
"splice() will not be used." % conf.get('splice'))
|
||||
elif conf_wants_splice and splice_available:
|
||||
elif conf_wants_splice and splice.available:
|
||||
try:
|
||||
sockfd = get_md5_socket()
|
||||
os.close(sockfd)
|
||||
|
|
@ -1006,8 +1004,8 @@ class DiskFileReader(object):
|
|||
try:
|
||||
while True:
|
||||
# Read data from disk to pipe
|
||||
bytes_in_pipe = self._threadpool.run_in_thread(
|
||||
splice, rfd, 0, client_wpipe, 0, pipe_size, 0)
|
||||
(bytes_in_pipe, _1, _2) = self._threadpool.run_in_thread(
|
||||
splice, rfd, None, client_wpipe, None, pipe_size, 0)
|
||||
if bytes_in_pipe == 0:
|
||||
self._read_to_eof = True
|
||||
self._drop_cache(rfd, dropped_cache,
|
||||
|
|
@ -1038,20 +1036,23 @@ class DiskFileReader(object):
|
|||
# bytes in it, so read won't block, and we're splicing it into
|
||||
# an MD5 socket, which synchronously hashes any data sent to
|
||||
# it, so writing won't block either.
|
||||
hashed = splice(hash_rpipe, 0, md5_sockfd, 0,
|
||||
bytes_in_pipe, SPLICE_F_MORE)
|
||||
(hashed, _1, _2) = splice(hash_rpipe, None, md5_sockfd, None,
|
||||
bytes_in_pipe, splice.SPLICE_F_MORE)
|
||||
if hashed != bytes_in_pipe:
|
||||
raise Exception("md5 socket didn't take all the data? "
|
||||
"(tried to write %d, but wrote %d)" %
|
||||
(bytes_in_pipe, hashed))
|
||||
|
||||
while bytes_in_pipe > 0:
|
||||
sent = splice(client_rpipe, 0, wsockfd, 0,
|
||||
bytes_in_pipe, 0)
|
||||
if sent is None: # would have blocked
|
||||
trampoline(wsockfd, write=True)
|
||||
else:
|
||||
bytes_in_pipe -= sent
|
||||
try:
|
||||
res = splice(client_rpipe, None, wsockfd, None,
|
||||
bytes_in_pipe, 0)
|
||||
bytes_in_pipe -= res[0]
|
||||
except IOError as exc:
|
||||
if exc.errno == errno.EWOULDBLOCK:
|
||||
trampoline(wsockfd, write=True)
|
||||
else:
|
||||
raise
|
||||
|
||||
if self._bytes_read - dropped_cache > DROP_CACHE_WINDOW:
|
||||
self._drop_cache(rfd, dropped_cache,
|
||||
|
|
|
|||
|
|
@ -726,7 +726,9 @@ class ObjectController(Controller):
|
|||
req.headers['X-Timestamp'] = Timestamp(time.time()).internal
|
||||
|
||||
if object_versions and not req.environ.get('swift_versioned_copy'):
|
||||
if hresp.status_int != HTTP_NOT_FOUND:
|
||||
is_manifest = 'X-Object-Manifest' in req.headers or \
|
||||
'X-Object-Manifest' in hresp.headers
|
||||
if hresp.status_int != HTTP_NOT_FOUND and not is_manifest:
|
||||
# This is a version manifest and needs to be handled
|
||||
# differently. First copy the existing data to a new object,
|
||||
# then write the data from this request to the version manifest
|
||||
|
|
|
|||
|
|
@ -2515,6 +2515,34 @@ class TestObjectVersioning(Base):
|
|||
versioned_obj.delete()
|
||||
self.assertRaises(ResponseError, versioned_obj.read)
|
||||
|
||||
def test_versioning_dlo(self):
|
||||
container = self.env.container
|
||||
versions_container = self.env.versions_container
|
||||
obj_name = Utils.create_name()
|
||||
|
||||
for i in ('1', '2', '3'):
|
||||
time.sleep(.01) # guarantee that the timestamp changes
|
||||
obj_name_seg = obj_name + '/' + i
|
||||
versioned_obj = container.file(obj_name_seg)
|
||||
versioned_obj.write(i)
|
||||
versioned_obj.write(i + i)
|
||||
|
||||
self.assertEqual(3, versions_container.info()['object_count'])
|
||||
|
||||
man_file = container.file(obj_name)
|
||||
man_file.write('', hdrs={"X-Object-Manifest": "%s/%s/" %
|
||||
(self.env.container.name, obj_name)})
|
||||
|
||||
# guarantee that the timestamp changes
|
||||
time.sleep(.01)
|
||||
|
||||
# write manifest file again
|
||||
man_file.write('', hdrs={"X-Object-Manifest": "%s/%s/" %
|
||||
(self.env.container.name, obj_name)})
|
||||
|
||||
self.assertEqual(3, versions_container.info()['object_count'])
|
||||
self.assertEqual("112233", man_file.read())
|
||||
|
||||
|
||||
class TestObjectVersioningUTF8(Base2, TestObjectVersioning):
|
||||
set_up = False
|
||||
|
|
|
|||
|
|
@ -0,0 +1,235 @@
|
|||
# Copyright (c) 2014 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
'''Tests for `swift.common.splice`'''
|
||||
|
||||
import os
|
||||
import errno
|
||||
import ctypes
|
||||
import logging
|
||||
import tempfile
|
||||
import unittest
|
||||
import contextlib
|
||||
|
||||
from mock import patch
|
||||
|
||||
from swift.common.splice import splice, tee
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def safe_close(fd):
|
||||
'''Close a file descriptor, ignoring any exceptions'''
|
||||
|
||||
try:
|
||||
os.close(fd)
|
||||
except Exception:
|
||||
LOGGER.exception('Error while closing FD')
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def pipe():
|
||||
'''Context-manager providing 2 ends of a pipe, closing them at exit'''
|
||||
|
||||
fds = os.pipe()
|
||||
|
||||
try:
|
||||
yield fds
|
||||
finally:
|
||||
safe_close(fds[0])
|
||||
safe_close(fds[1])
|
||||
|
||||
|
||||
@unittest.skipUnless(splice.available, 'splice not available')
|
||||
class TestSplice(unittest.TestCase):
|
||||
'''Tests for `splice`'''
|
||||
|
||||
def test_flags(self):
|
||||
'''Test flag attribute availability'''
|
||||
|
||||
self.assert_(hasattr(splice, 'SPLICE_F_MOVE'))
|
||||
self.assert_(hasattr(splice, 'SPLICE_F_NONBLOCK'))
|
||||
self.assert_(hasattr(splice, 'SPLICE_F_MORE'))
|
||||
self.assert_(hasattr(splice, 'SPLICE_F_GIFT'))
|
||||
|
||||
@patch('swift.common.splice.splice._c_splice', None)
|
||||
def test_available(self):
|
||||
'''Test `available` attribute correctness'''
|
||||
|
||||
self.assertFalse(splice.available)
|
||||
|
||||
def test_splice_pipe_to_pipe(self):
|
||||
'''Test `splice` from a pipe to a pipe'''
|
||||
|
||||
with pipe() as (p1a, p1b), pipe() as (p2a, p2b):
|
||||
os.write(p1b, 'abcdef')
|
||||
res = splice(p1a, None, p2b, None, 3, 0)
|
||||
self.assertEqual(res, (3, None, None))
|
||||
self.assertEqual(os.read(p2a, 3), 'abc')
|
||||
self.assertEqual(os.read(p1a, 3), 'def')
|
||||
|
||||
def test_splice_file_to_pipe(self):
|
||||
'''Test `splice` from a file to a pipe'''
|
||||
|
||||
with tempfile.NamedTemporaryFile(bufsize=0) as fd, pipe() as (pa, pb):
|
||||
fd.write('abcdef')
|
||||
fd.seek(0, os.SEEK_SET)
|
||||
|
||||
res = splice(fd, None, pb, None, 3, 0)
|
||||
self.assertEqual(res, (3, None, None))
|
||||
# `fd.tell()` isn't updated...
|
||||
self.assertEqual(os.lseek(fd.fileno(), 0, os.SEEK_CUR), 3)
|
||||
|
||||
fd.seek(0, os.SEEK_SET)
|
||||
res = splice(fd, 3, pb, None, 3, 0)
|
||||
self.assertEqual(res, (3, 6, None))
|
||||
self.assertEqual(os.lseek(fd.fileno(), 0, os.SEEK_CUR), 0)
|
||||
|
||||
self.assertEquals(os.read(pa, 6), 'abcdef')
|
||||
|
||||
def test_splice_pipe_to_file(self):
|
||||
'''Test `splice` from a pipe to a file'''
|
||||
|
||||
with tempfile.NamedTemporaryFile(bufsize=0) as fd, pipe() as (pa, pb):
|
||||
os.write(pb, 'abcdef')
|
||||
|
||||
res = splice(pa, None, fd, None, 3, 0)
|
||||
self.assertEqual(res, (3, None, None))
|
||||
self.assertEqual(fd.tell(), 3)
|
||||
|
||||
fd.seek(0, os.SEEK_SET)
|
||||
|
||||
res = splice(pa, None, fd, 3, 3, 0)
|
||||
self.assertEqual(res, (3, None, 6))
|
||||
self.assertEqual(fd.tell(), 0)
|
||||
|
||||
self.assertEqual(fd.read(6), 'abcdef')
|
||||
|
||||
@patch.object(splice, '_c_splice')
|
||||
def test_fileno(self, mock_splice):
|
||||
'''Test handling of file-descriptors'''
|
||||
|
||||
splice(1, None, 2, None, 3, 0)
|
||||
self.assertEqual(mock_splice.call_args,
|
||||
((1, None, 2, None, 3, 0), {}))
|
||||
|
||||
mock_splice.reset_mock()
|
||||
|
||||
with open('/dev/zero', 'r') as fd:
|
||||
splice(fd, None, fd, None, 3, 0)
|
||||
self.assertEqual(mock_splice.call_args,
|
||||
((fd.fileno(), None, fd.fileno(), None, 3, 0),
|
||||
{}))
|
||||
|
||||
@patch.object(splice, '_c_splice')
|
||||
def test_flags_list(self, mock_splice):
|
||||
'''Test handling of flag lists'''
|
||||
|
||||
splice(1, None, 2, None, 3,
|
||||
[splice.SPLICE_F_MOVE, splice.SPLICE_F_NONBLOCK])
|
||||
|
||||
flags = splice.SPLICE_F_MOVE | splice.SPLICE_F_NONBLOCK
|
||||
self.assertEqual(mock_splice.call_args,
|
||||
((1, None, 2, None, 3, flags), {}))
|
||||
|
||||
mock_splice.reset_mock()
|
||||
|
||||
splice(1, None, 2, None, 3, [])
|
||||
self.assertEqual(mock_splice.call_args,
|
||||
((1, None, 2, None, 3, 0), {}))
|
||||
|
||||
def test_errno(self):
|
||||
'''Test handling of failures'''
|
||||
|
||||
# Invoke EBADF by using a read-only FD as fd_out
|
||||
with open('/dev/null', 'r') as fd:
|
||||
err = errno.EBADF
|
||||
msg = r'\[Errno %d\] splice: %s' % (err, os.strerror(err))
|
||||
|
||||
self.assertRaisesRegexp(IOError, msg, splice, fd, None, fd, None,
|
||||
3, 0)
|
||||
|
||||
self.assertEqual(ctypes.get_errno(), 0)
|
||||
|
||||
@patch('swift.common.splice.splice._c_splice', None)
|
||||
def test_unavailable(self):
|
||||
'''Test exception when unavailable'''
|
||||
|
||||
self.assertRaises(EnvironmentError, splice, 1, None, 2, None, 2, 0)
|
||||
|
||||
|
||||
@unittest.skipUnless(tee.available, 'tee not available')
|
||||
class TestTee(unittest.TestCase):
|
||||
'''Tests for `tee`'''
|
||||
|
||||
@patch('swift.common.splice.tee._c_tee', None)
|
||||
def test_available(self):
|
||||
'''Test `available` attribute correctness'''
|
||||
|
||||
self.assertFalse(tee.available)
|
||||
|
||||
def test_tee_pipe_to_pipe(self):
|
||||
'''Test `tee` from a pipe to a pipe'''
|
||||
|
||||
with pipe() as (p1a, p1b), pipe() as (p2a, p2b):
|
||||
os.write(p1b, 'abcdef')
|
||||
res = tee(p1a, p2b, 3, 0)
|
||||
self.assertEqual(res, 3)
|
||||
self.assertEqual(os.read(p2a, 3), 'abc')
|
||||
self.assertEqual(os.read(p1a, 6), 'abcdef')
|
||||
|
||||
@patch.object(tee, '_c_tee')
|
||||
def test_fileno(self, mock_tee):
|
||||
'''Test handling of file-descriptors'''
|
||||
|
||||
with pipe() as (pa, pb):
|
||||
tee(pa, pb, 3, 0)
|
||||
self.assertEqual(mock_tee.call_args, ((pa, pb, 3, 0), {}))
|
||||
|
||||
mock_tee.reset_mock()
|
||||
|
||||
tee(os.fdopen(pa, 'r'), os.fdopen(pb, 'w'), 3, 0)
|
||||
self.assertEqual(mock_tee.call_args, ((pa, pb, 3, 0), {}))
|
||||
|
||||
@patch.object(tee, '_c_tee')
|
||||
def test_flags_list(self, mock_tee):
|
||||
'''Test handling of flag lists'''
|
||||
|
||||
tee(1, 2, 3, [splice.SPLICE_F_MOVE | splice.SPLICE_F_NONBLOCK])
|
||||
flags = splice.SPLICE_F_MOVE | splice.SPLICE_F_NONBLOCK
|
||||
self.assertEqual(mock_tee.call_args, ((1, 2, 3, flags), {}))
|
||||
|
||||
mock_tee.reset_mock()
|
||||
|
||||
tee(1, 2, 3, [])
|
||||
self.assertEqual(mock_tee.call_args, ((1, 2, 3, 0), {}))
|
||||
|
||||
def test_errno(self):
|
||||
'''Test handling of failures'''
|
||||
|
||||
# Invoke EBADF by using a read-only FD as fd_out
|
||||
with open('/dev/null', 'r') as fd:
|
||||
err = errno.EBADF
|
||||
msg = r'\[Errno %d\] tee: %s' % (err, os.strerror(err))
|
||||
|
||||
self.assertRaisesRegexp(IOError, msg, tee, fd, fd, 3, 0)
|
||||
|
||||
self.assertEqual(ctypes.get_errno(), 0)
|
||||
|
||||
@patch('swift.common.splice.tee._c_tee', None)
|
||||
def test_unavailable(self):
|
||||
'''Test exception when unavailable'''
|
||||
|
||||
self.assertRaises(EnvironmentError, tee, 1, 2, 2, 0)
|
||||
|
|
@ -41,6 +41,7 @@ from swift.obj import diskfile
|
|||
from swift.common import utils
|
||||
from swift.common.utils import hash_path, mkdirs, Timestamp
|
||||
from swift.common import ring
|
||||
from swift.common.splice import splice
|
||||
from swift.common.exceptions import DiskFileNotExist, DiskFileQuarantined, \
|
||||
DiskFileDeviceUnavailable, DiskFileDeleted, DiskFileNotOpen, \
|
||||
DiskFileError, ReplicationLockTimeout, PathNotDir, DiskFileCollision, \
|
||||
|
|
@ -954,8 +955,7 @@ class TestDiskFileManager(unittest.TestCase):
|
|||
|
||||
def test_missing_splice_warning(self):
|
||||
logger = FakeLogger()
|
||||
with mock.patch('swift.obj.diskfile.system_has_splice',
|
||||
lambda: False):
|
||||
with mock.patch('swift.common.splice.splice._c_splice', None):
|
||||
self.conf['splice'] = 'yes'
|
||||
mgr = diskfile.DiskFileManager(self.conf, logger)
|
||||
|
||||
|
|
@ -2242,7 +2242,7 @@ class TestDiskFile(unittest.TestCase):
|
|||
self.assertTrue(exp_name in set(dl))
|
||||
|
||||
def _system_can_zero_copy(self):
|
||||
if not utils.system_has_splice():
|
||||
if not splice.available:
|
||||
return False
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \
|
|||
NullLogger, storage_directory, public, replication
|
||||
from swift.common import constraints
|
||||
from swift.common.swob import Request, HeaderKeyDict, WsgiStringIO
|
||||
from swift.common.splice import splice
|
||||
from swift.common.storage_policy import POLICIES
|
||||
from swift.common.exceptions import DiskFileDeviceUnavailable
|
||||
|
||||
|
|
@ -1563,7 +1564,8 @@ class TestObjectController(unittest.TestCase):
|
|||
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={
|
||||
'X-Timestamp': normalize_timestamp(time()),
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'X-Object-Meta-Soup': 'gazpacho',
|
||||
'Content-Type': 'application/fizzbuzz',
|
||||
'Content-Length': '4'})
|
||||
req.body = 'test'
|
||||
resp = req.get_response(self.object_controller)
|
||||
|
|
@ -1580,6 +1582,8 @@ class TestObjectController(unittest.TestCase):
|
|||
resp = req.get_response(self.object_controller)
|
||||
self.assertEquals(resp.status_int, 304)
|
||||
self.assertEquals(resp.etag, etag)
|
||||
self.assertEquals(resp.headers['Content-Type'], 'application/fizzbuzz')
|
||||
self.assertEquals(resp.headers['X-Object-Meta-Soup'], 'gazpacho')
|
||||
|
||||
req = Request.blank('/sda1/p/a/c/o2',
|
||||
environ={'REQUEST_METHOD': 'GET'},
|
||||
|
|
@ -1807,7 +1811,8 @@ class TestObjectController(unittest.TestCase):
|
|||
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={
|
||||
'X-Timestamp': timestamp,
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'X-Object-Meta-Burr': 'ito',
|
||||
'Content-Type': 'application/cat-picture',
|
||||
'Content-Length': '4'})
|
||||
req.body = 'test'
|
||||
resp = req.get_response(self.object_controller)
|
||||
|
|
@ -1830,6 +1835,9 @@ class TestObjectController(unittest.TestCase):
|
|||
headers={'If-Unmodified-Since': since})
|
||||
resp = req.get_response(self.object_controller)
|
||||
self.assertEquals(resp.status_int, 412)
|
||||
self.assertEquals(resp.headers['Content-Type'],
|
||||
'application/cat-picture')
|
||||
self.assertEquals(resp.headers['X-Object-Meta-Burr'], 'ito')
|
||||
|
||||
since = \
|
||||
strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) + 9))
|
||||
|
|
@ -4680,7 +4688,7 @@ class TestZeroCopy(unittest.TestCase):
|
|||
"""Test the object server's zero-copy functionality"""
|
||||
|
||||
def _system_can_zero_copy(self):
|
||||
if not utils.system_has_splice():
|
||||
if not splice.available:
|
||||
return False
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -382,6 +382,25 @@ def _make_callback_func(calls):
|
|||
return callback
|
||||
|
||||
|
||||
def _limit_max_file_size(f):
|
||||
"""
|
||||
This will limit constraints.MAX_FILE_SIZE for the duration of the
|
||||
wrapped function, based on whether MAX_FILE_SIZE exceeds the
|
||||
sys.maxsize limit on the system running the tests.
|
||||
|
||||
This allows successful testing on 32 bit systems.
|
||||
"""
|
||||
@functools.wraps(f)
|
||||
def wrapper(*args, **kwargs):
|
||||
test_max_file_size = constraints.MAX_FILE_SIZE
|
||||
if constraints.MAX_FILE_SIZE >= sys.maxsize:
|
||||
test_max_file_size = (2 ** 30 + 2)
|
||||
with mock.patch.object(constraints, 'MAX_FILE_SIZE',
|
||||
test_max_file_size):
|
||||
return f(*args, **kwargs)
|
||||
return wrapper
|
||||
|
||||
|
||||
# tests
|
||||
class TestController(unittest.TestCase):
|
||||
|
||||
|
|
@ -3636,12 +3655,13 @@ class TestObjectController(unittest.TestCase):
|
|||
self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay')
|
||||
self.assertEquals(resp.headers.get('x-delete-at'), '9876543210')
|
||||
|
||||
@_limit_max_file_size
|
||||
def test_copy_source_larger_than_max_file_size(self):
|
||||
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
|
||||
headers={'Content-Length': '0',
|
||||
'X-Copy-From': '/c/o'})
|
||||
|
||||
# copy-from object is too large to fit in target object
|
||||
|
||||
class LargeResponseBody(object):
|
||||
|
||||
def __len__(self):
|
||||
|
|
@ -3880,6 +3900,7 @@ class TestObjectController(unittest.TestCase):
|
|||
self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay')
|
||||
self.assertEquals(resp.headers.get('x-delete-at'), '9876543210')
|
||||
|
||||
@_limit_max_file_size
|
||||
def test_COPY_source_larger_than_max_file_size(self):
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'COPY'},
|
||||
|
|
@ -3902,6 +3923,7 @@ class TestObjectController(unittest.TestCase):
|
|||
resp = controller.COPY(req)
|
||||
self.assertEquals(resp.status_int, 413)
|
||||
|
||||
@_limit_max_file_size
|
||||
def test_COPY_account_source_larger_than_max_file_size(self):
|
||||
req = Request.blank('/v1/a/c/o',
|
||||
environ={'REQUEST_METHOD': 'COPY'},
|
||||
|
|
@ -4539,18 +4561,22 @@ class TestObjectController(unittest.TestCase):
|
|||
exp = 'HTTP/1.1 404'
|
||||
self.assertEquals(headers[:len(exp)], exp)
|
||||
|
||||
# make sure manifest files don't get versioned
|
||||
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
|
||||
fd = sock.makefile()
|
||||
fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: '
|
||||
'localhost\r\nConnection: close\r\nX-Storage-Token: '
|
||||
't\r\nContent-Length: 0\r\nContent-Type: text/jibberish0\r\n'
|
||||
'Foo: barbaz\r\nX-Object-Manifest: %s/foo_\r\n\r\n'
|
||||
% (oc, vc, o))
|
||||
fd.flush()
|
||||
headers = readuntil2crlfs(fd)
|
||||
exp = 'HTTP/1.1 201'
|
||||
self.assertEquals(headers[:len(exp)], exp)
|
||||
# make sure dlo manifest files don't get versioned
|
||||
for _junk in xrange(1, versions_to_create):
|
||||
sleep(.01) # guarantee that the timestamp changes
|
||||
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
|
||||
fd = sock.makefile()
|
||||
fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: '
|
||||
'localhost\r\nConnection: close\r\nX-Storage-Token: '
|
||||
't\r\nContent-Length: 0\r\n'
|
||||
'Content-Type: text/jibberish0\r\n'
|
||||
'Foo: barbaz\r\nX-Object-Manifest: %s/%s/\r\n\r\n'
|
||||
% (oc, o, oc, o))
|
||||
fd.flush()
|
||||
headers = readuntil2crlfs(fd)
|
||||
exp = 'HTTP/1.1 201'
|
||||
self.assertEquals(headers[:len(exp)], exp)
|
||||
|
||||
# Ensure we have no saved versions
|
||||
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
|
||||
fd = sock.makefile()
|
||||
|
|
|
|||
Loading…
Reference in New Issue