Merge master to feature/ec
Change-Id: I3e1d725fa85bf68c3d5fb0a062c9a053064a9e66 Implements: blueprint swift-ec
This commit is contained in:
commit
06800cbe44
1
.mailmap
1
.mailmap
|
@ -67,3 +67,4 @@ Mauro Stettler <mauro.stettler@gmail.com> <mauro.stettler@gmail.com>
|
|||
Pawel Palucki <pawel.palucki@gmail.com> <pawel.palucki@gmail.com>
|
||||
Guang Yee <guang.yee@hp.com> <guang.yee@hp.com>
|
||||
Jing Liuqing <jing.liuqing@99cloud.net> <jing.liuqing@99cloud.net>
|
||||
Lorcan Browne <lorcan.browne@hp.com> <lorcan.browne@hp.com>
|
||||
|
|
16
AUTHORS
16
AUTHORS
|
@ -21,6 +21,7 @@ Joe Arnold (joe@swiftstack.com)
|
|||
Ionuț Arțăriși (iartarisi@suse.cz)
|
||||
Christian Berendt (berendt@b1-systems.de)
|
||||
Luis de Bethencourt (luis@debethencourt.com)
|
||||
Keshava Bharadwaj (kb.sankethi@gmail.com)
|
||||
Yummy Bian (yummy.bian@gmail.com)
|
||||
Darrell Bishop (darrell@swiftstack.com)
|
||||
James E. Blair (jeblair@openstack.org)
|
||||
|
@ -28,19 +29,24 @@ Fabien Boucher (fabien.boucher@enovance.com)
|
|||
Chmouel Boudjnah (chmouel@enovance.com)
|
||||
Clark Boylan (clark.boylan@gmail.com)
|
||||
Pádraig Brady (pbrady@redhat.com)
|
||||
Lorcan Browne (lorcan.browne@hp.com)
|
||||
Russell Bryant (rbryant@redhat.com)
|
||||
Jay S. Bryant (jsbryant@us.ibm.com)
|
||||
Brian D. Burns (iosctr@gmail.com)
|
||||
Devin Carlen (devin.carlen@gmail.com)
|
||||
Thierry Carrez (thierry@openstack.org)
|
||||
Mahati Chamarthy (mahati.chamarthy@gmail.com)
|
||||
Zap Chang (zapchang@gmail.com)
|
||||
François Charlier (francois.charlier@enovance.com)
|
||||
Ray Chen (oldsharp@163.com)
|
||||
Brian Cline (bcline@softlayer.com)
|
||||
Alistair Coles (alistair.coles@hp.com)
|
||||
Brian Curtin (brian.curtin@rackspace.com)
|
||||
Thiago da Silva (thiago@redhat.com)
|
||||
Julien Danjou (julien@danjou.info)
|
||||
Ksenia Demina (kdemina@mirantis.com)
|
||||
Dan Dillinger (dan.dillinger@sonian.net)
|
||||
Gerry Drudy (gerry.drudy@hp.com)
|
||||
Morgan Fainberg (morgan.fainberg@gmail.com)
|
||||
ZhiQiang Fan (aji.zqfan@gmail.com)
|
||||
Flaper Fesp (flaper87@gmail.com)
|
||||
|
@ -55,6 +61,7 @@ David Goetz (david.goetz@rackspace.com)
|
|||
Jonathan Gonzalez V (jonathan.abdiel@gmail.com)
|
||||
Joe Gordon (jogo@cloudscaling.com)
|
||||
David Hadas (davidh@il.ibm.com)
|
||||
Andrew Hale (andy@wwwdata.eu)
|
||||
Soren Hansen (soren@linux2go.dk)
|
||||
Richard (Rick) Hawkins (richard.hawkins@rackspace.com)
|
||||
Doug Hellmann (doug.hellmann@dreamhost.com)
|
||||
|
@ -67,6 +74,7 @@ Kun Huang (gareth@unitedstack.com)
|
|||
Matthieu Huin (mhu@enovance.com)
|
||||
Hodong Hwang (hodong.hwang@kt.com)
|
||||
Motonobu Ichimura (motonobu@gmail.com)
|
||||
Andreas Jaeger (aj@suse.de)
|
||||
Shri Javadekar (shrinand@maginatics.com)
|
||||
Iryoung Jeong (iryoung@gmail.com)
|
||||
Paul Jimenez (pj@place.org)
|
||||
|
@ -80,6 +88,7 @@ Morita Kazutaka (morita.kazutaka@gmail.com)
|
|||
Josh Kearney (josh@jk0.org)
|
||||
Ilya Kharin (ikharin@mirantis.com)
|
||||
Dae S. Kim (dae@velatum.com)
|
||||
Nathan Kinder (nkinder@redhat.com)
|
||||
Eugene Kirpichov (ekirpichov@gmail.com)
|
||||
Leah Klearman (lklrmn@gmail.com)
|
||||
Steve Kowalik (steven@wedontsleep.org)
|
||||
|
@ -88,6 +97,7 @@ Sushil Kumar (sushil.kumar2@globallogic.com)
|
|||
Madhuri Kumari (madhuri.rai07@gmail.com)
|
||||
Steven Lang (Steven.Lang@hgst.com)
|
||||
Gonéri Le Bouder (goneri.lebouder@enovance.com)
|
||||
John Leach (john@johnleach.co.uk)
|
||||
Ed Leafe (ed.leafe@rackspace.com)
|
||||
Thomas Leaman (thomas.leaman@hp.com)
|
||||
Eohyung Lee (liquid@kt.com)
|
||||
|
@ -104,6 +114,7 @@ Dragos Manolescu (dragosm@hp.com)
|
|||
Steve Martinelli (stevemar@ca.ibm.com)
|
||||
Juan J. Martinez (juan@memset.com)
|
||||
Marcelo Martins (btorch@gmail.com)
|
||||
Dolph Mathews (dolph.mathews@gmail.com)
|
||||
Donagh McCabe (donagh.mccabe@hp.com)
|
||||
Andy McCrae (andy.mccrae@gmail.com)
|
||||
Paul McMillan (paul.mcmillan@nebula.com)
|
||||
|
@ -117,6 +128,7 @@ Maru Newby (mnewby@internap.com)
|
|||
Newptone (xingchao@unitedstack.com)
|
||||
Colin Nicholson (colin.nicholson@iomart.com)
|
||||
Zhenguo Niu (zhenguo@unitedstack.com)
|
||||
Timothy Okwii (tokwii@cisco.com)
|
||||
Matthew Oliver (matt@oliver.net.au)
|
||||
Eamonn O'Toole (eamonn.otoole@hp.com)
|
||||
James Page (james.page@ubuntu.com)
|
||||
|
@ -129,17 +141,19 @@ Dieter Plaetinck (dieter@vimeo.com)
|
|||
Peter Portante (peter.portante@redhat.com)
|
||||
Dan Prince (dprince@redhat.com)
|
||||
Felipe Reyes (freyes@tty.cl)
|
||||
Matt Riedemann (mriedem@us.ibm.com)
|
||||
Li Riqiang (lrqrun@gmail.com)
|
||||
Rafael Rivero (rafael@cloudscaling.com)
|
||||
Victor Rodionov (victor.rodionov@nexenta.com)
|
||||
Aaron Rosen (arosen@nicira.com)
|
||||
Brent Roskos (broskos@internap.com)
|
||||
Cristian A Sanchez (cristian.a.sanchez@intel.com)
|
||||
saranjan (saranjan@cisco.com)
|
||||
Christian Schwede (info@cschwede.de)
|
||||
Mark Seger (Mark.Seger@hp.com)
|
||||
Andrew Clay Shafer (acs@parvuscaptus.com)
|
||||
Chuck Short (chuck.short@canonical.com)
|
||||
Michael Shuler (mshuler@gmail.com)
|
||||
Thiago da Silva (thiago@redhat.com)
|
||||
David Moreau Simard (dmsimard@iweb.com)
|
||||
Scott Simpson (sasimpson@gmail.com)
|
||||
Liu Siqi (meizu647@gmail.com)
|
||||
|
|
54
CHANGELOG
54
CHANGELOG
|
@ -1,3 +1,57 @@
|
|||
swift (2.2.0)
|
||||
|
||||
* Added support for Keystone v3 auth.
|
||||
|
||||
Keystone v3 introduced the concept of "domains" and user names
|
||||
are no longer unique across domains. Swift's Keystone integration
|
||||
now requires that ACLs be set on IDs, which are unique across
|
||||
domains, and further restricts setting new ACLs to only use IDs.
|
||||
|
||||
Please see http://swift.openstack.org/overview_auth.html for
|
||||
more information on configuring Swift and Keystone together.
|
||||
|
||||
* Swift now supports server-side account-to-account copy. Server-
|
||||
side copy in Swift requires the X-Copy-From header (on a PUT)
|
||||
or the Destination header (on a COPY). To initiate an account-to-
|
||||
account copy, the existing header value remains the same, but the
|
||||
X-Copy-From-Account header (on a PUT) or the Destination-Account
|
||||
(on a COPY) are used to indicate the proper account.
|
||||
|
||||
* Limit partition movement when adding a new placement tier.
|
||||
|
||||
When adding a new placement tier (server, zone, or region), Swift
|
||||
previously attempted to move all placement partitions, regardless
|
||||
of the space available on the new tier, to ensure the best possible
|
||||
durability. Unfortunately, this could result in too many partitions
|
||||
being moved all at once to a new tier. Swift's ring-builder now
|
||||
ensures that only the correct number of placement partitions are
|
||||
rebalanced, and thus makes adding capacity to the cluster more
|
||||
efficient.
|
||||
|
||||
* Per storage policy container counts are now reported in an
|
||||
account response headers.
|
||||
|
||||
* Swift will now reject, with a 4xx series response, GET requests
|
||||
with more than 50 ranges, more than 3 overlapping ranges, or more
|
||||
than 8 non-increasing ranges.
|
||||
|
||||
* The bind_port config setting is now required to be explicitly set.
|
||||
|
||||
* The object server can now use splice() for a zero-copy GET
|
||||
response. This feature is enabled with the "splice" config variable
|
||||
in the object server config and defaults to off. Also, this feature
|
||||
only works on recent Linux kernels (AF_ALG sockets must be
|
||||
supported). A zero-copy GET response can significantly reduce CPU
|
||||
requirements for object servers.
|
||||
|
||||
* Added "--no-overlap" option to swift-dispersion populate so that
|
||||
multiple runs of the tool can add coverage without overlapping
|
||||
existing monitored partitions.
|
||||
|
||||
* swift-recon now supports filtering by region.
|
||||
|
||||
* Various other minor bug fixes and improvements.
|
||||
|
||||
swift (2.1.0)
|
||||
|
||||
* swift-ring-builder placement was improved to allow gradual addition
|
||||
|
|
|
@ -156,7 +156,7 @@ class AccountController(object):
|
|||
for key, value in req.headers.iteritems()
|
||||
if is_sys_or_user_meta('account', key))
|
||||
if metadata:
|
||||
broker.update_metadata(metadata)
|
||||
broker.update_metadata(metadata, validate_metadata=True)
|
||||
if created:
|
||||
return HTTPCreated(request=req)
|
||||
else:
|
||||
|
@ -249,7 +249,7 @@ class AccountController(object):
|
|||
for key, value in req.headers.iteritems()
|
||||
if is_sys_or_user_meta('account', key))
|
||||
if metadata:
|
||||
broker.update_metadata(metadata)
|
||||
broker.update_metadata(metadata, validate_metadata=True)
|
||||
return HTTPNoContent(request=req)
|
||||
|
||||
def __call__(self, env, start_response):
|
||||
|
|
|
@ -830,10 +830,18 @@ def main(arguments=None):
|
|||
|
||||
builder_file, ring_file = parse_builder_ring_filename_args(argv)
|
||||
|
||||
if exists(builder_file):
|
||||
try:
|
||||
builder = RingBuilder.load(builder_file)
|
||||
elif len(argv) < 3 or argv[2] not in('create', 'write_builder'):
|
||||
print 'Ring Builder file does not exist: %s' % argv[1]
|
||||
except exceptions.UnPicklingError as e:
|
||||
print e
|
||||
exit(EXIT_ERROR)
|
||||
except (exceptions.FileNotFoundError, exceptions.PermissionError) as e:
|
||||
if len(argv) < 3 or argv[2] not in('create', 'write_builder'):
|
||||
print e
|
||||
exit(EXIT_ERROR)
|
||||
except Exception as e:
|
||||
print 'Problem occurred while reading builder file: %s. %s' % (
|
||||
argv[1], e.message)
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
backup_dir = pathjoin(dirname(argv[1]), 'backups')
|
||||
|
|
|
@ -101,7 +101,10 @@ FORMAT2CONTENT_TYPE = {'plain': 'text/plain', 'json': 'application/json',
|
|||
|
||||
def check_metadata(req, target_type):
|
||||
"""
|
||||
Check metadata sent in the request headers.
|
||||
Check metadata sent in the request headers. This should only check
|
||||
that the metadata in the request given is valid. Checks against
|
||||
account/container overall metadata should be forwarded on to its
|
||||
respective server to be checked.
|
||||
|
||||
:param req: request object
|
||||
:param target_type: str: one of: object, container, or account: indicates
|
||||
|
|
|
@ -30,9 +30,11 @@ from tempfile import mkstemp
|
|||
from eventlet import sleep, Timeout
|
||||
import sqlite3
|
||||
|
||||
from swift.common.constraints import MAX_META_COUNT, MAX_META_OVERALL_SIZE
|
||||
from swift.common.utils import json, Timestamp, renamer, \
|
||||
mkdirs, lock_parent_directory, fallocate
|
||||
from swift.common.exceptions import LockTimeout
|
||||
from swift.common.swob import HTTPBadRequest
|
||||
|
||||
|
||||
#: Whether calls will be made to preallocate disk space for database files.
|
||||
|
@ -719,7 +721,35 @@ class DatabaseBroker(object):
|
|||
metadata = {}
|
||||
return metadata
|
||||
|
||||
def update_metadata(self, metadata_updates):
|
||||
@staticmethod
|
||||
def validate_metadata(metadata):
|
||||
"""
|
||||
Validates that metadata_falls within acceptable limits.
|
||||
|
||||
:param metadata: to be validated
|
||||
:raises: HTTPBadRequest if MAX_META_COUNT or MAX_META_OVERALL_SIZE
|
||||
is exceeded
|
||||
"""
|
||||
meta_count = 0
|
||||
meta_size = 0
|
||||
for key, (value, timestamp) in metadata.iteritems():
|
||||
key = key.lower()
|
||||
if value != '' and (key.startswith('x-account-meta') or
|
||||
key.startswith('x-container-meta')):
|
||||
prefix = 'x-account-meta-'
|
||||
if key.startswith('x-container-meta-'):
|
||||
prefix = 'x-container-meta-'
|
||||
key = key[len(prefix):]
|
||||
meta_count = meta_count + 1
|
||||
meta_size = meta_size + len(key) + len(value)
|
||||
if meta_count > MAX_META_COUNT:
|
||||
raise HTTPBadRequest('Too many metadata items; max %d'
|
||||
% MAX_META_COUNT)
|
||||
if meta_size > MAX_META_OVERALL_SIZE:
|
||||
raise HTTPBadRequest('Total metadata too large; max %d'
|
||||
% MAX_META_OVERALL_SIZE)
|
||||
|
||||
def update_metadata(self, metadata_updates, validate_metadata=False):
|
||||
"""
|
||||
Updates the metadata dict for the database. The metadata dict values
|
||||
are tuples of (value, timestamp) where the timestamp indicates when
|
||||
|
@ -752,6 +782,8 @@ class DatabaseBroker(object):
|
|||
value, timestamp = value_timestamp
|
||||
if key not in md or timestamp > md[key][1]:
|
||||
md[key] = value_timestamp
|
||||
if validate_metadata:
|
||||
DatabaseBroker.validate_metadata(md)
|
||||
conn.execute('UPDATE %s_stat SET metadata = ?' % self.db_type,
|
||||
(json.dumps(md),))
|
||||
conn.commit()
|
||||
|
|
|
@ -108,7 +108,7 @@ def direct_get_account(node, part, account, marker=None, limit=None,
|
|||
:param marker: marker query
|
||||
:param limit: query limit
|
||||
:param prefix: prefix query
|
||||
:param delimeter: delimeter for the query
|
||||
:param delimiter: delimiter for the query
|
||||
:param conn_timeout: timeout in seconds for establishing the connection
|
||||
:param response_timeout: timeout in seconds for getting the response
|
||||
:returns: a tuple of (response headers, a list of containers) The response
|
||||
|
@ -116,11 +116,11 @@ def direct_get_account(node, part, account, marker=None, limit=None,
|
|||
"""
|
||||
path = '/' + account
|
||||
return _get_direct_account_container(path, "Account", node, part,
|
||||
account, marker=None,
|
||||
limit=None, prefix=None,
|
||||
delimiter=None,
|
||||
conn_timeout=5,
|
||||
response_timeout=15)
|
||||
account, marker=marker,
|
||||
limit=limit, prefix=prefix,
|
||||
delimiter=delimiter,
|
||||
conn_timeout=conn_timeout,
|
||||
response_timeout=response_timeout)
|
||||
|
||||
|
||||
def direct_delete_account(node, part, account, conn_timeout=5,
|
||||
|
@ -183,7 +183,7 @@ def direct_get_container(node, part, account, container, marker=None,
|
|||
:param marker: marker query
|
||||
:param limit: query limit
|
||||
:param prefix: prefix query
|
||||
:param delimeter: delimeter for the query
|
||||
:param delimiter: delimiter for the query
|
||||
:param conn_timeout: timeout in seconds for establishing the connection
|
||||
:param response_timeout: timeout in seconds for getting the response
|
||||
:returns: a tuple of (response headers, a list of objects) The response
|
||||
|
@ -191,11 +191,11 @@ def direct_get_container(node, part, account, container, marker=None,
|
|||
"""
|
||||
path = '/%s/%s' % (account, container)
|
||||
return _get_direct_account_container(path, "Container", node,
|
||||
part, account, marker=None,
|
||||
limit=None, prefix=None,
|
||||
delimiter=None,
|
||||
conn_timeout=5,
|
||||
response_timeout=15)
|
||||
part, account, marker=marker,
|
||||
limit=limit, prefix=prefix,
|
||||
delimiter=delimiter,
|
||||
conn_timeout=conn_timeout,
|
||||
response_timeout=response_timeout)
|
||||
|
||||
|
||||
def direct_delete_container(node, part, account, container, conn_timeout=5,
|
||||
|
|
|
@ -123,6 +123,18 @@ class DuplicateDeviceError(RingBuilderError):
|
|||
pass
|
||||
|
||||
|
||||
class UnPicklingError(SwiftException):
|
||||
pass
|
||||
|
||||
|
||||
class FileNotFoundError(SwiftException):
|
||||
pass
|
||||
|
||||
|
||||
class PermissionError(SwiftException):
|
||||
pass
|
||||
|
||||
|
||||
class ListingIterError(SwiftException):
|
||||
pass
|
||||
|
||||
|
|
|
@ -625,7 +625,7 @@ class Server(object):
|
|||
"""
|
||||
conf_files = self.conf_files(**kwargs)
|
||||
if not conf_files:
|
||||
return []
|
||||
return {}
|
||||
|
||||
pids = self.get_running_pids(**kwargs)
|
||||
|
||||
|
@ -645,7 +645,7 @@ class Server(object):
|
|||
|
||||
if already_started:
|
||||
print _("%s already started...") % self.server
|
||||
return []
|
||||
return {}
|
||||
|
||||
if self.server not in START_ONCE_SERVERS:
|
||||
kwargs['once'] = False
|
||||
|
|
|
@ -18,8 +18,7 @@ from swift import gettext_ as _
|
|||
import eventlet
|
||||
|
||||
from swift.common.utils import cache_from_env, get_logger, register_swift_info
|
||||
from swift.proxy.controllers.base import get_container_memcache_key, \
|
||||
get_account_info
|
||||
from swift.proxy.controllers.base import get_account_info, get_container_info
|
||||
from swift.common.memcached import MemcacheConnectionError
|
||||
from swift.common.swob import Request, Response
|
||||
|
||||
|
@ -118,11 +117,10 @@ class RateLimitMiddleware(object):
|
|||
self.container_listing_ratelimits = interpret_conf_limits(
|
||||
conf, 'container_listing_ratelimit_')
|
||||
|
||||
def get_container_size(self, account_name, container_name):
|
||||
def get_container_size(self, env):
|
||||
rv = 0
|
||||
memcache_key = get_container_memcache_key(account_name,
|
||||
container_name)
|
||||
container_info = self.memcache_client.get(memcache_key)
|
||||
container_info = get_container_info(
|
||||
env, self.app, swift_source='RL')
|
||||
if isinstance(container_info, dict):
|
||||
rv = container_info.get(
|
||||
'object_count', container_info.get('container_size', 0))
|
||||
|
@ -149,8 +147,7 @@ class RateLimitMiddleware(object):
|
|||
|
||||
if account_name and container_name and obj_name and \
|
||||
req.method in ('PUT', 'DELETE', 'POST', 'COPY'):
|
||||
container_size = self.get_container_size(
|
||||
account_name, container_name)
|
||||
container_size = self.get_container_size(req.environ)
|
||||
container_rate = get_maxrate(
|
||||
self.container_ratelimits, container_size)
|
||||
if container_rate:
|
||||
|
@ -160,8 +157,7 @@ class RateLimitMiddleware(object):
|
|||
|
||||
if account_name and container_name and not obj_name and \
|
||||
req.method == 'GET':
|
||||
container_size = self.get_container_size(
|
||||
account_name, container_name)
|
||||
container_size = self.get_container_size(req.environ)
|
||||
container_rate = get_maxrate(
|
||||
self.container_listing_ratelimits, container_size)
|
||||
if container_rate:
|
||||
|
|
|
@ -313,55 +313,55 @@ class HTMLViewer(object):
|
|||
return empty_description, headers
|
||||
try:
|
||||
stats = Stats2(*log_files)
|
||||
if not fulldirs:
|
||||
stats.strip_dirs()
|
||||
stats.sort_stats(sort)
|
||||
nfl_filter_esc =\
|
||||
nfl_filter.replace('(', '\(').replace(')', '\)')
|
||||
amount = [nfl_filter_esc, limit] if nfl_filter_esc else [limit]
|
||||
profile_html = self.generate_stats_html(stats, self.app_path,
|
||||
profile_id, *amount)
|
||||
description = "Profiling information is generated by using\
|
||||
'%s' profiler." % self.profile_module
|
||||
sort_repl = '<option value="%s">' % sort
|
||||
sort_selected = '<option value="%s" selected>' % sort
|
||||
sort = sort_tmpl.replace(sort_repl, sort_selected)
|
||||
plist = ''.join(['<option value="%s">%s</option>' % (p, p)
|
||||
for p in self.profile_log.get_all_pids()])
|
||||
profile_element = string.Template(profile_tmpl).substitute(
|
||||
{'profile_list': plist})
|
||||
profile_repl = '<option value="%s">' % profile_id
|
||||
profile_selected = '<option value="%s" selected>' % profile_id
|
||||
profile_element = profile_element.replace(profile_repl,
|
||||
profile_selected)
|
||||
limit_repl = '<option value="%s">' % limit
|
||||
limit_selected = '<option value="%s" selected>' % limit
|
||||
limit = limit_tmpl.replace(limit_repl, limit_selected)
|
||||
fulldirs_checked = 'checked' if fulldirs else ''
|
||||
fulldirs_element = string.Template(fulldirs_tmpl).substitute(
|
||||
{'fulldir_checked': fulldirs_checked})
|
||||
nfl_filter_element = string.Template(nfl_filter_tmpl).\
|
||||
substitute({'nfl_filter': nfl_filter})
|
||||
form_elements = string.Template(formelements_tmpl).substitute(
|
||||
{'description': description,
|
||||
'action': url,
|
||||
'profile': profile_element,
|
||||
'sort': sort,
|
||||
'limit': limit,
|
||||
'fulldirs': fulldirs_element,
|
||||
'nfl_filter': nfl_filter_element,
|
||||
}
|
||||
)
|
||||
content = string.Template(index_tmpl).substitute(
|
||||
{'formelements': form_elements,
|
||||
'action': url,
|
||||
'description': description,
|
||||
'profilehtml': profile_html,
|
||||
})
|
||||
return content, headers
|
||||
except:
|
||||
except (IOError, ValueError):
|
||||
raise DataLoadFailure(_('Can not load profile data from %s.')
|
||||
% log_files)
|
||||
if not fulldirs:
|
||||
stats.strip_dirs()
|
||||
stats.sort_stats(sort)
|
||||
nfl_filter_esc =\
|
||||
nfl_filter.replace('(', '\(').replace(')', '\)')
|
||||
amount = [nfl_filter_esc, limit] if nfl_filter_esc else [limit]
|
||||
profile_html = self.generate_stats_html(stats, self.app_path,
|
||||
profile_id, *amount)
|
||||
description = "Profiling information is generated by using\
|
||||
'%s' profiler." % self.profile_module
|
||||
sort_repl = '<option value="%s">' % sort
|
||||
sort_selected = '<option value="%s" selected>' % sort
|
||||
sort = sort_tmpl.replace(sort_repl, sort_selected)
|
||||
plist = ''.join(['<option value="%s">%s</option>' % (p, p)
|
||||
for p in self.profile_log.get_all_pids()])
|
||||
profile_element = string.Template(profile_tmpl).substitute(
|
||||
{'profile_list': plist})
|
||||
profile_repl = '<option value="%s">' % profile_id
|
||||
profile_selected = '<option value="%s" selected>' % profile_id
|
||||
profile_element = profile_element.replace(profile_repl,
|
||||
profile_selected)
|
||||
limit_repl = '<option value="%s">' % limit
|
||||
limit_selected = '<option value="%s" selected>' % limit
|
||||
limit = limit_tmpl.replace(limit_repl, limit_selected)
|
||||
fulldirs_checked = 'checked' if fulldirs else ''
|
||||
fulldirs_element = string.Template(fulldirs_tmpl).substitute(
|
||||
{'fulldir_checked': fulldirs_checked})
|
||||
nfl_filter_element = string.Template(nfl_filter_tmpl).\
|
||||
substitute({'nfl_filter': nfl_filter})
|
||||
form_elements = string.Template(formelements_tmpl).substitute(
|
||||
{'description': description,
|
||||
'action': url,
|
||||
'profile': profile_element,
|
||||
'sort': sort,
|
||||
'limit': limit,
|
||||
'fulldirs': fulldirs_element,
|
||||
'nfl_filter': nfl_filter_element,
|
||||
}
|
||||
)
|
||||
content = string.Template(index_tmpl).substitute(
|
||||
{'formelements': form_elements,
|
||||
'action': url,
|
||||
'description': description,
|
||||
'profilehtml': profile_html,
|
||||
})
|
||||
return content, headers
|
||||
|
||||
def download(self, log_files, sort='time', limit=-1, nfl_filter='',
|
||||
output_format='default'):
|
||||
|
@ -438,7 +438,7 @@ class HTMLViewer(object):
|
|||
file_path = nfls[0]
|
||||
try:
|
||||
lineno = int(nfls[1])
|
||||
except:
|
||||
except (TypeError, ValueError, IndexError):
|
||||
lineno = 0
|
||||
# for security reason, this need to be fixed.
|
||||
if not file_path.endswith('.py'):
|
||||
|
|
|
@ -242,18 +242,13 @@ class ProfileMiddleware(object):
|
|||
start_response('500 Internal Server Error', [])
|
||||
return _('Error on render profiling results: %s') % ex
|
||||
else:
|
||||
try:
|
||||
_locals = locals()
|
||||
code = self.unwind and PROFILE_EXEC_EAGER or\
|
||||
PROFILE_EXEC_LAZY
|
||||
self.profiler.runctx(code, globals(), _locals)
|
||||
app_iter = _locals['app_iter_']
|
||||
self.dump_checkpoint()
|
||||
return app_iter
|
||||
except:
|
||||
self.logger.exception(_('Error profiling code'))
|
||||
finally:
|
||||
pass
|
||||
_locals = locals()
|
||||
code = self.unwind and PROFILE_EXEC_EAGER or\
|
||||
PROFILE_EXEC_LAZY
|
||||
self.profiler.runctx(code, globals(), _locals)
|
||||
app_iter = _locals['app_iter_']
|
||||
self.dump_checkpoint()
|
||||
return app_iter
|
||||
|
||||
def renew_profile(self):
|
||||
self.profiler = get_profiler(self.profile_module)
|
||||
|
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
import bisect
|
||||
import copy
|
||||
import errno
|
||||
import itertools
|
||||
import math
|
||||
import random
|
||||
|
@ -623,6 +624,7 @@ class RingBuilder(object):
|
|||
"""
|
||||
self._last_part_moves = array('B', (0 for _junk in xrange(self.parts)))
|
||||
self._last_part_moves_epoch = int(time())
|
||||
self._set_parts_wanted()
|
||||
|
||||
self._reassign_parts(self._adjust_replica2part2dev_size()[0])
|
||||
|
||||
|
@ -643,6 +645,26 @@ class RingBuilder(object):
|
|||
self._last_part_moves[part] = 0xff
|
||||
self._last_part_moves_epoch = int(time())
|
||||
|
||||
def _get_available_parts(self):
|
||||
"""
|
||||
Returns a tuple (wanted_parts_total, dict of (tier: available parts in
|
||||
other tiers) for all tiers in the ring.
|
||||
|
||||
Devices that have too much partitions (negative parts_wanted) are
|
||||
ignored, otherwise the sum of all parts_wanted is 0 +/- rounding
|
||||
errors.
|
||||
|
||||
"""
|
||||
wanted_parts_total = 0
|
||||
wanted_parts_for_tier = {}
|
||||
for dev in self._iter_devs():
|
||||
wanted_parts_total += max(0, dev['parts_wanted'])
|
||||
for tier in tiers_for_dev(dev):
|
||||
if tier not in wanted_parts_for_tier:
|
||||
wanted_parts_for_tier[tier] = 0
|
||||
wanted_parts_for_tier[tier] += max(0, dev['parts_wanted'])
|
||||
return (wanted_parts_total, wanted_parts_for_tier)
|
||||
|
||||
def _gather_reassign_parts(self):
|
||||
"""
|
||||
Returns a list of (partition, replicas) pairs to be reassigned by
|
||||
|
@ -671,6 +693,9 @@ class RingBuilder(object):
|
|||
# currently sufficient spread out across the cluster.
|
||||
spread_out_parts = defaultdict(list)
|
||||
max_allowed_replicas = self._build_max_replicas_by_tier()
|
||||
wanted_parts_total, wanted_parts_for_tier = \
|
||||
self._get_available_parts()
|
||||
moved_parts = 0
|
||||
for part in xrange(self.parts):
|
||||
# Only move one replica at a time if possible.
|
||||
if part in removed_dev_parts:
|
||||
|
@ -701,14 +726,20 @@ class RingBuilder(object):
|
|||
rep_at_tier = 0
|
||||
if tier in replicas_at_tier:
|
||||
rep_at_tier = replicas_at_tier[tier]
|
||||
# Only allowing parts to be gathered if
|
||||
# there are wanted parts on other tiers
|
||||
available_parts_for_tier = wanted_parts_total - \
|
||||
wanted_parts_for_tier[tier] - moved_parts
|
||||
if (rep_at_tier > max_allowed_replicas[tier] and
|
||||
self._last_part_moves[part] >=
|
||||
self.min_part_hours):
|
||||
self.min_part_hours and
|
||||
available_parts_for_tier > 0):
|
||||
self._last_part_moves[part] = 0
|
||||
spread_out_parts[part].append(replica)
|
||||
dev['parts_wanted'] += 1
|
||||
dev['parts'] -= 1
|
||||
removed_replica = True
|
||||
moved_parts += 1
|
||||
break
|
||||
if removed_replica:
|
||||
if dev['id'] not in tfd:
|
||||
|
@ -1055,7 +1086,26 @@ class RingBuilder(object):
|
|||
:param builder_file: path to builder file to load
|
||||
:return: RingBuilder instance
|
||||
"""
|
||||
builder = pickle.load(open(builder_file, 'rb'))
|
||||
try:
|
||||
fp = open(builder_file, 'rb')
|
||||
except IOError as e:
|
||||
if e.errno == errno.ENOENT:
|
||||
raise exceptions.FileNotFoundError(
|
||||
'Ring Builder file does not exist: %s' % builder_file)
|
||||
elif e.errno in [errno.EPERM, errno.EACCES]:
|
||||
raise exceptions.PermissionError(
|
||||
'Ring Builder file cannot be accessed: %s' % builder_file)
|
||||
else:
|
||||
raise
|
||||
else:
|
||||
with fp:
|
||||
try:
|
||||
builder = pickle.load(fp)
|
||||
except Exception:
|
||||
# raise error during unpickling as UnPicklingError
|
||||
raise exceptions.UnPicklingError(
|
||||
'Ring Builder file is invalid: %s' % builder_file)
|
||||
|
||||
if not hasattr(builder, 'devs'):
|
||||
builder_dict = builder
|
||||
builder = RingBuilder(1, 1, 1)
|
||||
|
|
|
@ -380,6 +380,10 @@ def run_server(conf, logger, sock, global_conf=None):
|
|||
eventlet.patcher.monkey_patch(all=False, socket=True)
|
||||
eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
|
||||
eventlet.debug.hub_exceptions(eventlet_debug)
|
||||
wsgi_logger = NullLogger()
|
||||
if eventlet_debug:
|
||||
# let eventlet.wsgi.server log to stderr
|
||||
wsgi_logger = None
|
||||
# utils.LogAdapter stashes name in server; fallback on unadapted loggers
|
||||
if not global_conf:
|
||||
if hasattr(logger, 'server'):
|
||||
|
@ -395,10 +399,10 @@ def run_server(conf, logger, sock, global_conf=None):
|
|||
# necessary for the AWS SDK to work with swift3 middleware.
|
||||
argspec = inspect.getargspec(wsgi.server)
|
||||
if 'capitalize_response_headers' in argspec.args:
|
||||
wsgi.server(sock, app, NullLogger(), custom_pool=pool,
|
||||
wsgi.server(sock, app, wsgi_logger, custom_pool=pool,
|
||||
capitalize_response_headers=False)
|
||||
else:
|
||||
wsgi.server(sock, app, NullLogger(), custom_pool=pool)
|
||||
wsgi.server(sock, app, wsgi_logger, custom_pool=pool)
|
||||
except socket.error as err:
|
||||
if err[0] != errno.EINVAL:
|
||||
raise
|
||||
|
|
|
@ -365,6 +365,7 @@ class ContainerReconciler(Daemon):
|
|||
object queue entry.
|
||||
|
||||
:param container: the misplaced objects container
|
||||
:param obj: the name of the misplaced object
|
||||
:param q_ts: the timestamp of the misplaced object
|
||||
:param q_record: the timestamp of the queue entry
|
||||
|
||||
|
@ -387,7 +388,7 @@ class ContainerReconciler(Daemon):
|
|||
|
||||
:param account: the account name
|
||||
:param container: the container name
|
||||
:param account: the object name
|
||||
:param obj: the object name
|
||||
:param timestamp: the timestamp of the object to delete
|
||||
:param policy_index: the policy index to direct the request
|
||||
:param path: the path to be used for logging
|
||||
|
@ -732,7 +733,7 @@ class ContainerReconciler(Daemon):
|
|||
"""
|
||||
try:
|
||||
self.reconcile()
|
||||
except:
|
||||
except: # noqa
|
||||
self.logger.exception('Unhandled Exception trying to reconcile')
|
||||
self.log_stats(force=True)
|
||||
|
||||
|
|
|
@ -374,7 +374,7 @@ class ContainerController(object):
|
|||
metadata['X-Container-Sync-To'][0] != \
|
||||
broker.metadata['X-Container-Sync-To'][0]:
|
||||
broker.set_x_container_sync_points(-1, -1)
|
||||
broker.update_metadata(metadata)
|
||||
broker.update_metadata(metadata, validate_metadata=True)
|
||||
resp = self.account_update(req, account, container, broker)
|
||||
if resp:
|
||||
return resp
|
||||
|
@ -551,7 +551,7 @@ class ContainerController(object):
|
|||
metadata['X-Container-Sync-To'][0] != \
|
||||
broker.metadata['X-Container-Sync-To'][0]:
|
||||
broker.set_x_container_sync_points(-1, -1)
|
||||
broker.update_metadata(metadata)
|
||||
broker.update_metadata(metadata, validate_metadata=True)
|
||||
return HTTPNoContent(request=req)
|
||||
|
||||
def __call__(self, env, start_response):
|
||||
|
|
|
@ -1,21 +0,0 @@
|
|||
# Translations template for heat.
|
||||
# Copyright (C) 2014 ORGANIZATION
|
||||
# This file is distributed under the same license as the heat project.
|
||||
#
|
||||
# Translators:
|
||||
# Andi Chandler <andi@gowling.com>, 2014
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: Swift\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
|
||||
"PO-Revision-Date: 2014-07-25 15:03+0000\n"
|
||||
"Last-Translator: Andi Chandler <andi@gowling.com>\n"
|
||||
"Language-Team: English (United Kingdom) (http://www.transifex.com/projects/p/"
|
||||
"swift/language/en_GB/)\n"
|
||||
"Language: en_GB\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Generated-By: Babel 1.3\n"
|
||||
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
|
|
@ -1,21 +0,0 @@
|
|||
# Translations template for heat.
|
||||
# Copyright (C) 2014 ORGANIZATION
|
||||
# This file is distributed under the same license as the heat project.
|
||||
#
|
||||
# Translators:
|
||||
# Andi Chandler <andi@gowling.com>, 2014
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: Swift\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
|
||||
"PO-Revision-Date: 2014-07-25 23:08+0000\n"
|
||||
"Last-Translator: Andi Chandler <andi@gowling.com>\n"
|
||||
"Language-Team: English (United Kingdom) (http://www.transifex.com/projects/p/"
|
||||
"swift/language/en_GB/)\n"
|
||||
"Language: en_GB\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Generated-By: Babel 1.3\n"
|
||||
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
|
|
@ -1,21 +0,0 @@
|
|||
# Translations template for heat.
|
||||
# Copyright (C) 2014 ORGANIZATION
|
||||
# This file is distributed under the same license as the heat project.
|
||||
#
|
||||
# Translators:
|
||||
# Andi Chandler <andi@gowling.com>, 2014
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: Swift\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
|
||||
"PO-Revision-Date: 2014-07-25 15:03+0000\n"
|
||||
"Last-Translator: Andi Chandler <andi@gowling.com>\n"
|
||||
"Language-Team: English (United Kingdom) (http://www.transifex.com/projects/p/"
|
||||
"swift/language/en_GB/)\n"
|
||||
"Language: en_GB\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Generated-By: Babel 1.3\n"
|
||||
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
|
|
@ -1,21 +0,0 @@
|
|||
# Translations template for heat.
|
||||
# Copyright (C) 2014 ORGANIZATION
|
||||
# This file is distributed under the same license as the heat project.
|
||||
#
|
||||
# Translators:
|
||||
# Andi Chandler <andi@gowling.com>, 2014
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: Swift\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
|
||||
"PO-Revision-Date: 2014-07-25 15:02+0000\n"
|
||||
"Last-Translator: Andi Chandler <andi@gowling.com>\n"
|
||||
"Language-Team: English (United Kingdom) (http://www.transifex.com/projects/p/"
|
||||
"swift/language/en_GB/)\n"
|
||||
"Language: en_GB\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Generated-By: Babel 1.3\n"
|
||||
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
|
|
@ -1,21 +0,0 @@
|
|||
# Translations template for heat.
|
||||
# Copyright (C) 2014 ORGANIZATION
|
||||
# This file is distributed under the same license as the heat project.
|
||||
#
|
||||
# Translators:
|
||||
# Mario Cho <hephaex@gmail.com>, 2014
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: Swift\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
|
||||
"PO-Revision-Date: 2014-09-18 02:40+0000\n"
|
||||
"Last-Translator: Mario Cho <hephaex@gmail.com>\n"
|
||||
"Language-Team: Korean (Korea) (http://www.transifex.com/projects/p/swift/"
|
||||
"language/ko_KR/)\n"
|
||||
"Language: ko_KR\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Generated-By: Babel 1.3\n"
|
||||
"Plural-Forms: nplurals=1; plural=0;\n"
|
|
@ -1,21 +0,0 @@
|
|||
# Translations template for heat.
|
||||
# Copyright (C) 2014 ORGANIZATION
|
||||
# This file is distributed under the same license as the heat project.
|
||||
#
|
||||
# Translators:
|
||||
# Mario Cho <hephaex@gmail.com>, 2014
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: Swift\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
|
||||
"PO-Revision-Date: 2014-09-18 02:40+0000\n"
|
||||
"Last-Translator: Mario Cho <hephaex@gmail.com>\n"
|
||||
"Language-Team: Korean (Korea) (http://www.transifex.com/projects/p/swift/"
|
||||
"language/ko_KR/)\n"
|
||||
"Language: ko_KR\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Generated-By: Babel 1.3\n"
|
||||
"Plural-Forms: nplurals=1; plural=0;\n"
|
|
@ -1,21 +0,0 @@
|
|||
# Translations template for heat.
|
||||
# Copyright (C) 2014 ORGANIZATION
|
||||
# This file is distributed under the same license as the heat project.
|
||||
#
|
||||
# Translators:
|
||||
# Mario Cho <hephaex@gmail.com>, 2014
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: Swift\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
|
||||
"PO-Revision-Date: 2014-09-18 02:40+0000\n"
|
||||
"Last-Translator: Mario Cho <hephaex@gmail.com>\n"
|
||||
"Language-Team: Korean (Korea) (http://www.transifex.com/projects/p/swift/"
|
||||
"language/ko_KR/)\n"
|
||||
"Language: ko_KR\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Generated-By: Babel 1.3\n"
|
||||
"Plural-Forms: nplurals=1; plural=0;\n"
|
|
@ -1,21 +0,0 @@
|
|||
# Translations template for heat.
|
||||
# Copyright (C) 2014 ORGANIZATION
|
||||
# This file is distributed under the same license as the heat project.
|
||||
#
|
||||
# Translators:
|
||||
# Mario Cho <hephaex@gmail.com>, 2014
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: Swift\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
|
||||
"PO-Revision-Date: 2014-09-18 02:40+0000\n"
|
||||
"Last-Translator: Mario Cho <hephaex@gmail.com>\n"
|
||||
"Language-Team: Korean (Korea) (http://www.transifex.com/projects/p/swift/"
|
||||
"language/ko_KR/)\n"
|
||||
"Language: ko_KR\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Generated-By: Babel 1.3\n"
|
||||
"Plural-Forms: nplurals=1; plural=0;\n"
|
|
@ -6,9 +6,9 @@
|
|||
#, fuzzy
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: swift 2.1.0.77.g0d0c16d\n"
|
||||
"Project-Id-Version: swift 2.1.0.98.g6cd860b\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
|
||||
"POT-Creation-Date: 2014-09-28 06:08+0000\n"
|
||||
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
|
||||
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
|
||||
"Language-Team: LANGUAGE <LL@li.org>\n"
|
||||
|
@ -17,42 +17,54 @@ msgstr ""
|
|||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Generated-By: Babel 1.3\n"
|
||||
|
||||
#: swift/account/auditor.py:58
|
||||
#: swift/account/auditor.py:59
|
||||
#, python-format
|
||||
msgid ""
|
||||
"Since %(time)s: Account audits: %(passed)s passed audit,%(failed)s failed"
|
||||
" audit"
|
||||
msgstr ""
|
||||
|
||||
#: swift/account/auditor.py:81
|
||||
#: swift/account/auditor.py:82
|
||||
msgid "Begin account audit pass."
|
||||
msgstr ""
|
||||
|
||||
#: swift/account/auditor.py:87 swift/container/auditor.py:86
|
||||
#: swift/account/auditor.py:88 swift/container/auditor.py:86
|
||||
msgid "ERROR auditing"
|
||||
msgstr ""
|
||||
|
||||
#: swift/account/auditor.py:92
|
||||
#: swift/account/auditor.py:93
|
||||
#, python-format
|
||||
msgid "Account audit pass completed: %.02fs"
|
||||
msgstr ""
|
||||
|
||||
#: swift/account/auditor.py:98
|
||||
#: swift/account/auditor.py:99
|
||||
msgid "Begin account audit \"once\" mode"
|
||||
msgstr ""
|
||||
|
||||
#: swift/account/auditor.py:103
|
||||
#: swift/account/auditor.py:104
|
||||
#, python-format
|
||||
msgid "Account audit \"once\" mode completed: %.02fs"
|
||||
msgstr ""
|
||||
|
||||
#: swift/account/auditor.py:124
|
||||
#: swift/account/auditor.py:123
|
||||
#, python-format
|
||||
msgid ""
|
||||
"The total %(key)s for the container (%(total)s) does not match the sum of"
|
||||
" %(key)s across policies (%(sum)s)"
|
||||
msgstr ""
|
||||
|
||||
#: swift/account/auditor.py:149
|
||||
#, python-format
|
||||
msgid "Audit Failed for %s: %s"
|
||||
msgstr ""
|
||||
|
||||
#: swift/account/auditor.py:153
|
||||
#, python-format
|
||||
msgid "ERROR Could not get account info %s"
|
||||
msgstr ""
|
||||
|
||||
#: swift/account/reaper.py:132 swift/common/utils.py:1952
|
||||
#: swift/obj/diskfile.py:425 swift/obj/updater.py:78 swift/obj/updater.py:121
|
||||
#: swift/account/reaper.py:132 swift/common/utils.py:1970
|
||||
#: swift/obj/diskfile.py:432 swift/obj/updater.py:78 swift/obj/updater.py:121
|
||||
#, python-format
|
||||
msgid "Skipping %s as it is not mounted"
|
||||
msgstr ""
|
||||
|
@ -142,7 +154,7 @@ msgid "Exception with objects for container %(container)s for account %(account)
|
|||
msgstr ""
|
||||
|
||||
#: swift/account/server.py:278 swift/container/server.py:580
|
||||
#: swift/obj/server.py:697
|
||||
#: swift/obj/server.py:710
|
||||
#, python-format
|
||||
msgid "ERROR __call__ error with %(method)s %(path)s "
|
||||
msgstr ""
|
||||
|
@ -366,95 +378,95 @@ msgstr ""
|
|||
msgid "Error limiting server %s"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:306
|
||||
#: swift/common/utils.py:324
|
||||
#, python-format
|
||||
msgid "Unable to locate %s in libc. Leaving as a no-op."
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:480
|
||||
#: swift/common/utils.py:498
|
||||
msgid "Unable to locate fallocate, posix_fallocate in libc. Leaving as a no-op."
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:911
|
||||
#: swift/common/utils.py:929
|
||||
msgid "STDOUT: Connection reset by peer"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:913 swift/common/utils.py:916
|
||||
#: swift/common/utils.py:931 swift/common/utils.py:934
|
||||
#, python-format
|
||||
msgid "STDOUT: %s"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:1150
|
||||
#: swift/common/utils.py:1168
|
||||
msgid "Connection refused"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:1152
|
||||
#: swift/common/utils.py:1170
|
||||
msgid "Host unreachable"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:1154
|
||||
#: swift/common/utils.py:1172
|
||||
msgid "Connection timeout"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:1456
|
||||
#: swift/common/utils.py:1474
|
||||
msgid "UNCAUGHT EXCEPTION"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:1511
|
||||
#: swift/common/utils.py:1529
|
||||
msgid "Error: missing config path argument"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:1516
|
||||
#: swift/common/utils.py:1534
|
||||
#, python-format
|
||||
msgid "Error: unable to locate %s"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:1813
|
||||
#: swift/common/utils.py:1831
|
||||
#, python-format
|
||||
msgid "Unable to read config from %s"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:1819
|
||||
#: swift/common/utils.py:1837
|
||||
#, python-format
|
||||
msgid "Unable to find %s config section in %s"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:2173
|
||||
#: swift/common/utils.py:2191
|
||||
#, python-format
|
||||
msgid "Invalid X-Container-Sync-To format %r"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:2178
|
||||
#: swift/common/utils.py:2196
|
||||
#, python-format
|
||||
msgid "No realm key for %r"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:2182
|
||||
#: swift/common/utils.py:2200
|
||||
#, python-format
|
||||
msgid "No cluster endpoint for %r %r"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:2191
|
||||
#: swift/common/utils.py:2209
|
||||
#, python-format
|
||||
msgid ""
|
||||
"Invalid scheme %r in X-Container-Sync-To, must be \"//\", \"http\", or "
|
||||
"\"https\"."
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:2195
|
||||
#: swift/common/utils.py:2213
|
||||
msgid "Path required in X-Container-Sync-To"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:2198
|
||||
#: swift/common/utils.py:2216
|
||||
msgid "Params, queries, and fragments not allowed in X-Container-Sync-To"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:2203
|
||||
#: swift/common/utils.py:2221
|
||||
#, python-format
|
||||
msgid "Invalid host %r in X-Container-Sync-To"
|
||||
msgstr ""
|
||||
|
||||
#: swift/common/utils.py:2395
|
||||
#: swift/common/utils.py:2413
|
||||
msgid "Exception dumping recon cache"
|
||||
msgstr ""
|
||||
|
||||
|
@ -793,36 +805,36 @@ msgstr ""
|
|||
msgid "ERROR auditing: %s"
|
||||
msgstr ""
|
||||
|
||||
#: swift/obj/diskfile.py:275
|
||||
#: swift/obj/diskfile.py:282
|
||||
#, python-format
|
||||
msgid "Quarantined %(hsh_path)s to %(quar_path)s because it is not a directory"
|
||||
msgstr ""
|
||||
|
||||
#: swift/obj/diskfile.py:364
|
||||
#: swift/obj/diskfile.py:371
|
||||
msgid "Error hashing suffix"
|
||||
msgstr ""
|
||||
|
||||
#: swift/obj/diskfile.py:439 swift/obj/updater.py:160
|
||||
#: swift/obj/diskfile.py:446 swift/obj/updater.py:160
|
||||
#, python-format
|
||||
msgid "Directory %s does not map to a valid policy"
|
||||
msgstr ""
|
||||
|
||||
#: swift/obj/diskfile.py:602
|
||||
#: swift/obj/diskfile.py:642
|
||||
#, python-format
|
||||
msgid "Quarantined %(object_path)s to %(quar_path)s because it is not a directory"
|
||||
msgstr ""
|
||||
|
||||
#: swift/obj/diskfile.py:784
|
||||
#: swift/obj/diskfile.py:824
|
||||
#, python-format
|
||||
msgid "Problem cleaning up %s"
|
||||
msgstr ""
|
||||
|
||||
#: swift/obj/diskfile.py:969
|
||||
#: swift/obj/diskfile.py:1120
|
||||
#, python-format
|
||||
msgid "ERROR DiskFile %(data_file)s close failure: %(exc)s : %(stack)s"
|
||||
msgstr ""
|
||||
|
||||
#: swift/obj/diskfile.py:1246
|
||||
#: swift/obj/diskfile.py:1401
|
||||
#, python-format
|
||||
msgid ""
|
||||
"Client path %(client)s does not match path stored in object metadata "
|
||||
|
@ -971,21 +983,21 @@ msgstr ""
|
|||
msgid "Object replication complete. (%.02f minutes)"
|
||||
msgstr ""
|
||||
|
||||
#: swift/obj/server.py:188
|
||||
#: swift/obj/server.py:201
|
||||
#, python-format
|
||||
msgid ""
|
||||
"ERROR Container update failed (saving for async update later): %(status)d"
|
||||
" response from %(ip)s:%(port)s/%(dev)s"
|
||||
msgstr ""
|
||||
|
||||
#: swift/obj/server.py:195
|
||||
#: swift/obj/server.py:208
|
||||
#, python-format
|
||||
msgid ""
|
||||
"ERROR container update failed with %(ip)s:%(port)s/%(dev)s (saving for "
|
||||
"async update later)"
|
||||
msgstr ""
|
||||
|
||||
#: swift/obj/server.py:230
|
||||
#: swift/obj/server.py:243
|
||||
#, python-format
|
||||
msgid ""
|
||||
"ERROR Container update failed: different numbers of hosts and devices in "
|
||||
|
|
|
@ -257,8 +257,12 @@ class ObjectAuditor(Daemon):
|
|||
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
||||
if zero_byte_fps:
|
||||
kwargs['zero_byte_fps'] = self.conf_zero_byte_fps
|
||||
self.run_audit(**kwargs)
|
||||
sys.exit()
|
||||
try:
|
||||
self.run_audit(**kwargs)
|
||||
except Exception as e:
|
||||
self.logger.error(_("ERROR: Unable to run auditing: %s") % e)
|
||||
finally:
|
||||
sys.exit()
|
||||
|
||||
def audit_loop(self, parent, zbo_fps, override_devices=None, **kwargs):
|
||||
"""Parallel audit loop"""
|
||||
|
|
|
@ -358,7 +358,7 @@ def get_hashes(partition_dir, recalculate=None, do_listdir=False,
|
|||
if len(suff) == 3:
|
||||
hashes.setdefault(suff, None)
|
||||
modified = True
|
||||
hashes.update((hash_, None) for hash_ in recalculate)
|
||||
hashes.update((suffix, None) for suffix in recalculate)
|
||||
for suffix, hash_ in hashes.items():
|
||||
if not hash_:
|
||||
suffix_dir = join(partition_dir, suffix)
|
||||
|
|
|
@ -83,7 +83,6 @@ class ObjectReplicator(Daemon):
|
|||
self.node_timeout = float(conf.get('node_timeout', 10))
|
||||
self.sync_method = getattr(self, conf.get('sync_method') or 'rsync')
|
||||
self.network_chunk_size = int(conf.get('network_chunk_size', 65536))
|
||||
self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536))
|
||||
self.headers = {
|
||||
'Content-Length': '0',
|
||||
'user-agent': 'object-replicator %s' % os.getpid()}
|
||||
|
@ -284,7 +283,7 @@ class ObjectReplicator(Daemon):
|
|||
job['nodes'],
|
||||
job['object_ring'].get_more_nodes(int(job['partition'])))
|
||||
while attempts_left > 0:
|
||||
# If this throws StopIterator it will be caught way below
|
||||
# If this throws StopIteration it will be caught way below
|
||||
node = next(nodes)
|
||||
attempts_left -= 1
|
||||
try:
|
||||
|
|
|
@ -774,6 +774,21 @@ class TestAccount(unittest.TestCase):
|
|||
resp.read()
|
||||
self.assertEqual(resp.status, 400)
|
||||
|
||||
def test_bad_metadata2(self):
|
||||
if tf.skip:
|
||||
raise SkipTest
|
||||
|
||||
def post(url, token, parsed, conn, extra_headers):
|
||||
headers = {'X-Auth-Token': token}
|
||||
headers.update(extra_headers)
|
||||
conn.request('POST', parsed.path, '', headers)
|
||||
return check_response(conn)
|
||||
|
||||
# TODO: Find the test that adds these and remove them.
|
||||
headers = {'x-remove-account-meta-temp-url-key': 'remove',
|
||||
'x-remove-account-meta-temp-url-key-2': 'remove'}
|
||||
resp = retry(post, headers)
|
||||
|
||||
headers = {}
|
||||
for x in xrange(self.max_meta_count):
|
||||
headers['X-Account-Meta-%d' % x] = 'v'
|
||||
|
@ -787,6 +802,16 @@ class TestAccount(unittest.TestCase):
|
|||
resp.read()
|
||||
self.assertEqual(resp.status, 400)
|
||||
|
||||
def test_bad_metadata3(self):
|
||||
if tf.skip:
|
||||
raise SkipTest
|
||||
|
||||
def post(url, token, parsed, conn, extra_headers):
|
||||
headers = {'X-Auth-Token': token}
|
||||
headers.update(extra_headers)
|
||||
conn.request('POST', parsed.path, '', headers)
|
||||
return check_response(conn)
|
||||
|
||||
headers = {}
|
||||
header_value = 'k' * self.max_meta_value_length
|
||||
size = 0
|
||||
|
|
|
@ -401,6 +401,16 @@ class TestContainer(unittest.TestCase):
|
|||
resp.read()
|
||||
self.assertEqual(resp.status, 400)
|
||||
|
||||
def test_POST_bad_metadata2(self):
|
||||
if tf.skip:
|
||||
raise SkipTest
|
||||
|
||||
def post(url, token, parsed, conn, extra_headers):
|
||||
headers = {'X-Auth-Token': token}
|
||||
headers.update(extra_headers)
|
||||
conn.request('POST', parsed.path + '/' + self.name, '', headers)
|
||||
return check_response(conn)
|
||||
|
||||
headers = {}
|
||||
for x in xrange(self.max_meta_count):
|
||||
headers['X-Container-Meta-%d' % x] = 'v'
|
||||
|
@ -414,6 +424,16 @@ class TestContainer(unittest.TestCase):
|
|||
resp.read()
|
||||
self.assertEqual(resp.status, 400)
|
||||
|
||||
def test_POST_bad_metadata3(self):
|
||||
if tf.skip:
|
||||
raise SkipTest
|
||||
|
||||
def post(url, token, parsed, conn, extra_headers):
|
||||
headers = {'X-Auth-Token': token}
|
||||
headers.update(extra_headers)
|
||||
conn.request('POST', parsed.path + '/' + self.name, '', headers)
|
||||
return check_response(conn)
|
||||
|
||||
headers = {}
|
||||
header_value = 'k' * self.max_meta_value_length
|
||||
size = 0
|
||||
|
|
|
@ -99,7 +99,7 @@ class TestRecon(unittest.TestCase):
|
|||
def tearDown(self, *_args, **_kwargs):
|
||||
try:
|
||||
os.remove(self.tmpfile_name)
|
||||
except:
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def test_gen_stats(self):
|
||||
|
@ -208,6 +208,9 @@ class TestRecon(unittest.TestCase):
|
|||
self.fail('Did not find expected substring %r '
|
||||
'in output:\n%s' % (expected, output))
|
||||
|
||||
for ring in ('account', 'container', 'object', 'object-1'):
|
||||
os.remove(os.path.join(self.swift_dir, "%s.ring.gz" % ring))
|
||||
|
||||
|
||||
class TestReconCommands(unittest.TestCase):
|
||||
def setUp(self):
|
||||
|
|
|
@ -13,11 +13,14 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import mock
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
import swift.cli.ringbuilder
|
||||
from swift.common import exceptions
|
||||
from swift.common.ring import RingBuilder
|
||||
|
||||
|
||||
|
@ -199,6 +202,65 @@ class TestCommands(unittest.TestCase):
|
|||
ring = RingBuilder.load(self.tmpfile)
|
||||
self.assertEqual(ring.replicas, 3.14159265359)
|
||||
|
||||
def test_validate(self):
|
||||
self.create_sample_ring()
|
||||
ring = RingBuilder.load(self.tmpfile)
|
||||
ring.rebalance()
|
||||
ring.save(self.tmpfile)
|
||||
argv = ["", self.tmpfile, "validate"]
|
||||
self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
|
||||
|
||||
def test_validate_empty_file(self):
|
||||
open(self.tmpfile, 'a').close
|
||||
argv = ["", self.tmpfile, "validate"]
|
||||
try:
|
||||
swift.cli.ringbuilder.main(argv)
|
||||
except SystemExit as e:
|
||||
self.assertEquals(e.code, 2)
|
||||
|
||||
def test_validate_corrupted_file(self):
|
||||
self.create_sample_ring()
|
||||
ring = RingBuilder.load(self.tmpfile)
|
||||
ring.rebalance()
|
||||
self.assertTrue(ring.validate()) # ring is valid until now
|
||||
ring.save(self.tmpfile)
|
||||
argv = ["", self.tmpfile, "validate"]
|
||||
|
||||
# corrupt the file
|
||||
with open(self.tmpfile, 'wb') as f:
|
||||
f.write(os.urandom(1024))
|
||||
try:
|
||||
swift.cli.ringbuilder.main(argv)
|
||||
except SystemExit as e:
|
||||
self.assertEquals(e.code, 2)
|
||||
|
||||
def test_validate_non_existent_file(self):
|
||||
rand_file = '%s/%s' % ('/tmp', str(uuid.uuid4()))
|
||||
argv = ["", rand_file, "validate"]
|
||||
try:
|
||||
swift.cli.ringbuilder.main(argv)
|
||||
except SystemExit as e:
|
||||
self.assertEquals(e.code, 2)
|
||||
|
||||
def test_validate_non_accessible_file(self):
|
||||
with mock.patch.object(
|
||||
RingBuilder, 'load',
|
||||
mock.Mock(side_effect=exceptions.PermissionError)):
|
||||
argv = ["", self.tmpfile, "validate"]
|
||||
try:
|
||||
swift.cli.ringbuilder.main(argv)
|
||||
except SystemExit as e:
|
||||
self.assertEquals(e.code, 2)
|
||||
|
||||
def test_validate_generic_error(self):
|
||||
with mock.patch.object(RingBuilder, 'load',
|
||||
mock.Mock(side_effect=
|
||||
IOError('Generic error occurred'))):
|
||||
argv = ["", self.tmpfile, "validate"]
|
||||
try:
|
||||
swift.cli.ringbuilder.main(argv)
|
||||
except SystemExit as e:
|
||||
self.assertEquals(e.code, 2)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
@ -18,13 +18,16 @@ import contextlib
|
|||
import hashlib
|
||||
import json
|
||||
import mock
|
||||
import shutil
|
||||
import tempfile
|
||||
from textwrap import dedent
|
||||
import time
|
||||
import unittest
|
||||
|
||||
from swift.common import exceptions, swob
|
||||
from swift.common.middleware import dlo
|
||||
from test.unit.common.middleware.helpers import FakeSwift
|
||||
from textwrap import dedent
|
||||
|
||||
|
||||
LIMIT = 'swift.common.constraints.CONTAINER_LISTING_LIMIT'
|
||||
|
||||
|
@ -898,6 +901,12 @@ class TestDloConfiguration(unittest.TestCase):
|
|||
proxy's config section if we don't have any config values.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.tmpdir = tempfile.mkdtemp()
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmpdir)
|
||||
|
||||
def test_skip_defaults_if_configured(self):
|
||||
# The presence of even one config value in our config section means we
|
||||
# won't go looking for the proxy config at all.
|
||||
|
@ -984,7 +993,7 @@ class TestDloConfiguration(unittest.TestCase):
|
|||
max_get_time = 2900
|
||||
""")
|
||||
|
||||
conf_dir = tempfile.mkdtemp()
|
||||
conf_dir = self.tmpdir
|
||||
|
||||
conffile1 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf')
|
||||
conffile1.write(proxy_conf1)
|
||||
|
|
|
@ -189,7 +189,7 @@ class TestRateLimit(unittest.TestCase):
|
|||
the_app = ratelimit.filter_factory(conf_dict)(FakeApp())
|
||||
the_app.memcache_client = fake_memcache
|
||||
req = lambda: None
|
||||
req.environ = {}
|
||||
req.environ = {'swift.cache': fake_memcache, 'PATH_INFO': '/v1/a/c/o'}
|
||||
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
|
||||
lambda *args, **kwargs: {}):
|
||||
req.method = 'DELETE'
|
||||
|
@ -243,7 +243,7 @@ class TestRateLimit(unittest.TestCase):
|
|||
the_app.memcache_client = fake_memcache
|
||||
req = lambda: None
|
||||
req.method = 'PUT'
|
||||
req.environ = {}
|
||||
req.environ = {'PATH_INFO': '/v1/a/c/o', 'swift.cache': fake_memcache}
|
||||
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
|
||||
lambda *args, **kwargs: {}):
|
||||
tuples = the_app.get_ratelimitable_key_tuples(req, 'a', 'c', 'o')
|
||||
|
@ -255,20 +255,23 @@ class TestRateLimit(unittest.TestCase):
|
|||
conf_dict = {'account_ratelimit': current_rate}
|
||||
self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
|
||||
ratelimit.http_connect = mock_http_connect(204)
|
||||
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
|
||||
with mock.patch('swift.common.middleware.ratelimit.get_container_info',
|
||||
lambda *args, **kwargs: {}):
|
||||
for meth, exp_time in [
|
||||
('DELETE', 9.8), ('GET', 0), ('POST', 0), ('PUT', 9.8)]:
|
||||
req = Request.blank('/v/a%s/c' % meth)
|
||||
req.method = meth
|
||||
req.environ['swift.cache'] = FakeMemcache()
|
||||
make_app_call = lambda: self.test_ratelimit(req.environ,
|
||||
start_response)
|
||||
begin = time.time()
|
||||
self._run(make_app_call, num_calls, current_rate,
|
||||
check_time=bool(exp_time))
|
||||
self.assertEquals(round(time.time() - begin, 1), exp_time)
|
||||
self._reset_time()
|
||||
with mock.patch(
|
||||
'swift.common.middleware.ratelimit.get_account_info',
|
||||
lambda *args, **kwargs: {}):
|
||||
for meth, exp_time in [('DELETE', 9.8), ('GET', 0),
|
||||
('POST', 0), ('PUT', 9.8)]:
|
||||
req = Request.blank('/v/a%s/c' % meth)
|
||||
req.method = meth
|
||||
req.environ['swift.cache'] = FakeMemcache()
|
||||
make_app_call = lambda: self.test_ratelimit(req.environ,
|
||||
start_response)
|
||||
begin = time.time()
|
||||
self._run(make_app_call, num_calls, current_rate,
|
||||
check_time=bool(exp_time))
|
||||
self.assertEquals(round(time.time() - begin, 1), exp_time)
|
||||
self._reset_time()
|
||||
|
||||
def test_ratelimit_set_incr(self):
|
||||
current_rate = 5
|
||||
|
@ -403,7 +406,7 @@ class TestRateLimit(unittest.TestCase):
|
|||
req.method = 'PUT'
|
||||
req.environ['swift.cache'] = FakeMemcache()
|
||||
req.environ['swift.cache'].set(
|
||||
ratelimit.get_container_memcache_key('a', 'c'),
|
||||
get_container_memcache_key('a', 'c'),
|
||||
{'container_size': 1})
|
||||
|
||||
time_override = [0, 0, 0, 0, None]
|
||||
|
@ -437,7 +440,7 @@ class TestRateLimit(unittest.TestCase):
|
|||
req.method = 'GET'
|
||||
req.environ['swift.cache'] = FakeMemcache()
|
||||
req.environ['swift.cache'].set(
|
||||
ratelimit.get_container_memcache_key('a', 'c'),
|
||||
get_container_memcache_key('a', 'c'),
|
||||
{'container_size': 1})
|
||||
|
||||
time_override = [0, 0, 0, 0, None]
|
||||
|
|
|
@ -195,8 +195,9 @@ class Test_profile_log(unittest.TestCase):
|
|||
def setUp(self):
|
||||
if xprofile is None:
|
||||
raise SkipTest
|
||||
self.tempdirs = [tempfile.mkdtemp(), tempfile.mkdtemp()]
|
||||
self.log_filename_prefix1 = self.tempdirs[0] + '/unittest.profile'
|
||||
|
||||
self.dir1 = tempfile.mkdtemp()
|
||||
self.log_filename_prefix1 = self.dir1 + '/unittest.profile'
|
||||
self.profile_log1 = ProfileLog(self.log_filename_prefix1, False)
|
||||
self.pids1 = ['123', '456', str(os.getpid())]
|
||||
profiler1 = xprofile.get_profiler('eventlet.green.profile')
|
||||
|
@ -204,7 +205,8 @@ class Test_profile_log(unittest.TestCase):
|
|||
profiler1.runctx('import os;os.getcwd();', globals(), locals())
|
||||
self.profile_log1.dump_profile(profiler1, pid)
|
||||
|
||||
self.log_filename_prefix2 = self.tempdirs[1] + '/unittest.profile'
|
||||
self.dir2 = tempfile.mkdtemp()
|
||||
self.log_filename_prefix2 = self.dir2 + '/unittest.profile'
|
||||
self.profile_log2 = ProfileLog(self.log_filename_prefix2, True)
|
||||
self.pids2 = ['321', '654', str(os.getpid())]
|
||||
profiler2 = xprofile.get_profiler('eventlet.green.profile')
|
||||
|
@ -215,8 +217,8 @@ class Test_profile_log(unittest.TestCase):
|
|||
def tearDown(self):
|
||||
self.profile_log1.clear('all')
|
||||
self.profile_log2.clear('all')
|
||||
for tempdir in self.tempdirs:
|
||||
shutil.rmtree(tempdir, ignore_errors=True)
|
||||
shutil.rmtree(self.dir1, ignore_errors=True)
|
||||
shutil.rmtree(self.dir2, ignore_errors=True)
|
||||
|
||||
def test_get_all_pids(self):
|
||||
self.assertEquals(self.profile_log1.get_all_pids(),
|
||||
|
|
|
@ -13,12 +13,14 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import errno
|
||||
import mock
|
||||
import operator
|
||||
import os
|
||||
import unittest
|
||||
import cPickle as pickle
|
||||
from collections import defaultdict
|
||||
from math import ceil
|
||||
from tempfile import mkdtemp
|
||||
from shutil import rmtree
|
||||
|
||||
|
@ -718,9 +720,7 @@ class TestRingBuilder(unittest.TestCase):
|
|||
population_by_region = self._get_population_by_region(rb)
|
||||
self.assertEquals(population_by_region, {0: 682, 1: 86})
|
||||
|
||||
# Rebalancing will reassign 143 of the partitions, which is ~1/5
|
||||
# of the total amount of partitions (3*256)
|
||||
self.assertEqual(143, changed_parts)
|
||||
self.assertEqual(87, changed_parts)
|
||||
|
||||
# and since there's not enough room, subsequent rebalances will not
|
||||
# cause additional assignments to r1
|
||||
|
@ -744,6 +744,35 @@ class TestRingBuilder(unittest.TestCase):
|
|||
population_by_region = self._get_population_by_region(rb)
|
||||
self.assertEquals(population_by_region, {0: 512, 1: 256})
|
||||
|
||||
def test_avoid_tier_change_new_region(self):
|
||||
rb = ring.RingBuilder(8, 3, 1)
|
||||
for i in range(5):
|
||||
rb.add_dev({'id': i, 'region': 0, 'zone': 0, 'weight': 100,
|
||||
'ip': '127.0.0.1', 'port': i, 'device': 'sda1'})
|
||||
rb.rebalance(seed=2)
|
||||
|
||||
# Add a new device in new region to a balanced ring
|
||||
rb.add_dev({'id': 5, 'region': 1, 'zone': 0, 'weight': 0,
|
||||
'ip': '127.0.0.5', 'port': 10000, 'device': 'sda1'})
|
||||
|
||||
# Increase the weight of region 1 slowly
|
||||
moved_partitions = []
|
||||
for weight in range(0, 101, 10):
|
||||
rb.set_dev_weight(5, weight)
|
||||
rb.pretend_min_part_hours_passed()
|
||||
changed_parts, _balance = rb.rebalance(seed=2)
|
||||
moved_partitions.append(changed_parts)
|
||||
# Ensure that the second region has enough partitions
|
||||
# Otherwise there will be replicas at risk
|
||||
min_parts_for_r1 = ceil(weight / (500.0 + weight) * 768)
|
||||
parts_for_r1 = self._get_population_by_region(rb).get(1, 0)
|
||||
self.assertEqual(min_parts_for_r1, parts_for_r1)
|
||||
|
||||
# Number of partitions moved on each rebalance
|
||||
# 10/510 * 768 ~ 15.06 -> move at least 15 partitions in first step
|
||||
ref = [0, 17, 16, 16, 14, 15, 13, 13, 12, 12, 14]
|
||||
self.assertEqual(ref, moved_partitions)
|
||||
|
||||
def test_set_replicas_increase(self):
|
||||
rb = ring.RingBuilder(8, 2, 0)
|
||||
rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
|
||||
|
@ -801,6 +830,14 @@ class TestRingBuilder(unittest.TestCase):
|
|||
self.assertEqual([len(p2d) for p2d in rb._replica2part2dev],
|
||||
[256, 256, 128])
|
||||
|
||||
def test_create_add_dev_add_replica_rebalance(self):
|
||||
rb = ring.RingBuilder(8, 3, 1)
|
||||
rb.add_dev({'id': 0, 'region': 0, 'region': 0, 'zone': 0, 'weight': 3,
|
||||
'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'})
|
||||
rb.set_replicas(4)
|
||||
rb.rebalance() # this would crash since parts_wanted was not set
|
||||
rb.validate()
|
||||
|
||||
def test_add_replicas_then_rebalance_respects_weight(self):
|
||||
rb = ring.RingBuilder(8, 3, 1)
|
||||
rb.add_dev({'id': 0, 'region': 0, 'region': 0, 'zone': 0, 'weight': 3,
|
||||
|
@ -877,17 +914,25 @@ class TestRingBuilder(unittest.TestCase):
|
|||
rb.rebalance()
|
||||
|
||||
real_pickle = pickle.load
|
||||
fake_open = mock.mock_open()
|
||||
|
||||
io_error_not_found = IOError()
|
||||
io_error_not_found.errno = errno.ENOENT
|
||||
|
||||
io_error_no_perm = IOError()
|
||||
io_error_no_perm.errno = errno.EPERM
|
||||
|
||||
io_error_generic = IOError()
|
||||
io_error_generic.errno = errno.EOPNOTSUPP
|
||||
try:
|
||||
#test a legit builder
|
||||
fake_pickle = mock.Mock(return_value=rb)
|
||||
fake_open = mock.Mock(return_value=None)
|
||||
pickle.load = fake_pickle
|
||||
builder = ring.RingBuilder.load('fake.builder', open=fake_open)
|
||||
self.assertEquals(fake_pickle.call_count, 1)
|
||||
fake_open.assert_has_calls([mock.call('fake.builder', 'rb')])
|
||||
self.assertEquals(builder, rb)
|
||||
fake_pickle.reset_mock()
|
||||
fake_open.reset_mock()
|
||||
|
||||
#test old style builder
|
||||
fake_pickle.return_value = rb.to_dict()
|
||||
|
@ -896,7 +941,6 @@ class TestRingBuilder(unittest.TestCase):
|
|||
fake_open.assert_has_calls([mock.call('fake.builder', 'rb')])
|
||||
self.assertEquals(builder.devs, rb.devs)
|
||||
fake_pickle.reset_mock()
|
||||
fake_open.reset_mock()
|
||||
|
||||
#test old devs but no meta
|
||||
no_meta_builder = rb
|
||||
|
@ -907,10 +951,48 @@ class TestRingBuilder(unittest.TestCase):
|
|||
builder = ring.RingBuilder.load('fake.builder', open=fake_open)
|
||||
fake_open.assert_has_calls([mock.call('fake.builder', 'rb')])
|
||||
self.assertEquals(builder.devs, rb.devs)
|
||||
fake_pickle.reset_mock()
|
||||
|
||||
#test an empty builder
|
||||
fake_pickle.side_effect = EOFError
|
||||
pickle.load = fake_pickle
|
||||
self.assertRaises(exceptions.UnPicklingError,
|
||||
ring.RingBuilder.load, 'fake.builder',
|
||||
open=fake_open)
|
||||
|
||||
#test a corrupted builder
|
||||
fake_pickle.side_effect = pickle.UnpicklingError
|
||||
pickle.load = fake_pickle
|
||||
self.assertRaises(exceptions.UnPicklingError,
|
||||
ring.RingBuilder.load, 'fake.builder',
|
||||
open=fake_open)
|
||||
|
||||
#test some error
|
||||
fake_pickle.side_effect = AttributeError
|
||||
pickle.load = fake_pickle
|
||||
self.assertRaises(exceptions.UnPicklingError,
|
||||
ring.RingBuilder.load, 'fake.builder',
|
||||
open=fake_open)
|
||||
finally:
|
||||
pickle.load = real_pickle
|
||||
|
||||
#test non existent builder file
|
||||
fake_open.side_effect = io_error_not_found
|
||||
self.assertRaises(exceptions.FileNotFoundError,
|
||||
ring.RingBuilder.load, 'fake.builder',
|
||||
open=fake_open)
|
||||
|
||||
#test non accessible builder file
|
||||
fake_open.side_effect = io_error_no_perm
|
||||
self.assertRaises(exceptions.PermissionError,
|
||||
ring.RingBuilder.load, 'fake.builder',
|
||||
open=fake_open)
|
||||
|
||||
#test an error other then ENOENT and ENOPERM
|
||||
fake_open.side_effect = io_error_generic
|
||||
self.assertRaises(IOError,
|
||||
ring.RingBuilder.load, 'fake.builder',
|
||||
open=fake_open)
|
||||
|
||||
def test_save_load(self):
|
||||
rb = ring.RingBuilder(8, 3, 1)
|
||||
devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1,
|
||||
|
|
|
@ -32,11 +32,14 @@ from mock import patch, MagicMock
|
|||
from eventlet.timeout import Timeout
|
||||
|
||||
import swift.common.db
|
||||
from swift.common.constraints import \
|
||||
MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE
|
||||
from swift.common.db import chexor, dict_factory, get_db_connection, \
|
||||
DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists, \
|
||||
GreenDBConnection, PICKLE_PROTOCOL
|
||||
from swift.common.utils import normalize_timestamp, mkdirs, json, Timestamp
|
||||
from swift.common.exceptions import LockTimeout
|
||||
from swift.common.swob import HTTPException
|
||||
|
||||
from test.unit import with_tempdir
|
||||
|
||||
|
@ -655,7 +658,7 @@ class TestDatabaseBroker(unittest.TestCase):
|
|||
conn.execute('CREATE TABLE test (one TEXT)')
|
||||
conn.execute('CREATE TABLE test_stat (id TEXT)')
|
||||
conn.execute('INSERT INTO test_stat (id) VALUES (?)',
|
||||
(str(uuid4),))
|
||||
(str(uuid4),))
|
||||
conn.execute('INSERT INTO test (one) VALUES ("1")')
|
||||
conn.commit()
|
||||
stub_called = [False]
|
||||
|
@ -1107,6 +1110,91 @@ class TestDatabaseBroker(unittest.TestCase):
|
|||
[first_value, first_timestamp])
|
||||
self.assert_('Second' not in broker.metadata)
|
||||
|
||||
@patch.object(DatabaseBroker, 'validate_metadata')
|
||||
def test_validate_metadata_is_called_from_update_metadata(self, mock):
|
||||
broker = self.get_replication_info_tester(metadata=True)
|
||||
first_timestamp = normalize_timestamp(1)
|
||||
first_value = '1'
|
||||
metadata = {'First': [first_value, first_timestamp]}
|
||||
broker.update_metadata(metadata, validate_metadata=True)
|
||||
self.assertTrue(mock.called)
|
||||
|
||||
@patch.object(DatabaseBroker, 'validate_metadata')
|
||||
def test_validate_metadata_is_not_called_from_update_metadata(self, mock):
|
||||
broker = self.get_replication_info_tester(metadata=True)
|
||||
first_timestamp = normalize_timestamp(1)
|
||||
first_value = '1'
|
||||
metadata = {'First': [first_value, first_timestamp]}
|
||||
broker.update_metadata(metadata)
|
||||
self.assertFalse(mock.called)
|
||||
|
||||
def test_metadata_with_max_count(self):
|
||||
metadata = {}
|
||||
for c in xrange(MAX_META_COUNT):
|
||||
key = 'X-Account-Meta-F{0}'.format(c)
|
||||
metadata[key] = ('B', normalize_timestamp(1))
|
||||
key = 'X-Account-Meta-Foo'.format(c)
|
||||
metadata[key] = ('', normalize_timestamp(1))
|
||||
try:
|
||||
DatabaseBroker.validate_metadata(metadata)
|
||||
except HTTPException:
|
||||
self.fail('Unexpected HTTPException')
|
||||
|
||||
def test_metadata_raises_exception_over_max_count(self):
|
||||
metadata = {}
|
||||
for c in xrange(MAX_META_COUNT + 1):
|
||||
key = 'X-Account-Meta-F{0}'.format(c)
|
||||
metadata[key] = ('B', normalize_timestamp(1))
|
||||
message = ''
|
||||
try:
|
||||
DatabaseBroker.validate_metadata(metadata)
|
||||
except HTTPException as e:
|
||||
message = str(e)
|
||||
self.assertEqual(message, '400 Bad Request')
|
||||
|
||||
def test_metadata_with_max_overall_size(self):
|
||||
metadata = {}
|
||||
metadata_value = 'v' * MAX_META_VALUE_LENGTH
|
||||
size = 0
|
||||
x = 0
|
||||
while size < (MAX_META_OVERALL_SIZE - 4
|
||||
- MAX_META_VALUE_LENGTH):
|
||||
size += 4 + MAX_META_VALUE_LENGTH
|
||||
metadata['X-Account-Meta-%04d' % x] = (metadata_value,
|
||||
normalize_timestamp(1))
|
||||
x += 1
|
||||
if MAX_META_OVERALL_SIZE - size > 1:
|
||||
metadata['X-Account-Meta-k'] = (
|
||||
'v' * (MAX_META_OVERALL_SIZE - size - 1),
|
||||
normalize_timestamp(1))
|
||||
try:
|
||||
DatabaseBroker.validate_metadata(metadata)
|
||||
except HTTPException:
|
||||
self.fail('Unexpected HTTPException')
|
||||
|
||||
def test_metadata_raises_exception_over_max_overall_size(self):
|
||||
metadata = {}
|
||||
metadata_value = 'k' * MAX_META_VALUE_LENGTH
|
||||
size = 0
|
||||
x = 0
|
||||
while size < (MAX_META_OVERALL_SIZE - 4
|
||||
- MAX_META_VALUE_LENGTH):
|
||||
size += 4 + MAX_META_VALUE_LENGTH
|
||||
metadata['X-Account-Meta-%04d' % x] = (metadata_value,
|
||||
normalize_timestamp(1))
|
||||
x += 1
|
||||
if MAX_META_OVERALL_SIZE - size > 1:
|
||||
metadata['X-Account-Meta-k'] = (
|
||||
'v' * (MAX_META_OVERALL_SIZE - size - 1),
|
||||
normalize_timestamp(1))
|
||||
metadata['X-Account-Meta-k2'] = ('v', normalize_timestamp(1))
|
||||
message = ''
|
||||
try:
|
||||
DatabaseBroker.validate_metadata(metadata)
|
||||
except HTTPException as e:
|
||||
message = str(e)
|
||||
self.assertEqual(message, '400 Bad Request')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
@ -161,13 +161,19 @@ class TestDirectClient(unittest.TestCase):
|
|||
|
||||
with mocked_http_conn(200, stub_headers, body) as conn:
|
||||
resp_headers, resp = direct_client.direct_get_account(
|
||||
self.node, self.part, self.account)
|
||||
self.node, self.part, self.account, marker='marker',
|
||||
prefix='prefix', delimiter='delimiter', limit=1000)
|
||||
self.assertEqual(conn.method, 'GET')
|
||||
self.assertEqual(conn.path, self.account_path)
|
||||
|
||||
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
|
||||
self.assertEqual(resp_headers, stub_headers)
|
||||
self.assertEqual(json.loads(body), resp)
|
||||
self.assertTrue('marker=marker' in conn.query_string)
|
||||
self.assertTrue('delimiter=delimiter' in conn.query_string)
|
||||
self.assertTrue('limit=1000' in conn.query_string)
|
||||
self.assertTrue('prefix=prefix' in conn.query_string)
|
||||
self.assertTrue('format=json' in conn.query_string)
|
||||
|
||||
def test_direct_client_exception(self):
|
||||
stub_headers = {'X-Trans-Id': 'txb5f59485c578460f8be9e-0053478d09'}
|
||||
|
@ -302,12 +308,19 @@ class TestDirectClient(unittest.TestCase):
|
|||
|
||||
with mocked_http_conn(200, headers, body) as conn:
|
||||
resp_headers, resp = direct_client.direct_get_container(
|
||||
self.node, self.part, self.account, self.container)
|
||||
self.node, self.part, self.account, self.container,
|
||||
marker='marker', prefix='prefix', delimiter='delimiter',
|
||||
limit=1000)
|
||||
|
||||
self.assertEqual(conn.req_headers['user-agent'],
|
||||
'direct-client %s' % os.getpid())
|
||||
self.assertEqual(headers, resp_headers)
|
||||
self.assertEqual(json.loads(body), resp)
|
||||
self.assertTrue('marker=marker' in conn.query_string)
|
||||
self.assertTrue('delimiter=delimiter' in conn.query_string)
|
||||
self.assertTrue('limit=1000' in conn.query_string)
|
||||
self.assertTrue('prefix=prefix' in conn.query_string)
|
||||
self.assertTrue('format=json' in conn.query_string)
|
||||
|
||||
def test_direct_get_container_no_content_does_not_decode_body(self):
|
||||
headers = {}
|
||||
|
|
|
@ -1476,6 +1476,7 @@ class TestManager(unittest.TestCase):
|
|||
|
||||
def launch(self, **kwargs):
|
||||
self.called['launch'].append(kwargs)
|
||||
return {}
|
||||
|
||||
def wait(self, **kwargs):
|
||||
self.called['wait'].append(kwargs)
|
||||
|
@ -1538,6 +1539,7 @@ class TestManager(unittest.TestCase):
|
|||
|
||||
def launch(self, **kwargs):
|
||||
self.called['launch'].append(kwargs)
|
||||
return {}
|
||||
|
||||
def wait(self, **kwargs):
|
||||
self.called['wait'].append(kwargs)
|
||||
|
@ -1589,6 +1591,7 @@ class TestManager(unittest.TestCase):
|
|||
|
||||
def launch(self, **kwargs):
|
||||
self.called['launch'].append(kwargs)
|
||||
return {}
|
||||
|
||||
def interact(self, **kwargs):
|
||||
self.called['interact'].append(kwargs)
|
||||
|
@ -1630,7 +1633,8 @@ class TestManager(unittest.TestCase):
|
|||
return 0
|
||||
|
||||
def launch(self, **kwargs):
|
||||
return self.called['launch'].append(kwargs)
|
||||
self.called['launch'].append(kwargs)
|
||||
return {}
|
||||
|
||||
orig_swift_server = manager.Server
|
||||
try:
|
||||
|
|
|
@ -3863,6 +3863,7 @@ class TestAuditLocationGenerator(unittest.TestCase):
|
|||
audit = lambda: list(utils.audit_location_generator(
|
||||
tmpdir, "data", mount_check=False))
|
||||
self.assertRaises(OSError, audit)
|
||||
rmtree(tmpdir)
|
||||
|
||||
#Check Raise on Bad Suffix
|
||||
tmpdir = mkdtemp()
|
||||
|
@ -3881,6 +3882,7 @@ class TestAuditLocationGenerator(unittest.TestCase):
|
|||
audit = lambda: list(utils.audit_location_generator(
|
||||
tmpdir, "data", mount_check=False))
|
||||
self.assertRaises(OSError, audit)
|
||||
rmtree(tmpdir)
|
||||
|
||||
#Check Raise on Bad Hash
|
||||
tmpdir = mkdtemp()
|
||||
|
@ -3899,6 +3901,7 @@ class TestAuditLocationGenerator(unittest.TestCase):
|
|||
audit = lambda: list(utils.audit_location_generator(
|
||||
tmpdir, "data", mount_check=False))
|
||||
self.assertRaises(OSError, audit)
|
||||
rmtree(tmpdir)
|
||||
|
||||
def test_non_dir_drive(self):
|
||||
with temptree([]) as tmpdir:
|
||||
|
|
|
@ -317,7 +317,6 @@ class TestWSGI(unittest.TestCase):
|
|||
def test_run_server(self):
|
||||
config = """
|
||||
[DEFAULT]
|
||||
eventlet_debug = yes
|
||||
client_timeout = 30
|
||||
max_clients = 1000
|
||||
swift_dir = TEMPDIR
|
||||
|
@ -354,7 +353,7 @@ class TestWSGI(unittest.TestCase):
|
|||
_eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
|
||||
_eventlet.patcher.monkey_patch.assert_called_with(all=False,
|
||||
socket=True)
|
||||
_eventlet.debug.hub_exceptions.assert_called_with(True)
|
||||
_eventlet.debug.hub_exceptions.assert_called_with(False)
|
||||
_wsgi.server.assert_called()
|
||||
args, kwargs = _wsgi.server.call_args
|
||||
server_sock, server_app, server_logger = args
|
||||
|
@ -414,7 +413,6 @@ class TestWSGI(unittest.TestCase):
|
|||
""",
|
||||
'proxy-server.conf.d/default.conf': """
|
||||
[DEFAULT]
|
||||
eventlet_debug = yes
|
||||
client_timeout = 30
|
||||
"""
|
||||
}
|
||||
|
@ -443,7 +441,7 @@ class TestWSGI(unittest.TestCase):
|
|||
_eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
|
||||
_eventlet.patcher.monkey_patch.assert_called_with(all=False,
|
||||
socket=True)
|
||||
_eventlet.debug.hub_exceptions.assert_called_with(True)
|
||||
_eventlet.debug.hub_exceptions.assert_called_with(False)
|
||||
_wsgi.server.assert_called()
|
||||
args, kwargs = _wsgi.server.call_args
|
||||
server_sock, server_app, server_logger = args
|
||||
|
@ -452,6 +450,59 @@ class TestWSGI(unittest.TestCase):
|
|||
self.assert_(isinstance(server_logger, wsgi.NullLogger))
|
||||
self.assert_('custom_pool' in kwargs)
|
||||
|
||||
def test_run_server_debug(self):
|
||||
config = """
|
||||
[DEFAULT]
|
||||
eventlet_debug = yes
|
||||
client_timeout = 30
|
||||
max_clients = 1000
|
||||
swift_dir = TEMPDIR
|
||||
|
||||
[pipeline:main]
|
||||
pipeline = proxy-server
|
||||
|
||||
[app:proxy-server]
|
||||
use = egg:swift#proxy
|
||||
# while "set" values normally override default
|
||||
set client_timeout = 20
|
||||
# this section is not in conf during run_server
|
||||
set max_clients = 10
|
||||
"""
|
||||
|
||||
contents = dedent(config)
|
||||
with temptree(['proxy-server.conf']) as t:
|
||||
conf_file = os.path.join(t, 'proxy-server.conf')
|
||||
with open(conf_file, 'w') as f:
|
||||
f.write(contents.replace('TEMPDIR', t))
|
||||
_fake_rings(t)
|
||||
with mock.patch('swift.proxy.server.Application.'
|
||||
'modify_wsgi_pipeline'):
|
||||
with mock.patch('swift.common.wsgi.wsgi') as _wsgi:
|
||||
mock_server = _wsgi.server
|
||||
_wsgi.server = lambda *args, **kwargs: mock_server(
|
||||
*args, **kwargs)
|
||||
with mock.patch('swift.common.wsgi.eventlet') as _eventlet:
|
||||
conf = wsgi.appconfig(conf_file)
|
||||
logger = logging.getLogger('test')
|
||||
sock = listen(('localhost', 0))
|
||||
wsgi.run_server(conf, logger, sock)
|
||||
self.assertEquals('HTTP/1.0',
|
||||
_wsgi.HttpProtocol.default_request_version)
|
||||
self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
|
||||
_eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
|
||||
_eventlet.patcher.monkey_patch.assert_called_with(all=False,
|
||||
socket=True)
|
||||
_eventlet.debug.hub_exceptions.assert_called_with(True)
|
||||
mock_server.assert_called()
|
||||
args, kwargs = mock_server.call_args
|
||||
server_sock, server_app, server_logger = args
|
||||
self.assertEquals(sock, server_sock)
|
||||
self.assert_(isinstance(server_app, swift.proxy.server.Application))
|
||||
self.assertEquals(20, server_app.client_timeout)
|
||||
self.assertEqual(server_logger, None)
|
||||
self.assert_('custom_pool' in kwargs)
|
||||
self.assertEquals(1000, kwargs['custom_pool'].size)
|
||||
|
||||
def test_appconfig_dir_ignores_hidden_files(self):
|
||||
config_dir = {
|
||||
'server.conf.d/01.conf': """
|
||||
|
|
|
@ -491,6 +491,18 @@ class TestAuditor(unittest.TestCase):
|
|||
finally:
|
||||
auditor.diskfile.DiskFile = was_df
|
||||
|
||||
@mock.patch.object(auditor.ObjectAuditor, 'run_audit')
|
||||
@mock.patch('os.fork', return_value=0)
|
||||
def test_with_inaccessible_object_location(self, mock_os_fork,
|
||||
mock_run_audit):
|
||||
# Need to ensure that any failures in run_audit do
|
||||
# not prevent sys.exit() from running. Otherwise we get
|
||||
# zombie processes.
|
||||
e = OSError('permission denied')
|
||||
mock_run_audit.side_effect = e
|
||||
self.auditor = auditor.ObjectAuditor(self.conf)
|
||||
self.assertRaises(SystemExit, self.auditor.fork_child, self)
|
||||
|
||||
def test_with_tombstone(self):
|
||||
ts_file_path = self.setup_bad_zero_byte(with_ts=True)
|
||||
self.assertTrue(ts_file_path.endswith('ts'))
|
||||
|
|
|
@ -41,6 +41,7 @@ def not_sleep(seconds):
|
|||
|
||||
class TestObjectExpirer(TestCase):
|
||||
maxDiff = None
|
||||
internal_client = None
|
||||
|
||||
def setUp(self):
|
||||
global not_sleep
|
||||
|
@ -54,10 +55,10 @@ class TestObjectExpirer(TestCase):
|
|||
self.rcache = mkdtemp()
|
||||
self.logger = FakeLogger()
|
||||
|
||||
def teardown(self):
|
||||
def tearDown(self):
|
||||
rmtree(self.rcache)
|
||||
internal_client.sleep = self.old_sleep
|
||||
internal_client.loadapp = self.loadapp
|
||||
internal_client.loadapp = self.old_loadapp
|
||||
|
||||
def test_get_process_values_from_kwargs(self):
|
||||
x = expirer.ObjectExpirer({})
|
||||
|
|
|
@ -167,9 +167,9 @@ class TestObjectReplicator(unittest.TestCase):
|
|||
self.parts_1 = {}
|
||||
for part in ['0', '1', '2', '3']:
|
||||
self.parts[part] = os.path.join(self.objects, part)
|
||||
os.mkdir(os.path.join(self.objects, part))
|
||||
os.mkdir(self.parts[part])
|
||||
self.parts_1[part] = os.path.join(self.objects_1, part)
|
||||
os.mkdir(os.path.join(self.objects_1, part))
|
||||
os.mkdir(self.parts_1[part])
|
||||
_create_test_rings(self.testdir)
|
||||
self.conf = dict(
|
||||
swift_dir=self.testdir, devices=self.devices, mount_check='false',
|
||||
|
|
|
@ -63,8 +63,9 @@ class TestObjectController(unittest.TestCase):
|
|||
"""Set up for testing swift.object.server.ObjectController"""
|
||||
utils.HASH_PATH_SUFFIX = 'endcap'
|
||||
utils.HASH_PATH_PREFIX = 'startcap'
|
||||
self.testdir = \
|
||||
os.path.join(mkdtemp(), 'tmp_test_object_server_ObjectController')
|
||||
self.tmpdir = mkdtemp()
|
||||
self.testdir = os.path.join(self.tmpdir,
|
||||
'tmp_test_object_server_ObjectController')
|
||||
conf = {'devices': self.testdir, 'mount_check': 'false'}
|
||||
self.object_controller = object_server.ObjectController(
|
||||
conf, logger=debug_logger())
|
||||
|
@ -76,7 +77,7 @@ class TestObjectController(unittest.TestCase):
|
|||
|
||||
def tearDown(self):
|
||||
"""Tear down for testing swift.object.server.ObjectController"""
|
||||
rmtree(os.path.dirname(self.testdir))
|
||||
rmtree(self.tmpdir)
|
||||
tpool.execute = self._orig_tpool_exc
|
||||
|
||||
def _stage_tmp_dir(self, policy):
|
||||
|
@ -4318,7 +4319,8 @@ class TestObjectServer(unittest.TestCase):
|
|||
|
||||
def setUp(self):
|
||||
# dirs
|
||||
self.tempdir = os.path.join(tempfile.mkdtemp(), 'tmp_test_obj_server')
|
||||
self.tmpdir = tempfile.mkdtemp()
|
||||
self.tempdir = os.path.join(self.tmpdir, 'tmp_test_obj_server')
|
||||
|
||||
self.devices = os.path.join(self.tempdir, 'srv/node')
|
||||
for device in ('sda1', 'sdb1'):
|
||||
|
@ -4335,6 +4337,9 @@ class TestObjectServer(unittest.TestCase):
|
|||
self.server = spawn(wsgi.server, sock, app, utils.NullLogger())
|
||||
self.port = sock.getsockname()[1]
|
||||
|
||||
def tearDown(self):
|
||||
rmtree(self.tmpdir)
|
||||
|
||||
def test_not_found(self):
|
||||
conn = bufferedhttp.http_connect('127.0.0.1', self.port, 'sda1', '0',
|
||||
'GET', '/a/c/o')
|
||||
|
|
|
@ -14,6 +14,7 @@
|
|||
# limitations under the License.
|
||||
import unittest
|
||||
import os
|
||||
import shutil
|
||||
from tempfile import mkdtemp
|
||||
from urllib import quote
|
||||
from swift.common.storage_policy import StoragePolicy, REPL_POLICY
|
||||
|
@ -129,8 +130,9 @@ class TestObjectSysmeta(unittest.TestCase):
|
|||
account_ring=FakeRing(replicas=1),
|
||||
container_ring=FakeRing(replicas=1))
|
||||
monkey_patch_mimetools()
|
||||
self.testdir = \
|
||||
os.path.join(mkdtemp(), 'tmp_test_object_server_ObjectController')
|
||||
self.tmpdir = mkdtemp()
|
||||
self.testdir = os.path.join(self.tmpdir,
|
||||
'tmp_test_object_server_ObjectController')
|
||||
mkdirs(os.path.join(self.testdir, 'sda1', 'tmp'))
|
||||
conf = {'devices': self.testdir, 'mount_check': 'false'}
|
||||
self.obj_ctlr = object_server.ObjectController(
|
||||
|
@ -143,6 +145,9 @@ class TestObjectSysmeta(unittest.TestCase):
|
|||
swift.proxy.controllers.base.http_connect = http_connect
|
||||
swift.proxy.controllers.obj.http_connect = http_connect
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmpdir)
|
||||
|
||||
original_sysmeta_headers_1 = {'x-object-sysmeta-test0': 'val0',
|
||||
'x-object-sysmeta-test1': 'val1'}
|
||||
original_sysmeta_headers_2 = {'x-object-sysmeta-test2': 'val2'}
|
||||
|
|
5
tox.ini
5
tox.ini
|
@ -58,10 +58,11 @@ commands = python setup.py build_sphinx
|
|||
# it's not a bug that we aren't using all of hacking
|
||||
# H102 -> apache2 license exists
|
||||
# H103 -> license is apache
|
||||
# H201 -> no bare excepts # add when hacking supports noqa
|
||||
# H201 -> no bare excepts (unless marked with " # noqa")
|
||||
# H231 -> Check for except statements to be Python 3.x compatible
|
||||
# H501 -> don't use locals() for str formatting
|
||||
# H903 -> \n not \r\n
|
||||
ignore = H
|
||||
select = F,E,W,H102,H103,H501,H903,H231
|
||||
select = F,E,W,H102,H103,H201,H231,H501,H903
|
||||
exclude = .venv,.tox,dist,doc,*egg
|
||||
show-source = True
|
||||
|
|
Loading…
Reference in New Issue