openstack
/
swift
Code Issues Proposed changes

Merge storage policies feature commit chain into master 

Change-Id: I0a2cc5771e3f9d56dba5b40ba4ec83cd16f57eef
This commit is contained in:
John Dickinson 2014-06-19 20:59:17 -07:00 committed by Peter Portante
parent e00da6cabc e52e8bc917
commit 53d4d21300
125 changed files with 18287 additions and 2969 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
bin/swift-container-reconciler Executable file
View File

@ -0,0 +1,21 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from swift.container.reconciler import ContainerReconciler
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
if __name__ == '__main__':
conf_file, options = parse_options(once=True)
run_daemon(ContainerReconciler, conf_file, **options)

171
bin/swift-get-nodes
View File

@ -14,129 +14,70 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import optparse
import sys
import urllib
import os
from optparse import OptionParser
from swift.common.ring import Ring
from swift.common.utils import hash_path, storage_directory
from swift.cli.info import print_item_locations, InfoSystemExit
parser = optparse.OptionParser()
parser.add_option('-a', '--all', action='store_true',
help='Show all handoff nodes')
parser.add_option('-p', '--partition', metavar='PARTITION',
help='Show nodes for a given partition')
(options, args) = parser.parse_args()
if __name__ == '__main__':
if (len(args) < 2 or len(args) > 4) and \
(options.partition is None or not args):
print 'Usage: %s [-a] <ring.gz> <account> [<container>] [<object>]' \
% sys.argv[0]
print ' Or: %s [-a] <ring.gz> -p partition' % sys.argv[0]
print ' Note: account, container, object can also be a single arg ' \
'separated by /'
print 'Shows the nodes responsible for the item specified.'
print 'Example:'
print ' $ %s /etc/swift/account.ring.gz MyAccount' % sys.argv[0]
print ' Partition 5743883'
print ' Hash 96ae332a60b58910784e4417a03e1ad0'
print ' 10.1.1.7:8000 sdd1'
print ' 10.1.9.2:8000 sdb1'
print ' 10.1.5.5:8000 sdf1'
print ' 10.1.5.9:8000 sdt1 # [Handoff]'
sys.exit(1)
usage = '''
Shows the nodes responsible for the item specified.
Usage: %prog [-a] <ring.gz> <account> [<container>] [<object>]
Or: %prog [-a] <ring.gz> -p partition
Or: %prog [-a] -P policy_name <account> <container> <object>
Note: account, container, object can also be a single arg separated by /
Example:
$ %prog -a /etc/swift/account.ring.gz MyAccount
Partition 5743883
Hash 96ae332a60b58910784e4417a03e1ad0
10.1.1.7:8000 sdd1
10.1.9.2:8000 sdb1
10.1.5.5:8000 sdf1
10.1.5.9:8000 sdt1 # [Handoff]
'''
parser = OptionParser(usage)
parser.add_option('-a', '--all', action='store_true',
help='Show all handoff nodes')
parser.add_option('-p', '--partition', metavar='PARTITION',
help='Show nodes for a given partition')
parser.add_option('-P', '--policy-name', dest='policy_name',
help='Specify which policy to use')
parser.add_option('-d', '--swift-dir', default='/etc/swift',
dest='swift_dir', help='Path to swift directory')
options, args = parser.parse_args()
# swift-get-nodes -P nada -p 1
if len(args) == 0:
if not options.policy_name or not options.partition:
sys.exit(parser.print_help())
elif len(args) > 4 or len(args) < 1:
sys.exit(parser.print_help())
if len(args) == 2 and '/' in args[1]:
# Parse single path arg, as noted in above help text.
path = args[1].lstrip('/')
args = [args[0]] + [p for p in path.split('/', 2) if p]
# if len(args) == 1 and options.policy_name and '/' in args[0]:
if len(args) == 1 and not args[0].endswith('ring.gz'):
path = args[0].lstrip('/')
args = [p for p in path.split('/', 2) if p]
if len(args) == 2 and '/' in args[1]:
path = args[1].lstrip('/')
args = [args[0]] + [p for p in path.split('/', 2) if p]
ringloc = None
account = None
container = None
obj = None
ring = None
ring_name = None
if len(args) == 4:
# Account, Container and Object
ring_file, account, container, obj = args
ring = Ring(ring_file)
hash_str = hash_path(account, container, obj)
part, nodes = ring.get_nodes(account, container, obj)
target = "%s/%s/%s" % (account, container, obj)
loc = 'objects'
elif len(args) == 3:
# Account, Container
ring_file, account, container = args
ring = Ring(ring_file)
hash_str = hash_path(account, container)
part, nodes = ring.get_nodes(account, container)
target = "%s/%s" % (account, container)
loc = 'containers'
elif len(args) == 2:
# Account
ring_file, account = args
ring = Ring(ring_file)
hash_str = hash_path(account)
part, nodes = ring.get_nodes(account)
target = "%s" % (account)
loc = 'accounts'
elif len(args) == 1:
# Partition
ring_file = args[0]
ring = Ring(ring_file)
hash_str = None
part = int(options.partition)
nodes = ring.get_part_nodes(part)
target = ''
loc = ring_file.rsplit('/', 1)[-1].split('.', 1)[0]
if loc in ('account', 'container', 'object'):
loc += 's'
else:
loc = '<type>'
if len(args) >= 1 and args[0].endswith('ring.gz'):
if os.path.exists(args[0]):
ring_name = args[0].rsplit('/', 1)[-1].split('.', 1)[0]
ring = Ring(args[0])
else:
print 'Ring file does not exist'
args.pop(0)
more_nodes = []
for more_node in ring.get_more_nodes(part):
more_nodes.append(more_node)
if not options.all and len(more_nodes) >= len(nodes):
break
print '\nAccount \t%s' % account
print 'Container\t%s' % container
print 'Object \t%s\n' % obj
print '\nPartition\t%s' % part
print 'Hash \t%s\n' % hash_str
for node in nodes:
print 'Server:Port Device\t%s:%s %s' % (node['ip'], node['port'],
node['device'])
for mnode in more_nodes:
print 'Server:Port Device\t%s:%s %s\t [Handoff]' \
% (mnode['ip'], mnode['port'], mnode['device'])
print "\n"
for node in nodes:
print 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' \
% (node['ip'], node['port'], node['device'], part,
urllib.quote(target))
for mnode in more_nodes:
print 'curl -I -XHEAD "http://%s:%s/%s/%s/%s" # [Handoff]' \
% (mnode['ip'], mnode['port'], mnode['device'], part,
urllib.quote(target))
print "\n"
print 'Use your own device location of servers:'
print 'such as "export DEVICE=/srv/node"'
for node in nodes:
if hash_str:
print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/"' % (
node['ip'], node['device'], storage_directory(loc, part, hash_str))
else:
print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/%s/"' % (
node['ip'], node['device'], loc, part)
for mnode in more_nodes:
if hash_str:
print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/" '\
'# [Handoff]' % (mnode['ip'], mnode['device'],
storage_directory(loc, part, hash_str))
else:
print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/%s/" # [Handoff]' % (
mnode['ip'], mnode['device'], loc, part)
try:
print_item_locations(ring, ring_name, *args, **vars(options))
except InfoSystemExit:
sys.exit(1)

107
bin/swift-object-info
View File

@ -14,112 +14,31 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
from datetime import datetime
from hashlib import md5
from optparse import OptionParser
from swift.common.ring import Ring
from swift.obj.diskfile import read_metadata
from swift.common.utils import hash_path, storage_directory
def print_object_info(datafile, check_etag=True, swift_dir='/etc/swift'):
if not os.path.exists(datafile) or not datafile.endswith('.data'):
print "Data file doesn't exist"
sys.exit(1)
try:
ring = Ring(swift_dir, ring_name='object')
except Exception:
ring = None
fp = open(datafile, 'rb')
metadata = read_metadata(fp)
path = metadata.pop('name', '')
content_type = metadata.pop('Content-Type', '')
ts = metadata.pop('X-Timestamp', '')
etag = metadata.pop('ETag', '')
length = metadata.pop('Content-Length', '')
if path:
print 'Path: %s' % path
account, container, obj = path.split('/', 3)[1:]
print ' Account: %s' % account
print ' Container: %s' % container
print ' Object: %s' % obj
obj_hash = hash_path(account, container, obj)
print ' Object hash: %s' % obj_hash
else:
print 'Path: Not found in metadata'
if content_type:
print 'Content-Type: %s' % content_type
else:
print 'Content-Type: Not found in metadata'
if ts:
print 'Timestamp: %s (%s)' % (datetime.fromtimestamp(float(ts)), ts)
else:
print 'Timestamp: Not found in metadata'
file_len = None
if check_etag:
h = md5()
file_len = 0
while True:
data = fp.read(64 * 1024)
if not data:
break
h.update(data)
file_len += len(data)
h = h.hexdigest()
if etag:
if h == etag:
print 'ETag: %s (valid)' % etag
else:
print "Etag: %s doesn't match file hash of %s!" % (etag, h)
else:
print 'ETag: Not found in metadata'
else:
print 'ETag: %s (not checked)' % etag
file_len = os.fstat(fp.fileno()).st_size
if length:
if file_len == int(length):
print 'Content-Length: %s (valid)' % length
else:
print "Content-Length: %s doesn't match file length of %s" % (
length, file_len)
else:
print 'Content-Length: Not found in metadata'
print 'User Metadata: %s' % metadata
if ring is not None:
print 'Ring locations:'
part, nodes = ring.get_nodes(account, container, obj)
for node in nodes:
print (' %s:%s - /srv/node/%s/%s/%s.data' %
(node['ip'], node['port'], node['device'],
storage_directory('objects', part, obj_hash), ts))
print
print 'note: /srv/node is used as default value of `devices`, '\
'the real value is set in object-server.conf '\
'on each storage node.'
fp.close()
from swift.cli.info import print_obj, InfoSystemExit
if __name__ == '__main__':
parser = OptionParser()
parser.set_defaults(check_etag=True, swift_dir='/etc/swift')
parser = OptionParser('%prog [options] OBJECT_FILE')
parser.add_option(
'-n', '--no-check-etag',
'-n', '--no-check-etag', default=True,
action="store_false", dest="check_etag",
help="Don't verify file contents against stored etag")
parser.add_option(
'-d', '--swift-dir',
'-d', '--swift-dir', default='/etc/swift', dest='swift_dir',
help="Pass location of swift directory")
parser.add_option(
'-P', '--policy-name', dest='policy_name',
help="Specify storage policy name")
options, args = parser.parse_args()
if len(args) < 1:
print "Usage: %s [-n] [-d] OBJECT_FILE" % sys.argv[0]
sys.exit(1)
if len(args) != 1:
sys.exit(parser.print_help())
print_object_info(args[0], check_etag=options.check_etag,
swift_dir=options.swift_dir)
try:
print_obj(*args, **vars(options))
except InfoSystemExit:
sys.exit(1)

74
bin/swift-reconciler-enqueue Executable file
View File

@ -0,0 +1,74 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from optparse import OptionParser
import eventlet.debug
eventlet.debug.hub_exceptions(True)
from swift.common.ring import Ring
from swift.common.utils import split_path
from swift.common.storage_policy import POLICIES
from swift.container.reconciler import add_to_reconciler_queue
"""
This tool is primarly for debugging and development but can be used an example
of how an operator could enqueue objects manually if a problem is discovered -
might be particularlly useful if you need to hack a fix into the reconciler
and re-run it.
"""
USAGE = """
%prog <policy_index> </a/c/o> <timestamp> [options]
This script enqueues an object to be evaluated by the reconciler.
Arguments:
policy_index: the policy the object is currently stored in.
/a/c/o: the full path of the object - utf-8
timestamp: the timestamp of the datafile/tombstone.
""".strip()
parser = OptionParser(USAGE)
parser.add_option('-X', '--op', default='PUT', choices=('PUT', 'DELETE'),
help='the method of the misplaced operation')
parser.add_option('-f', '--force', action='store_true',
help='force an object to be re-enqueued')
def main():
options, args = parser.parse_args()
try:
policy_index, path, timestamp = args
except ValueError:
sys.exit(parser.print_help())
container_ring = Ring('/etc/swift/container.ring.gz')
policy = POLICIES.get_by_index(policy_index)
if not policy:
return 'ERROR: invalid storage policy index: %s' % policy
try:
account, container, obj = split_path(path, 3, 3, True)
except ValueError as e:
return 'ERROR: %s' % e
container_name = add_to_reconciler_queue(
container_ring, account, container, obj,
policy.idx, timestamp, options.op, force=options.force)
if not container_name:
return 'ERROR: unable to enqueue!'
print container_name
if __name__ == "__main__":
sys.exit(main())

6
doc/saio/bin/remakerings
View File

@ -10,6 +10,12 @@ swift-ring-builder object.builder add r1z2-127.0.0.1:6020/sdb2 1
swift-ring-builder object.builder add r1z3-127.0.0.1:6030/sdb3 1
swift-ring-builder object.builder add r1z4-127.0.0.1:6040/sdb4 1
swift-ring-builder object.builder rebalance
swift-ring-builder object-1.builder create 10 2 1
swift-ring-builder object-1.builder add r1z1-127.0.0.1:6010/sdb1 1
swift-ring-builder object-1.builder add r1z2-127.0.0.1:6020/sdb2 1
swift-ring-builder object-1.builder add r1z3-127.0.0.1:6030/sdb3 1
swift-ring-builder object-1.builder add r1z4-127.0.0.1:6040/sdb4 1
swift-ring-builder object-1.builder rebalance
swift-ring-builder container.builder create 10 3 1
swift-ring-builder container.builder add r1z1-127.0.0.1:6011/sdb1 1
swift-ring-builder container.builder add r1z2-127.0.0.1:6021/sdb2 1

47
doc/saio/swift/container-reconciler.conf Normal file
View File

@ -0,0 +1,47 @@
[DEFAULT]
# swift_dir = /etc/swift
user = <your-user-name>
# You can specify default log routing here if you want:
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# comma separated list of functions to call to setup custom log handlers.
# functions get passed: conf, name, log_to_console, log_route, fmt, logger,
# adapted_logger
# log_custom_handlers =
#
# If set, log_udp_host will override log_address
# log_udp_host =
# log_udp_port = 514
#
# You can enable StatsD logging here:
# log_statsd_host = localhost
# log_statsd_port = 8125
# log_statsd_default_sample_rate = 1.0
# log_statsd_sample_rate_factor = 1.0
# log_statsd_metric_prefix =
[container-reconciler]
# reclaim_age = 604800
# interval = 300
# request_tries = 3
[pipeline:main]
pipeline = catch_errors proxy-logging cache proxy-server
[app:proxy-server]
use = egg:swift#proxy
# See proxy-server.conf-sample for options
[filter:cache]
use = egg:swift#memcache
# See proxy-server.conf-sample for options
[filter:proxy-logging]
use = egg:swift#proxy_logging
[filter:catch_errors]
use = egg:swift#catch_errors
# See proxy-server.conf-sample for options

7
doc/saio/swift/swift.conf
View File

@ -2,3 +2,10 @@
# random unique strings that can never change (DO NOT LOSE)
swift_hash_path_prefix = changeme
swift_hash_path_suffix = changeme
[storage-policy:0]
name = gold
default = yes
[storage-policy:1]
name = silver

59
doc/source/admin_guide.rst
View File

@ -2,6 +2,33 @@
Administrator's Guide
=====================
-------------------------
Defining Storage Policies
-------------------------
Defining your Storage Policies is very easy to do with Swift. It is important
that the administrator understand the concepts behind Storage Policies
before actually creating and using them in order to get the most benefit out
of the feature and, more importantly, to avoid having to make unnecessary changes
once a set of policies have been deployed to a cluster.
It is highly recommended that the reader fully read and comprehend
:doc:`overview_policies` before proceeding with administration of
policies. Plan carefully and it is suggested that experimentation be
done first on a non-production cluster to be certain that the desired
configuration meets the needs of the users. See :ref:`upgrade-policy`
before planning the upgrade of your existing deployment.
Following is a high level view of the very few steps it takes to configure
policies once you have decided what you want to do:
#. Define your policies in ``/etc/swift/swift.conf``
#. Create the corresponding object rings
#. Communicate the names of the Storage Policies to cluster users
For a specific example that takes you through these steps, please see
:doc:`policies_saio`
------------------
Managing the Rings
------------------
@ -32,15 +59,15 @@ For more information see :doc:`overview_ring`.
Removing a device from the ring::
swift-ring-builder <builder-file> remove <ip_address>/<device_name>
Removing a server from the ring::
swift-ring-builder <builder-file> remove <ip_address>
Adding devices to the ring:
See :ref:`ring-preparing`
See what devices for a server are in the ring::
swift-ring-builder <builder-file> search <ip_address>
@ -49,7 +76,7 @@ Once you are done with all changes to the ring, the changes need to be
"committed"::
swift-ring-builder <builder-file> rebalance
Once the new rings are built, they should be pushed out to all the servers
in the cluster.
@ -126,7 +153,7 @@ is replaced. Once the drive is replaced, it can be re-added to the ring.
Handling Server Failure
-----------------------
If a server is having hardware issues, it is a good idea to make sure the
If a server is having hardware issues, it is a good idea to make sure the
swift services are not running. This will allow Swift to work around the
failure while you troubleshoot.
@ -149,7 +176,7 @@ Detecting Failed Drives
It has been our experience that when a drive is about to fail, error messages
will spew into `/var/log/kern.log`. There is a script called
`swift-drive-audit` that can be run via cron to watch for bad drives. If
`swift-drive-audit` that can be run via cron to watch for bad drives. If
errors are detected, it will unmount the bad drive, so that Swift can
work around it. The script takes a configuration file with the following
settings:
@ -170,7 +197,7 @@ log_file_pattern /var/log/kern* Location of the log file with globbing
pattern to check against device errors
regex_pattern_X (see below) Regular expression patterns to be used to
locate device blocks with errors in the
log file
log file
================== ============== ===========================================
The default regex pattern used to locate device blocks with errors are
@ -235,7 +262,7 @@ the cluster. Here is an example of a cluster in perfect health::
Queried 2621 containers for dispersion reporting, 19s, 0 retries
100.00% of container copies found (7863 of 7863)
Sample represents 1.00% of the container partition space
Queried 2619 objects for dispersion reporting, 7s, 0 retries
100.00% of object copies found (7857 of 7857)
Sample represents 1.00% of the object partition space
@ -251,7 +278,7 @@ that has::
Queried 2621 containers for dispersion reporting, 8s, 0 retries
100.00% of container copies found (7863 of 7863)
Sample represents 1.00% of the container partition space
Queried 2619 objects for dispersion reporting, 7s, 0 retries
There were 1763 partitions missing one copy.
77.56% of object copies found (6094 of 7857)
@ -285,7 +312,7 @@ You can also run the report for only containers or objects::
100.00% of object copies found (7857 of 7857)
Sample represents 1.00% of the object partition space
Alternatively, the dispersion report can also be output in json format. This
Alternatively, the dispersion report can also be output in json format. This
allows it to be more easily consumed by third party utilities::
$ swift-dispersion-report -j
@ -499,7 +526,7 @@ Request URI Description
This information can also be queried via the swift-recon command line utility::
fhines@ubuntu:~$ swift-recon -h
Usage:
Usage:
usage: swift-recon <server_type> [-v] [--suppress] [-a] [-r] [-u] [-d]
[-l] [--md5] [--auditor] [--updater] [--expirer] [--sockstat]
@ -893,8 +920,8 @@ Metric Name Description
`object-server.PUT.timing` Timing data for each PUT request not resulting in an
error.
`object-server.PUT.<device>.timing` Timing data per kB transferred (ms/kB) for each
non-zero-byte PUT request on each device.
Monitoring problematic devices, higher is bad.
non-zero-byte PUT request on each device.
Monitoring problematic devices, higher is bad.
`object-server.GET.errors.timing` Timing data for GET request errors: bad request,
not mounted, header timestamps before the epoch,
precondition failed.
@ -1046,7 +1073,7 @@ Managing Services
-----------------
Swift services are generally managed with `swift-init`. the general usage is
``swift-init <service> <command>``, where service is the swift service to
``swift-init <service> <command>``, where service is the swift service to
manage (for example object, container, account, proxy) and command is one of:
========== ===============================================
@ -1059,8 +1086,8 @@ shutdown Attempt to gracefully shutdown the service
reload Attempt to gracefully restart the service
========== ===============================================
A graceful shutdown or reload will finish any current requests before
completely stopping the old service. There is also a special case of
A graceful shutdown or reload will finish any current requests before
completely stopping the old service. There is also a special case of
`swift-init all <command>`, which will run the command for all swift services.
In cases where there are multiple configs for a service, a specific config

12
doc/source/container.rst
View File

@ -34,6 +34,18 @@ Container Server
:undoc-members:
:show-inheritance:
.. _container-replicator:
Container Replicator
====================
.. automodule:: swift.container.replicator
:members:
:undoc-members:
:show-inheritance:
.. _container-sync-daemon:
Container Sync
==============

15
doc/source/development_saio.rst
View File

@ -341,6 +341,10 @@ commands are as follows:
.. literalinclude:: /../saio/swift/object-expirer.conf
#. ``/etc/swift/container-reconciler.conf``
.. literalinclude:: /../saio/swift/container-reconciler.conf
#. ``/etc/swift/account-server/1.conf``
.. literalinclude:: /../saio/swift/account-server/1.conf
@ -447,8 +451,15 @@ Setting up scripts for running Swift
.. literalinclude:: /../saio/bin/remakerings
You can expect the ouptut from this command to produce the following::
You can expect the output from this command to produce the following (note
that 2 object rings are created in order to test storage policies in the
SAIO environment however they map to the same nodes)::
Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
Device d1r1z2-127.0.0.1:6020R127.0.0.1:6020/sdb2_"" with 1.0 weight got id 1
Device d2r1z3-127.0.0.1:6030R127.0.0.1:6030/sdb3_"" with 1.0 weight got id 2
Device d3r1z4-127.0.0.1:6040R127.0.0.1:6040/sdb4_"" with 1.0 weight got id 3
Reassigned 1024 (100.00%) partitions. Balance is now 0.00.
Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
Device d1r1z2-127.0.0.1:6020R127.0.0.1:6020/sdb2_"" with 1.0 weight got id 1
Device d2r1z3-127.0.0.1:6030R127.0.0.1:6030/sdb3_"" with 1.0 weight got id 2
@ -465,6 +476,8 @@ Setting up scripts for running Swift
Device d3r1z4-127.0.0.1:6042R127.0.0.1:6042/sdb4_"" with 1.0 weight got id 3
Reassigned 1024 (100.00%) partitions. Balance is now 0.00.
#. Read more about Storage Policies and your SAIO :doc:`policies_saio`
#. Verify the unit tests run::
$HOME/swift/.unittests

2
doc/source/index.rst
View File

@ -45,6 +45,7 @@ Overview and Concepts
Swift's API docs <http://docs.openstack.org/api/openstack-object-storage/1.0/content/>
overview_architecture
overview_ring
overview_policies
overview_reaper
overview_auth
overview_replication
@ -65,6 +66,7 @@ Developer Documentation
development_guidelines
development_saio
policies_saio
development_auth
development_middleware
development_ondisk_backends

2
doc/source/middleware.rst
View File

@ -130,6 +130,8 @@ KeystoneAuth
:members:
:show-inheritance:
.. _list_endpoints:
List Endpoints
==============

9
doc/source/misc.rst
View File

@ -120,3 +120,12 @@ WSGI
.. automodule:: swift.common.wsgi
:members:
:show-inheritance:
.. _storage_policy:
Storage Policy
==============
.. automodule:: swift.common.storage_policy
:members:
:show-inheritance:

33
doc/source/overview_architecture.rst
View File

@ -27,9 +27,9 @@ The Ring
A ring represents a mapping between the names of entities stored on disk and
their physical location. There are separate rings for accounts, containers, and
objects. When other components need to perform any operation on an object,
container, or account, they need to interact with the appropriate ring to
determine its location in the cluster.
one object ring per storage policy. When other components need to perform any
operation on an object, container, or account, they need to interact with the
appropriate ring to determine its location in the cluster.
The Ring maintains this mapping using zones, devices, partitions, and replicas.
Each partition in the ring is replicated, by default, 3 times across the
@ -54,6 +54,33 @@ drives are used in a cluster.
The ring is used by the Proxy server and several background processes
(like replication).
----------------
Storage Policies
----------------
Storage Policies provide a way for object storage providers to differentiate
service levels, features and behaviors of a Swift deployment. Each Storage
Policy configured in Swift is exposed to the client via an abstract name.
Each device in the system is assigned to one or more Storage Policies. This
is accomplished through the use of multiple object rings, where each Storage
Policy has an independent object ring, which may include a subset of hardware
implementing a particular differentiation.
For example, one might have the default policy with 3x replication, and create
a second policy which, when applied to new containers only uses 2x replication.
Another might add SSDs to a set of storage nodes and create a performance tier
storage policy for certain containers to have their objects stored there.
This mapping is then exposed on a per-container basis, where each container
can be assigned a specific storage policy when it is created, which remains in
effect for the lifetime of the container. Applications require minimal
awareness of storage policies to use them; once a container has been created
with a specific policy, all objects stored in it will be done so in accordance
with that policy.
Storage Policies are not implemented as a separate code module but are a core
abstraction of Swift architecture.
-------------
Object Server
-------------

603
doc/source/overview_policies.rst Executable file
View File

@ -0,0 +1,603 @@
================
Storage Policies
================
Storage Policies allow for some level of segmenting the cluster for various
purposes through the creation of multiple object rings. Storage Policies are
not implemented as a separate code module but are an important concept in
understanding Swift architecture.
As described in :doc:`overview_ring`, Swift uses modified hashing rings to
determine where data should reside in the cluster. There is a separate ring
for account databases, container databases, and there is also one object
ring per storage policy. Each object ring behaves exactly the same way
and is maintained in the same manner, but with policies, different devices
can belong to different rings with varying levels of replication. By supporting
multiple object rings, Swift allows the application and/or deployer to
essentially segregate the object storage within a single cluster. There are
many reasons why this might be desirable:
* Different levels of replication: If a provider wants to offer, for example,
2x replication and 3x replication but doesn't want to maintain 2 separate clusters,
they would setup a 2x policy and a 3x policy and assign the nodes to their
respective rings.
* Performance: Just as SSDs can be used as the exclusive members of an account or
database ring, an SSD-only object ring can be created as well and used to
implement a low-latency/high performance policy.
* Collecting nodes into group: Different object rings may have different
physical servers so that objects in specific storage policies are always
placed in a particular data center or geography.
* Different Storage implementations: Another example would be to collect
together a set of nodes that use a different Diskfile (e.g., Kinetic,
GlusterFS) and use a policy to direct traffic just to those nodes.
.. note::
Today, choosing a different storage policy allows the use of different
object rings, but future policies (such as Erasure Coding) will also
change some of the actual code paths when processing a request. Also note
that Diskfile refers to backend object storage plug-in architecture.
-----------------------
Containers and Policies
-----------------------
Policies are implemented at the container level. There are many advantages to
this approach, not the least of which is how easy it makes life on
applications that want to take advantage of them. It also ensures that
Storage Policies remain a core feature of swift independent of the auth
implementation. Policies were not implemented at the account/auth layer
because it would require changes to all auth systems in use by Swift
deployers. Each container has a new special immutable metadata element called
the storage policy index. Note that internally, Swift relies on policy
indexes and not policy names. Policy names exist for human readability and
translation is managed in the proxy. When a container is created, one new
optional header is supported to specify the policy name. If nothing is
specified, the default policy is used (and if no other policies defined,
Policy-0 is considered the default). We will be covering the difference
between default and Policy-0 in the next section.
Policies are assigned when a container is created. Once a container has been
assigned a policy, it cannot be changed until the container is deleted. The implications
on data placement/movement for large datasets would make this a task best left for
applications to perform. Therefore, if a container has an existing policy of,
for example 3x replication, and one wanted to migrate that data to a policy that specifies,
a different replication level, the application would create another container
specifying the other policy name and then simply move the data from one container
to the other. Policies apply on a per container basis allowing for minimal application
awareness; once a container has been created with a specific policy, all objects stored
in it will be done so in accordance with that policy. If a container with a
specific name is deleted (requires the container be empty) a new container may
be created with the same name without any restriction on storage policy
enforced by the deleted container which previously shared the same name.
Containers have a many-to-one relationship with policies meaning that any number
of containers can share one policy. There is no limit to how many containers can use
a specific policy.
The notion of associating a ring with a container introduces an interesting scenario:
What would happen if 2 containers of the same name were created with different
Storage Policies on either side of a network outage at the same time? Furthermore,
what would happen if objects were placed in those containers, a whole bunch of them,
and then later the network outage was restored? Well, without special care it would
be a big problem as an application could end up using the wrong ring to try and find
an object. Luckily there is a solution for this problem, a daemon covered in more
detail later, works tirelessly to identify and rectify this potential scenario.
--------------------
Container Reconciler
--------------------
Because atomicity of container creation cannot be enforced in a
distributed eventually consistent system, object writes into the wrong
storage policy must be eventually merged into the correct storage policy
by an asynchronous daemon. Recovery from object writes during a network
partition which resulted in a split brain container created with
different storage policies are handled by the
`swift-container-reconciler` daemon.
The container reconciler works off a queue similar to the
object-expirer. The queue is populated during container-replication.
It is never considered incorrect to enqueue an object to be evaluated by
the container-reconciler because if there is nothing wrong with the location
of the object the reconciler will simply dequeue it. The
container-reconciler queue is an indexed log for the real location of an
object for which a discrepancy in the storage policy of the container was
discovered.
To determine the correct storage policy of a container, it is necessary
to update the status_changed_at field in the container_stat table when a
container changes status from deleted to re-created. This transaction
log allows the container-replicator to update the correct storage policy
both when replicating a container and handling REPLICATE requests.
Because each object write is a separate distributed transaction it is
not possible to determine the correctness of the storage policy for each
object write with respect to the entire transaction log at a given
container database. As such, container databases will always record the
object write regardless of the storage policy on a per object row basis.
Object byte and count stats are tracked per storage policy in each
container and reconciled using normal object row merge semantics.
The object rows are ensured to be fully durable during replication using
the normal container replication. After the container
replicator pushes its object rows to available primary nodes any
misplaced object rows are bulk loaded into containers based off the
object timestamp under the ".misplaced_objects" system account. The
rows are initially written to a handoff container on the local node, and
at the end of the replication pass the .misplaced_object containers are
replicated to the correct primary nodes.
The container-reconciler processes the .misplaced_objects containers in
descending order and reaps its containers as the objects represented by
the rows are successfully reconciled. The container-reconciler will
always validate the correct storage policy for enqueued objects using
direct container HEAD requests which are accelerated via caching.
Because failure of individual storage nodes in aggregate is assumed to
be common at scale the container-reconciler will make forward progress
with a simple quorum majority. During a combination of failures and
rebalances it is possible that a quorum could provide an incomplete
record of the correct storage policy - so an object write may have to be
applied more than once. Because storage nodes and container databases
will not process writes with an ``X-Timestamp`` less than or equal to
their existing record when objects writes are re-applied their timestamp
is slightly incremented. In order for this increment to be applied
transparently to the client a second vector of time has been added to
Swift for internal use. See :class:`~swift.common.utils.Timestamp`.
As the reconciler applies object writes to the correct storage policy it
cleans up writes which no longer apply to the incorrect storage policy
and removes the rows from the ``.misplaced_objects`` containers. After all
rows have been successfully processed it sleeps and will periodically
check for newly enqueued rows to be discovered during container
replication.
.. _default-policy:
-------------------------
Default versus 'Policy-0'
-------------------------
Storage Policies is a versatile feature intended to support both new and
pre-existing clusters with the same level of flexibility. For that reason, we
introduce the ``Policy-0`` concept which is not the same as the "default"
policy. As you will see when we begin to configure policies, each policy has
both a name (human friendly, configurable) as well as an index (or simply
policy number). Swift reserves index 0 to map to the object ring that's
present in all installations (e.g., ``/etc/swift/object.ring.gz``). You can
name this policy anything you like, and if no policies are defined it will
report itself as ``Policy-0``, however you cannot change the index as there must
always be a policy with index 0.
Another important concept is the default policy which can be any policy
in the cluster. The default policy is the policy that is automatically
chosen when a container creation request is sent without a storage
policy being specified. :ref:`configure-policy` describes how to set the
default policy. The difference from ``Policy-0`` is subtle but
extremely important. ``Policy-0`` is what is used by Swift when
accessing pre-storage-policy containers which won't have a policy - in
this case we would not use the default as it might not have the same
policy as legacy containers. When no other policies are defined, Swift
will always choose ``Policy-0`` as the default.
In other words, default means "create using this policy if nothing else is specified"
and ``Policy-0`` means "use the legacy policy if a container doesn't have one" which
really means use ``object.ring.gz`` for lookups.
.. note::
With the Storage Policy based code, it's not possible to create a
container that doesn't have a policy. If nothing is provided, Swift will
still select the default and assign it to the container. For containers
created before Storage Policies were introduced, the legacy Policy-0 will
be used.
.. _deprecate-policy:
--------------------
Deprecating Policies
--------------------
There will be times when a policy is no longer desired; however simply
deleting the policy and associated rings would be problematic for existing
data. In order to ensure that resources are not orphaned in the cluster (left
on disk but no longer accessible) and to provide proper messaging to
applications when a policy needs to be retired, the notion of deprecation is
used. :ref:`configure-policy` describes how to deprecate a policy.
Swift's behavior with deprecated policies will change as follows:
* The deprecated policy will not appear in /info
* PUT/GET/DELETE/POST/HEAD are still allowed on the pre-existing containers
created with a deprecated policy
* Clients will get an ''400 Bad Request'' error when trying to create a new
container using the deprecated policy
* Clients still have access to policy statistics via HEAD on pre-existing
containers
.. note::
A policy can not be both the default and deprecated. If you deprecate the
default policy, you must specify a new default.
You can also use the deprecated feature to rollout new policies. If you
want to test a new storage policy before making it generally available
you could deprecate the policy when you initially roll it the new
configuration and rings to all nodes. Being deprecated will render it
innate and unable to be used. To test it you will need to create a
container with that storage policy; which will require a single proxy
instance (or a set of proxy-servers which are only internally
accessible) that has been one-off configured with the new policy NOT
marked deprecated. Once the container has been created with the new
storage policy any client authorized to use that container will be able
to add and access data stored in that container in the new storage
policy. When satisfied you can roll out a new ``swift.conf`` which does
not mark the policy as deprecated to all nodes.
.. _configure-policy:
--------------------
Configuring Policies
--------------------
Policies are configured in ``swift.conf`` and it is important that the deployer have a solid
understanding of the semantics for configuring policies. Recall that a policy must have
a corresponding ring file, so configuring a policy is a two-step process. First, edit
your ``/etc/swift/swift.conf`` file to add your new policy and, second, create the
corresponding policy object ring file.
See :doc:`policies_saio` for a step by step guide on adding a policy to the SAIO setup.
Note that each policy has a section starting with ``[storage-policy:N]`` where N is the
policy index. There's no reason other than readability that these be sequential but there
are a number of rules enforced by Swift when parsing this file:
* If a policy with index 0 is not declared and no other policies defined,
Swift will create one
* The policy index must be a non-negative integer
* If no policy is declared as the default and no other policies are
defined, the policy with index 0 is set as the default
* Policy indexes must be unique
* Policy names are required
* Policy names are case insensitive
* Policy names must contain only letters, digits or a dash
* Policy names must be unique
* The policy name 'Policy-0' can only be used for the policy with index 0
* If any policies are defined, exactly one policy must be declared default
* Deprecated policies can not be declared the default
The following is an example of a properly configured ''swift.conf'' file. See :doc:`policies_saio`
for full instructions on setting up an all-in-one with this example configuration.::
[swift-hash]
# random unique strings that can never change (DO NOT LOSE)
swift_hash_path_prefix = changeme
swift_hash_path_suffix = changeme
[storage-policy:0]
name = gold
default = yes
[storage-policy:1]
name = silver
deprecated = yes
Review :ref:`default-policy` and :ref:`deprecate-policy` for more
information about the ``default`` and ``deprecated`` options.
There are some other considerations when managing policies:
* Policy names can be changed (but be sure that users are aware, aliases are
not currently supported but could be implemented in custom middleware!)
* You cannot change the index of a policy once it has been created
* The default policy can be changed at any time, by adding the
default directive to the desired policy section
* Any policy may be deprecated by adding the deprecated directive to
the desired policy section, but a deprecated policy may not also
be declared the default, and you must specify a default - so you
must have policy which is not deprecated at all times.
There will be additional parameters for policies as new features are added
(e.g., Erasure Code), but for now only a section name/index and name are
required. Once ``swift.conf`` is configured for a new policy, a new ring must be
created. The ring tools are not policy name aware so it's critical that the
correct policy index be used when creating the new policy's ring file.
Additional object rings are created in the same manner as the legacy ring
except that '-N' is appended after the word ``object`` where N matches the
policy index used in ``swift.conf``. This naming convention follows the pattern
for per-policy storage node data directories as well. So, to create the ring
for policy 1::
swift-ring-builder object-1.builder create 10 3 1
<and add devices, rebalance using the same naming convention>
.. note::
The same drives can indeed be used for multiple policies and the details
of how that's managed on disk will be covered in a later section, it's
important to understand the implications of such a configuration before
setting one up. Make sure it's really what you want to do, in many cases
it will be, but in others maybe not.
--------------
Using Policies
--------------
Using policies is very simple, a policy is only specified when a container is
initially created, there are no other API changes. Creating a container can
be done without any special policy information::
curl -v -X PUT -H 'X-Auth-Token: <your auth token>' \
http://127.0.0.1:8080/v1/AUTH_test/myCont0
Which will result in a container created that is associated with the
policy name 'gold' assuming we're using the swift.conf example from
above. It would use 'gold' because it was specified as the default.
Now, when we put an object into this container, it will get placed on
nodes that are part of the ring we created for policy 'gold'.
If we wanted to explicitly state that we wanted policy 'gold' the command
would simply need to include a new header as shown below::
curl -v -X PUT -H 'X-Auth-Token: <your auth token>' \
-H 'X-Storage-Policy: gold' http://127.0.0.1:8080/v1/AUTH_test/myCont1
And that's it! The application does not need to specify the policy name ever
again. There are some illegal operations however:
* If an invalid (typo, non-existent) policy is specified: 400 Bad Request
* if you try to change the policy either via PUT or POST: 409 Conflict
If you'd like to see how the storage in the cluster is being used, simply HEAD
the account and you'll see not only the cumulative numbers, as before, but
per policy statistics as well. In the example below there's 3 objects total
with two of them in policy 'gold' and one in policy 'silver'::
curl -i -X HEAD -H 'X-Auth-Token: <your auth token>' \
http://127.0.0.1:8080/v1/AUTH_test
and your results will include (some output removed for readability)::
X-Account-Container-Count: 3
X-Account-Object-Count: 3
X-Account-Bytes-Used: 21
X-Storage-Policy-Gold-Object-Count: 2
X-Storage-Policy-Gold-Bytes-Used: 14
X-Storage-Policy-Silver-Object-Count: 1
X-Storage-Policy-Silver-Bytes-Used: 7
--------------
Under the Hood
--------------
Now that we've explained a little about what Policies are and how to
configure/use them, let's explore how Storage Policies fit in at the
nuts-n-bolts level.
Parsing and Configuring
-----------------------
The module, :ref:`storage_policy`, is responsible for parsing the
``swift.conf`` file, validating the input, and creating a global collection of
configured policies via class :class:`.StoragePolicyCollection`. This
collection is made up of policies of class :class:`.StoragePolicy`. The
collection class includes handy functions for getting to a policy either by
name or by index , getting info about the policies, etc. There's also one
very important function, :meth:`~.StoragePolicyCollection.get_object_ring`.
Object rings are now members of the :class:`.StoragePolicy` class and are
actually not instantiated until the :meth:`~.StoragePolicy.load_ring`
method is called. Any caller anywhere in the code base that needs to access
an object ring must use the :data:`.POLICIES` global singleton to access the
:meth:`~.StoragePolicyCollection.get_object_ring` function and provide the
policy index which will call :meth:`~.StoragePolicy.load_ring` if
needed; however, when starting request handling services such as the
:ref:`proxy-server` rings are proactively loaded to provide moderate
protection against a mis-configuration resulting in a run time error. The
global is instantiated when Swift starts and provides a mechanism to patch
policies for the test code.
Middleware
----------
Middleware can take advantage of policies through the :data:`.POLICIES` global
and by importing :func:`.get_container_info` to gain access to the policy
index associated with the container in question. From the index it
can then use the :data:`.POLICIES` singleton to grab the right ring. For example,
:ref:`list_endpoints` is policy aware using the means just described. Another
example is :ref:`recon` which will report the md5 sums for all object rings.
Proxy Server
------------
The :ref:`proxy-server` module's role in Storage Policies is essentially to make sure the
correct ring is used as its member element. Before policies, the one object ring
would be instantiated when the :class:`.Application` class was instantiated and could
be overridden by test code via init parameter. With policies, however, there is
no init parameter and the :class:`.Application` class instead depends on the :data:`.POLICIES`
global singleton to retrieve the ring which is instantiated the first time it's
needed. So, instead of an object ring member of the :class:`.Application` class, there is
an accessor function, :meth:`~.Application.get_object_ring`, that gets the ring from :data:`.POLICIES`.
In general, when any module running on the proxy requires an object ring, it
does so via first getting the policy index from the cached container info. The
exception is during container creation where it uses the policy name from the
request header to look up policy index from the :data:`.POLICIES` global. Once the
proxy has determined the policy index, it can use the :meth:`~.Application.get_object_ring` method
described earlier to gain access to the correct ring. It then has the responsibility
of passing the index information, not the policy name, on to the back-end servers
via the header ``X-Backend-Storage-Policy-Index``. Going the other way, the proxy also
strips the index out of headers that go back to clients, and makes sure they only
see the friendly policy names.
On Disk Storage
---------------
Policies each have their own directories on the back-end servers and are identified by
their storage policy indexes. Organizing the back-end directory structures by policy
index helps keep track of things and also allows for sharing of disks between policies
which may or may not make sense depending on the needs of the provider. More
on this later, but for now be aware of the following directory naming convention:
* ``/objects`` maps to objects associated with Policy-0
* ``/objects-N`` maps to storage policy index #N
* ``/async_pending`` maps to async pending update for Policy-0
* ``/async_pending-N`` maps to async pending update for storage policy index #N
* ``/tmp`` maps to the DiskFile temporary directory for Policy-0
* ``/tmp-N`` maps to the DiskFile temporary directory for policy index #N
* ``/quarantined/objects`` maps to the quarantine directory for Policy-0
* ``/quarantined/objects-N`` maps to the quarantine directory for policy index #N
Note that these directory names are actually owned by the specific Diskfile
Implementation, the names shown above are used by the default Diskfile.
Object Server
-------------
The :ref:`object-server` is not involved with selecting the storage policy
placement directly. However, because of how back-end directory structures are
setup for policies, as described earlier, the object server modules do play a
role. When the object server gets a :class:`.Diskfile`, it passes in the
policy index and leaves the actual directory naming/structure mechanisms to
:class:`.Diskfile`. By passing in the index, the instance of
:class:`.Diskfile` being used will assure that data is properly located in the
tree based on its policy.
For the same reason, the :ref:`object-updater` also is policy aware; as previously
described, different policies use different async pending directories so the
updater needs to know how to scan them appropriately.
The :ref:`object-replicator` is policy aware in that, depending on the policy, it may have to
do drastically different things, or maybe not. For example, the difference in
handling a replication job for 2x versus 3x is trivial; however, the difference in
handling replication between 3x and erasure code is most definitely not. In
fact, the term 'replication' really isn't appropriate for some policies
like erasure code; however, the majority of the framework for collecting and
processing jobs remains the same. Thus, those functions in the replicator are
leveraged for all policies and then there is policy specific code required for
each policy, added when the policy is defined if needed.
The ssync functionality is policy aware for the same reason. Some of the
other modules may not obviously be affected, but the back-end directory
structure owned by :class:`.Diskfile` requires the policy index
parameter. Therefore ssync being policy aware really means passing the
policy index along. See :class:`~swift.obj.ssync_sender` and
:class:`~swift.obj.ssync_receiver` for more information on ssync.
For :class:`.Diskfile` itself, being policy aware is all about managing the back-end
structure using the provided policy index. In other words, callers who get
a :class:`.Diskfile` instance provide a policy index and :class:`.Diskfile`'s job is to keep data
separated via this index (however it chooses) such that policies can share
the same media/nodes if desired. The included implementation of :class:`.Diskfile`
lays out the directory structure described earlier but that's owned within
:class:`.Diskfile`; external modules have no visibility into that detail. A common
function is provided to map various directory names and/or strings
based on their policy index. For example :class:`.Diskfile` defines :func:`.get_data_dir`
which builds off of a generic :func:`.get_policy_string` to consistently build
policy aware strings for various usage.
Container Server
----------------
The :ref:`container-server` plays a very important role in Storage Policies, it is
responsible for handling the assignment of a policy to a container and the
prevention of bad things like changing policies or picking the wrong policy
to use when nothing is specified (recall earlier discussion on Policy-0 versus
default).
The :ref:`container-updater` is policy aware, however its job is very simple, to
pass the policy index along to the :ref:`account-server` via a request header.
The :ref:`container-backend` is responsible for both altering existing DB
schema as well as assuring new DBs are created with a schema that supports
storage policies. The "on-demand" migration of container schemas allows Swift
to upgrade without downtime (sqlite's alter statements are fast regardless of
row count). To support rolling upgrades (and downgrades) the incompatible
schema changes to the ``container_stat`` table are made to a
``container_info`` table, and the ``container_stat`` table is replaced with a
view that includes an ``INSTEAD OF UPDATE`` trigger which makes it behave like
the old table.
The policy index is stored here for use in reporting information
about the container as well as managing split-brain scenario induced
discrepancies between containers and their storage policies. Furthermore,
during split-brain containers must be prepared to track object updates from
multiple policies, so the object table also includes a
``storage_policy_index`` column. Per-policy object counts and bytes are
updated in the ``policy_stat`` table using ``INSERT`` and ``DELETE`` triggers
similar to the pre-policy triggers that updated ``container_stat`` directly.
The :ref:`container-replicator` daemon will pro-actively migrate legacy
schemas as part of its normal consistency checking process when it updates the
``reconciler_sync_point`` entry in the ``container_info`` table. This ensures
that read heavy containers which do not encounter any writes will still get
migrated to be fully compatible with the post-storage-policy queries without
having to fall-back and retry queries with the legacy schema to service
container read requests.
The :ref:`container-sync-daemon` functionality only needs to be policy aware in that it
accesses the object rings. Therefore, it needs to pull the policy index
out of the container information and use it to select the appropriate
object ring from the :data:`.POLICIES` global.
Account Server
--------------
The :ref:`account-server`'s role in Storage Policies is really limited to reporting.
When a HEAD request is made on an account (see example provided earlier),
the account server is provided with the storage policy index and builds
the ``object_count`` and ``byte_count`` information for the client on a per
policy basis.
The account servers are able to report per-storage-policy object and byte
counts because of some policy specific DB schema changes. A policy specific
table, ``policy_stat``, maintains information on a per policy basis (one row
per policy) in the same manner in which the ``account_stat`` table does. The
``account_stat`` table still serves the same purpose and is not replaced by
``policy_stat``, it holds the total account stats whereas ``policy_stat`` just
has the break downs. The backend is also responsible for migrating
pre-storage-policy accounts by altering the DB schema and populating the
``policy_stat`` table for Policy-0 with current ``account_stat`` data at that
point in time.
The per-storage-policy object and byte counts are not updated with each object
PUT and DELETE request container updates to the account server is performed
asynchronously by the ``swift-container-updater``.
.. _upgrade-policy:
Upgrading and Confirming Functionality
--------------------------------------
Upgrading to a version of Swift that has Storage Policy support is not difficult,
in fact, the cluster administrator isn't required to make any special configuration
changes to get going. Swift will automatically begin using the existing object
ring as both the default ring and the Policy-0 ring. Adding the declaration of
policy 0 is totally optional and in its absence, the name given to the implicit
policy 0 will be 'Policy-0'. Let's say for testing purposes that you wanted to take
an existing cluster that already has lots of data on it and upgrade to Swift with
Storage Policies. From there you want to go ahead and create a policy and test a
few things out. All you need to do is:
#. Define your policies in ``/etc/swift/swift.conf``
#. Create the corresponding object rings
#. Create containers and objects and confirm their placement is as expected
For a specific example that takes you through these steps, please see
:doc:`policies_saio`
.. note::
If you downgrade from a Storage Policy enabled version of Swift to an
older version that doesn't support policies, you will not be able to
access any data stored in policies other than the policy with index 0 but
those objects WILL appear in container listings (possibly as duplicates if
there was a network partition and un-reconciled objects). It is EXTREMELY
important that you perform any necessary integration testing on the
upgraded deployment before enabling an additional storage policy to ensure
a consistent API experience for your clients. DO NOT downgrade to a
version of Swift that does not support storage policies once you expose
multiple storage policies.

2
doc/source/overview_replication.rst
View File

@ -93,6 +93,8 @@ systems, it was designed so that around 2% of the hash space on a normal node
will be invalidated per day, which has experimentally given us acceptable
replication speeds.
.. _ssync:
Work continues with a new ssync method where rsync is not used at all and
instead all-Swift code is used to transfer the objects. At first, this ssync
will just strive to emulate the rsync behavior. Once deemed stable it will open

146
doc/source/policies_saio.rst Normal file
View File

@ -0,0 +1,146 @@
===========================================
Adding Storage Policies to an Existing SAIO
===========================================
Depending on when you downloaded your SAIO environment, it may already
be prepared with two storage policies that enable some basic functional
tests. In the event that you are adding a storage policy to an existing
installation, however, the following section will walk you through the
steps for setting up Storage Policies. Note that configuring more than
one storage policy on your development environment is recommended but
optional. Enabling multiple Storage Policies is very easy regardless of
whether you are working with an existing installation or starting a
brand new one.
Now we will create two policies - the first one will be a standard triple
replication policy that we will also explicitly set as the default and
the second will be setup for reduced replication using a factor of 2x.
We will call the first one 'gold' and the second one 'silver'. In this
example both policies map to the same devices because it's also
important for this sample implementation to be simple and easy
to understand and adding a bunch of new devices isn't really required
to implement a usable set of policies.
1. To define your policies, add the following to your ``/etc/swift/swift.conf``
file::
[storage-policy:0]
name = gold
default = yes
[storage-policy:1]
name = silver
See :doc:`overview_policies` for detailed information on ``swift.conf`` policy
options.
2. To create the object ring for the silver policy (index 1), add the following
to your ``bin/remakerings`` script and re-run it (your script may already have
these changes)::
swift-ring-builder object-1.builder create 10 2 1
swift-ring-builder object-1.builder add r1z1-127.0.0.1:6010/sdb1 1
swift-ring-builder object-1.builder add r1z2-127.0.0.1:6020/sdb2 1
swift-ring-builder object-1.builder add r1z3-127.0.0.1:6030/sdb3 1
swift-ring-builder object-1.builder add r1z4-127.0.0.1:6040/sdb4 1
swift-ring-builder object-1.builder rebalance
Note that the reduced replication of the silver policy is only a function
of the replication parameter in the ``swift-ring-builder create`` command
and is not specified in ``/etc/swift/swift.conf``.
3. Copy ``etc/container-reconciler.conf-sample`` to
``/etc/swift/container-reconciler.conf`` and fix the user option::
cp etc/container-reconciler.conf-sample /etc/swift/container-reconciler.conf
sed -i "s/# user.*/user = $USER/g" /etc/swift/container-reconciler.conf
------------------
Using Policies
------------------
Setting up Storage Policies was very simple, and using them is even
simpler. In this section, we will run some commands to create a few
containers with different policies and store objects in them and see how
Storage Policies effect placement of data in Swift.
1. We will be using the list_endpoints middleware to confirm object locations,
so enable that now in your ``proxy-server.conf`` file by adding it to the pipeline
and including the filter section as shown below (be sure to restart your proxy
after making these changes)::
pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk \
slo dlo ratelimit crossdomain list-endpoints tempurl tempauth staticweb \
container-quotas account-quotas proxy-logging proxy-server
[filter:list-endpoints]
use = egg:swift#list_endpoints
2. Check to see that your policies are reported via /info::
swift -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing info
You should see this: (only showing the policy output here)::
policies: [{'default': True, 'name': 'gold'}, {'name': 'silver'}]
3. Now create a container without specifying a policy, it will use the
default, 'gold' and then put a test object in it (create the file ``file0.txt``
with your favorite editor with some content)::
curl -v -X PUT -H 'X-Auth-Token: <your auth token>' \
http://127.0.0.1:8080/v1/AUTH_test/myCont0
curl -X PUT -v -T file0.txt -H 'X-Auth-Token: <your auth token>' \
http://127.0.0.1:8080/v1/AUTH_test/myCont0/file0.txt
4. Now confirm placement of the object with the :ref:`list_endpoints` middleware::
curl -X GET -v http://127.0.0.1:8080/endpoints/AUTH_test/myCont0/file0.txt
You should see this: (note placement on expected devices)::
["http://127.0.0.1:6030/sdb3/761/AUTH_test/myCont0/file0.txt",
"http://127.0.0.1:6010/sdb1/761/AUTH_test/myCont0/file0.txt",
"http://127.0.0.1:6020/sdb2/761/AUTH_test/myCont0/file0.txt"]
5. Create a container using policy 'silver' and put a different file in it::
curl -v -X PUT -H 'X-Auth-Token: <your auth token>' -H \
"X-Storage-Policy: silver" \
http://127.0.0.1:8080/v1/AUTH_test/myCont1
curl -X PUT -v -T file1.txt -H 'X-Auth-Token: <your auth token>' \
http://127.0.0.1:8080/v1/AUTH_test/myCont1/
6. Confirm placement of the object for policy 'silver'::
curl -X GET -v http://127.0.0.1:8080/endpoints/AUTH_test/myCont1/file1.txt
You should see this: (note placement on expected devices)::
["http://127.0.0.1:6010/sdb1/32/AUTH_test/myCont1/file1.txt",
"http://127.0.0.1:6040/sdb4/32/AUTH_test/myCont1/file1.txt"]
7. Confirm account information with HEAD, make sure that your container-updater
service is running and has executed once since you performed the PUTs or the
account database won't be updated yet::
curl -i -X HEAD -H 'X-Auth-Token: <your auth token>' \
http://127.0.0.1:8080/v1/AUTH_test
You should see something like this (note that total and per policy stats
object sizes will vary)::
HTTP/1.1 204 No Content
Content-Length: 0
X-Account-Object-Count: 2
X-Account-Bytes-Used: 174
X-Account-Container-Count: 2
X-Account-Storage-Policy-Gold-Object-Count: 1
X-Account-Storage-Policy-Gold-Bytes-Used: 84
X-Account-Storage-Policy-Silver-Object-Count: 1
X-Account-Storage-Policy-Silver-Bytes-Used: 90
X-Timestamp: 1397230339.71525
Content-Type: text/plain; charset=utf-8
Accept-Ranges: bytes
X-Trans-Id: tx96e7496b19bb44abb55a3-0053482c75
Date: Fri, 11 Apr 2014 17:55:01 GMT

52
etc/container-reconciler.conf-sample Normal file
View File

@ -0,0 +1,52 @@
[DEFAULT]
# swift_dir = /etc/swift
# user = swift
# You can specify default log routing here if you want:
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# comma separated list of functions to call to setup custom log handlers.
# functions get passed: conf, name, log_to_console, log_route, fmt, logger,
# adapted_logger
# log_custom_handlers =
#
# If set, log_udp_host will override log_address
# log_udp_host =
# log_udp_port = 514
#
# You can enable StatsD logging here:
# log_statsd_host = localhost
# log_statsd_port = 8125
# log_statsd_default_sample_rate = 1.0
# log_statsd_sample_rate_factor = 1.0
# log_statsd_metric_prefix =
[container-reconciler]
# The reconciler will re-attempt reconciliation if the source object is not
# available up to reclaim_age seconds before it gives up and deletes the entry
# in the queue.
# reclaim_age = 604800
# The cycle time of the daemon
# interval = 300
# Server errors from requests will be retried by default
# request_tries = 3
[pipeline:main]
pipeline = catch_errors proxy-logging cache proxy-server
[app:proxy-server]
use = egg:swift#proxy
# See proxy-server.conf-sample for options
[filter:cache]
use = egg:swift#memcache
# See proxy-server.conf-sample for options
[filter:proxy-logging]
use = egg:swift#proxy_logging
[filter:catch_errors]
use = egg:swift#catch_errors
# See proxy-server.conf-sample for options

4
etc/object-expirer.conf-sample
View File

@ -46,6 +46,10 @@
# process is "zero based", if you want to use 3 processes, you should run
# processes with process set to 0, 1, and 2
# process = 0
# The expirer will re-attempt expiring if the source object is not available
# up to reclaim_age seconds before it gives up and deletes the entry in the
# queue.
# reclaim_age = 604800
[pipeline:main]
pipeline = catch_errors cache proxy-server

2
etc/proxy-server.conf-sample
View File

@ -578,6 +578,8 @@ use = egg:swift#container_sync
# Updating those will have to be done manually, as knowing what the true realm
# endpoint should be cannot always be guessed.
# allow_full_urls = true
# Set this to specify this clusters //realm/cluster as "current" in /info
# current = //REALM/CLUSTER
# Note: Put it at the beginning of the pipleline to profile all middleware. But
# it is safer to put this after catch_errors, gatekeeper and healthcheck.

31
etc/swift.conf-sample
View File

@ -8,6 +8,37 @@
swift_hash_path_suffix = changeme
swift_hash_path_prefix = changeme
# storage policies are defined here and determine various characteristics
# about how objects are stored and treated. Policies are specified by name on
# a per container basis. Names are case-insensitive. The policy index is
# specified in the section header and is used internally. The policy with
# index 0 is always used for legacy containers and can be given a name for use
# in metadata however the ring file name will always be 'object.ring.gz' for
# backwards compatibility. If no policies are defined a policy with index 0
# will be automatically created for backwards compatibility and given the name
# Policy-0. A default policy is used when creating new containers when no
# policy is specified in the request. If no other policies are defined the
# policy with index 0 will be declared the default. If multiple policies are
# defined you must define a policy with index 0 and you must specify a
# default. It is recommended you always define a section for
# storage-policy:0.
[storage-policy:0]
name = Policy-0
default = yes
# the following section would declare a policy called 'silver', the number of
# replicas will be determined by how the ring is built. In this example the
# 'silver' policy could have a lower or higher # of replicas than the
# 'Policy-0' policy above. The ring filename will be 'object-1.ring.gz'. You
# may only specify one storage policy section as the default. If you changed
# this section to specify 'silver' as the default, when a client created a new
# container w/o a policy specified, it will get the 'silver' policy because
# this config has specified it as the default. However if a legacy container
# (one created with a pre-policy version of swift) is accessed, it is known
# implicitly to be assigned to the policy with index 0 as opposed to the
# current default.
#[storage-policy:1]
#name = silver
# The swift-constraints section sets the basic constraints on data
# saved in the swift cluster. These constraints are automatically

2
setup.cfg
View File

@ -39,6 +39,8 @@ scripts =
bin/swift-container-server
bin/swift-container-sync
bin/swift-container-updater
bin/swift-container-reconciler
bin/swift-reconciler-enqueue
bin/swift-dispersion-populate
bin/swift-dispersion-report
bin/swift-drive-audit

193
swift/account/backend.py
View File

@ -24,20 +24,42 @@ import errno
import sqlite3
from swift.common.utils import normalize_timestamp, lock_parent_directory
from swift.common.utils import Timestamp, lock_parent_directory
from swift.common.db import DatabaseBroker, DatabaseConnectionError, \
PENDING_CAP, PICKLE_PROTOCOL, utf8encode
DATADIR = 'accounts'
POLICY_STAT_TRIGGER_SCRIPT = """
CREATE TRIGGER container_insert_ps AFTER INSERT ON container
BEGIN
INSERT OR IGNORE INTO policy_stat
(storage_policy_index, object_count, bytes_used)
VALUES (new.storage_policy_index, 0, 0);
UPDATE policy_stat
SET object_count = object_count + new.object_count,
bytes_used = bytes_used + new.bytes_used
WHERE storage_policy_index = new.storage_policy_index;
END;
CREATE TRIGGER container_delete_ps AFTER DELETE ON container
BEGIN
UPDATE policy_stat
SET object_count = object_count - old.object_count,
bytes_used = bytes_used - old.bytes_used
WHERE storage_policy_index = old.storage_policy_index;
END;
"""
class AccountBroker(DatabaseBroker):
"""Encapsulates working with an account database."""
db_type = 'account'
db_contains_type = 'container'
db_reclaim_timestamp = 'delete_timestamp'
def _initialize(self, conn, put_timestamp):
def _initialize(self, conn, put_timestamp, **kwargs):
"""
Create a brand new account database (tables, indices, triggers, etc.)
@ -49,6 +71,7 @@ class AccountBroker(DatabaseBroker):
'Attempting to create a new database with no account set')
self.create_container_table(conn)
self.create_account_stat_table(conn, put_timestamp)
self.create_policy_stat_table(conn)
def create_container_table(self, conn):
"""
@ -64,7 +87,8 @@ class AccountBroker(DatabaseBroker):
delete_timestamp TEXT,
object_count INTEGER,
bytes_used INTEGER,
deleted INTEGER DEFAULT 0
deleted INTEGER DEFAULT 0,
storage_policy_index INTEGER DEFAULT 0
);
CREATE INDEX ix_container_deleted_name ON
@ -99,7 +123,7 @@ class AccountBroker(DatabaseBroker):
old.delete_timestamp || '-' ||
old.object_count || '-' || old.bytes_used);
END;
""")
""" + POLICY_STAT_TRIGGER_SCRIPT)
def create_account_stat_table(self, conn, put_timestamp):
"""
@ -130,9 +154,30 @@ class AccountBroker(DatabaseBroker):
conn.execute('''
UPDATE account_stat SET account = ?, created_at = ?, id = ?,
put_timestamp = ?
''', (self.account, normalize_timestamp(time.time()), str(uuid4()),
put_timestamp))
put_timestamp = ?, status_changed_at = ?
''', (self.account, Timestamp(time.time()).internal, str(uuid4()),
put_timestamp, put_timestamp))
def create_policy_stat_table(self, conn):
"""
Create policy_stat table which is specific to the account DB.
Not a part of Pluggable Back-ends, internal to the baseline code.
:param conn: DB connection object
"""
conn.executescript("""
CREATE TABLE policy_stat (
storage_policy_index INTEGER PRIMARY KEY,
object_count INTEGER DEFAULT 0,
bytes_used INTEGER DEFAULT 0
);
INSERT OR IGNORE INTO policy_stat (
storage_policy_index, object_count, bytes_used
)
SELECT 0, object_count, bytes_used
FROM account_stat
WHERE container_count > 0;
""")
def get_db_version(self, conn):
if self._db_version == -1:
@ -159,16 +204,24 @@ class AccountBroker(DatabaseBroker):
def _commit_puts_load(self, item_list, entry):
"""See :func:`swift.common.db.DatabaseBroker._commit_puts_load`"""
(name, put_timestamp, delete_timestamp,
object_count, bytes_used, deleted) = \
pickle.loads(entry.decode('base64'))
loaded = pickle.loads(entry.decode('base64'))
# check to see if the update includes policy_index or not
(name, put_timestamp, delete_timestamp, object_count, bytes_used,
deleted) = loaded[:6]
if len(loaded) > 6:
storage_policy_index = loaded[6]
else:
# legacy support during upgrade until first non legacy storage
# policy is defined
storage_policy_index = 0
item_list.append(
{'name': name,
'put_timestamp': put_timestamp,
'delete_timestamp': delete_timestamp,
'object_count': object_count,
'bytes_used': bytes_used,
'deleted': deleted})
'deleted': deleted,
'storage_policy_index': storage_policy_index})
def empty(self):
"""
@ -183,7 +236,7 @@ class AccountBroker(DatabaseBroker):
return (row[0] == 0)
def put_container(self, name, put_timestamp, delete_timestamp,
object_count, bytes_used):
object_count, bytes_used, storage_policy_index):
"""
Create a container with the given attributes.
@ -192,6 +245,7 @@ class AccountBroker(DatabaseBroker):
:param delete_timestamp: delete_timestamp of the container to create
:param object_count: number of objects in the container
:param bytes_used: number of bytes used by the container
:param storage_policy_index: the storage policy for this container
"""
if delete_timestamp > put_timestamp and \
object_count in (None, '', 0, '0'):
@ -202,7 +256,8 @@ class AccountBroker(DatabaseBroker):
'delete_timestamp': delete_timestamp,
'object_count': object_count,
'bytes_used': bytes_used,
'deleted': deleted}
'deleted': deleted,
'storage_policy_index': storage_policy_index}
if self.db_file == ':memory:':
self.merge_items([record])
return
@ -225,27 +280,33 @@ class AccountBroker(DatabaseBroker):
fp.write(':')
fp.write(pickle.dumps(
(name, put_timestamp, delete_timestamp, object_count,
bytes_used, deleted),
bytes_used, deleted, storage_policy_index),
protocol=PICKLE_PROTOCOL).encode('base64'))
fp.flush()
def is_deleted(self):
def _is_deleted_info(self, status, container_count, delete_timestamp,
put_timestamp):
"""
Check if the account DB is considered to be deleted.
Apply delete logic to database info.
:returns: True if the account DB is considered to be deleted, False
otherwise
:returns: True if the DB is considered to be deleted, False otherwise
"""
if self.db_file != ':memory:' and not os.path.exists(self.db_file):
return True
self._commit_puts_stale_ok()
with self.get() as conn:
row = conn.execute('''
SELECT put_timestamp, delete_timestamp, container_count, status
FROM account_stat''').fetchone()
return row['status'] == 'DELETED' or (
row['container_count'] in (None, '', 0, '0') and
row['delete_timestamp'] > row['put_timestamp'])
return status == 'DELETED' or (
container_count in (None, '', 0, '0') and
Timestamp(delete_timestamp) > Timestamp(put_timestamp))
def _is_deleted(self, conn):
"""
Check account_stat table and evaluate info.
:param conn: database conn
:returns: True if the DB is considered to be deleted, False otherwise
"""
info = conn.execute('''
SELECT put_timestamp, delete_timestamp, container_count, status
FROM account_stat''').fetchone()
return self._is_deleted_info(**info)
def is_status_deleted(self):
"""Only returns true if the status field is set to DELETED."""
@ -255,19 +316,47 @@ class AccountBroker(DatabaseBroker):
FROM account_stat''').fetchone()
return (row['status'] == "DELETED")
def get_policy_stats(self):
"""
Get global policy stats for the account.
:returns: dict of policy stats where the key is the policy index and
the value is a dictionary like {'object_count': M,
'bytes_used': N}
"""
info = []
self._commit_puts_stale_ok()
with self.get() as conn:
try:
info = (conn.execute('''
SELECT storage_policy_index, object_count, bytes_used
FROM policy_stat
''').fetchall())
except sqlite3.OperationalError as err:
if "no such table: policy_stat" not in str(err):
raise
policy_stats = {}
for row in info:
stats = dict(row)
key = stats.pop('storage_policy_index')
policy_stats[key] = stats
return policy_stats
def get_info(self):
"""
Get global data for the account.
:returns: dict with keys: account, created_at, put_timestamp,
delete_timestamp, container_count, object_count,
bytes_used, hash, id
delete_timestamp, status_changed_at, container_count,
object_count, bytes_used, hash, id
"""
self._commit_puts_stale_ok()
with self.get() as conn:
return dict(conn.execute('''
SELECT account, created_at, put_timestamp, delete_timestamp,
container_count, object_count, bytes_used, hash, id
status_changed_at, container_count, object_count,
bytes_used, hash, id
FROM account_stat
''').fetchone())
@ -359,18 +448,20 @@ class AccountBroker(DatabaseBroker):
:param item_list: list of dictionaries of {'name', 'put_timestamp',
'delete_timestamp', 'object_count', 'bytes_used',
'deleted'}
'deleted', 'storage_policy_index'}
:param source: if defined, update incoming_sync with the source
"""
with self.get() as conn:
def _really_merge_items(conn):
max_rowid = -1
for rec in item_list:
record = [rec['name'], rec['put_timestamp'],
rec['delete_timestamp'], rec['object_count'],
rec['bytes_used'], rec['deleted']]
rec['bytes_used'], rec['deleted'],
rec['storage_policy_index']]
query = '''
SELECT name, put_timestamp, delete_timestamp,
object_count, bytes_used, deleted
object_count, bytes_used, deleted,
storage_policy_index
FROM container WHERE name = ?
'''
if self.get_db_version(conn) >= 1:
@ -400,8 +491,8 @@ class AccountBroker(DatabaseBroker):
conn.execute('''
INSERT INTO container (name, put_timestamp,
delete_timestamp, object_count, bytes_used,
deleted)
VALUES (?, ?, ?, ?, ?, ?)
deleted, storage_policy_index)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', record)
if source:
max_rowid = max(max_rowid, rec['ROWID'])
@ -413,7 +504,33 @@ class AccountBroker(DatabaseBroker):
''', (max_rowid, source))
except sqlite3.IntegrityError:
conn.execute('''
UPDATE incoming_sync SET sync_point=max(?, sync_point)
UPDATE incoming_sync
SET sync_point=max(?, sync_point)
WHERE remote_id=?
''', (max_rowid, source))
conn.commit()
with self.get() as conn:
# create the policy stat table if needed and add spi to container
try:
_really_merge_items(conn)
except sqlite3.OperationalError as err:
if 'no such column: storage_policy_index' not in str(err):
raise
self._migrate_add_storage_policy_index(conn)
_really_merge_items(conn)
def _migrate_add_storage_policy_index(self, conn):
"""
Add the storage_policy_index column to the 'container' table and
set up triggers, creating the policy_stat table if needed.
"""
try:
self.create_policy_stat_table(conn)
except sqlite3.OperationalError as err:
if 'table policy_stat already exists' not in str(err):
raise
conn.executescript('''
ALTER TABLE container
ADD COLUMN storage_policy_index INTEGER DEFAULT 0;
''' + POLICY_STAT_TRIGGER_SCRIPT)

63
swift/account/reaper.py
View File

@ -18,7 +18,7 @@ import random
from swift import gettext_ as _
from logging import DEBUG
from math import sqrt
from time import time, ctime
from time import time
from eventlet import GreenPool, sleep, Timeout
@ -29,8 +29,9 @@ from swift.common.direct_client import direct_delete_container, \
from swift.common.exceptions import ClientException
from swift.common.ring import Ring
from swift.common.utils import get_logger, whataremyips, ismount, \
config_true_value
config_true_value, Timestamp
from swift.common.daemon import Daemon
from swift.common.storage_policy import POLICIES, POLICY_INDEX
class AccountReaper(Daemon):
@ -54,9 +55,9 @@ class AccountReaper(Daemon):
configuration parameters.
"""
def __init__(self, conf):
def __init__(self, conf, logger=None):
self.conf = conf
self.logger = get_logger(conf, log_route='account-reaper')
self.logger = logger or get_logger(conf, log_route='account-reaper')
self.devices = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.interval = int(conf.get('interval', 3600))
@ -89,11 +90,14 @@ class AccountReaper(Daemon):
self.container_ring = Ring(self.swift_dir, ring_name='container')
return self.container_ring
def get_object_ring(self):
"""The object :class:`swift.common.ring.Ring` for the cluster."""
if not self.object_ring:
self.object_ring = Ring(self.swift_dir, ring_name='object')
return self.object_ring
def get_object_ring(self, policy_idx):
"""
Get the ring identified by the policy index
:param policy_idx: Storage policy index
:returns: A ring matching the storage policy
"""
return POLICIES.get_object_ring(policy_idx, self.swift_dir)
def run_forever(self, *args, **kwargs):
"""Main entry point when running the reaper in normal daemon mode.
@ -177,6 +181,15 @@ class AccountReaper(Daemon):
not broker.empty():
self.reap_account(broker, partition, nodes)
def reset_stats(self):
self.stats_return_codes = {}
self.stats_containers_deleted = 0
self.stats_objects_deleted = 0
self.stats_containers_remaining = 0
self.stats_objects_remaining = 0
self.stats_containers_possibly_remaining = 0
self.stats_objects_possibly_remaining = 0
def reap_account(self, broker, partition, nodes):
"""
Called once per pass for each account this server is the primary for
@ -216,17 +229,12 @@ class AccountReaper(Daemon):
"""
begin = time()
info = broker.get_info()
if time() - float(info['delete_timestamp']) <= self.delay_reaping:
if time() - float(Timestamp(info['delete_timestamp'])) <= \
self.delay_reaping:
return False
account = info['account']
self.logger.info(_('Beginning pass on account %s'), account)
self.stats_return_codes = {}
self.stats_containers_deleted = 0
self.stats_objects_deleted = 0
self.stats_containers_remaining = 0
self.stats_objects_remaining = 0
self.stats_containers_possibly_remaining = 0
self.stats_objects_possibly_remaining = 0
self.reset_stats()
try:
marker = ''
while True:
@ -274,10 +282,11 @@ class AccountReaper(Daemon):
log += _(', elapsed: %.02fs') % (time() - begin)
self.logger.info(log)
self.logger.timing_since('timing', self.start_time)
delete_timestamp = Timestamp(info['delete_timestamp'])
if self.stats_containers_remaining and \
begin - float(info['delete_timestamp']) >= self.reap_not_done_after:
begin - float(delete_timestamp) >= self.reap_not_done_after:
self.logger.warn(_('Account %s has not been reaped since %s') %
(account, ctime(float(info['delete_timestamp']))))
(account, delete_timestamp.isoformat))
return True
def reap_container(self, account, account_partition, account_nodes,
@ -324,11 +333,11 @@ class AccountReaper(Daemon):
while True:
objects = None
try:
objects = direct_get_container(
headers, objects = direct_get_container(
node, part, account, container,
marker=marker,
conn_timeout=self.conn_timeout,
response_timeout=self.node_timeout)[1]
response_timeout=self.node_timeout)
self.stats_return_codes[2] = \
self.stats_return_codes.get(2, 0) + 1
self.logger.increment('return_codes.2')
@ -343,11 +352,12 @@ class AccountReaper(Daemon):
if not objects:
break
try:
policy_index = headers.get(POLICY_INDEX, 0)
for obj in objects:
if isinstance(obj['name'], unicode):
obj['name'] = obj['name'].encode('utf8')
pool.spawn(self.reap_object, account, container, part,
nodes, obj['name'])
nodes, obj['name'], policy_index)
pool.waitall()
except (Exception, Timeout):
self.logger.exception(_('Exception with objects for container '
@ -396,7 +406,7 @@ class AccountReaper(Daemon):
self.logger.increment('containers_possibly_remaining')
def reap_object(self, account, container, container_partition,
container_nodes, obj):
container_nodes, obj, policy_index):
"""
Deletes the given object by issuing a delete request to each node for
the object. The format of the delete request is such that each object
@ -412,12 +422,14 @@ class AccountReaper(Daemon):
container ring.
:param container_nodes: The primary node dicts for the container.
:param obj: The name of the object to delete.
:param policy_index: The storage policy index of the object's container
* See also: :func:`swift.common.ring.Ring.get_nodes` for a description
of the container node dicts.
"""
container_nodes = list(container_nodes)
part, nodes = self.get_object_ring().get_nodes(account, container, obj)
ring = self.get_object_ring(policy_index)
part, nodes = ring.get_nodes(account, container, obj)
successes = 0
failures = 0
for node in nodes:
@ -429,7 +441,8 @@ class AccountReaper(Daemon):
response_timeout=self.node_timeout,
headers={'X-Container-Host': '%(ip)s:%(port)s' % cnode,
'X-Container-Partition': str(container_partition),
'X-Container-Device': cnode['device']})
'X-Container-Device': cnode['device'],
POLICY_INDEX: policy_index})
successes += 1
self.stats_return_codes[2] = \
self.stats_return_codes.get(2, 0) + 1

53
swift/account/server.py
View File

@ -22,14 +22,14 @@ from eventlet import Timeout
import swift.common.db
from swift.account.backend import AccountBroker, DATADIR
from swift.account.utils import account_listing_response
from swift.account.utils import account_listing_response, get_response_headers
from swift.common.db import DatabaseConnectionError, DatabaseAlreadyExists
from swift.common.request_helpers import get_param, get_listing_content_type, \
split_and_validate_path
from swift.common.utils import get_logger, hash_path, public, \
normalize_timestamp, storage_directory, config_true_value, \
Timestamp, storage_directory, config_true_value, \
json, timing_stats, replication, get_log_line
from swift.common.constraints import check_mount, check_float, check_utf8
from swift.common.constraints import check_mount, valid_timestamp, check_utf8
from swift.common import constraints
from swift.common.db_replicator import ReplicatorRpc
from swift.common.swob import HTTPAccepted, HTTPBadRequest, \
@ -38,6 +38,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, \
HTTPPreconditionFailed, HTTPConflict, Request, \
HTTPInsufficientStorage, HTTPException
from swift.common.request_helpers import is_sys_or_user_meta
from swift.common.storage_policy import POLICY_INDEX
class AccountController(object):
@ -89,14 +90,11 @@ class AccountController(object):
drive, part, account = split_and_validate_path(req, 3)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
if 'x-timestamp' not in req.headers or \
not check_float(req.headers['x-timestamp']):
return HTTPBadRequest(body='Missing timestamp', request=req,
content_type='text/plain')
req_timestamp = valid_timestamp(req)
broker = self._get_account_broker(drive, part, account)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
broker.delete_db(req.headers['x-timestamp'])
broker.delete_db(req_timestamp.internal)
return self._deleted_response(broker, req, HTTPNoContent)
@public
@ -107,7 +105,12 @@ class AccountController(object):
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
if container: # put account container
if 'x-timestamp' not in req.headers:
timestamp = Timestamp(time.time())
else:
timestamp = valid_timestamp(req)
pending_timeout = None
container_policy_index = req.headers.get(POLICY_INDEX, 0)
if 'x-trans-id' in req.headers:
pending_timeout = 3
broker = self._get_account_broker(drive, part, account,
@ -115,8 +118,7 @@ class AccountController(object):
if account.startswith(self.auto_create_account_prefix) and \
not os.path.exists(broker.db_file):
try:
broker.initialize(normalize_timestamp(
req.headers.get('x-timestamp') or time.time()))
broker.initialize(timestamp.internal)
except DatabaseAlreadyExists:
pass
if req.headers.get('x-account-override-deleted', 'no').lower() != \
@ -125,18 +127,19 @@ class AccountController(object):
broker.put_container(container, req.headers['x-put-timestamp'],
req.headers['x-delete-timestamp'],
req.headers['x-object-count'],
req.headers['x-bytes-used'])
req.headers['x-bytes-used'],
container_policy_index)
if req.headers['x-delete-timestamp'] > \
req.headers['x-put-timestamp']:
return HTTPNoContent(request=req)
else:
return HTTPCreated(request=req)
else: # put account
timestamp = valid_timestamp(req)
broker = self._get_account_broker(drive, part, account)
timestamp = normalize_timestamp(req.headers['x-timestamp'])
if not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp)
broker.initialize(timestamp.internal)
created = True
except DatabaseAlreadyExists:
created = False
@ -145,11 +148,11 @@ class AccountController(object):
body='Recently deleted')
else:
created = broker.is_deleted()
broker.update_put_timestamp(timestamp)
broker.update_put_timestamp(timestamp.internal)
if broker.is_deleted():
return HTTPConflict(request=req)
metadata = {}
metadata.update((key, (value, timestamp))
metadata.update((key, (value, timestamp.internal))
for key, value in req.headers.iteritems()
if is_sys_or_user_meta('account', key))
if metadata:
@ -172,16 +175,7 @@ class AccountController(object):
stale_reads_ok=True)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
info = broker.get_info()
headers = {
'X-Account-Container-Count': info['container_count'],
'X-Account-Object-Count': info['object_count'],
'X-Account-Bytes-Used': info['bytes_used'],
'X-Timestamp': info['created_at'],
'X-PUT-Timestamp': info['put_timestamp']}
headers.update((key, value)
for key, (value, timestamp) in
broker.metadata.iteritems() if value != '')
headers = get_response_headers(broker)
headers['Content-Type'] = out_content_type
return HTTPNoContent(request=req, headers=headers, charset='utf-8')
@ -244,19 +238,14 @@ class AccountController(object):
def POST(self, req):
"""Handle HTTP POST request."""
drive, part, account = split_and_validate_path(req, 3)
if 'x-timestamp' not in req.headers or \
not check_float(req.headers['x-timestamp']):
return HTTPBadRequest(body='Missing or bad timestamp',
request=req,
content_type='text/plain')
req_timestamp = valid_timestamp(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
broker = self._get_account_broker(drive, part, account)
if broker.is_deleted():
return self._deleted_response(broker, req, HTTPNotFound)
timestamp = normalize_timestamp(req.headers['x-timestamp'])
metadata = {}
metadata.update((key, (value, timestamp))
metadata.update((key, (value, req_timestamp.internal))
for key, value in req.headers.iteritems()
if is_sys_or_user_meta('account', key))
if metadata:

42
swift/account/utils.py
View File

@ -17,7 +17,8 @@ import time
from xml.sax import saxutils
from swift.common.swob import HTTPOk, HTTPNoContent
from swift.common.utils import json, normalize_timestamp
from swift.common.utils import json, Timestamp
from swift.common.storage_policy import POLICIES
class FakeAccountBroker(object):
@ -26,7 +27,7 @@ class FakeAccountBroker(object):
like an account broker would for a real, empty account with no metadata.
"""
def get_info(self):
now = normalize_timestamp(time.time())
now = Timestamp(time.time()).internal
return {'container_count': 0,
'object_count': 0,
'bytes_used': 0,
@ -40,6 +41,32 @@ class FakeAccountBroker(object):
def metadata(self):
return {}
def get_policy_stats(self):
return {}
def get_response_headers(broker):
info = broker.get_info()
resp_headers = {
'X-Account-Container-Count': info['container_count'],
'X-Account-Object-Count': info['object_count'],
'X-Account-Bytes-Used': info['bytes_used'],
'X-Timestamp': Timestamp(info['created_at']).normal,
'X-PUT-Timestamp': Timestamp(info['put_timestamp']).normal}
policy_stats = broker.get_policy_stats()
for policy_idx, stats in policy_stats.items():
policy = POLICIES.get_by_index(policy_idx)
if not policy:
continue
header_prefix = 'X-Account-Storage-Policy-%s-%%s' % policy.name
for key, value in stats.items():
header_name = header_prefix % key.replace('_', '-')
resp_headers[header_name] = value
resp_headers.update((key, value)
for key, (value, timestamp) in
broker.metadata.iteritems() if value != '')
return resp_headers
def account_listing_response(account, req, response_content_type, broker=None,
limit='', marker='', end_marker='', prefix='',
@ -47,16 +74,7 @@ def account_listing_response(account, req, response_content_type, broker=None,
if broker is None:
broker = FakeAccountBroker()
info = broker.get_info()
resp_headers = {
'X-Account-Container-Count': info['container_count'],
'X-Account-Object-Count': info['object_count'],
'X-Account-Bytes-Used': info['bytes_used'],
'X-Timestamp': info['created_at'],
'X-PUT-Timestamp': info['put_timestamp']}
resp_headers.update((key, value)
for key, (value, timestamp) in
broker.metadata.iteritems() if value != '')
resp_headers = get_response_headers(broker)
account_list = broker.list_containers_iter(limit, marker, end_marker,
prefix, delimiter)

391
swift/cli/info.py
View File

@ -10,16 +10,22 @@
# License for the specific language governing permissions and limitations
# under the License.
import itertools
import os
import sqlite3
from datetime import datetime
import urllib
from hashlib import md5
from swift.common.utils import hash_path, storage_directory
from swift.common.utils import hash_path, storage_directory, \
Timestamp
from swift.common.ring import Ring
from swift.common.request_helpers import is_sys_meta, is_user_meta, \
strip_sys_meta_prefix, strip_user_meta_prefix
from swift.account.backend import AccountBroker, DATADIR as ABDATADIR
from swift.container.backend import ContainerBroker, DATADIR as CBDATADIR
from swift.obj.diskfile import get_data_dir, read_metadata, DATADIR_BASE, \
extract_policy_index
from swift.common.storage_policy import POLICIES, POLICY_INDEX
class InfoSystemExit(Exception):
@ -29,35 +35,105 @@ class InfoSystemExit(Exception):
pass
def print_ring_locations(ring, datadir, account, container=None):
def print_ring_locations(ring, datadir, account, container=None, obj=None,
tpart=None, all_nodes=False, policy_index=None):
"""
print out ring locations of specified type
:param ring: ring instance
:param datadir: high level directory to store account/container/objects
:param datadir: name of directory where things are stored. Usually one of
"accounts", "containers", "objects", or "objects-N".
:param account: account name
:param container: container name
:param obj: object name
:param tpart: target partition in ring
:param all_nodes: include all handoff nodes. If false, only the N primary
nodes and first N handoffs will be printed.
:param policy_index: include policy_index in curl headers
"""
if ring is None or datadir is None or account is None:
raise ValueError('None type')
storage_type = 'account'
if container:
storage_type = 'container'
try:
part, nodes = ring.get_nodes(account, container, None)
except (ValueError, AttributeError):
raise ValueError('Ring error')
if not ring:
raise ValueError("No ring specified")
if not datadir:
raise ValueError("No datadir specified")
if tpart is None and not account:
raise ValueError("No partition or account/container/object specified")
if not account and (container or obj):
raise ValueError("Container/object specified without account")
if obj and not container:
raise ValueError('Object specified without container')
if obj:
target = '%s/%s/%s' % (account, container, obj)
elif container:
target = '%s/%s' % (account, container)
else:
path_hash = hash_path(account, container, None)
print '\nRing locations:'
for node in nodes:
print (' %s:%s - /srv/node/%s/%s/%s.db' %
(node['ip'], node['port'], node['device'],
storage_directory(datadir, part, path_hash),
path_hash))
print '\nnote: /srv/node is used as default value of `devices`, the ' \
'real value is set in the %s config file on each storage node.' % \
storage_type
target = '%s' % (account)
if tpart:
part = int(tpart)
else:
part = ring.get_part(account, container, obj)
primary_nodes = ring.get_part_nodes(part)
handoff_nodes = ring.get_more_nodes(part)
if not all_nodes:
handoff_nodes = itertools.islice(handoff_nodes, len(primary_nodes))
handoff_nodes = list(handoff_nodes)
if account and not tpart:
path_hash = hash_path(account, container, obj)
else:
path_hash = None
print 'Partition\t%s' % part
print 'Hash \t%s\n' % path_hash
for node in primary_nodes:
print 'Server:Port Device\t%s:%s %s' % (node['ip'], node['port'],
node['device'])
for node in handoff_nodes:
print 'Server:Port Device\t%s:%s %s\t [Handoff]' % (
node['ip'], node['port'], node['device'])
print "\n"
for node in primary_nodes:
cmd = 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' \
% (node['ip'], node['port'], node['device'], part,
urllib.quote(target))
if policy_index is not None:
cmd += ' -H "%s: %s"' % (POLICY_INDEX, policy_index)
print cmd
for node in handoff_nodes:
cmd = 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' \
% (node['ip'], node['port'], node['device'], part,
urllib.quote(target))
if policy_index is not None:
cmd += ' -H "%s: %s"' % (POLICY_INDEX, policy_index)
cmd += ' # [Handoff]'
print cmd
print "\n\nUse your own device location of servers:"
print "such as \"export DEVICE=/srv/node\""
if path_hash:
for node in primary_nodes:
print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s"' %
(node['ip'], node['device'],
storage_directory(datadir, part, path_hash)))
for node in handoff_nodes:
print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s" # [Handoff]' %
(node['ip'], node['device'],
storage_directory(datadir, part, path_hash)))
else:
for node in primary_nodes:
print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s/%d"' %
(node['ip'], node['device'], datadir, part))
for node in handoff_nodes:
print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s/%d"'
' # [Handoff]' %
(node['ip'], node['device'], datadir, part))
print '\nnote: `/srv/node*` is used as default value of `devices`, the ' \
'real value is set in the config file on each storage node.'
def print_db_info_metadata(db_type, info, metadata):
@ -98,33 +174,40 @@ def print_db_info_metadata(db_type, info, metadata):
print 'Metadata:'
print (' Created at: %s (%s)' %
(datetime.utcfromtimestamp(float(info['created_at'])),
(Timestamp(info['created_at']).isoformat,
info['created_at']))
print (' Put Timestamp: %s (%s)' %
(datetime.utcfromtimestamp(float(info['put_timestamp'])),
(Timestamp(info['put_timestamp']).isoformat,
info['put_timestamp']))
print (' Delete Timestamp: %s (%s)' %
(datetime.utcfromtimestamp(float(info['delete_timestamp'])),
(Timestamp(info['delete_timestamp']).isoformat,
info['delete_timestamp']))
print (' Status Timestamp: %s (%s)' %
(Timestamp(info['status_changed_at']).isoformat,
info['status_changed_at']))
if db_type == 'account':
print ' Container Count: %s' % info['container_count']
print ' Object Count: %s' % info['object_count']
print ' Bytes Used: %s' % info['bytes_used']
if db_type == 'container':
try:
policy_name = POLICIES[info['storage_policy_index']].name
except KeyError:
policy_name = 'Unknown'
print (' Storage Policy: %s (%s)' % (
policy_name, info['storage_policy_index']))
print (' Reported Put Timestamp: %s (%s)' %
(datetime.utcfromtimestamp(
float(info['reported_put_timestamp'])),
(Timestamp(info['reported_put_timestamp']).isoformat,
info['reported_put_timestamp']))
print (' Reported Delete Timestamp: %s (%s)' %
(datetime.utcfromtimestamp
(float(info['reported_delete_timestamp'])),
(Timestamp(info['reported_delete_timestamp']).isoformat,
info['reported_delete_timestamp']))
print ' Reported Object Count: %s' % info['reported_object_count']
print ' Reported Bytes Used: %s' % info['reported_bytes_used']
print ' Chexor: %s' % info['hash']
print ' UUID: %s' % info['id']
except KeyError:
raise ValueError('Info is incomplete')
except KeyError as e:
raise ValueError('Info is incomplete: %s' % e)
meta_prefix = 'x_' + db_type + '_'
for key, value in info.iteritems():
@ -152,6 +235,52 @@ def print_db_info_metadata(db_type, info, metadata):
print 'No user metadata found in db file'
def print_obj_metadata(metadata):
"""
Print out basic info and metadata from object, as returned from
:func:`swift.obj.diskfile.read_metadata`.
Metadata should include the keys: name, Content-Type, and
X-Timestamp.
Additional metadata is displayed unmodified.
:param metadata: dict of object metadata
:raises: ValueError
"""
if not metadata:
raise ValueError('Metadata is None')
path = metadata.pop('name', '')
content_type = metadata.pop('Content-Type', '')
ts = Timestamp(metadata.pop('X-Timestamp', 0))
account = container = obj = obj_hash = None
if path:
try:
account, container, obj = path.split('/', 3)[1:]
except ValueError:
raise ValueError('Path is invalid for object %r' % path)
else:
obj_hash = hash_path(account, container, obj)
print 'Path: %s' % path
print ' Account: %s' % account
print ' Container: %s' % container
print ' Object: %s' % obj
print ' Object hash: %s' % obj_hash
else:
print 'Path: Not found in metadata'
if content_type:
print 'Content-Type: %s' % content_type
else:
print 'Content-Type: Not found in metadata'
if ts:
print ('Timestamp: %s (%s)' % (ts.isoformat, ts.internal))
else:
print 'Timestamp: Not found in metadata'
print 'User Metadata: %s' % metadata
def print_info(db_type, db_file, swift_dir='/etc/swift'):
if db_type not in ('account', 'container'):
print "Unrecognized DB type: internal error"
@ -184,3 +313,201 @@ def print_info(db_type, db_file, swift_dir='/etc/swift'):
ring = None
else:
print_ring_locations(ring, datadir, account, container)
def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
policy_name=''):
"""
Display information about an object read from the datafile.
Optionally verify the datafile content matches the ETag metadata.
:param datafile: path on disk to object file
:param check_etag: boolean, will read datafile content and verify
computed checksum matches value stored in
metadata.
:param swift_dir: the path on disk to rings
:param policy_name: optionally the name to use when finding the ring
"""
if not os.path.exists(datafile) or not datafile.endswith('.data'):
print "Data file doesn't exist"
raise InfoSystemExit()
if not datafile.startswith(('/', './')):
datafile = './' + datafile
policy_index = None
ring = None
datadir = DATADIR_BASE
# try to extract policy index from datafile disk path
try:
policy_index = extract_policy_index(datafile)
except ValueError:
pass
try:
if policy_index:
datadir += '-' + str(policy_index)
ring = Ring(swift_dir, ring_name='object-' + str(policy_index))
elif policy_index == 0:
ring = Ring(swift_dir, ring_name='object')
except IOError:
# no such ring
pass
if policy_name:
policy = POLICIES.get_by_name(policy_name)
if policy:
policy_index_for_name = policy.idx
if (policy_index is not None and
policy_index_for_name is not None and
policy_index != policy_index_for_name):
print 'Attention: Ring does not match policy!'
print 'Double check your policy name!'
if not ring and policy_index_for_name:
ring = POLICIES.get_object_ring(policy_index_for_name,
swift_dir)
datadir = get_data_dir(policy_index_for_name)
with open(datafile, 'rb') as fp:
try:
metadata = read_metadata(fp)
except EOFError:
print "Invalid metadata"
raise InfoSystemExit()
etag = metadata.pop('ETag', '')
length = metadata.pop('Content-Length', '')
path = metadata.get('name', '')
print_obj_metadata(metadata)
# Optional integrity check; it's useful, but slow.
file_len = None
if check_etag:
h = md5()
file_len = 0
while True:
data = fp.read(64 * 1024)
if not data:
break
h.update(data)
file_len += len(data)
h = h.hexdigest()
if etag:
if h == etag:
print 'ETag: %s (valid)' % etag
else:
print ("ETag: %s doesn't match file hash of %s!" %
(etag, h))
else:
print 'ETag: Not found in metadata'
else:
print 'ETag: %s (not checked)' % etag
file_len = os.fstat(fp.fileno()).st_size
if length:
if file_len == int(length):
print 'Content-Length: %s (valid)' % length
else:
print ("Content-Length: %s doesn't match file length of %s"
% (length, file_len))
else:
print 'Content-Length: Not found in metadata'
account, container, obj = path.split('/', 3)[1:]
if ring:
print_ring_locations(ring, datadir, account, container, obj,
policy_index=policy_index)
def print_item_locations(ring, ring_name=None, account=None, container=None,
obj=None, **kwargs):
"""
Display placement information for an item based on ring lookup.
If a ring is provided it always takes precedence, but warnings will be
emitted if it doesn't match other optional arguments like the policy_name
or ring_name.
If no ring is provided the ring_name and/or policy_name will be used to
lookup the ring.
:param ring: a ring instance
:param ring_name: server type, or storage policy ring name if object ring
:param account: account name
:param container: container name
:param obj: object name
:param partition: part number for non path lookups
:param policy_name: name of storage policy to use to lookup the ring
:param all_nodes: include all handoff nodes. If false, only the N primary
nodes and first N handoffs will be printed.
"""
policy_name = kwargs.get('policy_name', None)
part = kwargs.get('partition', None)
all_nodes = kwargs.get('all', False)
swift_dir = kwargs.get('swift_dir', '/etc/swift')
if ring and policy_name:
policy = POLICIES.get_by_name(policy_name)
if policy:
if ring_name != policy.ring_name:
print 'Attention! mismatch between ring and policy detected!'
else:
print 'Attention! Policy %s is not valid' % policy_name
policy_index = None
if ring is None and (obj or part):
if not policy_name:
print 'Need a ring or policy'
raise InfoSystemExit()
policy = POLICIES.get_by_name(policy_name)
if not policy:
print 'No policy named %r' % policy_name
raise InfoSystemExit()
policy_index = int(policy)
ring = POLICIES.get_object_ring(policy_index, swift_dir)
ring_name = (POLICIES.get_by_name(policy_name)).ring_name
if account is None and (container is not None or obj is not None):
print 'No account specified'
raise InfoSystemExit()
if container is None and obj is not None:
print 'No container specified'
raise InfoSystemExit()
if account is None and part is None:
print 'No target specified'
raise InfoSystemExit()
loc = '<type>'
if part and ring_name:
if '-' in ring_name and ring_name.startswith('object'):
loc = 'objects-' + ring_name.split('-', 1)[1]
else:
loc = ring_name + 's'
if account and container and obj:
loc = 'objects'
if '-' in ring_name and ring_name.startswith('object'):
policy_index = int(ring_name.rsplit('-', 1)[1])
loc = 'objects-%d' % policy_index
if account and container and not obj:
loc = 'containers'
if not any([ring, ring_name]):
ring = Ring(swift_dir, ring_name='container')
else:
if ring_name != 'container':
print 'Attention! mismatch between ring and item detected!'
if account and not container and not obj:
loc = 'accounts'
if not any([ring, ring_name]):
ring = Ring(swift_dir, ring_name='account')
else:
if ring_name != 'account':
print 'Attention! mismatch between ring and item detected!'
print '\nAccount \t%s' % account
print 'Container\t%s' % container
print 'Object \t%s\n\n' % obj
print_ring_locations(ring, loc, account, container, obj, part, all_nodes,
policy_index=policy_index)

61
swift/cli/recon.py
View File

@ -192,35 +192,61 @@ class SwiftRecon(object):
ips = set((n['ip'], n['port']) for n in ring_data.devs if n)
return ips
def get_ringmd5(self, hosts, ringfile):
def get_ringmd5(self, hosts, swift_dir):
"""
Compare ring md5sum's with those on remote host
:param hosts: set of hosts to check. in the format of:
set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
:param ringfile: The local ring file to compare the md5sum with.
:param swift_dir: The local directory with the ring files.
"""
matches = 0
errors = 0
ring_sum = self._md5_file(ringfile)
ring_names = set()
for server_type in ('account', 'container'):
ring_name = '%s.ring.gz' % server_type
ring_names.add(ring_name)
# include any other object ring files
for ring_name in os.listdir(swift_dir):
if ring_name.startswith('object') and \
ring_name.endswith('ring.gz'):
ring_names.add(ring_name)
rings = {}
for ring_name in ring_names:
md5sum = md5()
with open(os.path.join(swift_dir, ring_name), 'rb') as f:
block = f.read(4096)
while block:
md5sum.update(block)
block = f.read(4096)
ring_sum = md5sum.hexdigest()
rings[ring_name] = ring_sum
recon = Scout("ringmd5", self.verbose, self.suppress_errors,
self.timeout)
print("[%s] Checking ring md5sums" % self._ptime())
if self.verbose:
print("-> On disk %s md5sum: %s" % (ringfile, ring_sum))
for ring_file, ring_sum in rings.items():
print("-> On disk %s md5sum: %s" % (ring_file, ring_sum))
for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200:
if response[ringfile] != ring_sum:
print("!! %s (%s) doesn't match on disk md5sum" %
(url, response[ringfile]))
else:
matches = matches + 1
if self.verbose:
print("-> %s matches." % url)
else:
if status != 200:
errors = errors + 1
print("%s/%s hosts matched, %s error[s] while checking hosts."
% (matches, len(hosts), errors))
continue
success = True
for remote_ring_file, remote_ring_sum in response.items():
remote_ring_name = os.path.basename(remote_ring_file)
ring_sum = rings.get(remote_ring_name, None)
if remote_ring_sum != ring_sum:
success = False
print("!! %s (%s => %s) doesn't match on disk md5sum" % (
url, remote_ring_name, remote_ring_sum))
if not success:
errors += 1
continue
matches += 1
if self.verbose:
print("-> %s matches." % url)
print("%s/%s hosts matched, %s error[s] while checking hosts." % (
matches, len(hosts), errors))
print("=" * 79)
def get_swiftconfmd5(self, hosts, printfn=print):
@ -867,7 +893,6 @@ class SwiftRecon(object):
self.server_type = 'object'
swift_dir = options.swiftdir
ring_file = os.path.join(swift_dir, '%s.ring.gz' % self.server_type)
self.verbose = options.verbose
self.suppress_errors = options.suppress
self.timeout = options.timeout
@ -897,7 +922,7 @@ class SwiftRecon(object):
self.umount_check(hosts)
self.load_check(hosts)
self.disk_usage(hosts)
self.get_ringmd5(hosts, ring_file)
self.get_ringmd5(hosts, swift_dir)
self.quarantine_check(hosts)
self.socket_usage(hosts)
else:
@ -933,7 +958,7 @@ class SwiftRecon(object):
if options.diskusage:
self.disk_usage(hosts, options.top, options.human_readable)
if options.md5:
self.get_ringmd5(hosts, ring_file)
self.get_ringmd5(hosts, swift_dir)
self.get_swiftconfmd5(hosts)
if options.quarantined:
self.quarantine_check(hosts)

18
swift/common/constraints.py
View File

@ -18,7 +18,7 @@ import urllib
from urllib import unquote
from ConfigParser import ConfigParser, NoSectionError, NoOptionError
from swift.common import utils
from swift.common import utils, exceptions
from swift.common.swob import HTTPBadRequest, HTTPLengthRequired, \
HTTPRequestEntityTooLarge, HTTPPreconditionFailed
@ -209,6 +209,22 @@ def check_float(string):
return False
def valid_timestamp(request):
"""
Helper function to extract a timestamp from requests that require one.
:param request: the swob request object
:returns: a valid Timestamp instance
:raises: HTTPBadRequest on missing or invalid X-Timestamp
"""
try:
return request.timestamp
except exceptions.InvalidTimestamp as e:
raise HTTPBadRequest(body=str(e), request=request,
content_type='text/plain')
def check_utf8(string):
"""
Validate if a string is valid UTF-8 str or unicode and that it

129
swift/common/db.py
View File

@ -29,7 +29,7 @@ from tempfile import mkstemp
from eventlet import sleep, Timeout
import sqlite3
from swift.common.utils import json, normalize_timestamp, renamer, \
from swift.common.utils import json, Timestamp, renamer, \
mkdirs, lock_parent_directory, fallocate
from swift.common.exceptions import LockTimeout
@ -144,7 +144,7 @@ def chexor(old, name, timestamp):
:param old: hex representation of the current DB hash
:param name: name of the object or container being inserted
:param timestamp: timestamp of the new record
:param timestamp: internalized timestamp of the new record
:returns: a hex representation of the new hash value
"""
if name is None:
@ -215,11 +215,15 @@ class DatabaseBroker(object):
"""
return self.db_file
def initialize(self, put_timestamp=None):
def initialize(self, put_timestamp=None, storage_policy_index=None):
"""
Create the DB
:param put_timestamp: timestamp of initial PUT request
The storage_policy_index is passed through to the subclass's
``_initialize`` method. It is ignored by ``AccountBroker``.
:param put_timestamp: internalized timestamp of initial PUT request
:param storage_policy_index: only required for containers
"""
if self.db_file == ':memory:':
tmp_db_file = None
@ -276,8 +280,9 @@ class DatabaseBroker(object):
END;
""")
if not put_timestamp:
put_timestamp = normalize_timestamp(0)
self._initialize(conn, put_timestamp)
put_timestamp = Timestamp(0).internal
self._initialize(conn, put_timestamp,
storage_policy_index=storage_policy_index)
conn.commit()
if tmp_db_file:
conn.close()
@ -297,9 +302,8 @@ class DatabaseBroker(object):
"""
Mark the DB as deleted
:param timestamp: delete timestamp
:param timestamp: internalized delete timestamp
"""
timestamp = normalize_timestamp(timestamp)
# first, clear the metadata
cleared_meta = {}
for k in self.metadata:
@ -420,6 +424,28 @@ class DatabaseBroker(object):
# Override for additional work when receiving an rsynced db.
pass
def _is_deleted(self, conn):
"""
Check if the database is considered deleted
:param conn: database conn
:returns: True if the DB is considered to be deleted, False otherwise
"""
raise NotImplementedError()
def is_deleted(self):
"""
Check if the DB is considered to be deleted.
:returns: True if the DB is considered to be deleted, False otherwise
"""
if self.db_file != ':memory:' and not os.path.exists(self.db_file):
return True
self._commit_puts_stale_ok()
with self.get() as conn:
return self._is_deleted(conn)
def merge_timestamps(self, created_at, put_timestamp, delete_timestamp):
"""
Used in replication to handle updating timestamps.
@ -429,11 +455,16 @@ class DatabaseBroker(object):
:param delete_timestamp: delete timestamp
"""
with self.get() as conn:
old_status = self._is_deleted(conn)
conn.execute('''
UPDATE %s_stat SET created_at=MIN(?, created_at),
put_timestamp=MAX(?, put_timestamp),
delete_timestamp=MAX(?, delete_timestamp)
''' % self.db_type, (created_at, put_timestamp, delete_timestamp))
if old_status != self._is_deleted(conn):
timestamp = Timestamp(time.time())
self._update_status_changed_at(conn, timestamp.internal)
conn.commit()
def get_items_since(self, start, count):
@ -480,38 +511,42 @@ class DatabaseBroker(object):
with self.get() as conn:
curs = conn.execute('''
SELECT remote_id, sync_point FROM %s_sync
''' % 'incoming' if incoming else 'outgoing')
''' % ('incoming' if incoming else 'outgoing'))
result = []
for row in curs:
result.append({'remote_id': row[0], 'sync_point': row[1]})
return result
def get_max_row(self):
query = '''
SELECT SQLITE_SEQUENCE.seq
FROM SQLITE_SEQUENCE
WHERE SQLITE_SEQUENCE.name == '%s'
LIMIT 1
''' % (self.db_contains_type)
with self.get() as conn:
row = conn.execute(query).fetchone()
return row[0] if row else -1
def get_replication_info(self):
"""
Get information about the DB required for replication.
:returns: dict containing keys: hash, id, created_at, put_timestamp,
delete_timestamp, count, max_row, and metadata
:returns: dict containing keys from get_info plus max_row and metadata
Note:: get_info's <db_contains_type>_count is translated to just
"count" and metadata is the raw string.
"""
info = self.get_info()
info['count'] = info.pop('%s_count' % self.db_contains_type)
info['metadata'] = self.get_raw_metadata()
info['max_row'] = self.get_max_row()
return info
def get_info(self):
self._commit_puts_stale_ok()
query_part1 = '''
SELECT hash, id, created_at, put_timestamp, delete_timestamp,
%s_count AS count,
CASE WHEN SQLITE_SEQUENCE.seq IS NOT NULL
THEN SQLITE_SEQUENCE.seq ELSE -1 END AS max_row, ''' % \
self.db_contains_type
query_part2 = '''
FROM (%s_stat LEFT JOIN SQLITE_SEQUENCE
ON SQLITE_SEQUENCE.name == '%s') LIMIT 1
''' % (self.db_type, self.db_contains_type)
with self.get() as conn:
try:
curs = conn.execute(query_part1 + 'metadata' + query_part2)
except sqlite3.OperationalError as err:
if 'no such column: metadata' not in str(err):
raise
curs = conn.execute(query_part1 + "'' as metadata" +
query_part2)
curs = conn.execute('SELECT * from %s_stat' % self.db_type)
curs.row_factory = dict_factory
return curs.fetchone()
@ -621,13 +656,7 @@ class DatabaseBroker(object):
with open(self.db_file, 'rb+') as fp:
fallocate(fp.fileno(), int(prealloc_size))
@property
def metadata(self):
"""
Returns the metadata dict for the database. The metadata dict values
are tuples of (value, timestamp) where the timestamp indicates when
that key was set to that value.
"""
def get_raw_metadata(self):
with self.get() as conn:
try:
metadata = conn.execute('SELECT metadata FROM %s_stat' %
@ -636,6 +665,16 @@ class DatabaseBroker(object):
if 'no such column: metadata' not in str(err):
raise
metadata = ''
return metadata
@property
def metadata(self):
"""
Returns the metadata dict for the database. The metadata dict values
are tuples of (value, timestamp) where the timestamp indicates when
that key was set to that value.
"""
metadata = self.get_raw_metadata()
if metadata:
metadata = json.loads(metadata)
utf8encodekeys(metadata)
@ -750,7 +789,7 @@ class DatabaseBroker(object):
Update the put_timestamp. Only modifies it if it is greater than
the current timestamp.
:param timestamp: put timestamp
:param timestamp: internalized put timestamp
"""
with self.get() as conn:
conn.execute(
@ -758,3 +797,21 @@ class DatabaseBroker(object):
' WHERE put_timestamp < ?' % self.db_type,
(timestamp, timestamp))
conn.commit()
def update_status_changed_at(self, timestamp):
"""
Update the status_changed_at field in the stat table. Only
modifies status_changed_at if the timestamp is greater than the
current status_changed_at timestamp.
:param timestamp: internalized timestamp
"""
with self.get() as conn:
self._update_status_changed_at(conn, timestamp)
conn.commit()
def _update_status_changed_at(self, conn, timestamp):
conn.execute(
'UPDATE %s_stat SET status_changed_at = ?'
' WHERE status_changed_at < ?' % self.db_type,
(timestamp, timestamp))

204
swift/common/db_replicator.py
View File

@ -21,17 +21,17 @@ import shutil
import uuid
import errno
import re
from contextlib import contextmanager
from swift import gettext_ as _
from eventlet import GreenPool, sleep, Timeout
from eventlet.green import subprocess
import simplejson
import swift.common.db
from swift.common.direct_client import quote
from swift.common.utils import get_logger, whataremyips, storage_directory, \
renamer, mkdirs, lock_parent_directory, config_true_value, \
unlink_older_than, dump_recon_cache, rsync_ip, ismount
unlink_older_than, dump_recon_cache, rsync_ip, ismount, json, Timestamp
from swift.common import ring
from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE
from swift.common.bufferedhttp import BufferedHTTPConnection
@ -129,7 +129,7 @@ class ReplConnection(BufferedHTTPConnection):
:returns: bufferedhttp response object
"""
try:
body = simplejson.dumps(args)
body = json.dumps(args)
self.request('REPLICATE', self.path, body,
{'Content-Type': 'application/json'})
response = self.getresponse()
@ -146,9 +146,9 @@ class Replicator(Daemon):
Implements the logic for directing db replication.
"""
def __init__(self, conf):
def __init__(self, conf, logger=None):
self.conf = conf
self.logger = get_logger(conf, log_route='replicator')
self.logger = logger or get_logger(conf, log_route='replicator')
self.root = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.port = int(conf.get('bind_port', self.default_port))
@ -156,6 +156,7 @@ class Replicator(Daemon):
self.cpool = GreenPool(size=concurrency)
swift_dir = conf.get('swift_dir', '/etc/swift')
self.ring = ring.Ring(swift_dir, ring_name=self.server_type)
self._local_device_ids = set()
self.per_diff = int(conf.get('per_diff', 1000))
self.max_diffs = int(conf.get('max_diffs') or 100)
self.interval = int(conf.get('interval') or
@ -348,6 +349,14 @@ class Replicator(Daemon):
os.path.basename(db_file).split('.', 1)[0],
self.logger)
def _gather_sync_args(self, info):
"""
Convert local replication_info to sync args tuple.
"""
sync_args_order = ('max_row', 'hash', 'id', 'created_at',
'put_timestamp', 'delete_timestamp', 'metadata')
return tuple(info[key] for key in sync_args_order)
def _repl_to_node(self, node, broker, partition, info):
"""
Replicate a database to a node.
@ -367,21 +376,22 @@ class Replicator(Daemon):
self.logger.error(
_('ERROR Unable to connect to remote server: %s'), node)
return False
sync_args = self._gather_sync_args(info)
with Timeout(self.node_timeout):
response = http.replicate(
'sync', info['max_row'], info['hash'], info['id'],
info['created_at'], info['put_timestamp'],
info['delete_timestamp'], info['metadata'])
response = http.replicate('sync', *sync_args)
if not response:
return False
elif response.status == HTTP_NOT_FOUND: # completely missing, rsync
return self._handle_sync_response(node, response, info, broker, http)
def _handle_sync_response(self, node, response, info, broker, http):
if response.status == HTTP_NOT_FOUND: # completely missing, rsync
self.stats['rsync'] += 1
self.logger.increment('rsyncs')
return self._rsync_db(broker, node, http, info['id'])
elif response.status == HTTP_INSUFFICIENT_STORAGE:
raise DriveNotMounted()
elif response.status >= 200 and response.status < 300:
rinfo = simplejson.loads(response.data)
rinfo = json.loads(response.data)
local_sync = broker.get_sync(rinfo['id'], incoming=False)
if self._in_sync(rinfo, info, broker, local_sync):
return True
@ -397,6 +407,14 @@ class Replicator(Daemon):
return self._usync_db(max(rinfo['point'], local_sync),
broker, http, rinfo['id'], info['id'])
def _post_replicate_hook(self, broker, info, responses):
"""
:param broker: the container that just replicated
:param info: pre-replication full info dict
:param responses: a list of bools indicating success from nodes
"""
pass
def _replicate_object(self, partition, object_file, node_id):
"""
Replicate the db, choosing method based on whether or not it
@ -416,17 +434,16 @@ class Replicator(Daemon):
broker.reclaim(now - self.reclaim_age,
now - (self.reclaim_age * 2))
info = broker.get_replication_info()
full_info = broker.get_info()
bpart = self.ring.get_part(
full_info['account'], full_info.get('container'))
info['account'], info.get('container'))
if bpart != int(partition):
partition = bpart
# Important to set this false here since the later check only
# checks if it's on the proper device, not partition.
shouldbehere = False
name = '/' + quote(full_info['account'])
if 'container' in full_info:
name += '/' + quote(full_info['container'])
name = '/' + quote(info['account'])
if 'container' in info:
name += '/' + quote(info['container'])
self.logger.error(
'Found %s for %s when it should be on partition %s; will '
'replicate out and remove.' % (object_file, name, bpart))
@ -441,20 +458,12 @@ class Replicator(Daemon):
return
# The db is considered deleted if the delete_timestamp value is greater
# than the put_timestamp, and there are no objects.
delete_timestamp = 0
try:
delete_timestamp = float(info['delete_timestamp'])
except ValueError:
pass
put_timestamp = 0
try:
put_timestamp = float(info['put_timestamp'])
except ValueError:
pass
delete_timestamp = Timestamp(info.get('delete_timestamp') or 0)
put_timestamp = Timestamp(info.get('put_timestamp') or 0)
if delete_timestamp < (now - self.reclaim_age) and \
delete_timestamp > put_timestamp and \
info['count'] in (None, '', 0, '0'):
if self.report_up_to_date(full_info):
if self.report_up_to_date(info):
self.delete_db(object_file)
self.logger.timing_since('timing', start_time)
return
@ -482,13 +491,19 @@ class Replicator(Daemon):
self.stats['success' if success else 'failure'] += 1
self.logger.increment('successes' if success else 'failures')
responses.append(success)
try:
self._post_replicate_hook(broker, info, responses)
except (Exception, Timeout):
self.logger.exception('UNHANDLED EXCEPTION: in post replicate '
'hook for %s', broker.db_file)
if not shouldbehere and all(responses):
# If the db shouldn't be on this node and has been successfully
# synced to all of its peers, it can be removed.
self.delete_db(object_file)
self.delete_db(broker)
self.logger.timing_since('timing', start_time)
def delete_db(self, object_file):
def delete_db(self, broker):
object_file = broker.db_file
hash_dir = os.path.dirname(object_file)
suf_dir = os.path.dirname(hash_dir)
with lock_parent_directory(object_file):
@ -526,6 +541,7 @@ class Replicator(Daemon):
if not ips:
self.logger.error(_('ERROR Failed to get my own IPs?'))
return
self._local_device_ids = set()
for node in self.ring.devs:
if (node and node['replication_ip'] in ips and
node['replication_port'] == self.port):
@ -539,6 +555,7 @@ class Replicator(Daemon):
time.time() - self.reclaim_age)
datadir = os.path.join(self.root, node['device'], self.datadir)
if os.path.isdir(datadir):
self._local_device_ids.add(node['id'])
dirs.append((datadir, node['id']))
self.logger.info(_('Beginning replication run'))
for part, object_file, node_id in roundrobin_datadirs(dirs):
@ -597,55 +614,90 @@ class ReplicatorRpc(object):
return HTTPNotFound()
return getattr(self, op)(self.broker_class(db_file), args)
def sync(self, broker, args):
@contextmanager
def debug_timing(self, name):
timemark = time.time()
yield
timespan = time.time() - timemark
if timespan > DEBUG_TIMINGS_THRESHOLD:
self.logger.debug(
'replicator-rpc-sync time for %s: %.02fs' % (
name, timespan))
def _parse_sync_args(self, args):
"""
Convert remote sync args to remote_info dictionary.
"""
(remote_sync, hash_, id_, created_at, put_timestamp,
delete_timestamp, metadata) = args
timemark = time.time()
try:
info = broker.get_replication_info()
except (Exception, Timeout) as e:
if 'no such table' in str(e):
self.logger.error(_("Quarantining DB %s"), broker)
quarantine_db(broker.db_file, broker.db_type)
return HTTPNotFound()
raise
timespan = time.time() - timemark
if timespan > DEBUG_TIMINGS_THRESHOLD:
self.logger.debug('replicator-rpc-sync time for info: %.02fs' %
timespan)
delete_timestamp, metadata) = args[:7]
remote_metadata = {}
if metadata:
timemark = time.time()
broker.update_metadata(simplejson.loads(metadata))
timespan = time.time() - timemark
if timespan > DEBUG_TIMINGS_THRESHOLD:
self.logger.debug('replicator-rpc-sync time for '
'update_metadata: %.02fs' % timespan)
if info['put_timestamp'] != put_timestamp or \
info['created_at'] != created_at or \
info['delete_timestamp'] != delete_timestamp:
timemark = time.time()
broker.merge_timestamps(
created_at, put_timestamp, delete_timestamp)
timespan = time.time() - timemark
if timespan > DEBUG_TIMINGS_THRESHOLD:
self.logger.debug('replicator-rpc-sync time for '
'merge_timestamps: %.02fs' % timespan)
timemark = time.time()
info['point'] = broker.get_sync(id_)
timespan = time.time() - timemark
if timespan > DEBUG_TIMINGS_THRESHOLD:
self.logger.debug('replicator-rpc-sync time for get_sync: '
'%.02fs' % timespan)
if hash_ == info['hash'] and info['point'] < remote_sync:
timemark = time.time()
broker.merge_syncs([{'remote_id': id_,
'sync_point': remote_sync}])
info['point'] = remote_sync
timespan = time.time() - timemark
if timespan > DEBUG_TIMINGS_THRESHOLD:
self.logger.debug('replicator-rpc-sync time for '
'merge_syncs: %.02fs' % timespan)
return Response(simplejson.dumps(info))
try:
remote_metadata = json.loads(metadata)
except ValueError:
self.logger.error("Unable to decode remote metadata %r",
metadata)
remote_info = {
'point': remote_sync,
'hash': hash_,
'id': id_,
'created_at': created_at,
'put_timestamp': put_timestamp,
'delete_timestamp': delete_timestamp,
'metadata': remote_metadata,
}
return remote_info
def sync(self, broker, args):
remote_info = self._parse_sync_args(args)
return self._handle_sync_request(broker, remote_info)
def _get_synced_replication_info(self, broker, remote_info):
"""
Apply any changes to the broker based on remote_info and return the
current replication info.
:param broker: the database broker
:param remote_info: the remote replication info
:returns: local broker replication info
"""
return broker.get_replication_info()
def _handle_sync_request(self, broker, remote_info):
"""
Update metadata, timestamps, sync points.
"""
with self.debug_timing('info'):
try:
info = self._get_synced_replication_info(broker, remote_info)
except (Exception, Timeout) as e:
if 'no such table' in str(e):
self.logger.error(_("Quarantining DB %s"), broker)
quarantine_db(broker.db_file, broker.db_type)
return HTTPNotFound()
raise
if remote_info['metadata']:
with self.debug_timing('update_metadata'):
broker.update_metadata(remote_info['metadata'])
sync_timestamps = ('created_at', 'put_timestamp', 'delete_timestamp')
if any(info[ts] != remote_info[ts] for ts in sync_timestamps):
with self.debug_timing('merge_timestamps'):
broker.merge_timestamps(*(remote_info[ts] for ts in
sync_timestamps))
with self.debug_timing('get_sync'):
info['point'] = broker.get_sync(remote_info['id'])
if remote_info['hash'] == info['hash'] and \
info['point'] < remote_info['point']:
with self.debug_timing('merge_syncs'):
translate = {
'remote_id': 'id',
'sync_point': 'point',
}
data = dict((k, remote_info[v]) for k, v in translate.items())
broker.merge_syncs([data])
info['point'] = remote_info['point']
return Response(json.dumps(info))
def merge_syncs(self, broker, args):
broker.merge_syncs(args[0])

199
swift/common/direct_client.py
View File

@ -27,7 +27,7 @@ from eventlet import sleep, Timeout
from swift.common.bufferedhttp import http_connect
from swift.common.exceptions import ClientException
from swift.common.utils import normalize_timestamp, FileLikeIter
from swift.common.utils import Timestamp, FileLikeIter
from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \
is_success, is_server_error
from swift.common.swob import HeaderKeyDict
@ -39,6 +39,19 @@ except ImportError:
import json
class DirectClientException(ClientException):
def __init__(self, stype, method, node, part, path, resp):
full_path = quote('/%s/%s%s' % (node['device'], part, path))
msg = '%s server %s:%s direct %s %r gave status %s' % (
stype, node['ip'], node['port'], method, full_path, resp.status)
headers = HeaderKeyDict(resp.getheaders())
super(DirectClientException, self).__init__(
msg, http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason, http_headers=headers)
def _get_direct_account_container(path, stype, node, part,
account, marker=None, limit=None,
prefix=None, delimiter=None, conn_timeout=5,
@ -65,17 +78,10 @@ def _get_direct_account_container(path, stype, node, part,
resp = conn.getresponse()
if not is_success(resp.status):
resp.read()
raise ClientException(
'%s server %s:%s direct GET %s gave stats %s' %
(stype, node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)),
resp.status),
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
resp_headers = {}
raise DirectClientException(stype, 'GET', node, part, path, resp)
resp_headers = HeaderKeyDict()
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
resp_headers[header] = value
if resp.status == HTTP_NO_CONTENT:
resp.read()
return resp_headers, []
@ -85,7 +91,7 @@ def _get_direct_account_container(path, stype, node, part,
def gen_headers(hdrs_in=None, add_ts=False):
hdrs_out = HeaderKeyDict(hdrs_in) if hdrs_in else HeaderKeyDict()
if add_ts:
hdrs_out['X-Timestamp'] = normalize_timestamp(time())
hdrs_out['X-Timestamp'] = Timestamp(time()).internal
hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid()
return hdrs_out
@ -106,7 +112,7 @@ def direct_get_account(node, part, account, marker=None, limit=None,
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: a tuple of (response headers, a list of containers) The response
headers will be a dict and all header names will be lowercase.
headers will HeaderKeyDict.
"""
path = '/' + account
return _get_direct_account_container(path, "Account", node, part,
@ -117,6 +123,24 @@ def direct_get_account(node, part, account, marker=None, limit=None,
response_timeout=15)
def direct_delete_account(node, part, account, conn_timeout=5,
response_timeout=15, headers=None):
if headers is None:
headers = {}
path = '/%s' % account
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'DELETE', path,
headers=gen_headers(headers, True))
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise DirectClientException('Account', 'DELETE',
node, part, path, resp)
def direct_head_container(node, part, account, container, conn_timeout=5,
response_timeout=15):
"""
@ -128,8 +152,7 @@ def direct_head_container(node, part, account, container, conn_timeout=5,
:param container: container name
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: a dict containing the response's headers (all header names will
be lowercase)
:returns: a dict containing the response's headers in a HeaderKeyDict
"""
path = '/%s/%s' % (account, container)
with Timeout(conn_timeout):
@ -139,17 +162,11 @@ def direct_head_container(node, part, account, container, conn_timeout=5,
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise ClientException(
'Container server %s:%s direct HEAD %s gave status %s' %
(node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)),
resp.status),
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
resp_headers = {}
raise DirectClientException('Container', 'HEAD',
node, part, path, resp)
resp_headers = HeaderKeyDict()
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
resp_headers[header] = value
return resp_headers
@ -170,7 +187,7 @@ def direct_get_container(node, part, account, container, marker=None,
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: a tuple of (response headers, a list of objects) The response
headers will be a dict and all header names will be lowercase.
headers will be a HeaderKeyDict.
"""
path = '/%s/%s' % (account, container)
return _get_direct_account_container(path, "Container", node,
@ -195,17 +212,56 @@ def direct_delete_container(node, part, account, container, conn_timeout=5,
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise ClientException(
'Container server %s:%s direct DELETE %s gave status %s' %
(node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)), resp.status),
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
raise DirectClientException('Container', 'DELETE',
node, part, path, resp)
def direct_put_container_object(node, part, account, container, obj,
conn_timeout=5, response_timeout=15,
headers=None):
if headers is None:
headers = {}
have_x_timestamp = 'x-timestamp' in (k.lower() for k in headers)
path = '/%s/%s/%s' % (account, container, obj)
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'PUT', path,
headers=gen_headers(headers,
add_ts=(not have_x_timestamp)))
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise DirectClientException('Container', 'PUT',
node, part, path, resp)
def direct_delete_container_object(node, part, account, container, obj,
conn_timeout=5, response_timeout=15,
headers=None):
if headers is None:
headers = {}
headers = gen_headers(headers, add_ts='x-timestamp' not in (
k.lower() for k in headers))
path = '/%s/%s/%s' % (account, container, obj)
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'DELETE', path, headers=headers)
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise DirectClientException('Container', 'DELETE',
node, part, path, resp)
def direct_head_object(node, part, account, container, obj, conn_timeout=5,
response_timeout=15):
response_timeout=15, headers=None):
"""
Request object information directly from the object server.
@ -216,28 +272,27 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5,
:param obj: object name
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: a dict containing the response's headers (all header names will
be lowercase)
:param headers: dict to be passed into HTTPConnection headers
:returns: a dict containing the response's headers in a HeaderKeyDict
"""
if headers is None:
headers = {}
headers = gen_headers(headers)
path = '/%s/%s/%s' % (account, container, obj)
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'HEAD', path, headers=gen_headers())
'HEAD', path, headers=headers)
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise ClientException(
'Object server %s:%s direct HEAD %s gave status %s' %
(node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)),
resp.status),
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
resp_headers = {}
raise DirectClientException('Object', 'HEAD',
node, part, path, resp)
resp_headers = HeaderKeyDict()
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
resp_headers[header] = value
return resp_headers
@ -256,7 +311,7 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5,
:param resp_chunk_size: if defined, chunk size of data to read.
:param headers: dict to be passed into HTTPConnection headers
:returns: a tuple of (response headers, the object's contents) The response
headers will be a dict and all header names will be lowercase.
headers will be a HeaderKeyDict.
"""
if headers is None:
headers = {}
@ -269,13 +324,8 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5,
resp = conn.getresponse()
if not is_success(resp.status):
resp.read()
raise ClientException(
'Object server %s:%s direct GET %s gave status %s' %
(node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)), resp.status),
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
raise DirectClientException('Object', 'GET',
node, part, path, resp)
if resp_chunk_size:
def _object_body():
@ -286,9 +336,9 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5,
object_body = _object_body()
else:
object_body = resp.read()
resp_headers = {}
resp_headers = HeaderKeyDict()
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
resp_headers[header] = value
return resp_headers, object_body
@ -368,14 +418,8 @@ def direct_put_object(node, part, account, container, name, contents,
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise ClientException(
'Object server %s:%s direct PUT %s gave status %s' %
(node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)),
resp.status),
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
raise DirectClientException('Object', 'PUT',
node, part, path, resp)
return resp.getheader('etag').strip('"')
@ -402,14 +446,8 @@ def direct_post_object(node, part, account, container, name, headers,
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise ClientException(
'Object server %s:%s direct POST %s gave status %s' %
(node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)),
resp.status),
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
raise DirectClientException('Object', 'POST',
node, part, path, resp)
def direct_delete_object(node, part, account, container, obj,
@ -429,22 +467,19 @@ def direct_delete_object(node, part, account, container, obj,
if headers is None:
headers = {}
headers = gen_headers(headers, add_ts='x-timestamp' not in (
k.lower() for k in headers))
path = '/%s/%s/%s' % (account, container, obj)
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'DELETE', path, headers=gen_headers(headers, True))
'DELETE', path, headers=headers)
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise ClientException(
'Object server %s:%s direct DELETE %s gave status %s' %
(node['ip'], node['port'],
repr('/%s/%s%s' % (node['device'], part, path)),
resp.status),
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
raise DirectClientException('Object', 'DELETE',
node, part, path, resp)
def retry(func, *args, **kwargs):

15
swift/common/exceptions.py
View File

@ -14,6 +14,7 @@
# limitations under the License.
from eventlet import Timeout
import swift.common.utils
class MessageTimeout(Timeout):
@ -30,6 +31,10 @@ class SwiftException(Exception):
pass
class InvalidTimestamp(SwiftException):
pass
class DiskFileError(SwiftException):
pass
@ -54,7 +59,8 @@ class DiskFileDeleted(DiskFileNotExist):
def __init__(self, metadata=None):
self.metadata = metadata or {}
self.timestamp = self.metadata.get('X-Timestamp', 0)
self.timestamp = swift.common.utils.Timestamp(
self.metadata.get('X-Timestamp', 0))
class DiskFileExpired(DiskFileDeleted):
@ -69,6 +75,10 @@ class DiskFileDeviceUnavailable(DiskFileError):
pass
class DeviceUnavailable(SwiftException):
pass
class PathNotDir(OSError):
pass
@ -139,7 +149,7 @@ class ClientException(Exception):
def __init__(self, msg, http_scheme='', http_host='', http_port='',
http_path='', http_query='', http_status=0, http_reason='',
http_device='', http_response_content=''):
http_device='', http_response_content='', http_headers=None):
Exception.__init__(self, msg)
self.msg = msg
self.http_scheme = http_scheme
@ -151,6 +161,7 @@ class ClientException(Exception):
self.http_reason = http_reason
self.http_device = http_device
self.http_response_content = http_response_content
self.http_headers = http_headers or {}
def __str__(self):
a = self.msg

40
swift/common/internal_client.py
View File

@ -27,7 +27,7 @@ from zlib import compressobj
from swift.common.utils import quote
from swift.common.http import HTTP_NOT_FOUND
from swift.common.swob import Request
from swift.common.wsgi import loadapp
from swift.common.wsgi import loadapp, pipeline_property
class UnexpectedResponse(Exception):
@ -142,6 +142,12 @@ class InternalClient(object):
self.user_agent = user_agent
self.request_tries = request_tries
get_object_ring = pipeline_property('get_object_ring')
container_ring = pipeline_property('container_ring')
account_ring = pipeline_property('account_ring')
auto_create_account_prefix = pipeline_property(
'auto_create_account_prefix', default='.')
def make_request(
self, method, path, headers, acceptable_statuses, body_file=None):
"""
@ -190,7 +196,8 @@ class InternalClient(object):
raise exc_type(*exc_value.args), None, exc_traceback
def _get_metadata(
self, path, metadata_prefix='', acceptable_statuses=(2,)):
self, path, metadata_prefix='', acceptable_statuses=(2,),
headers=None):
"""
Gets metadata by doing a HEAD on a path and using the metadata_prefix
to get values from the headers returned.
@ -201,6 +208,7 @@ class InternalClient(object):
keys in the dict returned. Defaults to ''.
:param acceptable_statuses: List of status for valid responses,
defaults to (2,).
:param headers: extra headers to send
:returns : A dict of metadata with metadata_prefix stripped from keys.
Keys will be lowercase.
@ -211,9 +219,8 @@ class InternalClient(object):
unexpected way.
"""
resp = self.make_request('HEAD', path, {}, acceptable_statuses)
if not resp.status_int // 100 == 2:
return {}
headers = headers or {}
resp = self.make_request('HEAD', path, headers, acceptable_statuses)
metadata_prefix = metadata_prefix.lower()
metadata = {}
for k, v in resp.headers.iteritems():
@ -544,7 +551,8 @@ class InternalClient(object):
def delete_object(
self, account, container, obj,
acceptable_statuses=(2, HTTP_NOT_FOUND)):
acceptable_statuses=(2, HTTP_NOT_FOUND),
headers=None):
"""
Deletes an object.
@ -553,6 +561,7 @@ class InternalClient(object):
:param obj: The object.
:param acceptable_statuses: List of status for valid responses,
defaults to (2, HTTP_NOT_FOUND).
:param headers: extra headers to send with request
:raises UnexpectedResponse: Exception raised when requests fail
to get a response with an acceptable status
@ -561,11 +570,11 @@ class InternalClient(object):
"""
path = self.make_path(account, container, obj)
self.make_request('DELETE', path, {}, acceptable_statuses)
self.make_request('DELETE', path, (headers or {}), acceptable_statuses)
def get_object_metadata(
self, account, container, obj, metadata_prefix='',
acceptable_statuses=(2,)):
acceptable_statuses=(2,), headers=None):
"""
Gets object metadata.
@ -577,6 +586,7 @@ class InternalClient(object):
keys in the dict returned. Defaults to ''.
:param acceptable_statuses: List of status for valid responses,
defaults to (2,).
:param headers: extra headers to send with request
:returns : Dict of object metadata.
@ -587,7 +597,19 @@ class InternalClient(object):
"""
path = self.make_path(account, container, obj)
return self._get_metadata(path, metadata_prefix, acceptable_statuses)
return self._get_metadata(path, metadata_prefix, acceptable_statuses,
headers=headers)
def get_object(self, account, container, obj, headers,
acceptable_statuses=(2,)):
"""
Returns a 3-tuple (status, headers, iterator of object body)
"""
headers = headers or {}
path = self.make_path(account, container, obj)
resp = self.make_request('GET', path, headers, acceptable_statuses)
return (resp.status_int, resp.headers, resp.app_iter)
def iter_object_lines(
self, account, container, obj, headers=None,

5
swift/common/manager.py
View File

@ -30,7 +30,8 @@ RUN_DIR = '/var/run/swift'
# auth-server has been removed from ALL_SERVERS, start it explicitly
ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor',
'container-replicator', 'container-server', 'container-sync',
'container-replicator', 'container-reconciler',
'container-server', 'container-sync',
'container-updater', 'object-auditor', 'object-server',
'object-expirer', 'object-replicator', 'object-updater',
'proxy-server', 'account-replicator', 'account-reaper']
@ -41,7 +42,7 @@ GRACEFUL_SHUTDOWN_SERVERS = MAIN_SERVERS + ['auth-server']
START_ONCE_SERVERS = REST_SERVERS
# These are servers that match a type (account-*, container-*, object-*) but
# don't use that type-server.conf file and instead use their own.
STANDALONE_SERVERS = ['object-expirer']
STANDALONE_SERVERS = ['object-expirer', 'container-reconciler']
KILL_WAIT = 15 # seconds to wait for servers to die (by default)
WARNING_WAIT = 3 # seconds to wait after message that may just be a warning

36
swift/common/middleware/container_sync.py
View File

@ -28,10 +28,10 @@ class ContainerSync(object):
using the container-sync-realms.conf style of container sync.
"""
def __init__(self, app, conf):
def __init__(self, app, conf, logger=None):
self.app = app
self.conf = conf
self.logger = get_logger(conf, log_route='container_sync')
self.logger = logger or get_logger(conf, log_route='container_sync')
self.realms_conf = ContainerSyncRealms(
os.path.join(
conf.get('swift_dir', '/etc/swift'),
@ -39,6 +39,31 @@ class ContainerSync(object):
self.logger)
self.allow_full_urls = config_true_value(
conf.get('allow_full_urls', 'true'))
# configure current realm/cluster for /info
self.realm = self.cluster = None
current = conf.get('current', None)
if current:
try:
self.realm, self.cluster = (p.upper() for p in
current.strip('/').split('/'))
except ValueError:
self.logger.error('Invalid current //REALM/CLUSTER (%s)',
current)
self.register_info()
def register_info(self):
dct = {}
for realm in self.realms_conf.realms():
clusters = self.realms_conf.clusters(realm)
if clusters:
dct[realm] = {'clusters': dict((c, {}) for c in clusters)}
if self.realm and self.cluster:
try:
dct[self.realm]['clusters'][self.cluster]['current'] = True
except KeyError:
self.logger.error('Unknown current //REALM/CLUSTER (%s)',
'//%s/%s' % (self.realm, self.cluster))
register_swift_info('container_sync', realms=dct)
@wsgify
def __call__(self, req):
@ -102,12 +127,7 @@ class ContainerSync(object):
req.environ['swift.authorize_override'] = True
if req.path == '/info':
# Ensure /info requests get the freshest results
dct = {}
for realm in self.realms_conf.realms():
clusters = self.realms_conf.clusters(realm)
if clusters:
dct[realm] = {'clusters': dict((c, {}) for c in clusters)}
register_swift_info('container_sync', realms=dct)
self.register_info()
return self.app

30
swift/common/middleware/list_endpoints.py
View File

@ -61,6 +61,8 @@ from swift.common.ring import Ring
from swift.common.utils import json, get_logger, split_path
from swift.common.swob import Request, Response
from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed
from swift.common.storage_policy import POLICIES
from swift.proxy.controllers.base import get_container_info
class ListEndpointsMiddleware(object):
@ -79,17 +81,24 @@ class ListEndpointsMiddleware(object):
def __init__(self, app, conf):
self.app = app
self.logger = get_logger(conf, log_route='endpoints')
swift_dir = conf.get('swift_dir', '/etc/swift')
self.account_ring = Ring(swift_dir, ring_name='account')
self.container_ring = Ring(swift_dir, ring_name='container')
self.object_ring = Ring(swift_dir, ring_name='object')
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.account_ring = Ring(self.swift_dir, ring_name='account')
self.container_ring = Ring(self.swift_dir, ring_name='container')
self.endpoints_path = conf.get('list_endpoints_path', '/endpoints/')
if not self.endpoints_path.endswith('/'):
self.endpoints_path += '/'
def get_object_ring(self, policy_idx):
"""
Get the ring object to use to handle a request based on its policy.
:policy_idx: policy index as defined in swift.conf
:returns: appropriate ring object
"""
return POLICIES.get_object_ring(policy_idx, self.swift_dir)
def __call__(self, env, start_response):
request = Request(env)
if not request.path.startswith(self.endpoints_path):
return self.app(env, start_response)
@ -112,7 +121,16 @@ class ListEndpointsMiddleware(object):
obj = unquote(obj)
if obj is not None:
partition, nodes = self.object_ring.get_nodes(
# remove 'endpoints' from call to get_container_info
stripped = request.environ
if stripped['PATH_INFO'][:len(self.endpoints_path)] == \
self.endpoints_path:
stripped['PATH_INFO'] = "/v1/" + \
stripped['PATH_INFO'][len(self.endpoints_path):]
container_info = get_container_info(
stripped, self.app, swift_source='LE')
obj_ring = self.get_object_ring(container_info['storage_policy'])
partition, nodes = obj_ring.get_nodes(
account, container, obj)
endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
'{account}/{container}/{obj}'

8
swift/common/middleware/recon.py
View File

@ -55,9 +55,11 @@ class ReconMiddleware(object):
'account.recon')
self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz')
self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz')
self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz')
self.rings = [self.account_ring_path, self.container_ring_path,
self.object_ring_path]
self.rings = [self.account_ring_path, self.container_ring_path]
# include all object ring files (for all policies)
for f in os.listdir(swift_dir):
if f.startswith('object') and f.endswith('ring.gz'):
self.rings.append(os.path.join(swift_dir, f))
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
def _from_recon_cache(self, cache_keys, cache_file, openr=open):

25
swift/common/request_helpers.py
View File

@ -30,6 +30,7 @@ from swift.common.exceptions import ListingIterError, SegmentError
from swift.common.http import is_success, HTTP_SERVICE_UNAVAILABLE
from swift.common.swob import HTTPBadRequest, HTTPNotAcceptable
from swift.common.utils import split_path, validate_device_partition
from swift.common.storage_policy import POLICY_INDEX
from swift.common.wsgi import make_subrequest
@ -78,12 +79,34 @@ def get_listing_content_type(req):
return out_content_type
def get_name_and_placement(request, minsegs=1, maxsegs=None,
rest_with_last=False):
"""
Utility function to split and validate the request path and
storage_policy_index. The storage_policy_index is extracted from
the headers of the request and converted to an integer, and then the
args are passed through to :meth:`split_and_validate_path`.
:returns: a list, result of :meth:`split_and_validate_path` with
storage_policy_index appended on the end
:raises: HTTPBadRequest
"""
policy_idx = request.headers.get(POLICY_INDEX, '0')
policy_idx = int(policy_idx)
results = split_and_validate_path(request, minsegs=minsegs,
maxsegs=maxsegs,
rest_with_last=rest_with_last)
results.append(policy_idx)
return results
def split_and_validate_path(request, minsegs=1, maxsegs=None,
rest_with_last=False):
"""
Utility function to split and validate the request path.
:returns: result of split_path if everything's okay
:returns: result of :meth:`~swift.common.utils.split_path` if
everything's okay
:raises: HTTPBadRequest if something's not okay
"""
try:

6
swift/common/ring/ring.py
View File

@ -97,11 +97,13 @@ class RingData(object):
for part2dev_id in ring['replica2part2dev_id']:
file_obj.write(part2dev_id.tostring())
def save(self, filename):
def save(self, filename, mtime=1300507380.0):
"""
Serialize this RingData instance to disk.
:param filename: File into which this instance should be serialized.
:param mtime: time used to override mtime for gzip, default or None
if the caller wants to include time
"""
# Override the timestamp so that the same ring data creates
# the same bytes on disk. This makes a checksum comparison a
@ -112,7 +114,7 @@ class RingData(object):
tempf = NamedTemporaryFile(dir=".", prefix=filename, delete=False)
try:
gz_file = GzipFile(filename, mode='wb', fileobj=tempf,
mtime=1300507380.0)
mtime=mtime)
except TypeError:
gz_file = GzipFile(filename, mode='wb', fileobj=tempf)
self.serialize_v1(gz_file)

354
swift/common/storage_policy.py Normal file
View File

@ -0,0 +1,354 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ConfigParser import ConfigParser
import textwrap
import string
from swift.common.utils import config_true_value, SWIFT_CONF_FILE
from swift.common.ring import Ring
POLICY = 'X-Storage-Policy'
POLICY_INDEX = 'X-Backend-Storage-Policy-Index'
LEGACY_POLICY_NAME = 'Policy-0'
VALID_CHARS = '-' + string.letters + string.digits
class PolicyError(ValueError):
def __init__(self, msg, index=None):
if index is not None:
msg += ', for index %r' % index
super(PolicyError, self).__init__(msg)
def _get_policy_string(base, policy_index):
if policy_index == 0 or policy_index is None:
return_string = base
else:
return_string = base + "-%d" % int(policy_index)
return return_string
def get_policy_string(base, policy_index):
"""
Helper function to construct a string from a base and the policy
index. Used to encode the policy index into either a file name
or a directory name by various modules.
:param base: the base string
:param policy_index: the storage policy index
:returns: base name with policy index added
"""
if POLICIES.get_by_index(policy_index) is None:
raise PolicyError("No policy with index %r" % policy_index)
return _get_policy_string(base, policy_index)
class StoragePolicy(object):
"""
Represents a storage policy.
Not meant to be instantiated directly; use
:func:`~swift.common.storage_policy.reload_storage_policies` to load
POLICIES from ``swift.conf``.
The object_ring property is lazy loaded once the service's ``swift_dir``
is known via :meth:`~StoragePolicyCollection.get_object_ring`, but it may
be over-ridden via object_ring kwarg at create time for testing or
actively loaded with :meth:`~StoragePolicy.load_ring`.
"""
def __init__(self, idx, name='', is_default=False, is_deprecated=False,
object_ring=None):
try:
self.idx = int(idx)
except ValueError:
raise PolicyError('Invalid index', idx)
if self.idx < 0:
raise PolicyError('Invalid index', idx)
if not name:
raise PolicyError('Invalid name %r' % name, idx)
# this is defensively restrictive, but could be expanded in the future
if not all(c in VALID_CHARS for c in name):
raise PolicyError('Names are used as HTTP headers, and can not '
'reliably contain any characters not in %r. '
'Invalid name %r' % (VALID_CHARS, name))
if name.upper() == LEGACY_POLICY_NAME.upper() and self.idx != 0:
msg = 'The name %s is reserved for policy index 0. ' \
'Invalid name %r' % (LEGACY_POLICY_NAME, name)
raise PolicyError(msg, idx)
self.name = name
self.is_deprecated = config_true_value(is_deprecated)
self.is_default = config_true_value(is_default)
if self.is_deprecated and self.is_default:
raise PolicyError('Deprecated policy can not be default. '
'Invalid config', self.idx)
self.ring_name = _get_policy_string('object', self.idx)
self.object_ring = object_ring
def __int__(self):
return self.idx
def __cmp__(self, other):
return cmp(self.idx, int(other))
def __repr__(self):
return ("StoragePolicy(%d, %r, is_default=%s, is_deprecated=%s)") % (
self.idx, self.name, self.is_default, self.is_deprecated)
def load_ring(self, swift_dir):
"""
Load the ring for this policy immediately.
:param swift_dir: path to rings
"""
if self.object_ring:
return
self.object_ring = Ring(swift_dir, ring_name=self.ring_name)
class StoragePolicyCollection(object):
"""
This class represents the collection of valid storage policies for the
cluster and is instantiated as :class:`StoragePolicy` objects are added to
the collection when ``swift.conf`` is parsed by
:func:`parse_storage_policies`.
When a StoragePolicyCollection is created, the following validation
is enforced:
* If a policy with index 0 is not declared and no other policies defined,
Swift will create one
* The policy index must be a non-negative integer
* If no policy is declared as the default and no other policies are
defined, the policy with index 0 is set as the default
* Policy indexes must be unique
* Policy names are required
* Policy names are case insensitive
* Policy names must contain only letters, digits or a dash
* Policy names must be unique
* The policy name 'Policy-0' can only be used for the policy with index 0
* If any policies are defined, exactly one policy must be declared default
* Deprecated policies can not be declared the default
"""
def __init__(self, pols):
self.default = []
self.by_name = {}
self.by_index = {}
self._validate_policies(pols)
def _add_policy(self, policy):
"""
Add pre-validated policies to internal indexes.
"""
self.by_name[policy.name.upper()] = policy
self.by_index[int(policy)] = policy
def __repr__(self):
return (textwrap.dedent("""
StoragePolicyCollection([
%s
])
""") % ',\n '.join(repr(p) for p in self)).strip()
def __len__(self):
return len(self.by_index)
def __getitem__(self, key):
return self.by_index[key]
def __iter__(self):
return iter(self.by_index.values())
def _validate_policies(self, policies):
"""
:param policies: list of policies
"""
for policy in policies:
if int(policy) in self.by_index:
raise PolicyError('Duplicate index %s conflicts with %s' % (
policy, self.get_by_index(int(policy))))
if policy.name.upper() in self.by_name:
raise PolicyError('Duplicate name %s conflicts with %s' % (
policy, self.get_by_name(policy.name)))
if policy.is_default:
if not self.default:
self.default = policy
else:
raise PolicyError(
'Duplicate default %s conflicts with %s' % (
policy, self.default))
self._add_policy(policy)
# If a 0 policy wasn't explicitly given, or nothing was
# provided, create the 0 policy now
if 0 not in self.by_index:
if len(self) != 0:
raise PolicyError('You must specify a storage policy '
'section for policy index 0 in order '
'to define multiple policies')
self._add_policy(StoragePolicy(0, name=LEGACY_POLICY_NAME))
# at least one policy must be enabled
enabled_policies = [p for p in self if not p.is_deprecated]
if not enabled_policies:
raise PolicyError("Unable to find policy that's not deprecated!")
# if needed, specify default
if not self.default:
if len(self) > 1:
raise PolicyError("Unable to find default policy")
self.default = self[0]
self.default.is_default = True
def get_by_name(self, name):
"""
Find a storage policy by its name.
:param name: name of the policy
:returns: storage policy, or None
"""
return self.by_name.get(name.upper())
def get_by_index(self, index):
"""
Find a storage policy by its index.
An index of None will be treated as 0.
:param index: numeric index of the storage policy
:returns: storage policy, or None if no such policy
"""
# makes it easier for callers to just pass in a header value
index = int(index) if index else 0
return self.by_index.get(index)
def get_object_ring(self, policy_idx, swift_dir):
"""
Get the ring object to use to handle a request based on its policy.
An index of None will be treated as 0.
:param policy_idx: policy index as defined in swift.conf
:param swift_dir: swift_dir used by the caller
:returns: appropriate ring object
"""
policy = self.get_by_index(policy_idx)
if not policy:
raise PolicyError("No policy with index %s" % policy_idx)
if not policy.object_ring:
policy.load_ring(swift_dir)
return policy.object_ring
def get_policy_info(self):
"""
Build info about policies for the /info endpoint
:returns: list of dicts containing relevant policy information
"""
policy_info = []
for pol in self:
# delete from /info if deprecated
if pol.is_deprecated:
continue
policy_entry = {}
policy_entry['name'] = pol.name
if pol.is_default:
policy_entry['default'] = pol.is_default
policy_info.append(policy_entry)
return policy_info
def parse_storage_policies(conf):
"""
Parse storage policies in ``swift.conf`` - note that validation
is done when the :class:`StoragePolicyCollection` is instantiated.
:param conf: ConfigParser parser object for swift.conf
"""
policies = []
for section in conf.sections():
if not section.startswith('storage-policy:'):
continue
policy_index = section.split(':', 1)[1]
# map config option name to StoragePolicy paramater name
config_to_policy_option_map = {
'name': 'name',
'default': 'is_default',
'deprecated': 'is_deprecated',
}
policy_options = {}
for config_option, value in conf.items(section):
try:
policy_option = config_to_policy_option_map[config_option]
except KeyError:
raise PolicyError('Invalid option %r in '
'storage-policy section %r' % (
config_option, section))
policy_options[policy_option] = value
policy = StoragePolicy(policy_index, **policy_options)
policies.append(policy)
return StoragePolicyCollection(policies)
class StoragePolicySingleton(object):
"""
An instance of this class is the primary interface to storage policies
exposed as a module level global named ``POLICIES``. This global
reference wraps ``_POLICIES`` which is normally instantiated by parsing
``swift.conf`` and will result in an instance of
:class:`StoragePolicyCollection`.
You should never patch this instance directly, instead patch the module
level ``_POLICIES`` instance so that swift code which imported
``POLICIES`` directly will reference the patched
:class:`StoragePolicyCollection`.
"""
def __iter__(self):
return iter(_POLICIES)
def __len__(self):
return len(_POLICIES)
def __getitem__(self, key):
return _POLICIES[key]
def __getattribute__(self, name):
return getattr(_POLICIES, name)
def __repr__(self):
return repr(_POLICIES)
def reload_storage_policies():
"""
Reload POLICIES from ``swift.conf``.
"""
global _POLICIES
policy_conf = ConfigParser()
policy_conf.read(SWIFT_CONF_FILE)
try:
_POLICIES = parse_storage_policies(policy_conf)
except PolicyError as e:
raise SystemExit('ERROR: Invalid Storage Policy Configuration '
'in %s (%s)' % (SWIFT_CONF_FILE, e))
# parse configuration and setup singleton
_POLICIES = None
reload_storage_policies()
POLICIES = StoragePolicySingleton()

20
swift/common/swob.py
View File

@ -49,7 +49,8 @@ import random
import functools
import inspect
from swift.common.utils import reiterate, split_path
from swift.common.utils import reiterate, split_path, Timestamp
from swift.common.exceptions import InvalidTimestamp
RESPONSE_REASONS = {
@ -762,6 +763,7 @@ class Request(object):
body = _req_body_property()
charset = None
_params_cache = None
_timestamp = None
acl = _req_environ_property('swob.ACL')
def __init__(self, environ):
@ -843,6 +845,22 @@ class Request(object):
return self._params_cache
str_params = params
@property
def timestamp(self):
"""
Provides HTTP_X_TIMESTAMP as a :class:`~swift.common.utils.Timestamp`
"""
if self._timestamp is None:
try:
raw_timestamp = self.environ['HTTP_X_TIMESTAMP']
except KeyError:
raise InvalidTimestamp('Missing X-Timestamp header')
try:
self._timestamp = Timestamp(raw_timestamp)
except ValueError:
raise InvalidTimestamp('Invalid X-Timestamp header')
return self._timestamp
@property
def path_qs(self):
"""The path of the request, without host but with query string."""

226
swift/common/utils.py
View File

@ -49,6 +49,7 @@ import glob
from urlparse import urlparse as stdlib_urlparse, ParseResult
import itertools
import stat
import datetime
import eventlet
import eventlet.semaphore
@ -62,7 +63,7 @@ utf8_decoder = codecs.getdecoder('utf-8')
utf8_encoder = codecs.getencoder('utf-8')
from swift import gettext_ as _
from swift.common.exceptions import LockTimeout, MessageTimeout
import swift.common.exceptions
from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND
# logging doesn't import patched as cleanly as one would like
@ -561,6 +562,120 @@ def drop_buffer_cache(fd, offset, length):
'length': length, 'ret': ret})
NORMAL_FORMAT = "%016.05f"
INTERNAL_FORMAT = NORMAL_FORMAT + '_%016x'
# Setting this to True will cause the internal format to always display
# extended digits - even when the value is equivalent to the normalized form.
# This isn't ideal during an upgrade when some servers might not understand
# the new time format - but flipping it to True works great for testing.
FORCE_INTERNAL = False # or True
class Timestamp(object):
"""
Internal Representation of Swift Time.
The normalized form of the X-Timestamp header looks like a float
with a fixed width to ensure stable string sorting - normalized
timestamps look like "1402464677.04188"
To support overwrites of existing data without modifying the original
timestamp but still maintain consistency a second internal offset vector
is append to the normalized timestamp form which compares and sorts
greater than the fixed width float format but less than a newer timestamp.
The internalized format of timestamps looks like
"1402464677.04188_0000000000000000" - the portion after the underscore is
the offset and is a formatted hexadecimal integer.
The internalized form is not exposed to clients in responses from
Swift. Normal client operations will not create a timestamp with an
offset.
The Timestamp class in common.utils supports internalized and
normalized formatting of timestamps and also comparison of timestamp
values. When the offset value of a Timestamp is 0 - it's considered
insignificant and need not be represented in the string format; to
support backwards compatibility during a Swift upgrade the
internalized and normalized form of a Timestamp with an
insignificant offset are identical. When a timestamp includes an
offset it will always be represented in the internalized form, but
is still excluded from the normalized form. Timestamps with an
equivalent timestamp portion (the float part) will compare and order
by their offset. Timestamps with a greater timestamp portion will
always compare and order greater than a Timestamp with a lesser
timestamp regardless of it's offset. String comparison and ordering
is guaranteed for the internalized string format, and is backwards
compatible for normalized timestamps which do not include an offset.
"""
def __init__(self, timestamp, offset=0):
if isinstance(timestamp, basestring):
parts = timestamp.split('_', 1)
self.timestamp = float(parts.pop(0))
if parts:
self.offset = int(parts[0], 16)
else:
self.offset = 0
else:
self.timestamp = float(timestamp)
self.offset = getattr(timestamp, 'offset', 0)
# increment offset
if offset >= 0:
self.offset += offset
else:
raise ValueError('offset must be non-negative')
def __repr__(self):
return INTERNAL_FORMAT % (self.timestamp, self.offset)
def __str__(self):
raise TypeError('You must specificy which string format is required')
def __float__(self):
return self.timestamp
def __int__(self):
return int(self.timestamp)
def __nonzero__(self):
return bool(self.timestamp or self.offset)
@property
def normal(self):
return NORMAL_FORMAT % self.timestamp
@property
def internal(self):
if self.offset or FORCE_INTERNAL:
return INTERNAL_FORMAT % (self.timestamp, self.offset)
else:
return self.normal
@property
def isoformat(self):
isoformat = datetime.datetime.utcfromtimestamp(
float(self.normal)).isoformat()
# python isoformat() doesn't include msecs when zero
if len(isoformat) < len("1970-01-01T00:00:00.000000"):
isoformat += ".000000"
return isoformat
def __eq__(self, other):
if not isinstance(other, Timestamp):
other = Timestamp(other)
return self.internal == other.internal
def __ne__(self, other):
if not isinstance(other, Timestamp):
other = Timestamp(other)
return self.internal != other.internal
def __cmp__(self, other):
if not isinstance(other, Timestamp):
other = Timestamp(other)
return cmp(self.internal, other.internal)
def normalize_timestamp(timestamp):
"""
Format a timestamp (string or numeric) into a standardized
@ -573,7 +688,19 @@ def normalize_timestamp(timestamp):
:param timestamp: unix timestamp
:returns: normalized timestamp as a string
"""
return "%016.05f" % (float(timestamp))
return Timestamp(timestamp).normal
def last_modified_date_to_timestamp(last_modified_date_str):
"""
Convert a last modified date (like you'd get from a container listing,
e.g. 2014-02-28T23:22:36.698390) to a float.
"""
return Timestamp(
datetime.datetime.strptime(
last_modified_date_str, '%Y-%m-%dT%H:%M:%S.%f'
).strftime('%s.%f')
)
def normalize_delete_at_timestamp(timestamp):
@ -994,7 +1121,7 @@ class LogAdapter(logging.LoggerAdapter, object):
emsg = exc.__class__.__name__
if hasattr(exc, 'seconds'):
emsg += ' (%ss)' % exc.seconds
if isinstance(exc, MessageTimeout):
if isinstance(exc, swift.common.exceptions.MessageTimeout):
if exc.msg:
emsg += ' %s' % exc.msg
else:
@ -1428,7 +1555,7 @@ def hash_path(account, container=None, object=None, raw_digest=False):
@contextmanager
def lock_path(directory, timeout=10, timeout_class=LockTimeout):
def lock_path(directory, timeout=10, timeout_class=None):
"""
Context manager that acquires a lock on a directory. This will block until
the lock can be acquired, or the timeout time has expired (whichever occurs
@ -1445,6 +1572,8 @@ def lock_path(directory, timeout=10, timeout_class=LockTimeout):
constructed as timeout_class(timeout, lockpath). Default:
LockTimeout
"""
if timeout_class is None:
timeout_class = swift.common.exceptions.LockTimeout
mkdirs(directory)
lockpath = '%s/.lock' % directory
fd = os.open(lockpath, os.O_WRONLY | os.O_CREAT)
@ -1484,7 +1613,7 @@ def lock_file(filename, timeout=10, append=False, unlink=True):
fd = os.open(filename, flags)
file_obj = os.fdopen(fd, mode)
try:
with LockTimeout(timeout, filename):
with swift.common.exceptions.LockTimeout(timeout, filename):
while True:
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
@ -2411,6 +2540,93 @@ class InputProxy(object):
return line
class LRUCache(object):
"""
Decorator for size/time bound memoization that evicts the least
recently used members.
"""
PREV, NEXT, KEY, CACHED_AT, VALUE = 0, 1, 2, 3, 4 # link fields
def __init__(self, maxsize=1000, maxtime=3600):
self.maxsize = maxsize
self.maxtime = maxtime
self.reset()
def reset(self):
self.mapping = {}
self.head = [None, None, None, None, None] # oldest
self.tail = [self.head, None, None, None, None] # newest
self.head[self.NEXT] = self.tail
def set_cache(self, value, *key):
while len(self.mapping) >= self.maxsize:
old_next, old_key = self.head[self.NEXT][self.NEXT:self.NEXT + 2]
self.head[self.NEXT], old_next[self.PREV] = old_next, self.head
del self.mapping[old_key]
last = self.tail[self.PREV]
link = [last, self.tail, key, time.time(), value]
self.mapping[key] = last[self.NEXT] = self.tail[self.PREV] = link
return value
def get_cached(self, link, *key):
link_prev, link_next, key, cached_at, value = link
if cached_at + self.maxtime < time.time():
raise KeyError('%r has timed out' % (key,))
link_prev[self.NEXT] = link_next
link_next[self.PREV] = link_prev
last = self.tail[self.PREV]
last[self.NEXT] = self.tail[self.PREV] = link
link[self.PREV] = last
link[self.NEXT] = self.tail
return value
def __call__(self, f):
class LRUCacheWrapped(object):
@functools.wraps(f)
def __call__(im_self, *key):
link = self.mapping.get(key, self.head)
if link is not self.head:
try:
return self.get_cached(link, *key)
except KeyError:
pass
value = f(*key)
self.set_cache(value, *key)
return value
def size(im_self):
"""
Return the size of the cache
"""
return len(self.mapping)
def reset(im_self):
return self.reset()
def get_maxsize(im_self):
return self.maxsize
def set_maxsize(im_self, i):
self.maxsize = i
def get_maxtime(im_self):
return self.maxtime
def set_maxtime(im_self, i):
self.maxtime = i
maxsize = property(get_maxsize, set_maxsize)
maxtime = property(get_maxtime, set_maxtime)
def __repr__(im_self):
return '<%s %r>' % (im_self.__class__.__name__, f)
return LRUCacheWrapped()
def tpool_reraise(func, *args, **kwargs):
"""
Hack to work around Eventlet's tpool not catching and reraising Timeouts.

48
swift/common/wsgi.py
View File

@ -201,6 +201,47 @@ class RestrictedGreenPool(GreenPool):
self.waitall()
def pipeline_property(name, **kwargs):
"""
Create a property accessor for the given name. The property will
dig through the bound instance on which it was accessed for an
attribute "app" and check that object for an attribute of the given
name. If the "app" object does not have such an attribute, it will
look for an attribute "app" on THAT object and continue it's search
from there. If the named attribute cannot be found accessing the
property will raise AttributeError.
If a default kwarg is provided you get that instead of the
AttributeError. When found the attribute will be cached on instance
with the property accessor using the same name as the attribute
prefixed with a leading underscore.
"""
cache_attr_name = '_%s' % name
def getter(self):
cached_value = getattr(self, cache_attr_name, None)
if cached_value:
return cached_value
app = self # first app is on self
while True:
app = getattr(app, 'app', None)
if not app:
break
try:
value = getattr(app, name)
except AttributeError:
continue
setattr(self, cache_attr_name, value)
return value
if 'default' in kwargs:
return kwargs['default']
raise AttributeError('No apps in pipeline have a '
'%s attribute' % name)
return property(getter)
class PipelineWrapper(object):
"""
This class provides a number of utility methods for
@ -292,6 +333,13 @@ def loadcontext(object_type, uri, name=None, relative_to=None,
global_conf=global_conf)
def _add_pipeline_properties(app, *names):
for property_name in names:
if not hasattr(app, property_name):
setattr(app.__class__, property_name,
pipeline_property(property_name))
def loadapp(conf_file, global_conf=None, allow_modify_pipeline=True):
"""
Loads a context from a config file, and if the context is a pipeline

4
swift/container/auditor.py
View File

@ -30,9 +30,9 @@ from swift.common.daemon import Daemon
class ContainerAuditor(Daemon):
"""Audit containers."""
def __init__(self, conf):
def __init__(self, conf, logger=None):
self.conf = conf
self.logger = get_logger(conf, log_route='container-auditor')
self.logger = logger or get_logger(conf, log_route='container-auditor')
self.devices = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.interval = int(conf.get('interval', 1800))

652
swift/container/backend.py
View File

@ -24,12 +24,115 @@ import errno
import sqlite3
from swift.common.utils import normalize_timestamp, lock_parent_directory
from swift.common.utils import Timestamp, lock_parent_directory
from swift.common.db import DatabaseBroker, DatabaseConnectionError, \
PENDING_CAP, PICKLE_PROTOCOL, utf8encode
DATADIR = 'containers'
POLICY_STAT_TABLE_CREATE = '''
CREATE TABLE policy_stat (
storage_policy_index INTEGER PRIMARY KEY,
object_count INTEGER DEFAULT 0,
bytes_used INTEGER DEFAULT 0
);
'''
POLICY_STAT_TRIGGER_SCRIPT = '''
CREATE TRIGGER object_insert_policy_stat AFTER INSERT ON object
BEGIN
UPDATE policy_stat
SET object_count = object_count + (1 - new.deleted),
bytes_used = bytes_used + new.size
WHERE storage_policy_index = new.storage_policy_index;
INSERT INTO policy_stat (
storage_policy_index, object_count, bytes_used)
SELECT new.storage_policy_index,
(1 - new.deleted),
new.size
WHERE NOT EXISTS(
SELECT changes() as change
FROM policy_stat
WHERE change <> 0
);
UPDATE container_info
SET hash = chexor(hash, new.name, new.created_at);
END;
CREATE TRIGGER object_delete_policy_stat AFTER DELETE ON object
BEGIN
UPDATE policy_stat
SET object_count = object_count - (1 - old.deleted),
bytes_used = bytes_used - old.size
WHERE storage_policy_index = old.storage_policy_index;
UPDATE container_info
SET hash = chexor(hash, old.name, old.created_at);
END;
'''
CONTAINER_INFO_TABLE_SCRIPT = '''
CREATE TABLE container_info (
account TEXT,
container TEXT,
created_at TEXT,
put_timestamp TEXT DEFAULT '0',
delete_timestamp TEXT DEFAULT '0',
reported_put_timestamp TEXT DEFAULT '0',
reported_delete_timestamp TEXT DEFAULT '0',
reported_object_count INTEGER DEFAULT 0,
reported_bytes_used INTEGER DEFAULT 0,
hash TEXT default '00000000000000000000000000000000',
id TEXT,
status TEXT DEFAULT '',
status_changed_at TEXT DEFAULT '0',
metadata TEXT DEFAULT '',
x_container_sync_point1 INTEGER DEFAULT -1,
x_container_sync_point2 INTEGER DEFAULT -1,
storage_policy_index INTEGER DEFAULT 0,
reconciler_sync_point INTEGER DEFAULT -1
);
'''
CONTAINER_STAT_VIEW_SCRIPT = '''
CREATE VIEW container_stat
AS SELECT ci.account, ci.container, ci.created_at,
ci.put_timestamp, ci.delete_timestamp,
ci.reported_put_timestamp, ci.reported_delete_timestamp,
ci.reported_object_count, ci.reported_bytes_used, ci.hash,
ci.id, ci.status, ci.status_changed_at, ci.metadata,
ci.x_container_sync_point1, ci.x_container_sync_point2,
ci.reconciler_sync_point,
ci.storage_policy_index,
coalesce(ps.object_count, 0) AS object_count,
coalesce(ps.bytes_used, 0) AS bytes_used
FROM container_info ci LEFT JOIN policy_stat ps
ON ci.storage_policy_index = ps.storage_policy_index;
CREATE TRIGGER container_stat_update
INSTEAD OF UPDATE ON container_stat
BEGIN
UPDATE container_info
SET account = NEW.account,
container = NEW.container,
created_at = NEW.created_at,
put_timestamp = NEW.put_timestamp,
delete_timestamp = NEW.delete_timestamp,
reported_put_timestamp = NEW.reported_put_timestamp,
reported_delete_timestamp = NEW.reported_delete_timestamp,
reported_object_count = NEW.reported_object_count,
reported_bytes_used = NEW.reported_bytes_used,
hash = NEW.hash,
id = NEW.id,
status = NEW.status,
status_changed_at = NEW.status_changed_at,
metadata = NEW.metadata,
x_container_sync_point1 = NEW.x_container_sync_point1,
x_container_sync_point2 = NEW.x_container_sync_point2,
storage_policy_index = NEW.storage_policy_index,
reconciler_sync_point = NEW.reconciler_sync_point;
END;
'''
class ContainerBroker(DatabaseBroker):
"""Encapsulates working with a container database."""
@ -37,7 +140,14 @@ class ContainerBroker(DatabaseBroker):
db_contains_type = 'object'
db_reclaim_timestamp = 'created_at'
def _initialize(self, conn, put_timestamp):
@property
def storage_policy_index(self):
if not hasattr(self, '_storage_policy_index'):
self._storage_policy_index = \
self.get_info()['storage_policy_index']
return self._storage_policy_index
def _initialize(self, conn, put_timestamp, storage_policy_index):
"""
Create a brand new container database (tables, indices, triggers, etc.)
"""
@ -48,7 +158,9 @@ class ContainerBroker(DatabaseBroker):
raise ValueError(
'Attempting to create a new database with no container set')
self.create_object_table(conn)
self.create_container_stat_table(conn, put_timestamp)
self.create_policy_stat_table(conn, storage_policy_index)
self.create_container_info_table(conn, put_timestamp,
storage_policy_index)
def create_object_table(self, conn):
"""
@ -65,74 +177,70 @@ class ContainerBroker(DatabaseBroker):
size INTEGER,
content_type TEXT,
etag TEXT,
deleted INTEGER DEFAULT 0
deleted INTEGER DEFAULT 0,
storage_policy_index INTEGER DEFAULT 0
);
CREATE INDEX ix_object_deleted_name ON object (deleted, name);
CREATE TRIGGER object_insert AFTER INSERT ON object
BEGIN
UPDATE container_stat
SET object_count = object_count + (1 - new.deleted),
bytes_used = bytes_used + new.size,
hash = chexor(hash, new.name, new.created_at);
END;
CREATE TRIGGER object_update BEFORE UPDATE ON object
BEGIN
SELECT RAISE(FAIL, 'UPDATE not allowed; DELETE and INSERT');
END;
CREATE TRIGGER object_delete AFTER DELETE ON object
BEGIN
UPDATE container_stat
SET object_count = object_count - (1 - old.deleted),
bytes_used = bytes_used - old.size,
hash = chexor(hash, old.name, old.created_at);
END;
""")
""" + POLICY_STAT_TRIGGER_SCRIPT)
def create_container_stat_table(self, conn, put_timestamp=None):
def create_container_info_table(self, conn, put_timestamp,
storage_policy_index):
"""
Create the container_stat table which is specific to the container DB.
Create the container_info table which is specific to the container DB.
Not a part of Pluggable Back-ends, internal to the baseline code.
Also creates the container_stat view.
:param conn: DB connection object
:param put_timestamp: put timestamp
:param storage_policy_index: storage policy index
"""
if put_timestamp is None:
put_timestamp = normalize_timestamp(0)
conn.executescript("""
CREATE TABLE container_stat (
account TEXT,
container TEXT,
created_at TEXT,
put_timestamp TEXT DEFAULT '0',
delete_timestamp TEXT DEFAULT '0',
object_count INTEGER,
bytes_used INTEGER,
reported_put_timestamp TEXT DEFAULT '0',
reported_delete_timestamp TEXT DEFAULT '0',
reported_object_count INTEGER DEFAULT 0,
reported_bytes_used INTEGER DEFAULT 0,
hash TEXT default '00000000000000000000000000000000',
id TEXT,
status TEXT DEFAULT '',
status_changed_at TEXT DEFAULT '0',
metadata TEXT DEFAULT '',
x_container_sync_point1 INTEGER DEFAULT -1,
x_container_sync_point2 INTEGER DEFAULT -1
);
put_timestamp = Timestamp(0).internal
# The container_stat view is for compatibility; old versions of Swift
# expected a container_stat table with columns "object_count" and
# "bytes_used", but when that stuff became per-storage-policy and
# moved to the policy_stat table, we stopped creating those columns in
# container_stat.
#
# To retain compatibility, we create the container_stat view with some
# triggers to make it behave like the old container_stat table. This
# way, if an old version of Swift encounters a database with the new
# schema, it can still work.
#
# Note that this can occur during a rolling Swift upgrade if a DB gets
# rsynced from an old node to a new, so it's necessary for
# availability during upgrades. The fact that it enables downgrades is
# a nice bonus.
conn.executescript(CONTAINER_INFO_TABLE_SCRIPT +
CONTAINER_STAT_VIEW_SCRIPT)
conn.execute("""
INSERT INTO container_info (account, container, created_at, id,
put_timestamp, status_changed_at, storage_policy_index)
VALUES (?, ?, ?, ?, ?, ?, ?);
""", (self.account, self.container, Timestamp(time.time()).internal,
str(uuid4()), put_timestamp, put_timestamp,
storage_policy_index))
INSERT INTO container_stat (object_count, bytes_used)
VALUES (0, 0);
""")
conn.execute('''
UPDATE container_stat
SET account = ?, container = ?, created_at = ?, id = ?,
put_timestamp = ?
''', (self.account, self.container, normalize_timestamp(time.time()),
str(uuid4()), put_timestamp))
def create_policy_stat_table(self, conn, storage_policy_index=0):
"""
Create policy_stat table.
:param conn: DB connection object
:param storage_policy_index: the policy_index the container is
being created with
"""
conn.executescript(POLICY_STAT_TABLE_CREATE)
conn.execute("""
INSERT INTO policy_stat (storage_policy_index)
VALUES (?)
""", (storage_policy_index,))
def get_db_version(self, conn):
if self._db_version == -1:
@ -165,14 +273,19 @@ class ContainerBroker(DatabaseBroker):
def _commit_puts_load(self, item_list, entry):
"""See :func:`swift.common.db.DatabaseBroker._commit_puts_load`"""
(name, timestamp, size, content_type, etag, deleted) = \
pickle.loads(entry.decode('base64'))
data = pickle.loads(entry.decode('base64'))
(name, timestamp, size, content_type, etag, deleted) = data[:6]
if len(data) > 6:
storage_policy_index = data[6]
else:
storage_policy_index = 0
item_list.append({'name': name,
'created_at': timestamp,
'size': size,
'content_type': content_type,
'etag': etag,
'deleted': deleted})
'deleted': deleted,
'storage_policy_index': storage_policy_index})
def empty(self):
"""
@ -182,20 +295,30 @@ class ContainerBroker(DatabaseBroker):
"""
self._commit_puts_stale_ok()
with self.get() as conn:
row = conn.execute(
'SELECT object_count from container_stat').fetchone()
try:
row = conn.execute(
'SELECT max(object_count) from policy_stat').fetchone()
except sqlite3.OperationalError as err:
if not any(msg in str(err) for msg in (
"no such column: storage_policy_index",
"no such table: policy_stat")):
raise
row = conn.execute(
'SELECT object_count from container_stat').fetchone()
return (row[0] == 0)
def delete_object(self, name, timestamp):
def delete_object(self, name, timestamp, storage_policy_index=0):
"""
Mark an object deleted.
:param name: object name to be deleted
:param timestamp: timestamp when the object was marked as deleted
"""
self.put_object(name, timestamp, 0, 'application/deleted', 'noetag', 1)
self.put_object(name, timestamp, 0, 'application/deleted', 'noetag',
deleted=1, storage_policy_index=storage_policy_index)
def put_object(self, name, timestamp, size, content_type, etag, deleted=0):
def put_object(self, name, timestamp, size, content_type, etag, deleted=0,
storage_policy_index=0):
"""
Creates an object in the DB with its metadata.
@ -206,10 +329,12 @@ class ContainerBroker(DatabaseBroker):
:param etag: object etag
:param deleted: if True, marks the object as deleted and sets the
deteleted_at timestamp to timestamp
:param storage_policy_index: the storage policy index for the object
"""
record = {'name': name, 'created_at': timestamp, 'size': size,
'content_type': content_type, 'etag': etag,
'deleted': deleted}
'deleted': deleted,
'storage_policy_index': storage_policy_index}
if self.db_file == ':memory:':
self.merge_items([record])
return
@ -231,93 +356,110 @@ class ContainerBroker(DatabaseBroker):
# delimiter
fp.write(':')
fp.write(pickle.dumps(
(name, timestamp, size, content_type, etag, deleted),
(name, timestamp, size, content_type, etag, deleted,
storage_policy_index),
protocol=PICKLE_PROTOCOL).encode('base64'))
fp.flush()
def is_deleted(self, timestamp=None):
def _is_deleted_info(self, object_count, put_timestamp, delete_timestamp,
**kwargs):
"""
Check if the DB is considered to be deleted.
Apply delete logic to database info.
:returns: True if the DB is considered to be deleted, False otherwise
"""
# The container is considered deleted if the delete_timestamp
# value is greater than the put_timestamp, and there are no
# objects in the container.
return (object_count in (None, '', 0, '0')) and (
Timestamp(delete_timestamp) > Timestamp(put_timestamp))
def _is_deleted(self, conn):
"""
Check container_stat view and evaluate info.
:param conn: database conn
:returns: True if the DB is considered to be deleted, False otherwise
"""
info = conn.execute('''
SELECT put_timestamp, delete_timestamp, object_count
FROM container_stat''').fetchone()
return self._is_deleted_info(**info)
def get_info_is_deleted(self):
"""
Get the is_deleted status and info for the container.
:returns: a tuple, in the form (info, is_deleted) info is a dict as
returned by get_info and is_deleted is a boolean.
"""
if self.db_file != ':memory:' and not os.path.exists(self.db_file):
return True
self._commit_puts_stale_ok()
with self.get() as conn:
row = conn.execute('''
SELECT put_timestamp, delete_timestamp, object_count
FROM container_stat''').fetchone()
# leave this db as a tombstone for a consistency window
if timestamp and row['delete_timestamp'] > timestamp:
return False
# The container is considered deleted if the delete_timestamp
# value is greater than the put_timestamp, and there are no
# objects in the container.
return (row['object_count'] in (None, '', 0, '0')) and \
(float(row['delete_timestamp']) > float(row['put_timestamp']))
return {}, True
info = self.get_info()
return info, self._is_deleted_info(**info)
def get_info(self):
"""
Get global data for the container.
:returns: dict with keys: account, container, created_at,
put_timestamp, delete_timestamp, object_count, bytes_used,
reported_put_timestamp, reported_delete_timestamp,
reported_object_count, reported_bytes_used, hash, id,
x_container_sync_point1, and x_container_sync_point2.
put_timestamp, delete_timestamp, status_changed_at,
object_count, bytes_used, reported_put_timestamp,
reported_delete_timestamp, reported_object_count,
reported_bytes_used, hash, id, x_container_sync_point1,
x_container_sync_point2, and storage_policy_index.
"""
self._commit_puts_stale_ok()
with self.get() as conn:
data = None
trailing = 'x_container_sync_point1, x_container_sync_point2'
trailing_sync = 'x_container_sync_point1, x_container_sync_point2'
trailing_pol = 'storage_policy_index'
errors = set()
while not data:
try:
data = conn.execute('''
data = conn.execute(('''
SELECT account, container, created_at, put_timestamp,
delete_timestamp, object_count, bytes_used,
delete_timestamp, status_changed_at,
object_count, bytes_used,
reported_put_timestamp, reported_delete_timestamp,
reported_object_count, reported_bytes_used, hash,
id, %s
FROM container_stat
''' % (trailing,)).fetchone()
id, %s, %s
FROM container_stat
''') % (trailing_sync, trailing_pol)).fetchone()
except sqlite3.OperationalError as err:
if 'no such column: x_container_sync_point' in str(err):
trailing = '-1 AS x_container_sync_point1, ' \
'-1 AS x_container_sync_point2'
err_msg = str(err)
if err_msg in errors:
# only attempt migration once
raise
errors.add(err_msg)
if 'no such column: storage_policy_index' in err_msg:
trailing_pol = '0 AS storage_policy_index'
elif 'no such column: x_container_sync_point' in err_msg:
trailing_sync = '-1 AS x_container_sync_point1, ' \
'-1 AS x_container_sync_point2'
else:
raise
data = dict(data)
# populate instance cache
self._storage_policy_index = data['storage_policy_index']
self.account = data['account']
self.container = data['container']
return data
def set_x_container_sync_points(self, sync_point1, sync_point2):
with self.get() as conn:
orig_isolation_level = conn.isolation_level
try:
# We turn off auto-transactions to ensure the alter table
# commands are part of the transaction.
conn.isolation_level = None
conn.execute('BEGIN')
try:
self._set_x_container_sync_points(conn, sync_point1,
sync_point2)
except sqlite3.OperationalError as err:
if 'no such column: x_container_sync_point' not in \
str(err):
raise
conn.execute('''
ALTER TABLE container_stat
ADD COLUMN x_container_sync_point1 INTEGER DEFAULT -1
''')
conn.execute('''
ALTER TABLE container_stat
ADD COLUMN x_container_sync_point2 INTEGER DEFAULT -1
''')
self._set_x_container_sync_points(conn, sync_point1,
sync_point2)
conn.execute('COMMIT')
finally:
conn.isolation_level = orig_isolation_level
self._set_x_container_sync_points(conn, sync_point1,
sync_point2)
except sqlite3.OperationalError as err:
if 'no such column: x_container_sync_point' not in \
str(err):
raise
self._migrate_add_container_sync_points(conn)
self._set_x_container_sync_points(conn, sync_point1,
sync_point2)
conn.commit()
def _set_x_container_sync_points(self, conn, sync_point1, sync_point2):
if sync_point1 is not None and sync_point2 is not None:
@ -337,6 +479,79 @@ class ContainerBroker(DatabaseBroker):
SET x_container_sync_point2 = ?
''', (sync_point2,))
def get_policy_stats(self):
with self.get() as conn:
try:
info = conn.execute('''
SELECT storage_policy_index, object_count, bytes_used
FROM policy_stat
''').fetchall()
except sqlite3.OperationalError as err:
if not any(msg in str(err) for msg in (
"no such column: storage_policy_index",
"no such table: policy_stat")):
raise
info = conn.execute('''
SELECT 0 as storage_policy_index, object_count, bytes_used
FROM container_stat
''').fetchall()
policy_stats = {}
for row in info:
stats = dict(row)
key = stats.pop('storage_policy_index')
policy_stats[key] = stats
return policy_stats
def has_multiple_policies(self):
with self.get() as conn:
try:
curs = conn.execute('''
SELECT count(storage_policy_index)
FROM policy_stat
''').fetchone()
except sqlite3.OperationalError as err:
if 'no such table: policy_stat' not in str(err):
raise
# no policy_stat row
return False
if curs and curs[0] > 1:
return True
# only one policy_stat row
return False
def set_storage_policy_index(self, policy_index, timestamp=None):
"""
Update the container_stat policy_index and status_changed_at.
"""
if timestamp is None:
timestamp = Timestamp(time.time()).internal
def _setit(conn):
conn.execute('''
INSERT OR IGNORE INTO policy_stat (storage_policy_index)
VALUES (?)
''', (policy_index,))
conn.execute('''
UPDATE container_stat
SET storage_policy_index = ?,
status_changed_at = MAX(?, status_changed_at)
WHERE storage_policy_index <> ?
''', (policy_index, timestamp, policy_index))
conn.commit()
with self.get() as conn:
try:
_setit(conn)
except sqlite3.OperationalError as err:
if not any(msg in str(err) for msg in (
"no such column: storage_policy_index",
"no such table: policy_stat")):
raise
self._migrate_add_storage_policy(conn)
_setit(conn)
self._storage_policy_index = policy_index
def reported(self, put_timestamp, delete_timestamp, object_count,
bytes_used):
"""
@ -356,7 +571,7 @@ class ContainerBroker(DatabaseBroker):
conn.commit()
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter,
path=None):
path=None, storage_policy_index=0):
"""
Get a list of objects sorted by name starting at marker onward, up
to limit entries. Entries will begin with the prefix and will not
@ -409,9 +624,27 @@ class ContainerBroker(DatabaseBroker):
query += ' +deleted = 0'
else:
query += ' deleted = 0'
query += ' ORDER BY name LIMIT ?'
query_args.append(limit - len(results))
curs = conn.execute(query, query_args)
orig_tail_query = '''
ORDER BY name LIMIT ?
'''
orig_tail_args = [limit - len(results)]
# storage policy filter
policy_tail_query = '''
AND storage_policy_index = ?
''' + orig_tail_query
policy_tail_args = [storage_policy_index] + orig_tail_args
tail_query, tail_args = \
policy_tail_query, policy_tail_args
try:
curs = conn.execute(query + tail_query,
tuple(query_args + tail_args))
except sqlite3.OperationalError as err:
if 'no such column: storage_policy_index' not in str(err):
raise
tail_query, tail_args = \
orig_tail_query, orig_tail_args
curs = conn.execute(query + tail_query,
tuple(query_args + tail_args))
curs.row_factory = None
if prefix is None:
@ -466,26 +699,34 @@ class ContainerBroker(DatabaseBroker):
'size', 'content_type', 'etag', 'deleted'}
:param source: if defined, update incoming_sync with the source
"""
with self.get() as conn:
def _really_merge_items(conn):
max_rowid = -1
for rec in item_list:
rec.setdefault('storage_policy_index', 0) # legacy
query = '''
DELETE FROM object
WHERE name = ? AND (created_at < ?)
AND storage_policy_index = ?
'''
if self.get_db_version(conn) >= 1:
query += ' AND deleted IN (0, 1)'
conn.execute(query, (rec['name'], rec['created_at']))
query = 'SELECT 1 FROM object WHERE name = ?'
conn.execute(query, (rec['name'], rec['created_at'],
rec['storage_policy_index']))
query = '''
SELECT 1 FROM object WHERE name = ?
AND storage_policy_index = ?
'''
if self.get_db_version(conn) >= 1:
query += ' AND deleted IN (0, 1)'
if not conn.execute(query, (rec['name'],)).fetchall():
if not conn.execute(query, (
rec['name'], rec['storage_policy_index'])).fetchall():
conn.execute('''
INSERT INTO object (name, created_at, size,
content_type, etag, deleted)
VALUES (?, ?, ?, ?, ?, ?)
content_type, etag, deleted, storage_policy_index)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', ([rec['name'], rec['created_at'], rec['size'],
rec['content_type'], rec['etag'], rec['deleted']]))
rec['content_type'], rec['etag'], rec['deleted'],
rec['storage_policy_index']]))
if source:
max_rowid = max(max_rowid, rec['ROWID'])
if source:
@ -500,3 +741,158 @@ class ContainerBroker(DatabaseBroker):
WHERE remote_id=?
''', (max_rowid, source))
conn.commit()
with self.get() as conn:
try:
return _really_merge_items(conn)
except sqlite3.OperationalError as err:
if 'no such column: storage_policy_index' not in str(err):
raise
self._migrate_add_storage_policy(conn)
return _really_merge_items(conn)
def get_reconciler_sync(self):
with self.get() as conn:
try:
return conn.execute('''
SELECT reconciler_sync_point FROM container_stat
''').fetchone()[0]
except sqlite3.OperationalError as err:
if "no such column: reconciler_sync_point" not in str(err):
raise
return -1
def update_reconciler_sync(self, point):
query = '''
UPDATE container_stat
SET reconciler_sync_point = ?
'''
with self.get() as conn:
try:
conn.execute(query, (point,))
except sqlite3.OperationalError as err:
if "no such column: reconciler_sync_point" not in str(err):
raise
self._migrate_add_storage_policy(conn)
conn.execute(query, (point,))
conn.commit()
def get_misplaced_since(self, start, count):
"""
Get a list of objects which are in a storage policy different
from the container's storage policy.
:param start: last reconciler sync point
:param count: maximum number of entries to get
:returns: list of dicts with keys: name, created_at, size,
content_type, etag, storage_policy_index
"""
qry = '''
SELECT ROWID, name, created_at, size, content_type, etag,
deleted, storage_policy_index
FROM object
WHERE ROWID > ?
AND storage_policy_index != (
SELECT storage_policy_index FROM container_stat LIMIT 1)
ORDER BY ROWID ASC LIMIT ?
'''
self._commit_puts_stale_ok()
with self.get() as conn:
try:
cur = conn.execute(qry, (start, count))
except sqlite3.OperationalError as err:
if "no such column: storage_policy_index" not in str(err):
raise
return []
return list(dict(row) for row in cur.fetchall())
def _migrate_add_container_sync_points(self, conn):
"""
Add the x_container_sync_point columns to the 'container_stat' table.
"""
conn.executescript('''
BEGIN;
ALTER TABLE container_stat
ADD COLUMN x_container_sync_point1 INTEGER DEFAULT -1;
ALTER TABLE container_stat
ADD COLUMN x_container_sync_point2 INTEGER DEFAULT -1;
COMMIT;
''')
def _migrate_add_storage_policy(self, conn):
"""
Migrate the container schema to support tracking objects from
multiple storage policies. If the container_stat table has any
pending migrations, they are applied now before copying into
container_info.
* create the 'policy_stat' table.
* copy the current 'object_count' and 'bytes_used' columns to a
row in the 'policy_stat' table.
* add the storage_policy_index column to the 'object' table.
* drop the 'object_insert' and 'object_delete' triggers.
* add the 'object_insert_policy_stat' and
'object_delete_policy_stat' triggers.
* create container_info table for non-policy container info
* insert values from container_stat into container_info
* drop container_stat table
* create container_stat view
"""
# I tried just getting the list of column names in the current
# container_stat table with a pragma table_info, but could never get
# it inside the same transaction as the DDL (non-DML) statements:
# https://docs.python.org/2/library/sqlite3.html
# #controlling-transactions
# So we just apply all pending migrations to container_stat and copy a
# static known list of column names into container_info.
try:
self._migrate_add_container_sync_points(conn)
except sqlite3.OperationalError as e:
if 'duplicate column' in str(e):
conn.execute('ROLLBACK;')
else:
raise
try:
conn.executescript("""
ALTER TABLE container_stat
ADD COLUMN metadata TEXT DEFAULT '';
""")
except sqlite3.OperationalError as e:
if 'duplicate column' not in str(e):
raise
column_names = ', '.join((
'account', 'container', 'created_at', 'put_timestamp',
'delete_timestamp', 'reported_put_timestamp',
'reported_object_count', 'reported_bytes_used', 'hash', 'id',
'status', 'status_changed_at', 'metadata',
'x_container_sync_point1', 'x_container_sync_point2'))
conn.executescript(
'BEGIN;' +
POLICY_STAT_TABLE_CREATE +
'''
INSERT INTO policy_stat (
storage_policy_index, object_count, bytes_used)
SELECT 0, object_count, bytes_used
FROM container_stat;
ALTER TABLE object
ADD COLUMN storage_policy_index INTEGER DEFAULT 0;
DROP TRIGGER object_insert;
DROP TRIGGER object_delete;
''' +
POLICY_STAT_TRIGGER_SCRIPT +
CONTAINER_INFO_TABLE_SCRIPT +
'''
INSERT INTO container_info (%s)
SELECT %s FROM container_stat;
DROP TABLE IF EXISTS container_stat;
''' % (column_names, column_names) +
CONTAINER_STAT_VIEW_SCRIPT +
'COMMIT;')

745
swift/container/reconciler.py Normal file
View File

@ -0,0 +1,745 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
from collections import defaultdict
import socket
import itertools
import logging
from eventlet import GreenPile, GreenPool, Timeout
from swift.common import constraints
from swift.common.daemon import Daemon
from swift.common.direct_client import (
direct_head_container, direct_delete_container_object,
direct_put_container_object, ClientException)
from swift.common.internal_client import InternalClient, UnexpectedResponse
from swift.common.storage_policy import POLICY_INDEX
from swift.common.utils import get_logger, split_path, quorum_size, \
FileLikeIter, Timestamp, last_modified_date_to_timestamp, \
LRUCache
MISPLACED_OBJECTS_ACCOUNT = '.misplaced_objects'
MISPLACED_OBJECTS_CONTAINER_DIVISOR = 3600 # 1 hour
CONTAINER_POLICY_TTL = 30
def cmp_policy_info(info, remote_info):
"""
You have to squint to see it, but the general strategy is just:
if either has been recreated:
return the newest (of the recreated)
else
return the oldest
I tried cleaning it up for awhile, but settled on just writing a bunch of
tests instead. Once you get an intuitive sense for the nuance here you
can try and see there's a better way to spell the boolean logic but it all
ends up looking sorta hairy.
:returns: -1 if info is correct, 1 if remote_info is better
"""
def is_deleted(info):
return (info['delete_timestamp'] > info['put_timestamp'] and
info.get('count', info.get('object_count', 0)) == 0)
deleted = is_deleted(info)
remote_deleted = is_deleted(remote_info)
if any([deleted, remote_deleted]):
if not deleted:
return -1
elif not remote_deleted:
return 1
return cmp(remote_info['status_changed_at'],
info['status_changed_at'])
def has_been_recreated(info):
return (info['put_timestamp'] > info['delete_timestamp'] >
Timestamp(0))
remote_recreated = has_been_recreated(remote_info)
recreated = has_been_recreated(info)
if any([remote_recreated, recreated]):
if not recreated:
return 1
elif not remote_recreated:
return -1
return cmp(remote_info['status_changed_at'],
info['status_changed_at'])
return cmp(info['status_changed_at'], remote_info['status_changed_at'])
def incorrect_policy_index(info, remote_info):
"""
Compare remote_info to info and decide if the remote storage policy index
should be used instead of ours.
"""
if 'storage_policy_index' not in remote_info:
return False
if remote_info['storage_policy_index'] == \
info['storage_policy_index']:
return False
return info['storage_policy_index'] != sorted(
[info, remote_info], cmp=cmp_policy_info)[0]['storage_policy_index']
def translate_container_headers_to_info(headers):
default_timestamp = Timestamp(0).internal
return {
'storage_policy_index': int(headers[POLICY_INDEX]),
'put_timestamp': headers.get('x-backend-put-timestamp',
default_timestamp),
'delete_timestamp': headers.get('x-backend-delete-timestamp',
default_timestamp),
'status_changed_at': headers.get('x-backend-status-changed-at',
default_timestamp),
}
def best_policy_index(headers):
container_info = map(translate_container_headers_to_info, headers)
container_info.sort(cmp=cmp_policy_info)
return container_info[0]['storage_policy_index']
def get_reconciler_container_name(obj_timestamp):
return str(int(Timestamp(obj_timestamp)) //
MISPLACED_OBJECTS_CONTAINER_DIVISOR *
MISPLACED_OBJECTS_CONTAINER_DIVISOR)
def get_reconciler_obj_name(policy_index, account, container, obj):
return "%(policy_index)d:/%(acc)s/%(con)s/%(obj)s" % {
'policy_index': policy_index, 'acc': account,
'con': container, 'obj': obj}
def get_reconciler_content_type(op):
try:
return {
'put': 'application/x-put',
'delete': 'application/x-delete',
}[op.lower()]
except KeyError:
raise ValueError('invalid operation type %r' % op)
def get_row_to_q_entry_translater(broker):
account = broker.account
container = broker.container
op_type = {
0: get_reconciler_content_type('put'),
1: get_reconciler_content_type('delete'),
}
def translater(obj_info):
name = get_reconciler_obj_name(obj_info['storage_policy_index'],
account, container,
obj_info['name'])
return {
'name': name,
'deleted': 0,
'created_at': obj_info['created_at'],
'etag': obj_info['created_at'],
'content_type': op_type[obj_info['deleted']],
'size': 0,
}
return translater
def add_to_reconciler_queue(container_ring, account, container, obj,
obj_policy_index, obj_timestamp, op,
force=False, conn_timeout=5, response_timeout=15):
"""
Add an object to the container reconciler's queue. This will cause the
container reconciler to move it from its current storage policy index to
the correct storage policy index.
:param container_ring: container ring
:param account: the misplaced object's account
:param container: the misplaced object's container
:param obj: the misplaced object
:param obj_policy_index: the policy index where the misplaced object
currently is
:param obj_timestamp: the misplaced object's X-Timestamp. We need this to
ensure that the reconciler doesn't overwrite a newer
object with an older one.
:param op: the method of the operation (DELETE or PUT)
:param force: over-write queue entries newer than obj_timestamp
:param conn_timeout: max time to wait for connection to container server
:param response_timeout: max time to wait for response from container
server
:returns: .misplaced_object container name, False on failure. "Success"
means a quorum of containers got the update.
"""
container_name = get_reconciler_container_name(obj_timestamp)
object_name = get_reconciler_obj_name(obj_policy_index, account,
container, obj)
if force:
# this allows an operator to re-enqueue an object that has
# already been popped from the queue to be reprocessed, but
# could potentially prevent out of order updates from making it
# into the queue
x_timestamp = Timestamp(time.time()).internal
else:
x_timestamp = obj_timestamp
q_op_type = get_reconciler_content_type(op)
headers = {
'X-Size': 0,
'X-Etag': obj_timestamp,
'X-Timestamp': x_timestamp,
'X-Content-Type': q_op_type,
}
def _check_success(*args, **kwargs):
try:
direct_put_container_object(*args, **kwargs)
return 1
except (ClientException, Timeout, socket.error):
return 0
pile = GreenPile()
part, nodes = container_ring.get_nodes(MISPLACED_OBJECTS_ACCOUNT,
container_name)
for node in nodes:
pile.spawn(_check_success, node, part, MISPLACED_OBJECTS_ACCOUNT,
container_name, object_name, headers=headers,
conn_timeout=conn_timeout,
response_timeout=response_timeout)
successes = sum(pile)
if successes >= quorum_size(len(nodes)):
return container_name
else:
return False
def slightly_later_timestamp(ts, offset=1):
return Timestamp(ts, offset=offset).internal
def parse_raw_obj(obj_info):
"""
Translate a reconciler container listing entry to a dictionary
containing the parts of the misplaced object queue entry.
:param obj_info: an entry in an a container listing with the
required keys: name, content_type, and hash
:returns: a queue entry dict with the keys: q_policy_index, account,
container, obj, q_op, q_ts, q_record, and path
"""
raw_obj_name = obj_info['name'].encode('utf-8')
policy_index, obj_name = raw_obj_name.split(':', 1)
q_policy_index = int(policy_index)
account, container, obj = split_path(obj_name, 3, 3, rest_with_last=True)
try:
q_op = {
'application/x-put': 'PUT',
'application/x-delete': 'DELETE',
}[obj_info['content_type']]
except KeyError:
raise ValueError('invalid operation type %r' %
obj_info.get('content_type', None))
return {
'q_policy_index': q_policy_index,
'account': account,
'container': container,
'obj': obj,
'q_op': q_op,
'q_ts': Timestamp(obj_info['hash']),
'q_record': last_modified_date_to_timestamp(
obj_info['last_modified']),
'path': '/%s/%s/%s' % (account, container, obj)
}
@LRUCache(maxtime=CONTAINER_POLICY_TTL)
def direct_get_container_policy_index(container_ring, account_name,
container_name):
"""
Talk directly to the primary container servers to figure out the storage
policy index for a given container.
:param container_ring: ring in which to look up the container locations
:param account_name: name of the container's account
:param container_name: name of the container
:returns: storage policy index, or None if it couldn't get a quorum
"""
def _eat_client_exception(*args):
try:
return direct_head_container(*args)
except ClientException as err:
if err.http_status == 404:
return err.http_headers
except (Timeout, socket.error):
pass
pile = GreenPile()
part, nodes = container_ring.get_nodes(account_name, container_name)
for node in nodes:
pile.spawn(_eat_client_exception, node, part, account_name,
container_name)
headers = [x for x in pile if x is not None]
if len(headers) < quorum_size(len(nodes)):
return
return best_policy_index(headers)
def direct_delete_container_entry(container_ring, account_name, container_name,
object_name, headers=None):
"""
Talk directly to the primary container servers to delete a particular
object listing. Does not talk to object servers; use this only when a
container entry does not actually have a corresponding object.
"""
pool = GreenPool()
part, nodes = container_ring.get_nodes(account_name, container_name)
for node in nodes:
pool.spawn_n(direct_delete_container_object, node, part, account_name,
container_name, object_name, headers=headers)
# This either worked or it didn't; if it didn't, we'll retry on the next
# reconciler loop when we see the queue entry again.
pool.waitall()
class ContainerReconciler(Daemon):
"""
Move objects that are in the wrong storage policy.
"""
def __init__(self, conf):
self.conf = conf
self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7))
self.interval = int(conf.get('interval', 30))
conf_path = conf.get('__file__') or \
'/etc/swift/container-reconciler.conf'
self.logger = get_logger(conf, log_route='container-reconciler')
request_tries = int(conf.get('request_tries') or 3)
self.swift = InternalClient(conf_path,
'Swift Container Reconciler',
request_tries)
self.stats = defaultdict(int)
self.last_stat_time = time.time()
def stats_log(self, metric, msg, *args, **kwargs):
"""
Update stats tracking for metric and emit log message.
"""
level = kwargs.pop('level', logging.DEBUG)
log_message = '%s: ' % metric + msg
self.logger.log(level, log_message, *args, **kwargs)
self.stats[metric] += 1
def log_stats(self, force=False):
"""
Dump stats to logger, noop when stats have been already been
logged in the last minute.
"""
now = time.time()
should_log = force or (now - self.last_stat_time > 60)
if should_log:
self.last_stat_time = now
self.logger.info('Reconciler Stats: %r', dict(**self.stats))
def pop_queue(self, container, obj, q_ts, q_record):
"""
Issue a delete object request to the container for the misplaced
object queue entry.
:param container: the misplaced objects container
:param q_ts: the timestamp of the misplaced object
:param q_record: the timestamp of the queue entry
N.B. q_ts will normally be the same time as q_record except when
an object was manually re-enqued.
"""
q_path = '/%s/%s/%s' % (MISPLACED_OBJECTS_ACCOUNT, container, obj)
x_timestamp = slightly_later_timestamp(q_record)
self.stats_log('pop_queue', 'remove %r (%f) from the queue (%s)',
q_path, q_ts, x_timestamp)
headers = {'X-Timestamp': x_timestamp}
direct_delete_container_entry(
self.swift.container_ring, MISPLACED_OBJECTS_ACCOUNT,
container, obj, headers=headers)
def throw_tombstones(self, account, container, obj, timestamp,
policy_index, path):
"""
Issue a delete object request to the given storage_policy.
:param account: the account name
:param container: the container name
:param account: the object name
:param timestamp: the timestamp of the object to delete
:param policy_index: the policy index to direct the request
:param path: the path to be used for logging
"""
x_timestamp = slightly_later_timestamp(timestamp)
self.stats_log('cleanup_attempt', '%r (%f) from policy_index '
'%s (%s) will be deleted',
path, timestamp, policy_index, x_timestamp)
headers = {
'X-Timestamp': x_timestamp,
'X-Backend-Storage-Policy-Index': policy_index,
}
success = False
try:
self.swift.delete_object(account, container, obj,
acceptable_statuses=(2, 404),
headers=headers)
except UnexpectedResponse as err:
self.stats_log('cleanup_failed', '%r (%f) was not cleaned up '
'in storage_policy %s (%s)', path, timestamp,
policy_index, err)
else:
success = True
self.stats_log('cleanup_success', '%r (%f) was successfully '
'removed from policy_index %s', path, timestamp,
policy_index)
return success
def _reconcile_object(self, account, container, obj, q_policy_index, q_ts,
q_op, path, **kwargs):
"""
Perform object reconciliation.
:param account: the account name of the misplaced object
:param container: the container name of the misplaced object
:param obj: the object name
:param q_policy_index: the policy index of the source indicated by the
queue entry.
:param q_ts: the timestamp of the misplaced object
:param q_op: the operation of the misplaced request
:param path: the full path of the misplaced object for logging
:returns: True to indicate the request is fully processed
successfully, otherwise False.
"""
container_policy_index = direct_get_container_policy_index(
self.swift.container_ring, account, container)
if container_policy_index is None:
self.stats_log('unavailable_container', '%r (%f) unable to '
'determine the destination policy_index',
path, q_ts)
return False
if container_policy_index == q_policy_index:
self.stats_log('noop_object', '%r (%f) container policy_index '
'%s matches queue policy index %s', path, q_ts,
container_policy_index, q_policy_index)
return True
# check if object exists in the destination already
self.logger.debug('checking for %r (%f) in destination '
'policy_index %s', path, q_ts,
container_policy_index)
headers = {
'X-Backend-Storage-Policy-Index': container_policy_index}
dest_obj = self.swift.get_object_metadata(account, container, obj,
headers=headers,
acceptable_statuses=(2, 4))
dest_ts = Timestamp(dest_obj.get('x-backend-timestamp', 0))
if dest_ts >= q_ts:
self.stats_log('found_object', '%r (%f) in policy_index %s '
'is newer than queue (%f)', path, dest_ts,
container_policy_index, q_ts)
return self.throw_tombstones(account, container, obj, q_ts,
q_policy_index, path)
# object is misplaced
self.stats_log('misplaced_object', '%r (%f) in policy_index %s '
'should be in policy_index %s', path, q_ts,
q_policy_index, container_policy_index)
# fetch object from the source location
self.logger.debug('fetching %r (%f) from storage policy %s', path,
q_ts, q_policy_index)
headers = {
'X-Backend-Storage-Policy-Index': q_policy_index}
try:
source_obj_status, source_obj_info, source_obj_iter = \
self.swift.get_object(account, container, obj,
headers=headers,
acceptable_statuses=(2, 4))
except UnexpectedResponse as err:
source_obj_status = err.resp.status_int
source_obj_info = {}
source_obj_iter = None
source_ts = Timestamp(source_obj_info.get('x-backend-timestamp', 0))
if source_obj_status == 404 and q_op == 'DELETE':
return self.ensure_tombstone_in_right_location(
q_policy_index, account, container, obj, q_ts, path,
container_policy_index, source_ts)
else:
return self.ensure_object_in_right_location(
q_policy_index, account, container, obj, q_ts, path,
container_policy_index, source_ts, source_obj_status,
source_obj_info, source_obj_iter)
def ensure_object_in_right_location(self, q_policy_index, account,
container, obj, q_ts, path,
container_policy_index, source_ts,
source_obj_status, source_obj_info,
source_obj_iter, **kwargs):
"""
Validate source object will satisfy the misplaced object queue entry
and move to destination.
:param q_policy_index: the policy_index for the source object
:param account: the account name of the misplaced object
:param container: the container name of the misplaced object
:param obj: the name of the misplaced object
:param q_ts: the timestamp of the misplaced object
:param path: the full path of the misplaced object for logging
:param container_policy_index: the policy_index of the destination
:param source_ts: the timestamp of the source object
:param source_obj_status: the HTTP status source object request
:param source_obj_info: the HTTP headers of the source object request
:param source_obj_iter: the body iter of the source object request
"""
if source_obj_status // 100 != 2 or source_ts < q_ts:
if q_ts < time.time() - self.reclaim_age:
# it's old and there are no tombstones or anything; give up
self.stats_log('lost_source', '%r (%s) was not available in '
'policy_index %s and has expired', path,
q_ts.internal, q_policy_index,
level=logging.CRITICAL)
return True
# the source object is unavailable or older than the queue
# entry; a version that will satisfy the queue entry hopefully
# exists somewhere in the cluster, so wait and try again
self.stats_log('unavailable_source', '%r (%s) in '
'policy_index %s responded %s (%s)', path,
q_ts.internal, q_policy_index, source_obj_status,
source_ts.internal, level=logging.WARNING)
return False
# optimistically move any source with a timestamp >= q_ts
ts = max(Timestamp(source_ts), q_ts)
# move the object
put_timestamp = slightly_later_timestamp(ts, offset=2)
self.stats_log('copy_attempt', '%r (%f) in policy_index %s will be '
'moved to policy_index %s (%s)', path, source_ts,
q_policy_index, container_policy_index, put_timestamp)
headers = source_obj_info.copy()
headers['X-Backend-Storage-Policy-Index'] = container_policy_index
headers['X-Timestamp'] = put_timestamp
try:
self.swift.upload_object(
FileLikeIter(source_obj_iter), account, container, obj,
headers=headers)
except UnexpectedResponse as err:
self.stats_log('copy_failed', 'upload %r (%f) from '
'policy_index %s to policy_index %s '
'returned %s', path, source_ts, q_policy_index,
container_policy_index, err, level=logging.WARNING)
return False
except: # noqa
self.stats_log('unhandled_error', 'unable to upload %r (%f) '
'from policy_index %s to policy_index %s ', path,
source_ts, q_policy_index, container_policy_index,
level=logging.ERROR, exc_info=True)
return False
self.stats_log('copy_success', '%r (%f) moved from policy_index %s '
'to policy_index %s (%s)', path, source_ts,
q_policy_index, container_policy_index, put_timestamp)
return self.throw_tombstones(account, container, obj, q_ts,
q_policy_index, path)
def ensure_tombstone_in_right_location(self, q_policy_index, account,
container, obj, q_ts, path,
container_policy_index, source_ts,
**kwargs):
"""
Issue a DELETE request against the destination to match the
misplaced DELETE against the source.
"""
delete_timestamp = slightly_later_timestamp(q_ts, offset=2)
self.stats_log('delete_attempt', '%r (%f) in policy_index %s '
'will be deleted from policy_index %s (%s)', path,
source_ts, q_policy_index, container_policy_index,
delete_timestamp)
headers = {
'X-Backend-Storage-Policy-Index': container_policy_index,
'X-Timestamp': delete_timestamp,
}
try:
self.swift.delete_object(account, container, obj,
headers=headers)
except UnexpectedResponse as err:
self.stats_log('delete_failed', 'delete %r (%f) from '
'policy_index %s (%s) returned %s', path,
source_ts, container_policy_index,
delete_timestamp, err, level=logging.WARNING)
return False
except: # noqa
self.stats_log('unhandled_error', 'unable to delete %r (%f) '
'from policy_index %s (%s)', path, source_ts,
container_policy_index, delete_timestamp,
level=logging.ERROR, exc_info=True)
return False
self.stats_log('delete_success', '%r (%f) deleted from '
'policy_index %s (%s)', path, source_ts,
container_policy_index, delete_timestamp,
level=logging.INFO)
return self.throw_tombstones(account, container, obj, q_ts,
q_policy_index, path)
def reconcile_object(self, info):
"""
Process a possibly misplaced object write request. Determine correct
destination storage policy by checking with primary containers. Check
source and destination, copying or deleting into destination and
cleaning up the source as needed.
This method wraps _reconcile_object for exception handling.
:param info: a queue entry dict
:returns: True to indicate the request is fully processed
successfully, otherwise False.
"""
self.logger.debug('checking placement for %r (%f) '
'in policy_index %s', info['path'],
info['q_ts'], info['q_policy_index'])
success = False
try:
success = self._reconcile_object(**info)
except: # noqa
self.logger.exception('Unhandled Exception trying to '
'reconcile %r (%f) in policy_index %s',
info['path'], info['q_ts'],
info['q_policy_index'])
if success:
metric = 'success'
msg = 'was handled successfully'
else:
metric = 'retry'
msg = 'must be retried'
msg = '%(path)r (%(q_ts)f) in policy_index %(q_policy_index)s ' + msg
self.stats_log(metric, msg, info, level=logging.INFO)
self.log_stats()
return success
def _iter_containers(self):
"""
Generate a list of containers to process.
"""
# hit most recent container first instead of waiting on the updaters
current_container = get_reconciler_container_name(time.time())
yield current_container
container_gen = self.swift.iter_containers(MISPLACED_OBJECTS_ACCOUNT)
self.logger.debug('looking for containers in %s',
MISPLACED_OBJECTS_ACCOUNT)
while True:
one_page = None
try:
one_page = list(itertools.islice(
container_gen, constraints.CONTAINER_LISTING_LIMIT))
except UnexpectedResponse as err:
self.logger.error('Error listing containers in '
'account %s (%s)',
MISPLACED_OBJECTS_ACCOUNT, err)
if not one_page:
# don't generally expect more than one page
break
# reversed order since we expect older containers to be empty
for c in reversed(one_page):
# encoding here is defensive
container = c['name'].encode('utf8')
if container == current_container:
continue # we've already hit this one this pass
yield container
def _iter_objects(self, container):
"""
Generate a list of objects to process.
:param container: the name of the container to process
If the given container is empty and older than reclaim_age this
processor will attempt to reap it.
"""
self.logger.debug('looking for objects in %s', container)
found_obj = False
try:
for raw_obj in self.swift.iter_objects(
MISPLACED_OBJECTS_ACCOUNT, container):
found_obj = True
yield raw_obj
except UnexpectedResponse as err:
self.logger.error('Error listing objects in container %s (%s)',
container, err)
if float(container) < time.time() - self.reclaim_age and \
not found_obj:
# Try to delete old empty containers so the queue doesn't
# grow without bound. It's ok if there's a conflict.
self.swift.delete_container(
MISPLACED_OBJECTS_ACCOUNT, container,
acceptable_statuses=(2, 404, 409, 412))
def reconcile(self):
"""
Main entry point for processing misplaced objects.
Iterate over all queue entries and delegate to reconcile_object.
"""
self.logger.debug('pulling items from the queue')
for container in self._iter_containers():
for raw_obj in self._iter_objects(container):
try:
obj_info = parse_raw_obj(raw_obj)
except Exception:
self.stats_log('invalid_record',
'invalid queue record: %r', raw_obj,
level=logging.ERROR, exc_info=True)
continue
finished = self.reconcile_object(obj_info)
if finished:
self.pop_queue(container, raw_obj['name'],
obj_info['q_ts'],
obj_info['q_record'])
self.log_stats()
self.logger.debug('finished container %s', container)
def run_once(self, *args, **kwargs):
"""
Process every entry in the queue.
"""
try:
self.reconcile()
except:
self.logger.exception('Unhandled Exception trying to reconcile')
self.log_stats(force=True)
def run_forever(self, *args, **kwargs):
while True:
self.run_once(*args, **kwargs)
self.stats = defaultdict(int)
self.logger.info('sleeping between intervals (%ss)', self.interval)
time.sleep(self.interval)

233
swift/container/replicator.py
View File

@ -13,8 +13,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import itertools
import time
from collections import defaultdict
from eventlet import Timeout
from swift.container.backend import ContainerBroker, DATADIR
from swift.container.reconciler import (
MISPLACED_OBJECTS_ACCOUNT, incorrect_policy_index,
get_reconciler_container_name, get_row_to_q_entry_translater)
from swift.common import db_replicator
from swift.common.storage_policy import POLICIES
from swift.common.exceptions import DeviceUnavailable
from swift.common.http import is_success
from swift.common.db import DatabaseAlreadyExists
from swift.common.utils import (json, Timestamp, hash_path,
storage_directory, quorum_size)
class ContainerReplicator(db_replicator.Replicator):
@ -29,3 +44,221 @@ class ContainerReplicator(db_replicator.Replicator):
if full_info['reported_' + key] != full_info[key]:
return False
return True
def _gather_sync_args(self, replication_info):
parent = super(ContainerReplicator, self)
sync_args = parent._gather_sync_args(replication_info)
if len(POLICIES) > 1:
sync_args += tuple(replication_info[k] for k in
('status_changed_at', 'count',
'storage_policy_index'))
return sync_args
def _handle_sync_response(self, node, response, info, broker, http):
parent = super(ContainerReplicator, self)
if is_success(response.status):
remote_info = json.loads(response.data)
if incorrect_policy_index(info, remote_info):
status_changed_at = Timestamp(time.time())
broker.set_storage_policy_index(
remote_info['storage_policy_index'],
timestamp=status_changed_at.internal)
broker.merge_timestamps(*(remote_info[key] for key in (
'created_at', 'put_timestamp', 'delete_timestamp')))
rv = parent._handle_sync_response(
node, response, info, broker, http)
return rv
def find_local_handoff_for_part(self, part):
"""
Look through devices in the ring for the first handoff devie that was
identified during job creation as available on this node.
:returns: a node entry from the ring
"""
nodes = self.ring.get_part_nodes(part)
more_nodes = self.ring.get_more_nodes(part)
for node in itertools.chain(nodes, more_nodes):
if node['id'] in self._local_device_ids:
return node
return None
def get_reconciler_broker(self, timestamp):
"""
Get a local instance of the reconciler container broker that is
appropriate to enqueue the given timestamp.
:param timestamp: the timestamp of the row to be enqueued
:returns: a local reconciler broker
"""
container = get_reconciler_container_name(timestamp)
if self.reconciler_containers and \
container in self.reconciler_containers:
return self.reconciler_containers[container][1]
account = MISPLACED_OBJECTS_ACCOUNT
part = self.ring.get_part(account, container)
node = self.find_local_handoff_for_part(part)
if not node:
raise DeviceUnavailable(
'No mounted devices found suitable to Handoff reconciler '
'container %s in partition %s' % (container, part))
hsh = hash_path(account, container)
db_dir = storage_directory(DATADIR, part, hsh)
db_path = os.path.join(self.root, node['device'], db_dir, hsh + '.db')
broker = ContainerBroker(db_path, account=account, container=container)
if not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp, 0)
except DatabaseAlreadyExists:
pass
if self.reconciler_containers is not None:
self.reconciler_containers[container] = part, broker, node['id']
return broker
def feed_reconciler(self, container, item_list):
"""
Add queue entries for rows in item_list to the local reconciler
container database.
:param container: the name of the reconciler container
:param item_list: the list of rows to enqueue
:returns: True if successfully enqueued
"""
try:
reconciler = self.get_reconciler_broker(container)
except DeviceUnavailable as e:
self.logger.warning('DeviceUnavailable: %s', e)
return False
self.logger.debug('Adding %d objects to the reconciler at %s',
len(item_list), reconciler.db_file)
try:
reconciler.merge_items(item_list)
except (Exception, Timeout):
self.logger.exception('UNHANDLED EXCEPTION: trying to merge '
'%d items to reconciler container %s',
len(item_list), reconciler.db_file)
return False
return True
def dump_to_reconciler(self, broker, point):
"""
Look for object rows for objects updates in the wrong storage policy
in broker with a ``ROWID`` greater than the rowid given as point.
:param broker: the container broker with misplaced objects
:param point: the last verified ``reconciler_sync_point``
:returns: the last successfull enqueued rowid
"""
max_sync = broker.get_max_row()
misplaced = broker.get_misplaced_since(point, self.per_diff)
if not misplaced:
return max_sync
translater = get_row_to_q_entry_translater(broker)
errors = False
low_sync = point
while misplaced:
batches = defaultdict(list)
for item in misplaced:
container = get_reconciler_container_name(item['created_at'])
batches[container].append(translater(item))
for container, item_list in batches.items():
success = self.feed_reconciler(container, item_list)
if not success:
errors = True
point = misplaced[-1]['ROWID']
if not errors:
low_sync = point
misplaced = broker.get_misplaced_since(point, self.per_diff)
return low_sync
def _post_replicate_hook(self, broker, info, responses):
if info['account'] == MISPLACED_OBJECTS_ACCOUNT:
return
if not broker.has_multiple_policies():
broker.update_reconciler_sync(info['max_row'])
return
point = broker.get_reconciler_sync()
max_sync = self.dump_to_reconciler(broker, point)
success = responses.count(True) >= quorum_size(len(responses))
if max_sync > point and success:
# to be safe, only slide up the sync point with a quorum on
# replication
broker.update_reconciler_sync(max_sync)
def delete_db(self, broker):
"""
Ensure that reconciler databases are only cleaned up at the end of the
replication run.
"""
if (self.reconciler_cleanups is not None and
broker.account == MISPLACED_OBJECTS_ACCOUNT):
# this container shouldn't be here, make sure it's cleaned up
self.reconciler_cleanups[broker.container] = broker
return
return super(ContainerReplicator, self).delete_db(broker)
def replicate_reconcilers(self):
"""
Ensure any items merged to reconciler containers during replication
are pushed out to correct nodes and any reconciler containers that do
not belong on this node are removed.
"""
self.logger.info('Replicating %d reconciler containers',
len(self.reconciler_containers))
for part, reconciler, node_id in self.reconciler_containers.values():
self.cpool.spawn_n(
self._replicate_object, part, reconciler.db_file, node_id)
self.cpool.waitall()
# wipe out the cache do disable bypass in delete_db
cleanups = self.reconciler_cleanups
self.reconciler_cleanups = self.reconciler_containers = None
self.logger.info('Cleaning up %d reconciler containers',
len(cleanups))
for reconciler in cleanups.values():
self.cpool.spawn_n(self.delete_db, reconciler)
self.cpool.waitall()
self.logger.info('Finished reconciler replication')
def run_once(self, *args, **kwargs):
self.reconciler_containers = {}
self.reconciler_cleanups = {}
rv = super(ContainerReplicator, self).run_once(*args, **kwargs)
if any([self.reconciler_containers, self.reconciler_cleanups]):
self.replicate_reconcilers()
return rv
class ContainerReplicatorRpc(db_replicator.ReplicatorRpc):
def _parse_sync_args(self, args):
parent = super(ContainerReplicatorRpc, self)
remote_info = parent._parse_sync_args(args)
if len(args) > 9:
remote_info['status_changed_at'] = args[7]
remote_info['count'] = args[8]
remote_info['storage_policy_index'] = args[9]
return remote_info
def _get_synced_replication_info(self, broker, remote_info):
"""
Sync the remote_info storage_policy_index if needed and return the
newly synced replication info.
:param broker: the database broker
:param remote_info: the remote replication info
:returns: local broker replication info
"""
info = broker.get_replication_info()
if incorrect_policy_index(info, remote_info):
status_changed_at = Timestamp(time.time()).internal
broker.set_storage_policy_index(
remote_info['storage_policy_index'],
timestamp=status_changed_at)
info = broker.get_replication_info()
return info

212
swift/container/server.py
View File

@ -16,7 +16,6 @@
import os
import time
import traceback
from datetime import datetime
from swift import gettext_ as _
from xml.etree.cElementTree import Element, SubElement, tostring
@ -24,26 +23,54 @@ from eventlet import Timeout
import swift.common.db
from swift.container.backend import ContainerBroker, DATADIR
from swift.container.replicator import ContainerReplicatorRpc
from swift.common.db import DatabaseAlreadyExists
from swift.common.container_sync_realms import ContainerSyncRealms
from swift.common.request_helpers import get_param, get_listing_content_type, \
split_and_validate_path, is_sys_or_user_meta
from swift.common.utils import get_logger, hash_path, public, \
normalize_timestamp, storage_directory, validate_sync_to, \
Timestamp, storage_directory, validate_sync_to, \
config_true_value, json, timing_stats, replication, \
override_bytes_from_content_type, get_log_line
from swift.common.constraints import check_mount, check_float, check_utf8
from swift.common.constraints import check_mount, valid_timestamp, check_utf8
from swift.common import constraints
from swift.common.bufferedhttp import http_connect
from swift.common.exceptions import ConnectionTimeout
from swift.common.db_replicator import ReplicatorRpc
from swift.common.http import HTTP_NOT_FOUND, is_success
from swift.common.storage_policy import POLICIES, POLICY_INDEX
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPConflict, \
HTTPCreated, HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \
HTTPPreconditionFailed, HTTPMethodNotAllowed, Request, Response, \
HTTPInsufficientStorage, HTTPException, HeaderKeyDict
def gen_resp_headers(info, is_deleted=False):
"""
Convert container info dict to headers.
"""
# backend headers are always included
headers = {
'X-Backend-Timestamp': Timestamp(info.get('created_at', 0)).internal,
'X-Backend-PUT-Timestamp': Timestamp(info.get(
'put_timestamp', 0)).internal,
'X-Backend-DELETE-Timestamp': Timestamp(
info.get('delete_timestamp', 0)).internal,
'X-Backend-Status-Changed-At': Timestamp(
info.get('status_changed_at', 0)).internal,
POLICY_INDEX: info.get('storage_policy_index', 0),
}
if not is_deleted:
# base container info on deleted containers is not exposed to client
headers.update({
'X-Container-Object-Count': info.get('object_count', 0),
'X-Container-Bytes-Used': info.get('bytes_used', 0),
'X-Timestamp': Timestamp(info.get('created_at', 0)).normal,
'X-PUT-Timestamp': Timestamp(
info.get('put_timestamp', 0)).normal,
})
return headers
class ContainerController(object):
"""WSGI Controller for the container server."""
@ -74,7 +101,7 @@ class ContainerController(object):
h.strip()
for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',')
if h.strip()]
self.replicator_rpc = ReplicatorRpc(
self.replicator_rpc = ContainerReplicatorRpc(
self.root, DATADIR, ContainerBroker, self.mount_check,
logger=self.logger)
self.auto_create_account_prefix = \
@ -102,6 +129,32 @@ class ContainerController(object):
kwargs.setdefault('logger', self.logger)
return ContainerBroker(db_path, **kwargs)
def get_and_validate_policy_index(self, req):
"""
Validate that the index supplied maps to a policy.
:returns: policy index from request, or None if not present
:raises: HTTPBadRequest if the supplied index is bogus
"""
policy_index = req.headers.get(POLICY_INDEX, None)
if policy_index is None:
return None
try:
policy_index = int(policy_index)
except ValueError:
raise HTTPBadRequest(
request=req, content_type="text/plain",
body=("Invalid X-Storage-Policy-Index %r" % policy_index))
policy = POLICIES.get_by_index(policy_index)
if policy is None:
raise HTTPBadRequest(
request=req, content_type="text/plain",
body=("Invalid X-Storage-Policy-Index %r" % policy_index))
return int(policy)
def account_update(self, req, account, container, broker):
"""
Update the account server(s) with latest container info.
@ -149,6 +202,7 @@ class ContainerController(object):
'x-object-count': info['object_count'],
'x-bytes-used': info['bytes_used'],
'x-trans-id': req.headers.get('x-trans-id', '-'),
POLICY_INDEX: info['storage_policy_index'],
'user-agent': 'container-server %s' % os.getpid(),
'referer': req.as_referer()})
if req.headers.get('x-account-override-deleted', 'no').lower() == \
@ -190,32 +244,32 @@ class ContainerController(object):
"""Handle HTTP DELETE request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
if 'x-timestamp' not in req.headers or \
not check_float(req.headers['x-timestamp']):
return HTTPBadRequest(body='Missing timestamp', request=req,
content_type='text/plain')
req_timestamp = valid_timestamp(req)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
# policy index is only relevant for delete_obj (and transitively for
# auto create accounts)
obj_policy_index = self.get_and_validate_policy_index(req) or 0
broker = self._get_container_broker(drive, part, account, container)
if account.startswith(self.auto_create_account_prefix) and obj and \
not os.path.exists(broker.db_file):
try:
broker.initialize(normalize_timestamp(
req.headers.get('x-timestamp') or time.time()))
broker.initialize(req_timestamp.internal, obj_policy_index)
except DatabaseAlreadyExists:
pass
if not os.path.exists(broker.db_file):
return HTTPNotFound()
if obj: # delete object
broker.delete_object(obj, req.headers.get('x-timestamp'))
broker.delete_object(obj, req.headers.get('x-timestamp'),
obj_policy_index)
return HTTPNoContent(request=req)
else:
# delete container
if not broker.empty():
return HTTPConflict(request=req)
existed = float(broker.get_info()['put_timestamp']) and \
existed = Timestamp(broker.get_info()['put_timestamp']) and \
not broker.is_deleted()
broker.delete_db(req.headers['X-Timestamp'])
broker.delete_db(req_timestamp.internal)
if not broker.is_deleted():
return HTTPConflict(request=req)
resp = self.account_update(req, account, container, broker)
@ -225,19 +279,43 @@ class ContainerController(object):
return HTTPNoContent(request=req)
return HTTPNotFound()
def _update_or_create(self, req, broker, timestamp):
def _update_or_create(self, req, broker, timestamp, new_container_policy,
requested_policy_index):
"""
Create new database broker or update timestamps for existing database.
:param req: the swob request object
:param broker: the broker instance for the container
:param timestamp: internalized timestamp
:param new_container_policy: the storage policy index to use
when creating the container
:param requested_policy_index: the storage policy index sent in the
request, may be None
:returns: created, a bool, if database did not previously exist
"""
if not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp)
broker.initialize(timestamp, new_container_policy)
except DatabaseAlreadyExists:
pass
else:
return True # created
created = broker.is_deleted()
recreated = broker.is_deleted()
if recreated:
# only set storage policy on deleted containers
broker.set_storage_policy_index(new_container_policy,
timestamp=timestamp)
elif requested_policy_index is not None:
# validate requested policy with existing container
if requested_policy_index != broker.storage_policy_index:
raise HTTPConflict(request=req)
broker.update_put_timestamp(timestamp)
if broker.is_deleted():
raise HTTPConflict(request=req)
return created
if recreated:
broker.update_status_changed_at(timestamp)
return recreated
@public
@timing_stats()
@ -245,10 +323,7 @@ class ContainerController(object):
"""Handle HTTP PUT request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
if 'x-timestamp' not in req.headers or \
not check_float(req.headers['x-timestamp']):
return HTTPBadRequest(body='Missing timestamp', request=req,
content_type='text/plain')
req_timestamp = valid_timestamp(req)
if 'x-container-sync-to' in req.headers:
err, sync_to, realm, realm_key = validate_sync_to(
req.headers['x-container-sync-to'], self.allowed_sync_hosts,
@ -257,36 +332,49 @@ class ContainerController(object):
return HTTPBadRequest(err)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
timestamp = normalize_timestamp(req.headers['x-timestamp'])
requested_policy_index = self.get_and_validate_policy_index(req)
broker = self._get_container_broker(drive, part, account, container)
if obj: # put container object
# obj put expects the policy_index header, default is for
# legacy support during upgrade.
obj_policy_index = requested_policy_index or 0
if account.startswith(self.auto_create_account_prefix) and \
not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp)
broker.initialize(req_timestamp.internal, obj_policy_index)
except DatabaseAlreadyExists:
pass
if not os.path.exists(broker.db_file):
return HTTPNotFound()
broker.put_object(obj, timestamp, int(req.headers['x-size']),
broker.put_object(obj, req_timestamp.internal,
int(req.headers['x-size']),
req.headers['x-content-type'],
req.headers['x-etag'])
req.headers['x-etag'], 0,
obj_policy_index)
return HTTPCreated(request=req)
else: # put container
created = self._update_or_create(req, broker, timestamp)
if requested_policy_index is None:
# use the default index sent by the proxy if available
new_container_policy = req.headers.get(
'X-Backend-Storage-Policy-Default', int(POLICIES.default))
else:
new_container_policy = requested_policy_index
created = self._update_or_create(req, broker,
req_timestamp.internal,
new_container_policy,
requested_policy_index)
metadata = {}
metadata.update(
(key, (value, timestamp))
(key, (value, req_timestamp.internal))
for key, value in req.headers.iteritems()
if key.lower() in self.save_headers or
is_sys_or_user_meta('container', key))
if metadata:
if 'X-Container-Sync-To' in metadata:
if 'X-Container-Sync-To' not in broker.metadata or \
metadata['X-Container-Sync-To'][0] != \
broker.metadata['X-Container-Sync-To'][0]:
broker.set_x_container_sync_points(-1, -1)
broker.update_metadata(metadata)
if 'X-Container-Sync-To' in metadata:
if 'X-Container-Sync-To' not in broker.metadata or \
metadata['X-Container-Sync-To'][0] != \
broker.metadata['X-Container-Sync-To'][0]:
broker.set_x_container_sync_points(-1, -1)
broker.update_metadata(metadata)
resp = self.account_update(req, account, container, broker)
if resp:
return resp
@ -307,15 +395,10 @@ class ContainerController(object):
broker = self._get_container_broker(drive, part, account, container,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return HTTPNotFound(request=req)
info = broker.get_info()
headers = {
'X-Container-Object-Count': info['object_count'],
'X-Container-Bytes-Used': info['bytes_used'],
'X-Timestamp': info['created_at'],
'X-PUT-Timestamp': info['put_timestamp'],
}
info, is_deleted = broker.get_info_is_deleted()
headers = gen_resp_headers(info, is_deleted=is_deleted)
if is_deleted:
return HTTPNotFound(request=req, headers=headers)
headers.update(
(key, value)
for key, (value, timestamp) in broker.metadata.iteritems()
@ -335,16 +418,12 @@ class ContainerController(object):
:params record: object entry record
:returns: modified record
"""
(name, created, size, content_type, etag) = record
(name, created, size, content_type, etag) = record[:5]
if content_type is None:
return {'subdir': name}
response = {'bytes': size, 'hash': etag, 'name': name,
'content_type': content_type}
last_modified = datetime.utcfromtimestamp(float(created)).isoformat()
# python isoformat() doesn't include msecs when zero
if len(last_modified) < len("1970-01-01T00:00:00.000000"):
last_modified += ".000000"
response['last_modified'] = last_modified
response['last_modified'] = Timestamp(created).isoformat
override_bytes_from_content_type(response, logger=self.logger)
return response
@ -377,22 +456,18 @@ class ContainerController(object):
broker = self._get_container_broker(drive, part, account, container,
pending_timeout=0.1,
stale_reads_ok=True)
if broker.is_deleted():
return HTTPNotFound(request=req)
info = broker.get_info()
container_list = broker.list_objects_iter(limit, marker, end_marker,
prefix, delimiter, path)
return self.create_listing(req, out_content_type, info,
info, is_deleted = broker.get_info_is_deleted()
resp_headers = gen_resp_headers(info, is_deleted=is_deleted)
if is_deleted:
return HTTPNotFound(request=req, headers=resp_headers)
container_list = broker.list_objects_iter(
limit, marker, end_marker, prefix, delimiter, path,
storage_policy_index=info['storage_policy_index'])
return self.create_listing(req, out_content_type, info, resp_headers,
broker.metadata, container_list, container)
def create_listing(self, req, out_content_type, info, metadata,
container_list, container):
resp_headers = {
'X-Container-Object-Count': info['object_count'],
'X-Container-Bytes-Used': info['bytes_used'],
'X-Timestamp': info['created_at'],
'X-PUT-Timestamp': info['put_timestamp'],
}
def create_listing(self, req, out_content_type, info, resp_headers,
metadata, container_list, container):
for key, (value, timestamp) in metadata.iteritems():
if value and (key.lower() in self.save_headers or
is_sys_or_user_meta('container', key)):
@ -452,10 +527,7 @@ class ContainerController(object):
def POST(self, req):
"""Handle HTTP POST request."""
drive, part, account, container = split_and_validate_path(req, 4)
if 'x-timestamp' not in req.headers or \
not check_float(req.headers['x-timestamp']):
return HTTPBadRequest(body='Missing or bad timestamp',
request=req, content_type='text/plain')
req_timestamp = valid_timestamp(req)
if 'x-container-sync-to' in req.headers:
err, sync_to, realm, realm_key = validate_sync_to(
req.headers['x-container-sync-to'], self.allowed_sync_hosts,
@ -467,10 +539,10 @@ class ContainerController(object):
broker = self._get_container_broker(drive, part, account, container)
if broker.is_deleted():
return HTTPNotFound(request=req)
timestamp = normalize_timestamp(req.headers['x-timestamp'])
metadata = {}
metadata.update(
(key, (value, timestamp)) for key, value in req.headers.iteritems()
(key, (value, req_timestamp.internal))
for key, value in req.headers.iteritems()
if key.lower() in self.save_headers or
is_sys_or_user_meta('container', key))
if metadata:

38
swift/container/sync.py
View File

@ -32,9 +32,10 @@ from swift.common.ring import Ring
from swift.common.utils import (
audit_location_generator, clean_content_type, config_true_value,
FileLikeIter, get_logger, hash_path, quote, urlparse, validate_sync_to,
whataremyips)
whataremyips, Timestamp)
from swift.common.daemon import Daemon
from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND
from swift.common.storage_policy import POLICIES, POLICY_INDEX
class ContainerSync(Daemon):
@ -99,11 +100,9 @@ class ContainerSync(Daemon):
section of the container-server.conf
:param container_ring: If None, the <swift_dir>/container.ring.gz will be
loaded. This is overridden by unit tests.
:param object_ring: If None, the <swift_dir>/object.ring.gz will be loaded.
This is overridden by unit tests.
"""
def __init__(self, conf, container_ring=None, object_ring=None):
def __init__(self, conf, container_ring=None):
#: The dict of configuration values from the [container-sync] section
#: of the container-server.conf.
self.conf = conf
@ -150,17 +149,24 @@ class ContainerSync(Daemon):
self.container_failures = 0
#: Time of last stats report.
self.reported = time()
swift_dir = conf.get('swift_dir', '/etc/swift')
self.swift_dir = conf.get('swift_dir', '/etc/swift')
#: swift.common.ring.Ring for locating containers.
self.container_ring = container_ring or Ring(swift_dir,
self.container_ring = container_ring or Ring(self.swift_dir,
ring_name='container')
#: swift.common.ring.Ring for locating objects.
self.object_ring = object_ring or Ring(swift_dir, ring_name='object')
self._myips = whataremyips()
self._myport = int(conf.get('bind_port', 6001))
swift.common.db.DB_PREALLOCATION = \
config_true_value(conf.get('db_preallocation', 'f'))
def get_object_ring(self, policy_idx):
"""
Get the ring object to use based on its policy.
:policy_idx: policy index as defined in swift.conf
:returns: appropriate ring object
"""
return POLICIES.get_object_ring(policy_idx, self.swift_dir)
def run_forever(self, *args, **kwargs):
"""
Runs container sync scans until stopped.
@ -361,20 +367,24 @@ class ContainerSync(Daemon):
self.logger.increment('deletes')
self.logger.timing_since('deletes.timing', start_time)
else:
part, nodes = self.object_ring.get_nodes(
info['account'], info['container'],
row['name'])
part, nodes = \
self.get_object_ring(info['storage_policy_index']). \
get_nodes(info['account'], info['container'],
row['name'])
shuffle(nodes)
exc = None
looking_for_timestamp = float(row['created_at'])
looking_for_timestamp = Timestamp(row['created_at'])
timestamp = -1
headers = body = None
headers_out = {POLICY_INDEX: str(info['storage_policy_index'])}
for node in nodes:
try:
these_headers, this_body = direct_get_object(
node, part, info['account'], info['container'],
row['name'], resp_chunk_size=65536)
this_timestamp = float(these_headers['x-timestamp'])
row['name'], headers=headers_out,
resp_chunk_size=65536)
this_timestamp = Timestamp(
these_headers['x-timestamp'])
if this_timestamp > timestamp:
timestamp = this_timestamp
headers = these_headers

13
swift/container/updater.py
View File

@ -30,9 +30,10 @@ from swift.common.bufferedhttp import http_connect
from swift.common.exceptions import ConnectionTimeout
from swift.common.ring import Ring
from swift.common.utils import get_logger, config_true_value, ismount, \
dump_recon_cache, quorum_size
dump_recon_cache, quorum_size, Timestamp
from swift.common.daemon import Daemon
from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR
from swift.common.storage_policy import POLICY_INDEX
class ContainerUpdater(Daemon):
@ -209,7 +210,7 @@ class ContainerUpdater(Daemon):
info = broker.get_info()
# Don't send updates if the container was auto-created since it
# definitely doesn't have up to date statistics.
if float(info['put_timestamp']) <= 0:
if Timestamp(info['put_timestamp']) <= 0:
return
if self.account_suppressions.get(info['account'], 0) > time.time():
return
@ -221,7 +222,8 @@ class ContainerUpdater(Daemon):
part, nodes = self.get_account_ring().get_nodes(info['account'])
events = [spawn(self.container_report, node, part, container,
info['put_timestamp'], info['delete_timestamp'],
info['object_count'], info['bytes_used'])
info['object_count'], info['bytes_used'],
info['storage_policy_index'])
for node in nodes]
successes = 0
for event in events:
@ -254,7 +256,8 @@ class ContainerUpdater(Daemon):
self.no_changes += 1
def container_report(self, node, part, container, put_timestamp,
delete_timestamp, count, bytes):
delete_timestamp, count, bytes,
storage_policy_index):
"""
Report container info to an account server.
@ -265,6 +268,7 @@ class ContainerUpdater(Daemon):
:param delete_timestamp: delete timestamp
:param count: object count in the container
:param bytes: bytes used in the container
:param storage_policy_index: the policy index for the container
"""
with ConnectionTimeout(self.conn_timeout):
try:
@ -274,6 +278,7 @@ class ContainerUpdater(Daemon):
'X-Object-Count': count,
'X-Bytes-Used': bytes,
'X-Account-Override-Deleted': 'yes',
POLICY_INDEX: storage_policy_index,
'user-agent': self.user_agent}
conn = http_connect(
node['ip'], node['port'], node['device'], part,

151
swift/obj/diskfile.py
View File

@ -49,7 +49,7 @@ from eventlet import Timeout
from swift import gettext_ as _
from swift.common.constraints import check_mount
from swift.common.utils import mkdirs, normalize_timestamp, \
from swift.common.utils import mkdirs, Timestamp, \
storage_directory, hash_path, renamer, fallocate, fsync, \
fdatasync, drop_buffer_cache, ThreadPool, lock_path, write_pickle, \
config_true_value, listdir, split_path, ismount, remove_file
@ -58,7 +58,8 @@ from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \
DiskFileDeleted, DiskFileError, DiskFileNotOpen, PathNotDir, \
ReplicationLockTimeout, DiskFileExpired
from swift.common.swob import multi_range_iterator
from swift.common.storage_policy import get_policy_string, POLICIES
from functools import partial
PICKLE_PROTOCOL = 2
ONE_WEEK = 604800
@ -67,8 +68,12 @@ METADATA_KEY = 'user.swift.metadata'
# These are system-set metadata keys that cannot be changed with a POST.
# They should be lowercase.
DATAFILE_SYSTEM_META = set('content-length content-type deleted etag'.split())
DATADIR = 'objects'
ASYNCDIR = 'async_pending'
DATADIR_BASE = 'objects'
ASYNCDIR_BASE = 'async_pending'
TMP_BASE = 'tmp'
get_data_dir = partial(get_policy_string, DATADIR_BASE)
get_async_dir = partial(get_policy_string, ASYNCDIR_BASE)
get_tmp_dir = partial(get_policy_string, TMP_BASE)
def read_metadata(fd):
@ -105,6 +110,37 @@ def write_metadata(fd, metadata):
key += 1
def extract_policy_index(obj_path):
"""
Extracts the policy index for an object (based on the name of the objects
directory) given the device-relative path to the object. Returns 0 in
the event that the path is malformed in some way.
The device-relative path is everything after the mount point; for example:
/srv/node/d42/objects-5/179/
485dc017205a81df3af616d917c90179/1401811134.873649.data
would have device-relative path:
objects-5/179/485dc017205a81df3af616d917c90179/1401811134.873649.data
:param obj_path: device-relative path of an object
:returns: storage policy index
"""
policy_idx = 0
try:
obj_portion = obj_path[obj_path.index(DATADIR_BASE):]
obj_dirname = obj_portion[:obj_portion.index('/')]
except Exception:
return policy_idx
if '-' in obj_dirname:
base, policy_idx = obj_dirname.split('-', 1)
if POLICIES.get_by_index(policy_idx) is None:
policy_idx = 0
return int(policy_idx)
def quarantine_renamer(device_path, corrupted_file_path):
"""
In the case that a file is corrupted, move it to a quarantined
@ -118,7 +154,9 @@ def quarantine_renamer(device_path, corrupted_file_path):
exceptions from rename
"""
from_dir = dirname(corrupted_file_path)
to_dir = join(device_path, 'quarantined', 'objects', basename(from_dir))
to_dir = join(device_path, 'quarantined',
get_data_dir(extract_policy_index(corrupted_file_path)),
basename(from_dir))
invalidate_hash(dirname(from_dir))
try:
renamer(from_dir, to_dir)
@ -190,7 +228,7 @@ def hash_cleanup_listdir(hsh_path, reclaim_age=ONE_WEEK):
if files[0].endswith('.ts'):
# remove tombstones older than reclaim_age
ts = files[0].rsplit('.', 1)[0]
if (time.time() - float(ts)) > reclaim_age:
if (time.time() - float(Timestamp(ts))) > reclaim_age:
remove_file(join(hsh_path, files[0]))
files.remove(files[0])
elif files:
@ -385,27 +423,40 @@ def object_audit_location_generator(devices, mount_check=True, logger=None,
logger.debug(
_('Skipping %s as it is not mounted'), device)
continue
datadir_path = os.path.join(devices, device, DATADIR)
partitions = listdir(datadir_path)
for partition in partitions:
part_path = os.path.join(datadir_path, partition)
# loop through object dirs for all policies
for dir in [dir for dir in os.listdir(os.path.join(devices, device))
if dir.startswith(DATADIR_BASE)]:
datadir_path = os.path.join(devices, device, dir)
# warn if the object dir doesn't match with a policy
policy_idx = 0
if '-' in dir:
base, policy_idx = dir.split('-', 1)
try:
suffixes = listdir(part_path)
except OSError as e:
if e.errno != errno.ENOTDIR:
raise
continue
for asuffix in suffixes:
suff_path = os.path.join(part_path, asuffix)
get_data_dir(policy_idx)
except ValueError:
if logger:
logger.warn(_('Directory %s does not map to a '
'valid policy') % dir)
partitions = listdir(datadir_path)
for partition in partitions:
part_path = os.path.join(datadir_path, partition)
try:
hashes = listdir(suff_path)
suffixes = listdir(part_path)
except OSError as e:
if e.errno != errno.ENOTDIR:
raise
continue
for hsh in hashes:
hsh_path = os.path.join(suff_path, hsh)
yield AuditLocation(hsh_path, device, partition)
for asuffix in suffixes:
suff_path = os.path.join(part_path, asuffix)
try:
hashes = listdir(suff_path)
except OSError as e:
if e.errno != errno.ENOTDIR:
raise
continue
for hsh in hashes:
hsh_path = os.path.join(suff_path, hsh)
yield AuditLocation(hsh_path, device, partition)
class DiskFileManager(object):
@ -493,25 +544,26 @@ class DiskFileManager(object):
yield True
def pickle_async_update(self, device, account, container, obj, data,
timestamp):
timestamp, policy_idx):
device_path = self.construct_dev_path(device)
async_dir = os.path.join(device_path, ASYNCDIR)
async_dir = os.path.join(device_path, get_async_dir(policy_idx))
ohash = hash_path(account, container, obj)
self.threadpools[device].run_in_thread(
write_pickle,
data,
os.path.join(async_dir, ohash[-3:], ohash + '-' +
normalize_timestamp(timestamp)),
os.path.join(device_path, 'tmp'))
Timestamp(timestamp).internal),
os.path.join(device_path, get_tmp_dir(policy_idx)))
self.logger.increment('async_pendings')
def get_diskfile(self, device, partition, account, container, obj,
**kwargs):
policy_idx=0, **kwargs):
dev_path = self.get_dev_path(device)
if not dev_path:
raise DiskFileDeviceUnavailable()
return DiskFile(self, dev_path, self.threadpools[device],
partition, account, container, obj, **kwargs)
partition, account, container, obj,
policy_idx=policy_idx, **kwargs)
def object_audit_location_generator(self, device_dirs=None):
return object_audit_location_generator(self.devices, self.mount_check,
@ -523,7 +575,8 @@ class DiskFileManager(object):
self, audit_location.path, dev_path,
audit_location.partition)
def get_diskfile_from_hash(self, device, partition, object_hash, **kwargs):
def get_diskfile_from_hash(self, device, partition, object_hash,
policy_idx, **kwargs):
"""
Returns a DiskFile instance for an object at the given
object_hash. Just in case someone thinks of refactoring, be
@ -537,7 +590,8 @@ class DiskFileManager(object):
if not dev_path:
raise DiskFileDeviceUnavailable()
object_path = os.path.join(
dev_path, DATADIR, partition, object_hash[-3:], object_hash)
dev_path, get_data_dir(policy_idx), partition, object_hash[-3:],
object_hash)
try:
filenames = hash_cleanup_listdir(object_path, self.reclaim_age)
except OSError as err:
@ -563,13 +617,15 @@ class DiskFileManager(object):
except ValueError:
raise DiskFileNotExist()
return DiskFile(self, dev_path, self.threadpools[device],
partition, account, container, obj, **kwargs)
partition, account, container, obj,
policy_idx=policy_idx, **kwargs)
def get_hashes(self, device, partition, suffix):
def get_hashes(self, device, partition, suffix, policy_idx):
dev_path = self.get_dev_path(device)
if not dev_path:
raise DiskFileDeviceUnavailable()
partition_path = os.path.join(dev_path, DATADIR, partition)
partition_path = os.path.join(dev_path, get_data_dir(policy_idx),
partition)
if not os.path.exists(partition_path):
mkdirs(partition_path)
suffixes = suffix.split('-') if suffix else []
@ -587,7 +643,7 @@ class DiskFileManager(object):
path, err)
return []
def yield_suffixes(self, device, partition):
def yield_suffixes(self, device, partition, policy_idx):
"""
Yields tuples of (full_path, suffix_only) for suffixes stored
on the given device and partition.
@ -595,7 +651,8 @@ class DiskFileManager(object):
dev_path = self.get_dev_path(device)
if not dev_path:
raise DiskFileDeviceUnavailable()
partition_path = os.path.join(dev_path, DATADIR, partition)
partition_path = os.path.join(dev_path, get_data_dir(policy_idx),
partition)
for suffix in self._listdir(partition_path):
if len(suffix) != 3:
continue
@ -605,7 +662,7 @@ class DiskFileManager(object):
continue
yield (os.path.join(partition_path, suffix), suffix)
def yield_hashes(self, device, partition, suffixes=None):
def yield_hashes(self, device, partition, policy_idx, suffixes=None):
"""
Yields tuples of (full_path, hash_only, timestamp) for object
information stored for the given device, partition, and
@ -618,9 +675,10 @@ class DiskFileManager(object):
if not dev_path:
raise DiskFileDeviceUnavailable()
if suffixes is None:
suffixes = self.yield_suffixes(device, partition)
suffixes = self.yield_suffixes(device, partition, policy_idx)
else:
partition_path = os.path.join(dev_path, DATADIR, partition)
partition_path = os.path.join(dev_path, get_data_dir(policy_idx),
partition)
suffixes = (
(os.path.join(partition_path, suffix), suffix)
for suffix in suffixes)
@ -736,7 +794,7 @@ class DiskFileWriter(object):
:param metadata: dictionary of metadata to be associated with the
object
"""
timestamp = normalize_timestamp(metadata['X-Timestamp'])
timestamp = Timestamp(metadata['X-Timestamp']).internal
metadata['name'] = self._name
target_path = join(self._datadir, timestamp + self._extension)
@ -937,10 +995,13 @@ class DiskFile(object):
:param account: account name for the object
:param container: container name for the object
:param obj: object name for the object
:param _datadir: override the full datadir otherwise constructed here
:param policy_idx: used to get the data dir when constructing it here
"""
def __init__(self, mgr, device_path, threadpool, partition,
account=None, container=None, obj=None, _datadir=None):
account=None, container=None, obj=None, _datadir=None,
policy_idx=0):
self._mgr = mgr
self._device_path = device_path
self._threadpool = threadpool or ThreadPool(nthreads=0)
@ -954,7 +1015,8 @@ class DiskFile(object):
self._obj = obj
name_hash = hash_path(account, container, obj)
self._datadir = join(
device_path, storage_directory(DATADIR, partition, name_hash))
device_path, storage_directory(get_data_dir(policy_idx),
partition, name_hash))
else:
# gets populated when we read the metadata
self._name = None
@ -962,7 +1024,7 @@ class DiskFile(object):
self._container = None
self._obj = None
self._datadir = None
self._tmpdir = join(device_path, 'tmp')
self._tmpdir = join(device_path, get_tmp_dir(policy_idx))
self._metadata = None
self._data_file = None
self._fp = None
@ -973,7 +1035,8 @@ class DiskFile(object):
else:
name_hash = hash_path(account, container, obj)
self._datadir = join(
device_path, storage_directory(DATADIR, partition, name_hash))
device_path, storage_directory(get_data_dir(policy_idx),
partition, name_hash))
@property
def account(self):
@ -997,7 +1060,7 @@ class DiskFile(object):
def timestamp(self):
if self._metadata is None:
raise DiskFileNotOpen()
return self._metadata.get('X-Timestamp')
return Timestamp(self._metadata.get('X-Timestamp'))
@classmethod
def from_hash_dir(cls, mgr, hash_dir_path, device_path, partition):
@ -1386,7 +1449,7 @@ class DiskFile(object):
:raises DiskFileError: this implementation will raise the same
errors as the `create()` method.
"""
timestamp = normalize_timestamp(timestamp)
timestamp = Timestamp(timestamp).internal
with self.create() as deleter:
deleter._extension = '.ts'

37
swift/obj/expirer.py
View File

@ -24,11 +24,13 @@ from eventlet import sleep, Timeout
from eventlet.greenpool import GreenPool
from swift.common.daemon import Daemon
from swift.common.internal_client import InternalClient
from swift.common.internal_client import InternalClient, UnexpectedResponse
from swift.common.utils import get_logger, dump_recon_cache
from swift.common.http import HTTP_NOT_FOUND, HTTP_CONFLICT, \
HTTP_PRECONDITION_FAILED
from swift.container.reconciler import direct_delete_container_entry
class ObjectExpirer(Daemon):
"""
@ -38,18 +40,17 @@ class ObjectExpirer(Daemon):
:param conf: The daemon configuration.
"""
def __init__(self, conf):
def __init__(self, conf, logger=None, swift=None):
self.conf = conf
self.logger = get_logger(conf, log_route='object-expirer')
self.logger = logger or get_logger(conf, log_route='object-expirer')
self.interval = int(conf.get('interval') or 300)
self.expiring_objects_account = \
(conf.get('auto_create_account_prefix') or '.') + \
(conf.get('expiring_objects_account_name') or 'expiring_objects')
conf_path = conf.get('__file__') or '/etc/swift/object-expirer.conf'
request_tries = int(conf.get('request_tries') or 3)
self.swift = InternalClient(conf_path,
'Swift Object Expirer',
request_tries)
self.swift = swift or InternalClient(
conf_path, 'Swift Object Expirer', request_tries)
self.report_interval = int(conf.get('report_interval') or 300)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
@ -61,6 +62,7 @@ class ObjectExpirer(Daemon):
raise ValueError("concurrency must be set to at least 1")
self.processes = int(self.conf.get('processes', 0))
self.process = int(self.conf.get('process', 0))
self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7))
def report(self, final=False):
"""
@ -200,9 +202,15 @@ class ObjectExpirer(Daemon):
def delete_object(self, actual_obj, timestamp, container, obj):
start_time = time()
try:
self.delete_actual_object(actual_obj, timestamp)
self.swift.delete_object(self.expiring_objects_account,
container, obj)
try:
self.delete_actual_object(actual_obj, timestamp)
except UnexpectedResponse as err:
if err.resp.status_int != HTTP_NOT_FOUND:
raise
if float(timestamp) > time() - self.reclaim_age:
# we'll have to retry the DELETE later
raise
self.pop_queue(container, obj)
self.report_objects += 1
self.logger.increment('objects')
except (Exception, Timeout) as err:
@ -213,6 +221,15 @@ class ObjectExpirer(Daemon):
self.logger.timing_since('timing', start_time)
self.report()
def pop_queue(self, container, obj):
"""
Issue a delete object request to the container for the expiring object
queue entry.
"""
direct_delete_container_entry(self.swift.container_ring,
self.expiring_objects_account,
container, obj)
def delete_actual_object(self, actual_obj, timestamp):
"""
Deletes the end-user object indicated by the actual object name given
@ -227,4 +244,4 @@ class ObjectExpirer(Daemon):
path = '/v1/' + urllib.quote(actual_obj.lstrip('/'))
self.swift.make_request('DELETE', path,
{'X-If-Delete-At': str(timestamp)},
(2, HTTP_NOT_FOUND, HTTP_PRECONDITION_FAILED))
(2, HTTP_PRECONDITION_FAILED))

5
swift/obj/mem_diskfile.py
View File

@ -22,7 +22,7 @@ from contextlib import contextmanager
from eventlet import Timeout
from swift.common.utils import normalize_timestamp
from swift.common.utils import Timestamp
from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \
DiskFileCollision, DiskFileDeleted, DiskFileNotOpen
from swift.common.swob import multi_range_iterator
@ -394,7 +394,6 @@ class DiskFile(object):
:param timestamp: timestamp to compare with each file
"""
timestamp = normalize_timestamp(timestamp)
fp, md = self._filesystem.get_object(self._name)
if md['X-Timestamp'] < timestamp:
if md['X-Timestamp'] < Timestamp(timestamp):
self._filesystem.del_object(self._name)

3
swift/obj/mem_server.py
View File

@ -54,7 +54,7 @@ class ObjectController(server.ObjectController):
return self._filesystem.get_diskfile(account, container, obj, **kwargs)
def async_update(self, op, account, container, obj, host, partition,
contdevice, headers_out, objdevice):
contdevice, headers_out, objdevice, policy_idx):
"""
Sends or saves an async update.
@ -68,6 +68,7 @@ class ObjectController(server.ObjectController):
:param headers_out: dictionary of headers to send in the container
request
:param objdevice: device name that the object is in
:param policy_idx: the associated storage policy index
"""
headers_out['user-agent'] = 'obj-server %s' % os.getpid()
full_path = '/%s/%s/%s' % (account, container, obj)

69
swift/obj/replicator.py
View File

@ -27,7 +27,6 @@ from eventlet import GreenPool, tpool, Timeout, sleep, hubs
from eventlet.green import subprocess
from eventlet.support.greenlets import GreenletExit
from swift.common.ring import Ring
from swift.common.utils import whataremyips, unlink_older_than, \
compute_eta, get_logger, dump_recon_cache, ismount, \
rsync_ip, mkdirs, config_true_value, list_from_csv, get_hub, \
@ -36,7 +35,9 @@ from swift.common.bufferedhttp import http_connect
from swift.common.daemon import Daemon
from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE
from swift.obj import ssync_sender
from swift.obj.diskfile import DiskFileManager, get_hashes
from swift.obj.diskfile import (DiskFileManager, get_hashes, get_data_dir,
get_tmp_dir)
from swift.common.storage_policy import POLICY_INDEX, POLICIES
hubs.use_hub(get_hub())
@ -65,7 +66,6 @@ class ObjectReplicator(Daemon):
self.port = int(conf.get('bind_port', 6000))
self.concurrency = int(conf.get('concurrency', 1))
self.stats_interval = int(conf.get('stats_interval', '300'))
self.object_ring = Ring(self.swift_dir, ring_name='object')
self.ring_check_interval = int(conf.get('ring_check_interval', 15))
self.next_check = time.time() + self.ring_check_interval
self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7))
@ -108,6 +108,15 @@ class ObjectReplicator(Daemon):
"""
return self.sync_method(node, job, suffixes)
def get_object_ring(self, policy_idx):
"""
Get the ring object to use to handle a request based on its policy.
:policy_idx: policy index as defined in swift.conf
:returns: appropriate ring object
"""
return POLICIES.get_object_ring(policy_idx, self.swift_dir)
def _rsync(self, args):
"""
Execute the rsync binary to replicate a partition.
@ -185,22 +194,24 @@ class ObjectReplicator(Daemon):
had_any = True
if not had_any:
return False
data_dir = get_data_dir(job['policy_idx'])
args.append(join(rsync_module, node['device'],
'objects', job['partition']))
data_dir, job['partition']))
return self._rsync(args) == 0
def ssync(self, node, job, suffixes):
return ssync_sender.Sender(self, node, job, suffixes)()
def check_ring(self):
def check_ring(self, object_ring):
"""
Check to see if the ring has been updated
:param object_ring: the ring to check
:returns: boolean indicating whether or not the ring has changed
"""
if time.time() > self.next_check:
self.next_check = time.time() + self.ring_check_interval
if self.object_ring.has_changed():
if object_ring.has_changed():
return False
return True
@ -217,6 +228,7 @@ class ObjectReplicator(Daemon):
if len(suff) == 3 and isdir(join(path, suff))]
self.replication_count += 1
self.logger.increment('partition.delete.count.%s' % (job['device'],))
self.headers[POLICY_INDEX] = job['policy_idx']
begin = time.time()
try:
responses = []
@ -258,6 +270,7 @@ class ObjectReplicator(Daemon):
"""
self.replication_count += 1
self.logger.increment('partition.update.count.%s' % (job['device'],))
self.headers[POLICY_INDEX] = job['policy_idx']
begin = time.time()
try:
hashed, local_hash = tpool_reraise(
@ -269,7 +282,7 @@ class ObjectReplicator(Daemon):
attempts_left = len(job['nodes'])
nodes = itertools.chain(
job['nodes'],
self.object_ring.get_more_nodes(int(job['partition'])))
job['object_ring'].get_more_nodes(int(job['partition'])))
while attempts_left > 0:
# If this throws StopIterator it will be caught way below
node = next(nodes)
@ -394,19 +407,19 @@ class ObjectReplicator(Daemon):
self.kill_coros()
self.last_replication_count = self.replication_count
def collect_jobs(self):
def process_repl(self, policy, jobs, ips):
"""
Returns a sorted list of jobs (dictionaries) that specify the
partitions, nodes, etc to be synced.
Helper function for collect_jobs to build jobs for replication
using replication style storage policy
"""
jobs = []
ips = whataremyips()
for local_dev in [dev for dev in self.object_ring.devs
obj_ring = self.get_object_ring(policy.idx)
data_dir = get_data_dir(policy.idx)
for local_dev in [dev for dev in obj_ring.devs
if dev and dev['replication_ip'] in ips and
dev['replication_port'] == self.port]:
dev_path = join(self.devices_dir, local_dev['device'])
obj_path = join(dev_path, 'objects')
tmp_path = join(dev_path, 'tmp')
obj_path = join(dev_path, data_dir)
tmp_path = join(dev_path, get_tmp_dir(int(policy)))
if self.mount_check and not ismount(dev_path):
self.logger.warn(_('%s is not mounted'), local_dev['device'])
continue
@ -423,12 +436,12 @@ class ObjectReplicator(Daemon):
if isfile(job_path):
# Clean up any (probably zero-byte) files where a
# partition should be.
self.logger.warning('Removing partition directory '
'which was a file: %s', job_path)
self.logger.warning(
'Removing partition directory '
'which was a file: %s', job_path)
os.remove(job_path)
continue
part_nodes = \
self.object_ring.get_part_nodes(int(partition))
part_nodes = obj_ring.get_part_nodes(int(partition))
nodes = [node for node in part_nodes
if node['id'] != local_dev['id']]
jobs.append(
@ -436,9 +449,23 @@ class ObjectReplicator(Daemon):
device=local_dev['device'],
nodes=nodes,
delete=len(nodes) > len(part_nodes) - 1,
partition=partition))
policy_idx=policy.idx,
partition=partition,
object_ring=obj_ring))
except (ValueError, OSError):
continue
def collect_jobs(self):
"""
Returns a sorted list of jobs (dictionaries) that specify the
partitions, nodes, etc to be rsynced.
"""
jobs = []
ips = whataremyips()
for policy in POLICIES:
# may need to branch here for future policy types
self.process_repl(policy, jobs, ips)
random.shuffle(jobs)
if self.handoffs_first:
# Move the handoff parts to the front of the list
@ -478,7 +505,7 @@ class ObjectReplicator(Daemon):
if self.mount_check and not ismount(dev_path):
self.logger.warn(_('%s is not mounted'), job['device'])
continue
if not self.check_ring():
if not self.check_ring(job['object_ring']):
self.logger.info(_("Ring change detected. Aborting "
"current replication pass."))
return

171
swift/obj/server.py
View File

@ -29,16 +29,16 @@ from eventlet import sleep, Timeout
from swift.common.utils import public, get_logger, \
config_true_value, timing_stats, replication, \
normalize_delete_at_timestamp, get_log_line
normalize_delete_at_timestamp, get_log_line, Timestamp
from swift.common.bufferedhttp import http_connect
from swift.common.constraints import check_object_creation, \
check_float, check_utf8
valid_timestamp, check_utf8
from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \
DiskFileNotExist, DiskFileCollision, DiskFileNoSpace, DiskFileDeleted, \
DiskFileDeviceUnavailable, DiskFileExpired, ChunkReadTimeout
from swift.obj import ssync_receiver
from swift.common.http import is_success
from swift.common.request_helpers import split_and_validate_path, is_user_meta
from swift.common.request_helpers import get_name_and_placement, is_user_meta
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \
HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
@ -46,6 +46,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
HTTPInsufficientStorage, HTTPForbidden, HTTPException, HeaderKeyDict, \
HTTPConflict
from swift.obj.diskfile import DATAFILE_SYSTEM_META, DiskFileManager
from swift.common.storage_policy import POLICY_INDEX
class ObjectController(object):
@ -90,8 +91,9 @@ class ObjectController(object):
for header in extra_allowed_headers:
if header not in DATAFILE_SYSTEM_META:
self.allowed_headers.add(header)
self.expiring_objects_account = \
(conf.get('auto_create_account_prefix') or '.') + \
self.auto_create_account_prefix = \
conf.get('auto_create_account_prefix') or '.'
self.expiring_objects_account = self.auto_create_account_prefix + \
(conf.get('expiring_objects_account_name') or 'expiring_objects')
self.expiring_objects_container_divisor = \
int(conf.get('expiring_objects_container_divisor') or 86400)
@ -139,7 +141,7 @@ class ObjectController(object):
conf.get('replication_failure_ratio') or 1.0)
def get_diskfile(self, device, partition, account, container, obj,
**kwargs):
policy_idx, **kwargs):
"""
Utility method for instantiating a DiskFile object supporting a given
REST API.
@ -149,10 +151,10 @@ class ObjectController(object):
behavior.
"""
return self._diskfile_mgr.get_diskfile(
device, partition, account, container, obj, **kwargs)
device, partition, account, container, obj, policy_idx, **kwargs)
def async_update(self, op, account, container, obj, host, partition,
contdevice, headers_out, objdevice):
contdevice, headers_out, objdevice, policy_index):
"""
Sends or saves an async update.
@ -166,6 +168,7 @@ class ObjectController(object):
:param headers_out: dictionary of headers to send in the container
request
:param objdevice: device name that the object is in
:param policy_index: the associated storage policy index
"""
headers_out['user-agent'] = 'obj-server %s' % os.getpid()
full_path = '/%s/%s/%s' % (account, container, obj)
@ -196,10 +199,11 @@ class ObjectController(object):
'obj': obj, 'headers': headers_out}
timestamp = headers_out['x-timestamp']
self._diskfile_mgr.pickle_async_update(objdevice, account, container,
obj, data, timestamp)
obj, data, timestamp,
policy_index)
def container_update(self, op, account, container, obj, request,
headers_out, objdevice):
headers_out, objdevice, policy_idx):
"""
Update the container when objects are updated.
@ -222,7 +226,7 @@ class ObjectController(object):
if len(conthosts) != len(contdevices):
# This shouldn't happen unless there's a bug in the proxy,
# but if there is, we want to know about it.
self.logger.error(_('ERROR Container update failed: different '
self.logger.error(_('ERROR Container update failed: different '
'numbers of hosts and devices in request: '
'"%s" vs "%s"') %
(headers_in.get('X-Container-Host', ''),
@ -236,13 +240,14 @@ class ObjectController(object):
headers_out['x-trans-id'] = headers_in.get('x-trans-id', '-')
headers_out['referer'] = request.as_referer()
headers_out[POLICY_INDEX] = policy_idx
for conthost, contdevice in updates:
self.async_update(op, account, container, obj, conthost,
contpartition, contdevice, headers_out,
objdevice)
objdevice, policy_idx)
def delete_at_update(self, op, delete_at, account, container, obj,
request, objdevice):
request, objdevice, policy_index):
"""
Update the expiring objects container when objects are updated.
@ -253,6 +258,7 @@ class ObjectController(object):
:param obj: object name
:param request: the original request driving the update
:param objdevice: device name that the object is in
:param policy_index: the policy index to be used for tmp dir
"""
if config_true_value(
request.headers.get('x-backend-replication', 'f')):
@ -264,7 +270,8 @@ class ObjectController(object):
hosts = contdevices = [None]
headers_in = request.headers
headers_out = HeaderKeyDict({
'x-timestamp': headers_in['x-timestamp'],
POLICY_INDEX: 0, # system accounts are always Policy-0
'x-timestamp': request.timestamp.internal,
'x-trans-id': headers_in.get('x-trans-id', '-'),
'referer': request.as_referer()})
if op != 'DELETE':
@ -309,36 +316,34 @@ class ObjectController(object):
self.async_update(
op, self.expiring_objects_account, delete_at_container,
'%s-%s/%s/%s' % (delete_at, account, container, obj),
host, partition, contdevice, headers_out, objdevice)
host, partition, contdevice, headers_out, objdevice,
policy_index)
@public
@timing_stats()
def POST(self, request):
"""Handle HTTP POST requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
if 'x-timestamp' not in request.headers or \
not check_float(request.headers['x-timestamp']):
return HTTPBadRequest(body='Missing timestamp', request=request,
content_type='text/plain')
device, partition, account, container, obj, policy_idx = \
get_name_and_placement(request, 5, 5, True)
req_timestamp = valid_timestamp(request)
new_delete_at = int(request.headers.get('X-Delete-At') or 0)
if new_delete_at and new_delete_at < time.time():
return HTTPBadRequest(body='X-Delete-At in past', request=request,
content_type='text/plain')
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj)
device, partition, account, container, obj,
policy_idx=policy_idx)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
orig_metadata = disk_file.read_metadata()
except (DiskFileNotExist, DiskFileQuarantined):
return HTTPNotFound(request=request)
orig_timestamp = orig_metadata.get('X-Timestamp', '0')
if orig_timestamp >= request.headers['x-timestamp']:
orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0))
if orig_timestamp >= req_timestamp:
return HTTPConflict(request=request)
metadata = {'X-Timestamp': request.headers['x-timestamp']}
metadata = {'X-Timestamp': req_timestamp.internal}
metadata.update(val for val in request.headers.iteritems()
if is_user_meta('object', val[0]))
for header_key in self.allowed_headers:
@ -349,10 +354,11 @@ class ObjectController(object):
if orig_delete_at != new_delete_at:
if new_delete_at:
self.delete_at_update('PUT', new_delete_at, account, container,
obj, request, device)
obj, request, device, policy_idx)
if orig_delete_at:
self.delete_at_update('DELETE', orig_delete_at, account,
container, obj, request, device)
container, obj, request, device,
policy_idx)
disk_file.write_metadata(metadata)
return HTTPAccepted(request=request)
@ -360,13 +366,9 @@ class ObjectController(object):
@timing_stats()
def PUT(self, request):
"""Handle HTTP PUT requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
if 'x-timestamp' not in request.headers or \
not check_float(request.headers['x-timestamp']):
return HTTPBadRequest(body='Missing timestamp', request=request,
content_type='text/plain')
device, partition, account, container, obj, policy_idx = \
get_name_and_placement(request, 5, 5, True)
req_timestamp = valid_timestamp(request)
error_response = check_object_creation(request, obj)
if error_response:
return error_response
@ -381,7 +383,8 @@ class ObjectController(object):
content_type='text/plain')
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj)
device, partition, account, container, obj,
policy_idx=policy_idx)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
@ -398,8 +401,8 @@ class ObjectController(object):
# The current ETag matches, so return 412
return HTTPPreconditionFailed(request=request)
orig_timestamp = orig_metadata.get('X-Timestamp')
if orig_timestamp and orig_timestamp >= request.headers['x-timestamp']:
orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0))
if orig_timestamp and orig_timestamp >= req_timestamp:
return HTTPConflict(request=request)
orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
upload_expiration = time.time() + self.max_upload_time
@ -436,7 +439,7 @@ class ObjectController(object):
request.headers['etag'].lower() != etag:
return HTTPUnprocessableEntity(request=request)
metadata = {
'X-Timestamp': request.headers['x-timestamp'],
'X-Timestamp': request.timestamp.internal,
'Content-Type': request.headers['content-type'],
'ETag': etag,
'Content-Length': str(upload_size),
@ -455,12 +458,12 @@ class ObjectController(object):
if orig_delete_at != new_delete_at:
if new_delete_at:
self.delete_at_update(
'PUT', new_delete_at, account, container, obj,
request, device)
'PUT', new_delete_at, account, container, obj, request,
device, policy_idx)
if orig_delete_at:
self.delete_at_update(
'DELETE', orig_delete_at, account, container, obj,
request, device)
request, device, policy_idx)
self.container_update(
'PUT', account, container, obj, request,
HeaderKeyDict({
@ -468,29 +471,29 @@ class ObjectController(object):
'x-content-type': metadata['Content-Type'],
'x-timestamp': metadata['X-Timestamp'],
'x-etag': metadata['ETag']}),
device)
device, policy_idx)
return HTTPCreated(request=request, etag=etag)
@public
@timing_stats()
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
device, partition, account, container, obj, policy_idx = \
get_name_and_placement(request, 5, 5, True)
keep_cache = self.keep_cache_private or (
'X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers)
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj)
device, partition, account, container, obj,
policy_idx=policy_idx)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
with disk_file.open():
metadata = disk_file.get_metadata()
obj_size = int(metadata['Content-Length'])
file_x_ts = metadata['X-Timestamp']
file_x_ts_flt = float(file_x_ts)
file_x_ts = Timestamp(metadata['X-Timestamp'])
keep_cache = (self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers))
@ -504,34 +507,44 @@ class ObjectController(object):
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = math.ceil(file_x_ts_flt)
response.last_modified = math.ceil(float(file_x_ts))
response.content_length = obj_size
try:
response.content_encoding = metadata[
'Content-Encoding']
except KeyError:
pass
response.headers['X-Timestamp'] = file_x_ts
response.headers['X-Timestamp'] = file_x_ts.normal
response.headers['X-Backend-Timestamp'] = file_x_ts.internal
resp = request.get_response(response)
except (DiskFileNotExist, DiskFileQuarantined):
resp = HTTPNotFound(request=request, conditional_response=True)
except (DiskFileNotExist, DiskFileQuarantined) as e:
headers = {}
if hasattr(e, 'timestamp'):
headers['X-Backend-Timestamp'] = e.timestamp.internal
resp = HTTPNotFound(request=request, headers=headers,
conditional_response=True)
return resp
@public
@timing_stats(sample_rate=0.8)
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
device, partition, account, container, obj, policy_idx = \
get_name_and_placement(request, 5, 5, True)
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj)
device, partition, account, container, obj,
policy_idx=policy_idx)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
metadata = disk_file.read_metadata()
except (DiskFileNotExist, DiskFileQuarantined):
return HTTPNotFound(request=request, conditional_response=True)
except (DiskFileNotExist, DiskFileQuarantined) as e:
headers = {}
if hasattr(e, 'timestamp'):
headers['X-Backend-Timestamp'] = e.timestamp.internal
return HTTPNotFound(request=request, headers=headers,
conditional_response=True)
response = Response(request=request, conditional_response=True)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
@ -540,10 +553,11 @@ class ObjectController(object):
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
ts = metadata['X-Timestamp']
ts = Timestamp(metadata['X-Timestamp'])
response.last_modified = math.ceil(float(ts))
# Needed for container sync feature
response.headers['X-Timestamp'] = ts
response.headers['X-Timestamp'] = ts.normal
response.headers['X-Backend-Timestamp'] = ts.internal
response.content_length = int(metadata['Content-Length'])
try:
response.content_encoding = metadata['Content-Encoding']
@ -555,15 +569,13 @@ class ObjectController(object):
@timing_stats()
def DELETE(self, request):
"""Handle HTTP DELETE requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
if 'x-timestamp' not in request.headers or \
not check_float(request.headers['x-timestamp']):
return HTTPBadRequest(body='Missing timestamp', request=request,
content_type='text/plain')
device, partition, account, container, obj, policy_idx = \
get_name_and_placement(request, 5, 5, True)
req_timestamp = valid_timestamp(request)
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj)
device, partition, account, container, obj,
policy_idx=policy_idx)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
@ -581,8 +593,8 @@ class ObjectController(object):
orig_metadata = {}
response_class = HTTPNotFound
else:
orig_timestamp = orig_metadata.get('X-Timestamp', 0)
if orig_timestamp < request.headers['x-timestamp']:
orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0))
if orig_timestamp < req_timestamp:
response_class = HTTPNoContent
else:
response_class = HTTPConflict
@ -597,20 +609,28 @@ class ObjectController(object):
request=request,
body='Bad X-If-Delete-At header value')
else:
# request includes x-if-delete-at; we must not place a tombstone
# if we can not verify the x-if-delete-at time
if not orig_timestamp:
# no object found at all
return HTTPNotFound()
if orig_delete_at != req_if_delete_at:
return HTTPPreconditionFailed(
request=request,
body='X-If-Delete-At and X-Delete-At do not match')
else:
# differentiate success from no object at all
response_class = HTTPNoContent
if orig_delete_at:
self.delete_at_update('DELETE', orig_delete_at, account,
container, obj, request, device)
req_timestamp = request.headers['X-Timestamp']
container, obj, request, device,
policy_idx)
if orig_timestamp < req_timestamp:
disk_file.delete(req_timestamp)
self.container_update(
'DELETE', account, container, obj, request,
HeaderKeyDict({'x-timestamp': req_timestamp}),
device)
HeaderKeyDict({'x-timestamp': req_timestamp.internal}),
device, policy_idx)
return response_class(request=request)
@public
@ -621,10 +641,11 @@ class ObjectController(object):
Handle REPLICATE requests for the Swift Object Server. This is used
by the object replicator to get hashes for directories.
"""
device, partition, suffix = split_and_validate_path(
request, 2, 3, True)
device, partition, suffix, policy_idx = \
get_name_and_placement(request, 2, 3, True)
try:
hashes = self._diskfile_mgr.get_hashes(device, partition, suffix)
hashes = self._diskfile_mgr.get_hashes(device, partition, suffix,
policy_idx)
except DiskFileDeviceUnavailable:
resp = HTTPInsufficientStorage(drive=device, request=request)
else:

6
swift/obj/ssync_receiver.py
View File

@ -25,6 +25,8 @@ from swift.common import http
from swift.common import swob
from swift.common import utils
from swift.common.storage_policy import POLICY_INDEX
class Receiver(object):
"""
@ -168,6 +170,7 @@ class Receiver(object):
self.request.environ['eventlet.minimum_write_chunk_size'] = 0
self.device, self.partition = utils.split_path(
urllib.unquote(self.request.path), 2, 2, False)
self.policy_idx = int(self.request.headers.get(POLICY_INDEX, 0))
utils.validate_device_partition(self.device, self.partition)
if self.app._diskfile_mgr.mount_check and \
not constraints.check_mount(
@ -228,7 +231,7 @@ class Receiver(object):
want = False
try:
df = self.app._diskfile_mgr.get_diskfile_from_hash(
self.device, self.partition, object_hash)
self.device, self.partition, object_hash, self.policy_idx)
except exceptions.DiskFileNotExist:
want = True
else:
@ -351,6 +354,7 @@ class Receiver(object):
subreq_iter())
else:
raise Exception('Invalid subrequest method %s' % method)
subreq.headers[POLICY_INDEX] = self.policy_idx
subreq.headers['X-Backend-Replication'] = 'True'
if replication_headers:
subreq.headers['X-Backend-Replication-Headers'] = \

13
swift/obj/ssync_sender.py
View File

@ -18,6 +18,8 @@ from swift.common import bufferedhttp
from swift.common import exceptions
from swift.common import http
from swift.common.storage_policy import POLICY_INDEX
class Sender(object):
"""
@ -40,6 +42,10 @@ class Sender(object):
self.send_list = None
self.failures = 0
@property
def policy_idx(self):
return int(self.job.get('policy_idx', 0))
def __call__(self):
if not self.suffixes:
return True
@ -94,6 +100,7 @@ class Sender(object):
self.connection.putrequest('REPLICATION', '/%s/%s' % (
self.node['device'], self.job['partition']))
self.connection.putheader('Transfer-Encoding', 'chunked')
self.connection.putheader(POLICY_INDEX, self.policy_idx)
self.connection.endheaders()
with exceptions.MessageTimeout(
self.daemon.node_timeout, 'connect receive'):
@ -163,7 +170,8 @@ class Sender(object):
self.connection.send('%x\r\n%s\r\n' % (len(msg), msg))
for path, object_hash, timestamp in \
self.daemon._diskfile_mgr.yield_hashes(
self.job['device'], self.job['partition'], self.suffixes):
self.job['device'], self.job['partition'],
self.policy_idx, self.suffixes):
with exceptions.MessageTimeout(
self.daemon.node_timeout,
'missing_check send line'):
@ -217,7 +225,8 @@ class Sender(object):
for object_hash in self.send_list:
try:
df = self.daemon._diskfile_mgr.get_diskfile_from_hash(
self.job['device'], self.job['partition'], object_hash)
self.job['device'], self.job['partition'], object_hash,
self.policy_idx)
except exceptions.DiskFileNotExist:
continue
url_path = urllib.quote(

102
swift/obj/updater.py
View File

@ -26,10 +26,11 @@ from eventlet import patcher, Timeout
from swift.common.bufferedhttp import http_connect
from swift.common.exceptions import ConnectionTimeout
from swift.common.ring import Ring
from swift.common.storage_policy import POLICY_INDEX
from swift.common.utils import get_logger, renamer, write_pickle, \
dump_recon_cache, config_true_value, ismount
from swift.common.daemon import Daemon
from swift.obj.diskfile import ASYNCDIR
from swift.obj.diskfile import get_tmp_dir, get_async_dir, ASYNCDIR_BASE
from swift.common.http import is_success, HTTP_NOT_FOUND, \
HTTP_INTERNAL_SERVER_ERROR
@ -37,9 +38,9 @@ from swift.common.http import is_success, HTTP_NOT_FOUND, \
class ObjectUpdater(Daemon):
"""Update object information in container listings."""
def __init__(self, conf):
def __init__(self, conf, logger=None):
self.conf = conf
self.logger = get_logger(conf, log_route='object-updater')
self.logger = logger or get_logger(conf, log_route='object-updater')
self.devices = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.swift_dir = conf.get('swift_dir', '/etc/swift')
@ -137,45 +138,69 @@ class ObjectUpdater(Daemon):
:param device: path to device
"""
start_time = time.time()
async_pending = os.path.join(device, ASYNCDIR)
if not os.path.isdir(async_pending):
return
for prefix in os.listdir(async_pending):
prefix_path = os.path.join(async_pending, prefix)
if not os.path.isdir(prefix_path):
# loop through async pending dirs for all policies
for asyncdir in os.listdir(device):
# skip stuff like "accounts", "containers", etc.
if not (asyncdir == ASYNCDIR_BASE or
asyncdir.startswith(ASYNCDIR_BASE + '-')):
continue
last_obj_hash = None
for update in sorted(os.listdir(prefix_path), reverse=True):
update_path = os.path.join(prefix_path, update)
if not os.path.isfile(update_path):
continue
try:
obj_hash, timestamp = update.split('-')
except ValueError:
self.logger.increment('errors')
self.logger.error(
_('ERROR async pending file with unexpected name %s')
% (update_path))
continue
if obj_hash == last_obj_hash:
self.logger.increment("unlinks")
os.unlink(update_path)
else:
self.process_object_update(update_path, device)
last_obj_hash = obj_hash
time.sleep(self.slowdown)
try:
os.rmdir(prefix_path)
except OSError:
pass
self.logger.timing_since('timing', start_time)
def process_object_update(self, update_path, device):
# we only care about directories
async_pending = os.path.join(device, asyncdir)
if not os.path.isdir(async_pending):
continue
if asyncdir == ASYNCDIR_BASE:
policy_idx = 0
else:
_junk, policy_idx = asyncdir.split('-', 1)
try:
policy_idx = int(policy_idx)
get_async_dir(policy_idx)
except ValueError:
self.logger.warn(_('Directory %s does not map to a '
'valid policy') % asyncdir)
continue
for prefix in os.listdir(async_pending):
prefix_path = os.path.join(async_pending, prefix)
if not os.path.isdir(prefix_path):
continue
last_obj_hash = None
for update in sorted(os.listdir(prefix_path), reverse=True):
update_path = os.path.join(prefix_path, update)
if not os.path.isfile(update_path):
continue
try:
obj_hash, timestamp = update.split('-')
except ValueError:
self.logger.increment('errors')
self.logger.error(
_('ERROR async pending file with unexpected '
'name %s')
% (update_path))
continue
if obj_hash == last_obj_hash:
self.logger.increment("unlinks")
os.unlink(update_path)
else:
self.process_object_update(update_path, device,
policy_idx)
last_obj_hash = obj_hash
time.sleep(self.slowdown)
try:
os.rmdir(prefix_path)
except OSError:
pass
self.logger.timing_since('timing', start_time)
def process_object_update(self, update_path, device, policy_idx):
"""
Process the object information to be updated and update.
:param update_path: path to pickled object update file
:param device: path to device
:param policy_idx: storage policy index of object update
"""
try:
update = pickle.load(open(update_path, 'rb'))
@ -196,8 +221,10 @@ class ObjectUpdater(Daemon):
new_successes = False
for node in nodes:
if node['id'] not in successes:
headers = update['headers'].copy()
headers.setdefault(POLICY_INDEX, str(policy_idx))
status = self.object_update(node, part, update['op'], obj,
update['headers'])
headers)
if not is_success(status) and status != HTTP_NOT_FOUND:
success = False
else:
@ -217,7 +244,8 @@ class ObjectUpdater(Daemon):
{'obj': obj, 'path': update_path})
if new_successes:
update['successes'] = successes
write_pickle(update, update_path, os.path.join(device, 'tmp'))
write_pickle(update, update_path, os.path.join(
device, get_tmp_dir(policy_idx)))
def object_update(self, node, part, op, obj, headers):
"""

21
swift/proxy/controllers/base.py
View File

@ -36,7 +36,7 @@ from eventlet import sleep
from eventlet.timeout import Timeout
from swift.common.wsgi import make_pre_authed_env
from swift.common.utils import normalize_timestamp, config_true_value, \
from swift.common.utils import Timestamp, config_true_value, \
public, split_path, list_from_csv, GreenthreadSafeIterator, \
quorum_size, GreenAsyncPile
from swift.common.bufferedhttp import http_connect
@ -50,6 +50,7 @@ from swift.common.swob import Request, Response, HeaderKeyDict, Range, \
HTTPException, HTTPRequestedRangeNotSatisfiable
from swift.common.request_helpers import strip_sys_meta_prefix, \
strip_user_meta_prefix, is_user_meta, is_sys_meta, is_sys_or_user_meta
from swift.common.storage_policy import POLICY_INDEX, POLICY, POLICIES
def update_headers(response, headers):
@ -161,6 +162,7 @@ def headers_to_container_info(headers, status_int=HTTP_OK):
'object_count': headers.get('x-container-object-count'),
'bytes': headers.get('x-container-bytes-used'),
'versions': headers.get('x-versions-location'),
'storage_policy': headers.get(POLICY_INDEX.lower(), '0'),
'cors': {
'allow_origin': meta.get('access-control-allow-origin'),
'expose_headers': meta.get('access-control-expose-headers'),
@ -507,7 +509,8 @@ def get_info(app, env, account, container=None, ret_not_found=False,
path = '/v1/%s' % account
if container:
# Stop and check if we have an account?
if not get_info(app, env, account):
if not get_info(app, env, account) and not account.startswith(
getattr(app, 'auto_create_account_prefix', '.')):
return None
path += '/' + container
@ -923,7 +926,7 @@ class Controller(object):
headers = HeaderKeyDict(additional) if additional else HeaderKeyDict()
if transfer:
self.transfer_headers(orig_req.headers, headers)
headers.setdefault('x-timestamp', normalize_timestamp(time.time()))
headers.setdefault('x-timestamp', Timestamp(time.time()).internal)
if orig_req:
referer = orig_req.as_referer()
else:
@ -1155,7 +1158,7 @@ class Controller(object):
"""
partition, nodes = self.app.account_ring.get_nodes(account)
path = '/%s' % account
headers = {'X-Timestamp': normalize_timestamp(time.time()),
headers = {'X-Timestamp': Timestamp(time.time()).internal,
'X-Trans-Id': self.trans_id,
'Connection': 'close'}
resp = self.make_requests(Request.blank('/v1' + path),
@ -1201,6 +1204,16 @@ class Controller(object):
container, obj, res)
except ValueError:
pass
# if a backend policy index is present in resp headers, translate it
# here with the friendly policy name
if POLICY_INDEX in res.headers and is_success(res.status_int):
policy = POLICIES.get_by_index(res.headers[POLICY_INDEX])
if policy:
res.headers[POLICY] = policy.name
else:
self.app.logger.error(
'Could not translate %s (%r) from %r to policy',
POLICY_INDEX, res.headers[POLICY_INDEX], path)
return res
def is_origin_allowed(self, cors_info, origin):

39
swift/proxy/controllers/container.py
View File

@ -17,12 +17,13 @@ from swift import gettext_ as _
from urllib import unquote
import time
from swift.common.utils import public, csv_append, normalize_timestamp
from swift.common.utils import public, csv_append, Timestamp
from swift.common.constraints import check_metadata
from swift.common import constraints
from swift.common.http import HTTP_ACCEPTED
from swift.proxy.controllers.base import Controller, delay_denial, \
cors_validation, clear_info_cache
from swift.common.storage_policy import POLICIES, POLICY, POLICY_INDEX
from swift.common.swob import HTTPBadRequest, HTTPForbidden, \
HTTPNotFound
@ -47,6 +48,27 @@ class ContainerController(Controller):
'x-remove-%s-write' % st,
'x-remove-versions-location']
def _convert_policy_to_index(self, req):
"""
Helper method to convert a policy name (from a request from a client)
to a policy index (for a request to a backend).
:param req: incoming request
"""
policy_name = req.headers.get(POLICY)
if not policy_name:
return
policy = POLICIES.get_by_name(policy_name)
if not policy:
raise HTTPBadRequest(request=req,
content_type="text/plain",
body=("Invalid %s '%s'"
% (POLICY, policy_name)))
if policy.is_deprecated:
body = 'Storage Policy %r is deprecated' % (policy.name)
raise HTTPBadRequest(request=req, body=body)
return int(policy)
def clean_acls(self, req):
if 'swift.clean_acl' in req.environ:
for header in ('x-container-read', 'x-container-write'):
@ -101,6 +123,7 @@ class ContainerController(Controller):
self.clean_acls(req) or check_metadata(req, 'container')
if error_response:
return error_response
policy_index = self._convert_policy_to_index(req)
if not req.environ.get('swift_owner'):
for key in self.app.swift_owner_headers:
req.headers.pop(key, None)
@ -128,7 +151,8 @@ class ContainerController(Controller):
container_partition, containers = self.app.container_ring.get_nodes(
self.account_name, self.container_name)
headers = self._backend_requests(req, len(containers),
account_partition, accounts)
account_partition, accounts,
policy_index)
clear_info_cache(self.app, req.environ,
self.account_name, self.container_name)
resp = self.make_requests(
@ -183,9 +207,14 @@ class ContainerController(Controller):
return HTTPNotFound(request=req)
return resp
def _backend_requests(self, req, n_outgoing,
account_partition, accounts):
additional = {'X-Timestamp': normalize_timestamp(time.time())}
def _backend_requests(self, req, n_outgoing, account_partition, accounts,
policy_index=None):
additional = {'X-Timestamp': Timestamp(time.time()).internal}
if policy_index is None:
additional['X-Backend-Storage-Policy-Default'] = \
int(POLICIES.default)
else:
additional[POLICY_INDEX] = str(policy_index)
headers = [self.generate_request_headers(req, transfer=True,
additional=additional)
for _junk in range(n_outgoing)]

78
swift/proxy/controllers/obj.py
View File

@ -37,8 +37,8 @@ from eventlet.timeout import Timeout
from swift.common.utils import (
clean_content_type, config_true_value, ContextPool, csv_append,
GreenAsyncPile, GreenthreadSafeIterator, json,
normalize_delete_at_timestamp, normalize_timestamp, public, quorum_size)
GreenAsyncPile, GreenthreadSafeIterator, json, Timestamp,
normalize_delete_at_timestamp, public, quorum_size)
from swift.common.bufferedhttp import http_connect
from swift.common.constraints import check_metadata, check_object_creation, \
check_copy_from_header
@ -56,6 +56,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \
HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \
HTTPServerError, HTTPServiceUnavailable, Request, \
HTTPClientDisconnect, HTTPNotImplemented
from swift.common.storage_policy import POLICY_INDEX
from swift.common.request_helpers import is_user_meta
@ -195,15 +196,19 @@ class ObjectController(Controller):
container_info = self.container_info(
self.account_name, self.container_name, req)
req.acl = container_info['read_acl']
# pass the policy index to storage nodes via req header
policy_index = req.headers.get(POLICY_INDEX,
container_info['storage_policy'])
obj_ring = self.app.get_object_ring(policy_index)
req.headers[POLICY_INDEX] = policy_index
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
partition = self.app.object_ring.get_part(
partition = obj_ring.get_part(
self.account_name, self.container_name, self.object_name)
resp = self.GETorHEAD_base(
req, _('Object'), self.app.object_ring, partition,
req, _('Object'), obj_ring, partition,
req.swift_entity_path)
if ';' in resp.headers.get('content-type', ''):
@ -297,15 +302,20 @@ class ObjectController(Controller):
self.app.expiring_objects_account, delete_at_container)
else:
delete_at_container = delete_at_part = delete_at_nodes = None
partition, nodes = self.app.object_ring.get_nodes(
# pass the policy index to storage nodes via req header
policy_index = req.headers.get(POLICY_INDEX,
container_info['storage_policy'])
obj_ring = self.app.get_object_ring(policy_index)
req.headers[POLICY_INDEX] = policy_index
partition, nodes = obj_ring.get_nodes(
self.account_name, self.container_name, self.object_name)
req.headers['X-Timestamp'] = normalize_timestamp(time.time())
req.headers['X-Timestamp'] = Timestamp(time.time()).internal
headers = self._backend_requests(
req, len(nodes), container_partition, containers,
delete_at_container, delete_at_part, delete_at_nodes)
resp = self.make_requests(req, self.app.object_ring, partition,
resp = self.make_requests(req, obj_ring, partition,
'POST', req.swift_entity_path, headers)
return resp
@ -448,6 +458,11 @@ class ObjectController(Controller):
body='If-None-Match only supports *')
container_info = self.container_info(
self.account_name, self.container_name, req)
policy_index = req.headers.get(POLICY_INDEX,
container_info['storage_policy'])
obj_ring = self.app.get_object_ring(policy_index)
# pass the policy index to storage nodes via req header
req.headers[POLICY_INDEX] = policy_index
container_partition = container_info['partition']
containers = container_info['nodes']
req.acl = container_info['write_acl']
@ -478,33 +493,35 @@ class ObjectController(Controller):
body='Non-integer X-Delete-After')
req.headers['x-delete-at'] = normalize_delete_at_timestamp(
time.time() + x_delete_after)
partition, nodes = self.app.object_ring.get_nodes(
partition, nodes = obj_ring.get_nodes(
self.account_name, self.container_name, self.object_name)
# do a HEAD request for container sync and checking object versions
if 'x-timestamp' in req.headers or \
(object_versions and not
req.environ.get('swift_versioned_copy')):
hreq = Request.blank(req.path_info, headers={'X-Newest': 'True'},
# make sure proxy-server uses the right policy index
_headers = {POLICY_INDEX: req.headers[POLICY_INDEX],
'X-Newest': 'True'}
hreq = Request.blank(req.path_info, headers=_headers,
environ={'REQUEST_METHOD': 'HEAD'})
hresp = self.GETorHEAD_base(
hreq, _('Object'), self.app.object_ring, partition,
hreq, _('Object'), obj_ring, partition,
hreq.swift_entity_path)
# Used by container sync feature
if 'x-timestamp' in req.headers:
try:
req.headers['X-Timestamp'] = \
normalize_timestamp(req.headers['x-timestamp'])
req_timestamp = Timestamp(req.headers['X-Timestamp'])
if hresp.environ and 'swift_x_timestamp' in hresp.environ and \
float(hresp.environ['swift_x_timestamp']) >= \
float(req.headers['x-timestamp']):
hresp.environ['swift_x_timestamp'] >= req_timestamp:
return HTTPAccepted(request=req)
except ValueError:
return HTTPBadRequest(
request=req, content_type='text/plain',
body='X-Timestamp should be a UNIX timestamp float value; '
'was %r' % req.headers['x-timestamp'])
req.headers['X-Timestamp'] = req_timestamp.internal
else:
req.headers['X-Timestamp'] = normalize_timestamp(time.time())
req.headers['X-Timestamp'] = Timestamp(time.time()).internal
# Sometimes the 'content-type' header exists, but is set to None.
content_type_manually_set = True
detect_content_type = \
@ -536,7 +553,7 @@ class ObjectController(Controller):
ts_source = time.mktime(time.strptime(
hresp.headers['last-modified'],
'%a, %d %b %Y %H:%M:%S GMT'))
new_ts = normalize_timestamp(ts_source)
new_ts = Timestamp(ts_source).internal
vers_obj_name = lprefix + new_ts
copy_headers = {
'Destination': '%s/%s' % (lcontainer, vers_obj_name)}
@ -568,6 +585,8 @@ class ObjectController(Controller):
source_header = '/%s/%s/%s/%s' % (ver, acct,
src_container_name, src_obj_name)
source_req = req.copy_get()
# make sure the source request uses it's container_info
source_req.headers.pop(POLICY_INDEX, None)
source_req.path_info = source_header
source_req.headers['X-Newest'] = 'true'
orig_obj_name = self.object_name
@ -642,7 +661,7 @@ class ObjectController(Controller):
delete_at_container = delete_at_part = delete_at_nodes = None
node_iter = GreenthreadSafeIterator(
self.iter_nodes_local_first(self.app.object_ring, partition))
self.iter_nodes_local_first(obj_ring, partition))
pile = GreenPile(len(nodes))
te = req.headers.get('transfer-encoding', '')
chunked = ('chunked' in te)
@ -746,7 +765,8 @@ class ObjectController(Controller):
resp.headers['X-Copied-From-Last-Modified'] = \
source_resp.headers['last-modified']
copy_headers_into(req, resp)
resp.last_modified = math.ceil(float(req.headers['X-Timestamp']))
resp.last_modified = math.ceil(
float(Timestamp(req.headers['X-Timestamp'])))
return resp
@public
@ -756,6 +776,12 @@ class ObjectController(Controller):
"""HTTP DELETE request handler."""
container_info = self.container_info(
self.account_name, self.container_name, req)
# pass the policy index to storage nodes via req header
policy_index = req.headers.get(POLICY_INDEX,
container_info['storage_policy'])
obj_ring = self.app.get_object_ring(policy_index)
# pass the policy index to storage nodes via req header
req.headers[POLICY_INDEX] = policy_index
container_partition = container_info['partition']
containers = container_info['nodes']
req.acl = container_info['write_acl']
@ -809,6 +835,10 @@ class ObjectController(Controller):
new_del_req = Request.blank(copy_path, environ=req.environ)
container_info = self.container_info(
self.account_name, self.container_name, req)
policy_idx = container_info['storage_policy']
obj_ring = self.app.get_object_ring(policy_idx)
# pass the policy index to storage nodes via req header
new_del_req.headers[POLICY_INDEX] = policy_idx
container_partition = container_info['partition']
containers = container_info['nodes']
new_del_req.acl = container_info['write_acl']
@ -823,24 +853,24 @@ class ObjectController(Controller):
return aresp
if not containers:
return HTTPNotFound(request=req)
partition, nodes = self.app.object_ring.get_nodes(
partition, nodes = obj_ring.get_nodes(
self.account_name, self.container_name, self.object_name)
# Used by container sync feature
if 'x-timestamp' in req.headers:
try:
req.headers['X-Timestamp'] = \
normalize_timestamp(req.headers['x-timestamp'])
req_timestamp = Timestamp(req.headers['X-Timestamp'])
except ValueError:
return HTTPBadRequest(
request=req, content_type='text/plain',
body='X-Timestamp should be a UNIX timestamp float value; '
'was %r' % req.headers['x-timestamp'])
req.headers['X-Timestamp'] = req_timestamp.internal
else:
req.headers['X-Timestamp'] = normalize_timestamp(time.time())
req.headers['X-Timestamp'] = Timestamp(time.time()).internal
headers = self._backend_requests(
req, len(nodes), container_partition, containers)
resp = self.make_requests(req, self.app.object_ring,
resp = self.make_requests(req, obj_ring,
partition, 'DELETE', req.swift_entity_path,
headers)
return resp

24
swift/proxy/server.py
View File

@ -25,6 +25,7 @@ from eventlet import Timeout
from swift import __canonical_version__ as swift_version
from swift.common import constraints
from swift.common.storage_policy import POLICIES
from swift.common.ring import Ring
from swift.common.utils import cache_from_env, get_logger, \
get_remote_client, split_path, config_true_value, generate_trans_id, \
@ -67,7 +68,7 @@ class Application(object):
"""WSGI application for the proxy server."""
def __init__(self, conf, memcache=None, logger=None, account_ring=None,
container_ring=None, object_ring=None):
container_ring=None):
if conf is None:
conf = {}
if logger is None:
@ -76,6 +77,7 @@ class Application(object):
self.logger = logger
swift_dir = conf.get('swift_dir', '/etc/swift')
self.swift_dir = swift_dir
self.node_timeout = int(conf.get('node_timeout', 10))
self.recoverable_node_timeout = int(
conf.get('recoverable_node_timeout', self.node_timeout))
@ -98,18 +100,21 @@ class Application(object):
config_true_value(conf.get('allow_account_management', 'no'))
self.object_post_as_copy = \
config_true_value(conf.get('object_post_as_copy', 'true'))
self.object_ring = object_ring or Ring(swift_dir, ring_name='object')
self.container_ring = container_ring or Ring(swift_dir,
ring_name='container')
self.account_ring = account_ring or Ring(swift_dir,
ring_name='account')
# ensure rings are loaded for all configured storage policies
for policy in POLICIES:
policy.load_ring(swift_dir)
self.memcache = memcache
mimetypes.init(mimetypes.knownfiles +
[os.path.join(swift_dir, 'mime.types')])
self.account_autocreate = \
config_true_value(conf.get('account_autocreate', 'no'))
self.expiring_objects_account = \
(conf.get('auto_create_account_prefix') or '.') + \
self.auto_create_account_prefix = (
conf.get('auto_create_account_prefix') or '.')
self.expiring_objects_account = self.auto_create_account_prefix + \
(conf.get('expiring_objects_account_name') or 'expiring_objects')
self.expiring_objects_container_divisor = \
int(conf.get('expiring_objects_container_divisor') or 86400)
@ -207,6 +212,7 @@ class Application(object):
register_swift_info(
version=swift_version,
strict_cors_mode=self.strict_cors_mode,
policies=POLICIES.get_policy_info(),
**constraints.EFFECTIVE_CONSTRAINTS)
def check_config(self):
@ -218,6 +224,16 @@ class Application(object):
"read_affinity setting will have no effect." %
self.sorting_method)
def get_object_ring(self, policy_idx):
"""
Get the ring object to use to handle a request based on its policy.
:param policy_idx: policy index as defined in swift.conf
:returns: appropriate ring object
"""
return POLICIES.get_object_ring(policy_idx, self.swift_dir)
def get_controller(self, path):
"""
Get the controller to handle a request.

81
test/functional/__init__.py
View File

@ -21,6 +21,7 @@ import locale
import eventlet
import eventlet.debug
import functools
import random
from time import time, sleep
from httplib import HTTPException
from urlparse import urlparse
@ -37,7 +38,7 @@ from test.functional.swift_test_client import Connection, ResponseError
# on file systems that don't support extended attributes.
from test.unit import debug_logger, FakeMemcache
from swift.common import constraints, utils, ring
from swift.common import constraints, utils, ring, storage_policy
from swift.common.wsgi import monkey_patch_mimetools
from swift.common.middleware import catch_errors, gatekeeper, healthcheck, \
proxy_logging, container_sync, bulk, tempurl, slo, dlo, ratelimit, \
@ -151,6 +152,8 @@ def in_process_setup(the_object_server=object_server):
orig_swift_conf_name = utils.SWIFT_CONF_FILE
utils.SWIFT_CONF_FILE = swift_conf
constraints.reload_constraints()
storage_policy.SWIFT_CONF_FILE = swift_conf
storage_policy.reload_storage_policies()
global config
if constraints.SWIFT_CONSTRAINTS_LOADED:
# Use the swift constraints that are loaded for the test framework
@ -344,7 +347,7 @@ def get_cluster_info():
# test.conf data
pass
else:
eff_constraints.update(cluster_info['swift'])
eff_constraints.update(cluster_info.get('swift', {}))
# Finally, we'll allow any constraint present in the swift-constraints
# section of test.conf to override everything. Note that only those
@ -620,6 +623,18 @@ def load_constraint(name):
return c
def get_storage_policy_from_cluster_info(info):
policies = info['swift'].get('policies', {})
default_policy = []
non_default_policies = []
for p in policies:
if p.get('default', {}):
default_policy.append(p)
else:
non_default_policies.append(p)
return default_policy, non_default_policies
def reset_acl():
def post(url, token, parsed, conn):
conn.request('POST', parsed.path, '', {
@ -650,3 +665,65 @@ def requires_acls(f):
reset_acl()
return rv
return wrapper
class FunctionalStoragePolicyCollection(object):
def __init__(self, policies):
self._all = policies
self.default = None
for p in self:
if p.get('default', False):
assert self.default is None, 'Found multiple default ' \
'policies %r and %r' % (self.default, p)
self.default = p
@classmethod
def from_info(cls, info=None):
if not (info or cluster_info):
get_cluster_info()
info = info or cluster_info
try:
policy_info = info['swift']['policies']
except KeyError:
raise AssertionError('Did not find any policy info in %r' % info)
policies = cls(policy_info)
assert policies.default, \
'Did not find default policy in %r' % policy_info
return policies
def __len__(self):
return len(self._all)
def __iter__(self):
return iter(self._all)
def __getitem__(self, index):
return self._all[index]
def filter(self, **kwargs):
return self.__class__([p for p in self if all(
p.get(k) == v for k, v in kwargs.items())])
def exclude(self, **kwargs):
return self.__class__([p for p in self if all(
p.get(k) != v for k, v in kwargs.items())])
def select(self):
return random.choice(self)
def requires_policies(f):
@functools.wraps(f)
def wrapper(self, *args, **kwargs):
if skip:
raise SkipTest
try:
self.policies = FunctionalStoragePolicyCollection.from_info()
except AssertionError:
raise SkipTest("Unable to determine available policies")
if len(self.policies) < 2:
raise SkipTest("Multiple policies not enabled")
return f(self, *args, **kwargs)
return wrapper

2
test/functional/swift_test_client.py
View File

@ -186,7 +186,7 @@ class Connection(object):
"""
status = self.make_request('GET', '/info',
cfg={'absolute_path': True})
if status == 404:
if status // 100 == 4:
return {}
if not 200 <= status <= 299:
raise ResponseError(self.response, 'GET', '/info')

30
test/functional/test_account.py
View File

@ -36,6 +36,36 @@ class TestAccount(unittest.TestCase):
self.max_meta_overall_size = load_constraint('max_meta_overall_size')
self.max_meta_value_length = load_constraint('max_meta_value_length')
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(head)
self.existing_metadata = set([
k for k, v in resp.getheaders() if
k.lower().startswith('x-account-meta')])
def tearDown(self):
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(head)
resp.read()
new_metadata = set(
[k for k, v in resp.getheaders() if
k.lower().startswith('x-account-meta')])
def clear_meta(url, token, parsed, conn, remove_metadata_keys):
headers = {'X-Auth-Token': token}
headers.update((k, '') for k in remove_metadata_keys)
conn.request('POST', parsed.path, '', headers)
return check_response(conn)
extra_metadata = list(self.existing_metadata ^ new_metadata)
for i in range(0, len(extra_metadata), 90):
batch = extra_metadata[i:i + 90]
resp = retry(clear_meta, batch)
resp.read()
self.assertEqual(resp.status // 100, 2)
def test_metadata(self):
if tf.skip:
raise SkipTest

218
test/functional/test_container.py
View File

@ -21,7 +21,7 @@ from nose import SkipTest
from uuid import uuid4
from test.functional import check_response, retry, requires_acls, \
load_constraint
load_constraint, requires_policies
import test.functional as tf
@ -31,6 +31,8 @@ class TestContainer(unittest.TestCase):
if tf.skip:
raise SkipTest
self.name = uuid4().hex
# this container isn't created by default, but will be cleaned up
self.container = uuid4().hex
def put(url, token, parsed, conn):
conn.request('PUT', parsed.path + '/' + self.name, '',
@ -50,38 +52,47 @@ class TestContainer(unittest.TestCase):
if tf.skip:
raise SkipTest
def get(url, token, parsed, conn):
conn.request('GET', parsed.path + '/' + self.name + '?format=json',
'', {'X-Auth-Token': token})
def get(url, token, parsed, conn, container):
conn.request(
'GET', parsed.path + '/' + container + '?format=json', '',
{'X-Auth-Token': token})
return check_response(conn)
def delete(url, token, parsed, conn, obj):
conn.request('DELETE',
'/'.join([parsed.path, self.name, obj['name']]), '',
def delete(url, token, parsed, conn, container, obj):
conn.request(
'DELETE', '/'.join([parsed.path, container, obj['name']]), '',
{'X-Auth-Token': token})
return check_response(conn)
for container in (self.name, self.container):
while True:
resp = retry(get, container)
body = resp.read()
if resp.status == 404:
break
self.assert_(resp.status // 100 == 2, resp.status)
objs = json.loads(body)
if not objs:
break
for obj in objs:
resp = retry(delete, container, obj)
resp.read()
self.assertEqual(resp.status, 204)
def delete(url, token, parsed, conn, container):
conn.request('DELETE', parsed.path + '/' + container, '',
{'X-Auth-Token': token})
return check_response(conn)
while True:
resp = retry(get)
body = resp.read()
self.assert_(resp.status // 100 == 2, resp.status)
objs = json.loads(body)
if not objs:
break
for obj in objs:
resp = retry(delete, obj)
resp.read()
self.assertEqual(resp.status, 204)
def delete(url, token, parsed, conn):
conn.request('DELETE', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(delete)
resp = retry(delete, self.name)
resp.read()
self.assertEqual(resp.status, 204)
# container may have not been created
resp = retry(delete, self.container)
resp.read()
self.assert_(resp.status in (204, 404))
def test_multi_metadata(self):
if tf.skip:
raise SkipTest
@ -1342,6 +1353,163 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.read(), 'Invalid UTF8 or contains NULL')
self.assertEqual(resp.status, 412)
def test_create_container_gets_default_policy_by_default(self):
try:
default_policy = \
tf.FunctionalStoragePolicyCollection.from_info().default
except AssertionError:
raise SkipTest()
def put(url, token, parsed, conn):
conn.request('PUT', parsed.path + '/' + self.container, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(put)
resp.read()
self.assertEqual(resp.status // 100, 2)
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path + '/' + self.container, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(head)
resp.read()
headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('x-storage-policy'),
default_policy['name'])
def test_error_invalid_storage_policy_name(self):
def put(url, token, parsed, conn, headers):
new_headers = dict({'X-Auth-Token': token}, **headers)
conn.request('PUT', parsed.path + '/' + self.container, '',
new_headers)
return check_response(conn)
# create
resp = retry(put, {'X-Storage-Policy': uuid4().hex})
resp.read()
self.assertEqual(resp.status, 400)
@requires_policies
def test_create_non_default_storage_policy_container(self):
policy = self.policies.exclude(default=True).select()
def put(url, token, parsed, conn, headers=None):
base_headers = {'X-Auth-Token': token}
if headers:
base_headers.update(headers)
conn.request('PUT', parsed.path + '/' + self.container, '',
base_headers)
return check_response(conn)
headers = {'X-Storage-Policy': policy['name']}
resp = retry(put, headers=headers)
resp.read()
self.assertEqual(resp.status, 201)
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path + '/' + self.container, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(head)
resp.read()
headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('x-storage-policy'),
policy['name'])
# and test recreate with-out specifiying Storage Policy
resp = retry(put)
resp.read()
self.assertEqual(resp.status, 202)
# should still be original storage policy
resp = retry(head)
resp.read()
headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('x-storage-policy'),
policy['name'])
# delete it
def delete(url, token, parsed, conn):
conn.request('DELETE', parsed.path + '/' + self.container, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(delete)
resp.read()
self.assertEqual(resp.status, 204)
# verify no policy header
resp = retry(head)
resp.read()
headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('x-storage-policy'), None)
@requires_policies
def test_conflict_change_storage_policy_with_put(self):
def put(url, token, parsed, conn, headers):
new_headers = dict({'X-Auth-Token': token}, **headers)
conn.request('PUT', parsed.path + '/' + self.container, '',
new_headers)
return check_response(conn)
# create
policy = self.policies.select()
resp = retry(put, {'X-Storage-Policy': policy['name']})
resp.read()
self.assertEqual(resp.status, 201)
# can't change it
other_policy = self.policies.exclude(name=policy['name']).select()
resp = retry(put, {'X-Storage-Policy': other_policy['name']})
resp.read()
self.assertEqual(resp.status, 409)
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path + '/' + self.container, '',
{'X-Auth-Token': token})
return check_response(conn)
# still original policy
resp = retry(head)
resp.read()
headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('x-storage-policy'),
policy['name'])
@requires_policies
def test_noop_change_storage_policy_with_post(self):
def put(url, token, parsed, conn, headers):
new_headers = dict({'X-Auth-Token': token}, **headers)
conn.request('PUT', parsed.path + '/' + self.container, '',
new_headers)
return check_response(conn)
# create
policy = self.policies.select()
resp = retry(put, {'X-Storage-Policy': policy['name']})
resp.read()
self.assertEqual(resp.status, 201)
def post(url, token, parsed, conn, headers):
new_headers = dict({'X-Auth-Token': token}, **headers)
conn.request('POST', parsed.path + '/' + self.container, '',
new_headers)
return check_response(conn)
# attempt update
for header in ('X-Storage-Policy', 'X-Storage-Policy-Index'):
other_policy = self.policies.exclude(name=policy['name']).select()
resp = retry(post, {header: other_policy['name']})
resp.read()
self.assertEqual(resp.status, 204)
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path + '/' + self.container, '',
{'X-Auth-Token': token})
return check_response(conn)
# still original policy
resp = retry(head)
resp.read()
headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('x-storage-policy'),
policy['name'])
if __name__ == '__main__':
unittest.main()

142
test/functional/test_object.py
View File

@ -21,7 +21,8 @@ from uuid import uuid4
from swift.common.utils import json
from test.functional import check_response, retry, requires_acls
from test.functional import check_response, retry, requires_acls, \
requires_policies
import test.functional as tf
@ -32,13 +33,9 @@ class TestObject(unittest.TestCase):
raise SkipTest
self.container = uuid4().hex
def put(url, token, parsed, conn):
conn.request('PUT', parsed.path + '/' + self.container, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(put)
resp.read()
self.assertEqual(resp.status, 201)
self.containers = []
self._create_container(self.container)
self.obj = uuid4().hex
def put(url, token, parsed, conn):
@ -50,40 +47,65 @@ class TestObject(unittest.TestCase):
resp.read()
self.assertEqual(resp.status, 201)
def _create_container(self, name=None, headers=None):
if not name:
name = uuid4().hex
self.containers.append(name)
headers = headers or {}
def put(url, token, parsed, conn, name):
new_headers = dict({'X-Auth-Token': token}, **headers)
conn.request('PUT', parsed.path + '/' + name, '',
new_headers)
return check_response(conn)
resp = retry(put, name)
resp.read()
self.assertEqual(resp.status, 201)
return name
def tearDown(self):
if tf.skip:
raise SkipTest
def delete(url, token, parsed, conn, obj):
conn.request('DELETE',
'%s/%s/%s' % (parsed.path, self.container, obj),
'', {'X-Auth-Token': token})
return check_response(conn)
# get list of objects in container
def list(url, token, parsed, conn):
conn.request('GET',
'%s/%s' % (parsed.path, self.container),
'', {'X-Auth-Token': token})
def get(url, token, parsed, conn, container):
conn.request(
'GET', parsed.path + '/' + container + '?format=json', '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(list)
object_listing = resp.read()
self.assertEqual(resp.status, 200)
# iterate over object listing and delete all objects
for obj in object_listing.splitlines():
resp = retry(delete, obj)
resp.read()
self.assertEqual(resp.status, 204)
# delete an object
def delete(url, token, parsed, conn, container, obj):
conn.request(
'DELETE', '/'.join([parsed.path, container, obj['name']]), '',
{'X-Auth-Token': token})
return check_response(conn)
for container in self.containers:
while True:
resp = retry(get, container)
body = resp.read()
if resp.status == 404:
break
self.assert_(resp.status // 100 == 2, resp.status)
objs = json.loads(body)
if not objs:
break
for obj in objs:
resp = retry(delete, container, obj)
resp.read()
self.assertEqual(resp.status, 204)
# delete the container
def delete(url, token, parsed, conn):
conn.request('DELETE', parsed.path + '/' + self.container, '',
def delete(url, token, parsed, conn, name):
conn.request('DELETE', parsed.path + '/' + name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(delete)
resp.read()
self.assertEqual(resp.status, 204)
for container in self.containers:
resp = retry(delete, container)
resp.read()
self.assert_(resp.status in (204, 404))
def test_if_none_match(self):
def put(url, token, parsed, conn):
@ -996,6 +1018,64 @@ class TestObject(unittest.TestCase):
self.assertEquals(headers.get('access-control-allow-origin'),
'http://m.com')
@requires_policies
def test_cross_policy_copy(self):
# create container in first policy
policy = self.policies.select()
container = self._create_container(
headers={'X-Storage-Policy': policy['name']})
obj = uuid4().hex
# create a container in second policy
other_policy = self.policies.exclude(name=policy['name']).select()
other_container = self._create_container(
headers={'X-Storage-Policy': other_policy['name']})
other_obj = uuid4().hex
def put_obj(url, token, parsed, conn, container, obj):
# to keep track of things, use the original path as the body
content = '%s/%s' % (container, obj)
path = '%s/%s' % (parsed.path, content)
conn.request('PUT', path, content, {'X-Auth-Token': token})
return check_response(conn)
# create objects
for c, o in zip((container, other_container), (obj, other_obj)):
resp = retry(put_obj, c, o)
resp.read()
self.assertEqual(resp.status, 201)
def put_copy_from(url, token, parsed, conn, container, obj, source):
dest_path = '%s/%s/%s' % (parsed.path, container, obj)
conn.request('PUT', dest_path, '',
{'X-Auth-Token': token,
'Content-Length': '0',
'X-Copy-From': source})
return check_response(conn)
copy_requests = (
(container, other_obj, '%s/%s' % (other_container, other_obj)),
(other_container, obj, '%s/%s' % (container, obj)),
)
# copy objects
for c, o, source in copy_requests:
resp = retry(put_copy_from, c, o, source)
resp.read()
self.assertEqual(resp.status, 201)
def get_obj(url, token, parsed, conn, container, obj):
path = '%s/%s/%s' % (parsed.path, container, obj)
conn.request('GET', path, '', {'X-Auth-Token': token})
return check_response(conn)
# validate contents, contents should be source
validate_requests = copy_requests
for c, o, body in validate_requests:
resp = retry(get_obj, c, o)
self.assertEqual(resp.status, 200)
self.assertEqual(body, resp.read())
if __name__ == '__main__':
unittest.main()

72
test/functional/tests.py
View File

@ -28,6 +28,8 @@ import uuid
import eventlet
from nose import SkipTest
from swift.common.storage_policy import POLICY
from test.functional import normalized_urls, load_constraint, cluster_info
import test.functional as tf
from test.functional.swift_test_client import Account, Connection, File, \
@ -2077,6 +2079,61 @@ class TestObjectVersioningEnv(object):
cls.versioning_enabled = 'versions' in container_info
class TestCrossPolicyObjectVersioningEnv(object):
# tri-state: None initially, then True/False
versioning_enabled = None
multiple_policies_enabled = None
policies = None
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
if cls.multiple_policies_enabled is None:
try:
cls.policies = tf.FunctionalStoragePolicyCollection.from_info()
except AssertionError:
pass
if cls.policies and len(cls.policies) > 1:
cls.multiple_policies_enabled = True
else:
cls.multiple_policies_enabled = False
# We have to lie here that versioning is enabled. We actually
# don't know, but it does not matter. We know these tests cannot
# run without multiple policies present. If multiple policies are
# present, we won't be setting this field to any value, so it
# should all still work.
cls.versioning_enabled = True
return
policy = cls.policies.select()
version_policy = cls.policies.exclude(name=policy['name']).select()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
# avoid getting a prefix that stops halfway through an encoded
# character
prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
cls.versions_container = cls.account.container(prefix + "-versions")
if not cls.versions_container.create(
{POLICY: policy['name']}):
raise ResponseError(cls.conn.response)
cls.container = cls.account.container(prefix + "-objs")
if not cls.container.create(
hdrs={'X-Versions-Location': cls.versions_container.name,
POLICY: version_policy['name']}):
raise ResponseError(cls.conn.response)
container_info = cls.container.info()
# if versioning is off, then X-Versions-Location won't persist
cls.versioning_enabled = 'versions' in container_info
class TestObjectVersioning(Base):
env = TestObjectVersioningEnv
set_up = False
@ -2127,6 +2184,21 @@ class TestObjectVersioningUTF8(Base2, TestObjectVersioning):
set_up = False
class TestCrossPolicyObjectVersioning(TestObjectVersioning):
env = TestCrossPolicyObjectVersioningEnv
set_up = False
def setUp(self):
super(TestCrossPolicyObjectVersioning, self).setUp()
if self.env.multiple_policies_enabled is False:
raise SkipTest('Cross policy test requires multiple policies')
elif self.env.multiple_policies_enabled is not True:
# just some sanity checking
raise Exception("Expected multiple_policies_enabled "
"to be True/False, got %r" % (
self.env.versioning_enabled,))
class TestTempurlEnv(object):
tempurl_enabled = None # tri-state: None initially, then True/False

36
test/probe/common.py
View File

@ -19,12 +19,14 @@ from subprocess import Popen, PIPE
import sys
from time import sleep, time
from collections import defaultdict
from nose import SkipTest
from swiftclient import get_auth, head_account
from swift.common.ring import Ring
from swift.common.utils import readconf
from swift.common.manager import Manager
from swift.common.storage_policy import POLICIES
from test.probe import CHECK_SERVER_TIMEOUT, VALIDATE_RSYNC
@ -136,13 +138,16 @@ def kill_nonprimary_server(primary_nodes, port2server, pids):
return port
def get_ring(server, force_validate=None):
ring = Ring('/etc/swift/%s.ring.gz' % server)
def get_ring(ring_name, server=None, force_validate=None):
if not server:
server = ring_name
ring = Ring('/etc/swift', ring_name=ring_name)
if not VALIDATE_RSYNC and not force_validate:
return ring
# easy sanity checks
assert 3 == ring.replica_count, '%s has %s replicas instead of 3' % (
ring.serialized_path, ring.replica_count)
if ring.replica_count != 3:
print 'WARNING: %s has %s replicas instead of 3' % (
ring.serialized_path, ring.replica_count)
assert 4 == len(ring.devs), '%s has %s devices instead of 4' % (
ring.serialized_path, len(ring.devs))
# map server to config by port
@ -197,7 +202,8 @@ def reset_environment():
try:
account_ring = get_ring('account')
container_ring = get_ring('container')
object_ring = get_ring('object')
policy = POLICIES.default
object_ring = get_ring(policy.ring_name, 'object')
Manager(['main']).start(wait=False)
port2server = {}
for server, port in [('account', 6002), ('container', 6001),
@ -218,15 +224,14 @@ def reset_environment():
try:
raise
except AssertionError as e:
print >>sys.stderr, 'ERROR: %s' % e
os._exit(1)
raise SkipTest(e)
finally:
try:
kill_servers(port2server, pids)
except Exception:
pass
return pids, port2server, account_ring, container_ring, object_ring, url, \
token, account, config_dict
return pids, port2server, account_ring, container_ring, object_ring, \
policy, url, token, account, config_dict
def get_to_final_state():
@ -242,6 +247,15 @@ def get_to_final_state():
if __name__ == "__main__":
for server in ('account', 'container', 'object'):
get_ring(server, force_validate=True)
for server in ('account', 'container'):
try:
get_ring(server, force_validate=True)
except AssertionError as err:
sys.exit('%s ERROR: %s' % (server, err))
print '%s OK' % server
for policy in POLICIES:
try:
get_ring(policy.ring_name, server='object', force_validate=True)
except AssertionError as err:
sys.exit('object ERROR (%s): %s' % (policy.name, err))
print 'object OK (%s)' % policy.name

2
test/probe/test_account_failures.py
View File

@ -28,7 +28,7 @@ class TestAccountFailures(TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
def tearDown(self):

2
test/probe/test_account_get_fake_responses_match.py
View File

@ -27,7 +27,7 @@ class TestAccountGetFakeResponsesMatch(unittest.TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
self.url, self.token = get_auth(
'http://127.0.0.1:8080/auth/v1.0', 'admin:admin', 'admin')

95
test/probe/test_account_reaper.py Normal file
View File

@ -0,0 +1,95 @@
#!/usr/bin/python -u
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import uuid
from swiftclient import client
from swift.common.storage_policy import POLICIES
from swift.common.manager import Manager
from swift.common.direct_client import direct_delete_account, \
direct_get_object, direct_head_container, ClientException
from test.probe.common import kill_servers, reset_environment, \
get_to_final_state
class TestAccountReaper(unittest.TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
def tearDown(self):
kill_servers(self.port2server, self.pids)
def test_sync(self):
all_objects = []
# upload some containers
for policy in POLICIES:
container = 'container-%s-%s' % (policy.name, uuid.uuid4())
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy': policy.name})
obj = 'object-%s' % uuid.uuid4()
body = 'test-body'
client.put_object(self.url, self.token, container, obj, body)
all_objects.append((policy, container, obj))
Manager(['container-updater']).once()
headers = client.head_account(self.url, self.token)
self.assertEqual(int(headers['x-account-container-count']),
len(POLICIES))
self.assertEqual(int(headers['x-account-object-count']),
len(POLICIES))
self.assertEqual(int(headers['x-account-bytes-used']),
len(POLICIES) * len(body))
part, nodes = self.account_ring.get_nodes(self.account)
for node in nodes:
direct_delete_account(node, part, self.account)
Manager(['account-reaper']).once()
get_to_final_state()
for policy, container, obj in all_objects:
cpart, cnodes = self.container_ring.get_nodes(
self.account, container)
for cnode in cnodes:
try:
direct_head_container(cnode, cpart, self.account,
container)
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail('Found un-reaped /%s/%s on %r' %
(self.account, container, node))
object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/')
part, nodes = object_ring.get_nodes(self.account, container, obj)
for node in nodes:
try:
direct_get_object(node, part, self.account,
container, obj)
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail('Found un-reaped /%s/%s/%s on %r in %s!' %
(self.account, container, obj, node, policy))
if __name__ == "__main__":
unittest.main()

2
test/probe/test_container_failures.py
View File

@ -44,7 +44,7 @@ class TestContainerFailures(TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
def tearDown(self):

612
test/probe/test_container_merge_policy_index.py Normal file
View File

@ -0,0 +1,612 @@
#!/usr/bin/python -u
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from hashlib import md5
import sys
import itertools
import time
import unittest
import uuid
from optparse import OptionParser
from urlparse import urlparse
import random
from nose import SkipTest
from swift.common.manager import Manager
from swift.common.internal_client import InternalClient
from swift.common import utils, direct_client, ring
from swift.common.storage_policy import POLICIES, POLICY_INDEX
from swift.common.http import HTTP_NOT_FOUND
from test.probe.common import reset_environment, get_to_final_state
from swiftclient import client, get_auth, ClientException
TIMEOUT = 60
def meta_command(name, bases, attrs):
"""
Look for attrs with a truthy attribute __command__ and add them to an
attribute __commands__ on the type that maps names to decorated methods.
The decorated methods' doc strings also get mapped in __docs__.
Also adds a method run(command_name, *args, **kwargs) that will
execute the method mapped to the name in __commands__.
"""
commands = {}
docs = {}
for attr, value in attrs.items():
if getattr(value, '__command__', False):
commands[attr] = value
# methods have always have a __doc__ attribute, sometimes empty
docs[attr] = (getattr(value, '__doc__', None) or
'perform the %s command' % attr).strip()
attrs['__commands__'] = commands
attrs['__docs__'] = docs
def run(self, command, *args, **kwargs):
return self.__commands__[command](self, *args, **kwargs)
attrs.setdefault('run', run)
return type(name, bases, attrs)
def command(f):
f.__command__ = True
return f
class BrainSplitter(object):
__metaclass__ = meta_command
def __init__(self, url, token, container_name='test', object_name='test'):
self.url = url
self.token = token
self.account = utils.split_path(urlparse(url).path, 2, 2)[1]
self.container_name = container_name
self.object_name = object_name
self.servers = Manager(['container-server'])
policies = list(POLICIES)
random.shuffle(policies)
self.policies = itertools.cycle(policies)
container_part, container_nodes = ring.Ring(
'/etc/swift/container.ring.gz').get_nodes(
self.account, self.container_name)
container_node_ids = [n['id'] for n in container_nodes]
if all(n_id in container_node_ids for n_id in (0, 1)):
self.primary_numbers = (1, 2)
self.handoff_numbers = (3, 4)
else:
self.primary_numbers = (3, 4)
self.handoff_numbers = (1, 2)
@command
def start_primary_half(self):
"""
start container servers 1 & 2
"""
tuple(self.servers.start(number=n) for n in self.primary_numbers)
@command
def stop_primary_half(self):
"""
stop container servers 1 & 2
"""
tuple(self.servers.stop(number=n) for n in self.primary_numbers)
@command
def start_handoff_half(self):
"""
start container servers 3 & 4
"""
tuple(self.servers.start(number=n) for n in self.handoff_numbers)
@command
def stop_handoff_half(self):
"""
stop container servers 3 & 4
"""
tuple(self.servers.stop(number=n) for n in self.handoff_numbers)
@command
def put_container(self, policy_index=None):
"""
put container with next storage policy
"""
policy = self.policies.next()
if policy_index is not None:
policy = POLICIES.get_by_index(int(policy_index))
if not policy:
raise ValueError('Unknown policy with index %s' % policy)
headers = {'X-Storage-Policy': policy.name}
client.put_container(self.url, self.token, self.container_name,
headers=headers)
@command
def delete_container(self):
"""
delete container
"""
client.delete_container(self.url, self.token, self.container_name)
@command
def put_object(self, headers=None):
"""
issue put for zero byte test object
"""
client.put_object(self.url, self.token, self.container_name,
self.object_name, headers=headers)
@command
def delete_object(self):
"""
issue delete for test object
"""
try:
client.delete_object(self.url, self.token, self.container_name,
self.object_name)
except ClientException as err:
if err.http_status != HTTP_NOT_FOUND:
raise
parser = OptionParser('%prog split-brain [options] '
'<command>[:<args>[,<args>...]] [<command>...]')
parser.usage += '\n\nCommands:\n\t' + \
'\n\t'.join("%s - %s" % (name, doc) for name, doc in
BrainSplitter.__docs__.items())
parser.add_option('-c', '--container', default='container-%s' % uuid.uuid4(),
help='set container name')
parser.add_option('-o', '--object', default='object-%s' % uuid.uuid4(),
help='set object name')
class TestContainerMergePolicyIndex(unittest.TestCase):
def setUp(self):
if len(POLICIES) < 2:
raise SkipTest()
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
self.container_name = 'container-%s' % uuid.uuid4()
self.object_name = 'object-%s' % uuid.uuid4()
self.brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name)
def test_merge_storage_policy_index(self):
# generic split brain
self.brain.stop_primary_half()
self.brain.put_container()
self.brain.start_primary_half()
self.brain.stop_handoff_half()
self.brain.put_container()
self.brain.put_object()
self.brain.start_handoff_half()
# make sure we have some manner of split brain
container_part, container_nodes = self.container_ring.get_nodes(
self.account, self.container_name)
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = set(metadata[POLICY_INDEX] for
node, metadata in head_responses)
self.assert_(len(found_policy_indexes) > 1,
'primary nodes did not disagree about policy index %r' %
head_responses)
# find our object
orig_policy_index = None
for policy_index in found_policy_indexes:
object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift')
part, nodes = object_ring.get_nodes(
self.account, self.container_name, self.object_name)
for node in nodes:
try:
direct_client.direct_head_object(
node, part, self.account, self.container_name,
self.object_name, headers={POLICY_INDEX: policy_index})
except direct_client.ClientException as err:
continue
orig_policy_index = policy_index
break
if orig_policy_index is not None:
break
else:
self.fail('Unable to find /%s/%s/%s in %r' % (
self.account, self.container_name, self.object_name,
found_policy_indexes))
get_to_final_state()
Manager(['container-reconciler']).once()
# validate containers
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = set(metadata[POLICY_INDEX] for
node, metadata in head_responses)
self.assert_(len(found_policy_indexes) == 1,
'primary nodes disagree about policy index %r' %
head_responses)
expected_policy_index = found_policy_indexes.pop()
self.assertNotEqual(orig_policy_index, expected_policy_index)
# validate object placement
orig_policy_ring = POLICIES.get_object_ring(orig_policy_index,
'/etc/swift')
for node in orig_policy_ring.devs:
try:
direct_client.direct_head_object(
node, part, self.account, self.container_name,
self.object_name, headers={
POLICY_INDEX: orig_policy_index})
except direct_client.ClientException as err:
if err.http_status == HTTP_NOT_FOUND:
continue
raise
else:
self.fail('Found /%s/%s/%s in %s' % (
self.account, self.container_name, self.object_name,
orig_policy_index))
# use proxy to access object (bad container info might be cached...)
timeout = time.time() + TIMEOUT
while time.time() < timeout:
try:
metadata = client.head_object(self.url, self.token,
self.container_name,
self.object_name)
except ClientException as err:
if err.http_status != HTTP_NOT_FOUND:
raise
time.sleep(1)
else:
break
else:
self.fail('could not HEAD /%s/%s/%s/ from policy %s '
'after %s seconds.' % (
self.account, self.container_name, self.object_name,
expected_policy_index, TIMEOUT))
def test_reconcile_delete(self):
# generic split brain
self.brain.stop_primary_half()
self.brain.put_container()
self.brain.put_object()
self.brain.start_primary_half()
self.brain.stop_handoff_half()
self.brain.put_container()
self.brain.delete_object()
self.brain.start_handoff_half()
# make sure we have some manner of split brain
container_part, container_nodes = self.container_ring.get_nodes(
self.account, self.container_name)
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = set(metadata[POLICY_INDEX] for
node, metadata in head_responses)
self.assert_(len(found_policy_indexes) > 1,
'primary nodes did not disagree about policy index %r' %
head_responses)
# find our object
orig_policy_index = ts_policy_index = None
for policy_index in found_policy_indexes:
object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift')
part, nodes = object_ring.get_nodes(
self.account, self.container_name, self.object_name)
for node in nodes:
try:
direct_client.direct_head_object(
node, part, self.account, self.container_name,
self.object_name, headers={POLICY_INDEX: policy_index})
except direct_client.ClientException as err:
if 'x-backend-timestamp' in err.http_headers:
ts_policy_index = policy_index
break
else:
orig_policy_index = policy_index
break
if not orig_policy_index:
self.fail('Unable to find /%s/%s/%s in %r' % (
self.account, self.container_name, self.object_name,
found_policy_indexes))
if not ts_policy_index:
self.fail('Unable to find tombstone /%s/%s/%s in %r' % (
self.account, self.container_name, self.object_name,
found_policy_indexes))
get_to_final_state()
Manager(['container-reconciler']).once()
# validate containers
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
new_found_policy_indexes = set(metadata[POLICY_INDEX] for node,
metadata in head_responses)
self.assert_(len(new_found_policy_indexes) == 1,
'primary nodes disagree about policy index %r' %
dict((node['port'], metadata[POLICY_INDEX])
for node, metadata in head_responses))
expected_policy_index = new_found_policy_indexes.pop()
self.assertEqual(orig_policy_index, expected_policy_index)
# validate object fully deleted
for policy_index in found_policy_indexes:
object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift')
part, nodes = object_ring.get_nodes(
self.account, self.container_name, self.object_name)
for node in nodes:
try:
direct_client.direct_head_object(
node, part, self.account, self.container_name,
self.object_name, headers={POLICY_INDEX: policy_index})
except direct_client.ClientException as err:
if err.http_status == HTTP_NOT_FOUND:
continue
else:
self.fail('Found /%s/%s/%s in %s on %s' % (
self.account, self.container_name, self.object_name,
orig_policy_index, node))
def test_reconcile_manifest(self):
manifest_data = []
def write_part(i):
body = 'VERIFY%0.2d' % i + '\x00' * 1048576
part_name = 'manifest_part_%0.2d' % i
manifest_entry = {
"path": "/%s/%s" % (self.container_name, part_name),
"etag": md5(body).hexdigest(),
"size_bytes": len(body),
}
client.put_object(self.url, self.token, self.container_name,
part_name, contents=body)
manifest_data.append(manifest_entry)
# get an old container stashed
self.brain.stop_primary_half()
policy = random.choice(list(POLICIES))
self.brain.put_container(policy.idx)
self.brain.start_primary_half()
# write some parts
for i in range(10):
write_part(i)
self.brain.stop_handoff_half()
wrong_policy = random.choice([p for p in POLICIES if p is not policy])
self.brain.put_container(wrong_policy.idx)
# write some more parts
for i in range(10, 20):
write_part(i)
# write manifest
try:
client.put_object(self.url, self.token, self.container_name,
self.object_name,
contents=utils.json.dumps(manifest_data),
query_string='multipart-manifest=put')
except ClientException as err:
# so as it works out, you can't really upload a multi-part
# manifest for objects that are currently misplaced - you have to
# wait until they're all available - which is about the same as
# some other failure that causes data to be unavailable to the
# proxy at the time of upload
self.assertEqual(err.http_status, 400)
# but what the heck, we'll sneak one in just to see what happens...
direct_manifest_name = self.object_name + '-direct-test'
object_ring = POLICIES.get_object_ring(wrong_policy.idx, '/etc/swift')
part, nodes = object_ring.get_nodes(
self.account, self.container_name, direct_manifest_name)
container_part = self.container_ring.get_part(self.account,
self.container_name)
def translate_direct(data):
return {
'hash': data['etag'],
'bytes': data['size_bytes'],
'name': data['path'],
}
direct_manifest_data = map(translate_direct, manifest_data)
headers = {
'x-container-host': ','.join('%s:%s' % (n['ip'], n['port']) for n
in self.container_ring.devs),
'x-container-device': ','.join(n['device'] for n in
self.container_ring.devs),
'x-container-partition': container_part,
POLICY_INDEX: wrong_policy.idx,
'X-Static-Large-Object': 'True',
}
for node in nodes:
direct_client.direct_put_object(
node, part, self.account, self.container_name,
direct_manifest_name,
contents=utils.json.dumps(direct_manifest_data),
headers=headers)
break # one should do it...
self.brain.start_handoff_half()
get_to_final_state()
Manager(['container-reconciler']).once()
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
# let's see how that direct upload worked out...
metadata, body = client.get_object(
self.url, self.token, self.container_name, direct_manifest_name,
query_string='multipart-manifest=get')
self.assertEqual(metadata['x-static-large-object'].lower(), 'true')
for i, entry in enumerate(utils.json.loads(body)):
for key in ('hash', 'bytes', 'name'):
self.assertEquals(entry[key], direct_manifest_data[i][key])
metadata, body = client.get_object(
self.url, self.token, self.container_name, direct_manifest_name)
self.assertEqual(metadata['x-static-large-object'].lower(), 'true')
self.assertEqual(int(metadata['content-length']),
sum(part['size_bytes'] for part in manifest_data))
self.assertEqual(body, ''.join('VERIFY%0.2d' % i + '\x00' * 1048576
for i in range(20)))
# and regular upload should work now too
client.put_object(self.url, self.token, self.container_name,
self.object_name,
contents=utils.json.dumps(manifest_data),
query_string='multipart-manifest=put')
metadata = client.head_object(self.url, self.token,
self.container_name,
self.object_name)
self.assertEqual(int(metadata['content-length']),
sum(part['size_bytes'] for part in manifest_data))
def test_reconciler_move_object_twice(self):
# select some policies
old_policy = random.choice(list(POLICIES))
new_policy = random.choice([p for p in POLICIES if p != old_policy])
# setup a split brain
self.brain.stop_handoff_half()
# get old_policy on two primaries
self.brain.put_container(policy_index=int(old_policy))
self.brain.start_handoff_half()
self.brain.stop_primary_half()
# force a recreate on handoffs
self.brain.put_container(policy_index=int(old_policy))
self.brain.delete_container()
self.brain.put_container(policy_index=int(new_policy))
self.brain.put_object() # populate memcache with new_policy
self.brain.start_primary_half()
# at this point two primaries have old policy
container_part, container_nodes = self.container_ring.get_nodes(
self.account, self.container_name)
head_responses = []
for node in container_nodes:
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
old_container_node_ids = [
node['id'] for node, metadata in head_responses
if int(old_policy) ==
int(metadata['X-Backend-Storage-Policy-Index'])]
self.assertEqual(2, len(old_container_node_ids))
# hopefully memcache still has the new policy cached
self.brain.put_object()
# double-check object correctly written to new policy
conf_files = []
for server in Manager(['container-reconciler']).servers:
conf_files.extend(server.conf_files())
conf_file = conf_files[0]
client = InternalClient(conf_file, 'probe-test', 3)
client.get_object_metadata(
self.account, self.container_name, self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(new_policy)})
client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,),
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
# shutdown the containers that know about the new policy
self.brain.stop_handoff_half()
# and get rows enqueued from old nodes
for server_type in ('container-replicator', 'container-updater'):
server = Manager([server_type])
tuple(server.once(number=n + 1) for n in old_container_node_ids)
# verify entry in the queue for the "misplaced" new_policy
for container in client.iter_containers('.misplaced_objects'):
for obj in client.iter_objects('.misplaced_objects',
container['name']):
expected = '%d:/%s/%s/%s' % (new_policy, self.account,
self.container_name,
self.object_name)
self.assertEqual(obj['name'], expected)
Manager(['container-reconciler']).once()
# verify object in old_policy
client.get_object_metadata(
self.account, self.container_name, self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
# verify object is *not* in new_policy
client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,),
headers={'X-Backend-Storage-Policy-Index': int(new_policy)})
get_to_final_state()
# verify entry in the queue
client = InternalClient(conf_file, 'probe-test', 3)
for container in client.iter_containers('.misplaced_objects'):
for obj in client.iter_objects('.misplaced_objects',
container['name']):
expected = '%d:/%s/%s/%s' % (old_policy, self.account,
self.container_name,
self.object_name)
self.assertEqual(obj['name'], expected)
Manager(['container-reconciler']).once()
# and now it flops back
client.get_object_metadata(
self.account, self.container_name, self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(new_policy)})
client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,),
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
def main():
options, commands = parser.parse_args()
commands.remove('split-brain')
if not commands:
parser.print_help()
return 'ERROR: must specify at least one command'
for cmd_args in commands:
cmd = cmd_args.split(':', 1)[0]
if cmd not in BrainSplitter.__commands__:
parser.print_help()
return 'ERROR: unknown command %s' % cmd
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',
'test:tester', 'testing')
brain = BrainSplitter(url, token, options.container, options.object)
for cmd_args in commands:
parts = cmd_args.split(':', 1)
command = parts[0]
if len(parts) > 1:
args = utils.list_from_csv(parts[1])
else:
args = ()
try:
brain.run(command, *args)
except ClientException as e:
print '**WARNING**: %s raised %s' % (command, e)
print 'STATUS'.join(['*' * 25] * 2)
brain.servers.status()
sys.exit()
if __name__ == "__main__":
if any('split-brain' in arg for arg in sys.argv):
sys.exit(main())
unittest.main()

100
test/probe/test_container_sync.py Normal file
View File

@ -0,0 +1,100 @@
#!/usr/bin/python -u
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import uuid
from urlparse import urlparse
import random
from nose import SkipTest
from swiftclient import client
from swift.common.storage_policy import POLICIES
from swift.common.manager import Manager
from test.probe.common import kill_servers, reset_environment
def get_current_realm_cluster(url):
parts = urlparse(url)
url = parts.scheme + '://' + parts.netloc + '/info'
http_conn = client.http_connection(url)
try:
info = client.get_capabilities(http_conn)
except client.ClientException:
raise SkipTest('Unable to retrieve cluster info')
try:
realms = info['container_sync']['realms']
except KeyError:
raise SkipTest('Unable to find container sync realms')
for realm, realm_info in realms.items():
for cluster, options in realm_info['clusters'].items():
if options.get('current', False):
return realm, cluster
raise SkipTest('Unable find current realm cluster')
class TestContainerSync(unittest.TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
self.realm, self.cluster = get_current_realm_cluster(self.url)
def tearDown(self):
kill_servers(self.port2server, self.pids)
def test_sync(self):
base_headers = {'X-Container-Sync-Key': 'secret'}
# setup dest container
dest_container = 'dest-container-%s' % uuid.uuid4()
dest_headers = base_headers.copy()
dest_policy = None
if len(POLICIES) > 1:
dest_policy = random.choice(list(POLICIES))
dest_headers['X-Storage-Policy'] = dest_policy.name
client.put_container(self.url, self.token, dest_container,
headers=dest_headers)
# setup source container
source_container = 'source-container-%s' % uuid.uuid4()
source_headers = base_headers.copy()
sync_to = '//%s/%s/%s/%s' % (self.realm, self.cluster, self.account,
dest_container)
source_headers['X-Container-Sync-To'] = sync_to
if dest_policy:
source_policy = random.choice([p for p in POLICIES
if p is not dest_policy])
source_headers['X-Storage-Policy'] = source_policy.name
client.put_container(self.url, self.token, source_container,
headers=source_headers)
# upload to source
object_name = 'object-%s' % uuid.uuid4()
client.put_object(self.url, self.token, source_container, object_name,
'test-body')
# cycle container-sync
Manager(['container-sync']).once()
# retrieve from sync'd container
headers, body = client.get_object(self.url, self.token,
dest_container, object_name)
self.assertEqual(body, 'test-body')
if __name__ == "__main__":
get_current_realm_cluster('http://localhost:8080')
unittest.main()

27
test/probe/test_empty_device_handoff.py
View File

@ -24,6 +24,8 @@ from uuid import uuid4
from swiftclient import client
from swift.common import direct_client
from swift.common.storage_policy import POLICY_INDEX
from swift.obj.diskfile import get_data_dir
from swift.common.exceptions import ClientException
from test.probe.common import kill_server, kill_servers, reset_environment,\
start_server
@ -35,7 +37,7 @@ class TestEmptyDevice(TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
def tearDown(self):
@ -52,7 +54,7 @@ class TestEmptyDevice(TestCase):
def test_main(self):
# Create container
# Kill one container/obj primary server
# Delete the "objects" directory on the primary server
# Delete the default data directory for objects on the primary server
# Create container/obj (goes to two primary servers and one handoff)
# Kill other two container/obj primary servers
# Indirectly through proxy assert we can get container/obj
@ -76,7 +78,8 @@ class TestEmptyDevice(TestCase):
self.account, container, obj)
onode = onodes[0]
kill_server(onode['port'], self.port2server, self.pids)
obj_dir = '%s/objects' % self._get_objects_dir(onode)
obj_dir = '%s/%s' % (self._get_objects_dir(onode),
get_data_dir(self.policy.idx))
shutil.rmtree(obj_dir, True)
self.assertFalse(os.path.exists(obj_dir))
client.put_object(self.url, self.token, container, obj, 'VERIFY')
@ -98,7 +101,8 @@ class TestEmptyDevice(TestCase):
# let's directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj)[-1]
another_onode, opart, self.account, container, obj,
headers={POLICY_INDEX: self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
@ -128,8 +132,9 @@ class TestEmptyDevice(TestCase):
self.assertFalse(os.path.exists(obj_dir))
exc = None
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
@ -150,15 +155,17 @@ class TestEmptyDevice(TestCase):
another_num = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_num)
odata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-1]
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
exc = None
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)

2
test/probe/test_object_async_update.py
View File

@ -29,7 +29,7 @@ class TestObjectAsyncUpdate(TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
def tearDown(self):

132
test/probe/test_object_expirer.py Normal file
View File

@ -0,0 +1,132 @@
#!/usr/bin/python -u
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import random
import unittest
import uuid
from nose import SkipTest
from swift.common.internal_client import InternalClient
from swift.common.manager import Manager
from swift.common.storage_policy import POLICIES
from swift.common.utils import Timestamp
from test.probe.common import reset_environment, get_to_final_state
from test.probe.test_container_merge_policy_index import BrainSplitter
from swiftclient import client
class TestObjectExpirer(unittest.TestCase):
def setUp(self):
if len(POLICIES) < 2:
raise SkipTest('Need more than one policy')
self.expirer = Manager(['object-expirer'])
self.expirer.start()
err = self.expirer.stop()
if err:
raise SkipTest('Unable to verify object-expirer service')
conf_files = []
for server in self.expirer.servers:
conf_files.extend(server.conf_files())
conf_file = conf_files[0]
self.client = InternalClient(conf_file, 'probe-test', 3)
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
self.container_name = 'container-%s' % uuid.uuid4()
self.object_name = 'object-%s' % uuid.uuid4()
self.brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name)
def test_expirer_object_split_brain(self):
old_policy = random.choice(list(POLICIES))
wrong_policy = random.choice([p for p in POLICIES if p != old_policy])
# create an expiring object and a container with the wrong policy
self.brain.stop_primary_half()
self.brain.put_container(int(old_policy))
self.brain.put_object(headers={'X-Delete-After': 2})
# get the object timestamp
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
create_timestamp = Timestamp(metadata['x-timestamp'])
self.brain.start_primary_half()
# get the expiring object updates in their queue, while we have all
# the servers up
Manager(['object-updater']).once()
self.brain.stop_handoff_half()
self.brain.put_container(int(wrong_policy))
# don't start handoff servers, only wrong policy is available
# make sure auto-created containers get in the account listing
Manager(['container-updater']).once()
# this guy should no-op since it's unable to expire the object
self.expirer.once()
self.brain.start_handoff_half()
get_to_final_state()
# validate object is expired
found_in_policy = None
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,),
headers={'X-Backend-Storage-Policy-Index': int(old_policy)})
self.assert_('x-backend-timestamp' in metadata)
self.assertEqual(Timestamp(metadata['x-backend-timestamp']),
create_timestamp)
# but it is still in the listing
for obj in self.client.iter_objects(self.account,
self.container_name):
if self.object_name == obj['name']:
break
else:
self.fail('Did not find listing for %s' % self.object_name)
# clear proxy cache
client.post_container(self.url, self.token, self.container_name, {})
# run the expirier again after replication
self.expirer.once()
# object is not in the listing
for obj in self.client.iter_objects(self.account,
self.container_name):
if self.object_name == obj['name']:
self.fail('Found listing for %s' % self.object_name)
# and validate object is tombstoned
found_in_policy = None
for policy in POLICIES:
metadata = self.client.get_object_metadata(
self.account, self.container_name, self.object_name,
acceptable_statuses=(4,),
headers={'X-Backend-Storage-Policy-Index': int(policy)})
if 'x-backend-timestamp' in metadata:
if found_in_policy:
self.fail('found object in %s and also %s' %
(found_in_policy, policy))
found_in_policy = policy
self.assert_('x-backend-timestamp' in metadata)
self.assert_(Timestamp(metadata['x-backend-timestamp']) >
create_timestamp)
if __name__ == "__main__":
unittest.main()

46
test/probe/test_object_failures.py
View File

@ -23,9 +23,10 @@ from uuid import uuid4
from swiftclient import client
from swift.common import direct_client
from swift.common.storage_policy import POLICY_INDEX
from swift.common.exceptions import ClientException
from swift.common.utils import hash_path, readconf
from swift.obj.diskfile import write_metadata, read_metadata
from swift.obj.diskfile import write_metadata, read_metadata, get_data_dir
from test.probe.common import kill_servers, reset_environment
@ -53,7 +54,7 @@ class TestObjectFailures(TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
def tearDown(self):
@ -72,9 +73,9 @@ class TestObjectFailures(TestCase):
hash_str = hash_path(self.account, container, obj)
obj_server_conf = readconf(self.configs['object-server'][node_id])
devices = obj_server_conf['app:object-server']['devices']
obj_dir = '%s/%s/objects/%s/%s/%s/' % (devices,
device, opart,
hash_str[-3:], hash_str)
obj_dir = '%s/%s/%s/%s/%s/%s/' % (devices, device,
get_data_dir(self.policy.idx),
opart, hash_str[-3:], hash_str)
data_file = get_data_file_path(obj_dir)
return onode, opart, data_file
@ -88,11 +89,13 @@ class TestObjectFailures(TestCase):
write_metadata(data_file, metadata)
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj)[-1]
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})[-1]
self.assertEquals(odata, 'VERIFY')
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
raise Exception("Did not quarantine object")
except ClientException as err:
self.assertEquals(err.http_status, 404)
@ -106,16 +109,21 @@ class TestObjectFailures(TestCase):
metadata = read_metadata(data_file)
metadata['ETag'] = 'badetag'
write_metadata(data_file, metadata)
base_headers = {POLICY_INDEX: self.policy.idx}
for header, result in [({'Range': 'bytes=0-2'}, 'RAN'),
({'Range': 'bytes=1-11'}, 'ANGE'),
({'Range': 'bytes=0-11'}, 'RANGE')]:
req_headers = base_headers.copy()
req_headers.update(header)
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers=header)[-1]
onode, opart, self.account, container, obj,
headers=req_headers)[-1]
self.assertEquals(odata, result)
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
raise Exception("Did not quarantine object")
except ClientException as err:
self.assertEquals(err.http_status, 404)
@ -130,9 +138,9 @@ class TestObjectFailures(TestCase):
with open(data_file, 'w') as fpointer:
write_metadata(fpointer, metadata)
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj, conn_timeout=1,
response_timeout=1)
direct_client.direct_get_object(
onode, opart, self.account, container, obj, conn_timeout=1,
response_timeout=1, headers={POLICY_INDEX: self.policy.idx})
raise Exception("Did not quarantine object")
except ClientException as err:
self.assertEquals(err.http_status, 404)
@ -147,9 +155,9 @@ class TestObjectFailures(TestCase):
with open(data_file, 'w') as fpointer:
write_metadata(fpointer, metadata)
try:
direct_client.direct_head_object(onode, opart, self.account,
container, obj, conn_timeout=1,
response_timeout=1)
direct_client.direct_head_object(
onode, opart, self.account, container, obj, conn_timeout=1,
response_timeout=1, headers={POLICY_INDEX: self.policy.idx})
raise Exception("Did not quarantine object")
except ClientException as err:
self.assertEquals(err.http_status, 404)
@ -164,10 +172,12 @@ class TestObjectFailures(TestCase):
with open(data_file, 'w') as fpointer:
write_metadata(fpointer, metadata)
try:
headers = {'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two',
POLICY_INDEX: self.policy.idx}
direct_client.direct_post_object(
onode, opart, self.account,
container, obj,
{'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'},
headers=headers,
conn_timeout=1,
response_timeout=1)
raise Exception("Did not quarantine object")

41
test/probe/test_object_handoff.py
View File

@ -20,6 +20,7 @@ from uuid import uuid4
from swiftclient import client
from swift.common import direct_client
from swift.common.storage_policy import POLICY_INDEX
from swift.common.exceptions import ClientException
from swift.common.manager import Manager
from test.probe.common import kill_server, kill_servers, reset_environment, \
@ -30,7 +31,7 @@ class TestObjectHandoff(TestCase):
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
def tearDown(self):
@ -90,7 +91,8 @@ class TestObjectHandoff(TestCase):
# directly verify it.
another_onode = self.object_ring.get_more_nodes(opart).next()
odata = direct_client.direct_get_object(
another_onode, opart, self.account, container, obj)[-1]
another_onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
@ -109,8 +111,9 @@ class TestObjectHandoff(TestCase):
start_server(onode['port'], self.port2server, self.pids)
exc = None
try:
direct_client.direct_get_object(onode, opart, self.account,
container, obj)
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
@ -128,21 +131,31 @@ class TestObjectHandoff(TestCase):
another_port_num = another_onode['port']
another_num = (another_port_num - 6000) / 10
Manager(['object-replicator']).once(number=another_num)
odata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-1]
odata = direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})[-1]
if odata != 'VERIFY':
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
exc = None
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)
kill_server(onode['port'], self.port2server, self.pids)
client.delete_object(self.url, self.token, container, obj)
try:
client.delete_object(self.url, self.token, container, obj)
except client.ClientException as err:
if self.object_ring.replica_count > 2:
raise
# Object DELETE returning 503 for (404, 204)
# remove this with fix for
# https://bugs.launchpad.net/swift/+bug/1318375
self.assertEqual(503, err.http_status)
exc = None
try:
client.head_object(self.url, self.token, container, obj)
@ -162,8 +175,9 @@ class TestObjectHandoff(TestCase):
'Container server %s:%s still knew about object' %
(cnode['ip'], cnode['port']))
start_server(onode['port'], self.port2server, self.pids)
direct_client.direct_get_object(onode, opart, self.account, container,
obj)
direct_client.direct_get_object(
onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
# Run the extra server last so it'll remove its extra partition
for node in onodes:
try:
@ -176,8 +190,9 @@ class TestObjectHandoff(TestCase):
Manager(['object-replicator']).once(number=another_node_id)
exc = None
try:
direct_client.direct_get_object(another_onode, opart, self.account,
container, obj)
direct_client.direct_get_object(
another_onode, opart, self.account, container, obj, headers={
POLICY_INDEX: self.policy.idx})
except ClientException as err:
exc = err
self.assertEquals(exc.http_status, 404)

28
test/probe/test_replication_servers_working.py
View File

@ -21,6 +21,8 @@ import time
import shutil
from swiftclient import client
from swift.common.storage_policy import POLICIES
from swift.obj.diskfile import get_data_dir
from test.probe.common import kill_servers, reset_environment
from swift.common.utils import readconf
@ -80,7 +82,7 @@ class TestReplicatorFunctions(TestCase):
Reset all environment and start all servers.
"""
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.url, self.token,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
def tearDown(self):
@ -100,6 +102,7 @@ class TestReplicatorFunctions(TestCase):
# Delete file "hashes.pkl".
# Check, that all files were replicated.
path_list = []
data_dir = get_data_dir(POLICIES.default.idx)
# Figure out where the devices are
for node_id in range(1, 5):
conf = readconf(self.configs['object-server'][node_id])
@ -124,7 +127,10 @@ class TestReplicatorFunctions(TestCase):
for files in files_list[num]:
if not files.endswith('.pending'):
test_node_files_list.append(files)
test_node_dir_list = dir_list[num]
test_node_dir_list = []
for d in dir_list[num]:
if not d.startswith('tmp'):
test_node_dir_list.append(d)
# Run all replicators
try:
Manager(['object-replicator', 'container-replicator',
@ -155,24 +161,24 @@ class TestReplicatorFunctions(TestCase):
time.sleep(1)
# Check behavior by deleting hashes.pkl file
for directory in os.listdir(os.path.join(test_node, 'objects')):
for directory in os.listdir(os.path.join(test_node, data_dir)):
for input_dir in os.listdir(os.path.join(
test_node, 'objects', directory)):
test_node, data_dir, directory)):
if os.path.isdir(os.path.join(
test_node, 'objects', directory, input_dir)):
test_node, data_dir, directory, input_dir)):
shutil.rmtree(os.path.join(
test_node, 'objects', directory, input_dir))
test_node, data_dir, directory, input_dir))
# We will keep trying these tests until they pass for up to 60s
begin = time.time()
while True:
try:
for directory in os.listdir(os.path.join(
test_node, 'objects')):
test_node, data_dir)):
for input_dir in os.listdir(os.path.join(
test_node, 'objects', directory)):
test_node, data_dir, directory)):
self.assertFalse(os.path.isdir(
os.path.join(test_node, 'objects',
os.path.join(test_node, data_dir,
directory, '/', input_dir)))
break
except Exception:
@ -180,9 +186,9 @@ class TestReplicatorFunctions(TestCase):
raise
time.sleep(1)
for directory in os.listdir(os.path.join(test_node, 'objects')):
for directory in os.listdir(os.path.join(test_node, data_dir)):
os.remove(os.path.join(
test_node, 'objects', directory, 'hashes.pkl'))
test_node, data_dir, directory, 'hashes.pkl'))
# We will keep trying these tests until they pass for up to 60s
begin = time.time()

257
test/unit/__init__.py
View File

@ -20,8 +20,9 @@ import copy
import logging
import errno
import sys
from contextlib import contextmanager
from collections import defaultdict
from contextlib import contextmanager, closing
from collections import defaultdict, Iterable
from numbers import Number
from tempfile import NamedTemporaryFile
import time
from eventlet.green import socket
@ -29,49 +30,153 @@ from tempfile import mkdtemp
from shutil import rmtree
from test import get_config
from swift.common.utils import config_true_value, LogAdapter
from swift.common.ring import Ring, RingData
from hashlib import md5
from eventlet import sleep, Timeout
import logging.handlers
from httplib import HTTPException
from numbers import Number
from swift.common import storage_policy
import functools
import cPickle as pickle
from gzip import GzipFile
import mock as mocklib
DEFAULT_PATCH_POLICIES = [storage_policy.StoragePolicy(0, 'nulo', True),
storage_policy.StoragePolicy(1, 'unu')]
LEGACY_PATCH_POLICIES = [storage_policy.StoragePolicy(0, 'legacy', True)]
class FakeRing(object):
def patch_policies(thing_or_policies=None, legacy_only=False):
if legacy_only:
default_policies = LEGACY_PATCH_POLICIES
else:
default_policies = DEFAULT_PATCH_POLICIES
def __init__(self, replicas=3, max_more_nodes=0):
thing_or_policies = thing_or_policies or default_policies
if isinstance(thing_or_policies, (
Iterable, storage_policy.StoragePolicyCollection)):
return PatchPolicies(thing_or_policies)
else:
# it's a thing!
return PatchPolicies(default_policies)(thing_or_policies)
class PatchPolicies(object):
"""
Why not mock.patch? In my case, when used as a decorator on the class it
seemed to patch setUp at the wrong time (i.e. in setup the global wasn't
patched yet)
"""
def __init__(self, policies):
if isinstance(policies, storage_policy.StoragePolicyCollection):
self.policies = policies
else:
self.policies = storage_policy.StoragePolicyCollection(policies)
def __call__(self, thing):
if isinstance(thing, type):
return self._patch_class(thing)
else:
return self._patch_method(thing)
def _patch_class(self, cls):
class NewClass(cls):
already_patched = False
def setUp(cls_self):
self._orig_POLICIES = storage_policy._POLICIES
if not cls_self.already_patched:
storage_policy._POLICIES = self.policies
cls_self.already_patched = True
super(NewClass, cls_self).setUp()
def tearDown(cls_self):
super(NewClass, cls_self).tearDown()
storage_policy._POLICIES = self._orig_POLICIES
NewClass.__name__ = cls.__name__
return NewClass
def _patch_method(self, f):
@functools.wraps(f)
def mywrapper(*args, **kwargs):
self._orig_POLICIES = storage_policy._POLICIES
try:
storage_policy._POLICIES = self.policies
return f(*args, **kwargs)
finally:
storage_policy._POLICIES = self._orig_POLICIES
return mywrapper
def __enter__(self):
self._orig_POLICIES = storage_policy._POLICIES
storage_policy._POLICIES = self.policies
def __exit__(self, *args):
storage_policy._POLICIES = self._orig_POLICIES
class FakeRing(Ring):
def __init__(self, replicas=3, max_more_nodes=0, part_power=0):
"""
:param part_power: make part calculation based on the path
If you set a part_power when you setup your FakeRing the parts you get
out of ring methods will actually be based on the path - otherwise we
exercise the real ring code, but ignore the result and return 1.
"""
# 9 total nodes (6 more past the initial 3) is the cap, no matter if
# this is set higher, or R^2 for R replicas
self.replicas = replicas
self.set_replicas(replicas)
self.max_more_nodes = max_more_nodes
self.devs = {}
self._part_shift = 32 - part_power
self._reload()
def get_part(self, *args, **kwargs):
real_part = super(FakeRing, self).get_part(*args, **kwargs)
if self._part_shift == 32:
return 1
return real_part
def _reload(self):
self._rtime = time.time()
def clear_errors(self):
for dev in self.devs:
for key in ('errors', 'last_error'):
try:
del dev[key]
except KeyError:
pass
def set_replicas(self, replicas):
self.replicas = replicas
self.devs = {}
self._devs = []
for x in range(self.replicas):
ip = '10.0.0.%s' % x
port = 1000 + x
self._devs.append({
'ip': ip,
'replication_ip': ip,
'port': port,
'replication_port': port,
'device': 'sd' + (chr(ord('a') + x)),
'zone': x % 3,
'region': x % 2,
'id': x,
})
@property
def replica_count(self):
return self.replicas
def get_part(self, account, container=None, obj=None):
return 1
def get_nodes(self, account, container=None, obj=None):
devs = []
for x in xrange(self.replicas):
devs.append(self.devs.get(x))
if devs[x] is None:
self.devs[x] = devs[x] = \
{'ip': '10.0.0.%s' % x,
'port': 1000 + x,
'device': 'sd' + (chr(ord('a') + x)),
'zone': x % 3,
'region': x % 2,
'id': x}
return 1, devs
def get_part_nodes(self, part):
return self.get_nodes('blah')[1]
def _get_part_nodes(self, part):
return list(self._devs)
def get_more_nodes(self, part):
# replicas^2 is the true cap
@ -85,6 +190,27 @@ class FakeRing(object):
'id': x}
def write_fake_ring(path, *devs):
"""
Pretty much just a two node, two replica, 2 part power ring...
"""
dev1 = {'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': 6000}
dev2 = {'id': 0, 'zone': 0, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': 6000}
dev1_updates, dev2_updates = devs or ({}, {})
dev1.update(dev1_updates)
dev2.update(dev2_updates)
replica2part2dev_id = [[0, 1, 0, 1], [1, 0, 1, 0]]
devs = [dev1, dev2]
part_shift = 30
with closing(GzipFile(path, 'wb')) as f:
pickle.dump(RingData(replica2part2dev_id, devs, part_shift), f)
class FakeMemcache(object):
def __init__(self):
@ -201,6 +327,22 @@ def temptree(files, contents=''):
rmtree(tempdir)
def with_tempdir(f):
"""
Decorator to give a single test a tempdir as argument to test method.
"""
@functools.wraps(f)
def wrapped(*args, **kwargs):
tempdir = mkdtemp()
args = list(args)
args.append(tempdir)
try:
return f(*args, **kwargs)
finally:
rmtree(tempdir)
return wrapped
class NullLoggingHandler(logging.Handler):
def emit(self, record):
@ -237,6 +379,7 @@ class FakeLogger(logging.Logger):
self.facility = kwargs['facility']
self.statsd_client = None
self.thread_locals = None
self.parent = None
def _clear(self):
self.log_dict = defaultdict(list)
@ -247,20 +390,20 @@ class FakeLogger(logging.Logger):
self.log_dict[store_name].append((args, kwargs))
return stub_fn
def _store_and_log_in(store_name):
def _store_and_log_in(store_name, level):
def stub_fn(self, *args, **kwargs):
self.log_dict[store_name].append((args, kwargs))
self._log(store_name, args[0], args[1:], **kwargs)
self._log(level, args[0], args[1:], **kwargs)
return stub_fn
def get_lines_for_level(self, level):
return self.lines_dict[level]
error = _store_and_log_in('error')
info = _store_and_log_in('info')
warning = _store_and_log_in('warning')
warn = _store_and_log_in('warning')
debug = _store_and_log_in('debug')
error = _store_and_log_in('error', logging.ERROR)
info = _store_and_log_in('info', logging.INFO)
warning = _store_and_log_in('warning', logging.WARNING)
warn = _store_and_log_in('warning', logging.WARNING)
debug = _store_and_log_in('debug', logging.DEBUG)
def exception(self, *args, **kwargs):
self.log_dict['exception'].append((args, kwargs,
@ -268,11 +411,12 @@ class FakeLogger(logging.Logger):
print 'FakeLogger Exception: %s' % self.log_dict
# mock out the StatsD logging methods:
update_stats = _store_in('update_stats')
increment = _store_in('increment')
decrement = _store_in('decrement')
timing = _store_in('timing')
timing_since = _store_in('timing_since')
update_stats = _store_in('update_stats')
transfer_rate = _store_in('transfer_rate')
set_statsd_prefix = _store_in('set_statsd_prefix')
def get_increments(self):
@ -315,7 +459,7 @@ class FakeLogger(logging.Logger):
print 'WARNING: unable to format log message %r %% %r' % (
record.msg, record.args)
raise
self.lines_dict[record.levelno].append(line)
self.lines_dict[record.levelname.lower()].append(line)
def handle(self, record):
self._handle(record)
@ -332,16 +476,40 @@ class DebugLogger(FakeLogger):
def __init__(self, *args, **kwargs):
FakeLogger.__init__(self, *args, **kwargs)
self.formatter = logging.Formatter("%(server)s: %(message)s")
self.formatter = logging.Formatter(
"%(server)s %(levelname)s: %(message)s")
def handle(self, record):
self._handle(record)
print self.formatter.format(record)
class DebugLogAdapter(LogAdapter):
def _send_to_logger(name):
def stub_fn(self, *args, **kwargs):
return getattr(self.logger, name)(*args, **kwargs)
return stub_fn
# delegate to FakeLogger's mocks
update_stats = _send_to_logger('update_stats')
increment = _send_to_logger('increment')
decrement = _send_to_logger('decrement')
timing = _send_to_logger('timing')
timing_since = _send_to_logger('timing_since')
transfer_rate = _send_to_logger('transfer_rate')
set_statsd_prefix = _send_to_logger('set_statsd_prefix')
def __getattribute__(self, name):
try:
return object.__getattribute__(self, name)
except AttributeError:
return getattr(self.__dict__['logger'], name)
def debug_logger(name='test'):
"""get a named adapted debug logger"""
return LogAdapter(DebugLogger(), name)
return DebugLogAdapter(DebugLogger(), name)
original_syslog_handler = logging.handlers.SysLogHandler
@ -458,7 +626,10 @@ def fake_http_connect(*code_iter, **kwargs):
self._next_sleep = None
def getresponse(self):
if kwargs.get('raise_exc'):
exc = kwargs.get('raise_exc')
if exc:
if isinstance(exc, Exception):
raise exc
raise Exception('test')
if kwargs.get('raise_timeout_exc'):
raise Timeout()
@ -586,3 +757,11 @@ def fake_http_connect(*code_iter, **kwargs):
connect.code_iter = code_iter
return connect
@contextmanager
def mocked_http_conn(*args, **kwargs):
fake_conn = fake_http_connect(*args, **kwargs)
with mocklib.patch('swift.common.bufferedhttp.http_connect_raw',
new=fake_conn):
yield fake_conn

662
test/unit/account/test_backend.py
View File

@ -17,14 +17,27 @@
import hashlib
import unittest
import pickle
import os
from time import sleep, time
from uuid import uuid4
from tempfile import mkdtemp
from shutil import rmtree
import sqlite3
import itertools
from contextlib import contextmanager
import random
from swift.account.backend import AccountBroker
from swift.common.utils import normalize_timestamp
from swift.common.utils import Timestamp
from test.unit import patch_policies, with_tempdir
from swift.common.db import DatabaseConnectionError
from swift.common.storage_policy import StoragePolicy, POLICIES
from test.unit.common.test_db import TestExampleBroker
@patch_policies
class TestAccountBroker(unittest.TestCase):
"""Tests for AccountBroker"""
@ -44,7 +57,7 @@ class TestAccountBroker(unittest.TestCase):
self.fail("Unexpected exception raised: %r" % e)
else:
self.fail("Expected a DatabaseConnectionError exception")
broker.initialize(normalize_timestamp('1'))
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
curs = conn.cursor()
curs.execute('SELECT 1')
@ -54,7 +67,7 @@ class TestAccountBroker(unittest.TestCase):
# Test AccountBroker throwing a conn away after exception
first_conn = None
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
first_conn = conn
try:
@ -68,18 +81,21 @@ class TestAccountBroker(unittest.TestCase):
def test_empty(self):
# Test AccountBroker.empty
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
broker.initialize(Timestamp('1').internal)
self.assert_(broker.empty())
broker.put_container('o', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('o', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
self.assert_(not broker.empty())
sleep(.00001)
broker.put_container('o', 0, normalize_timestamp(time()), 0, 0)
broker.put_container('o', 0, Timestamp(time()).internal, 0, 0,
POLICIES.default.idx)
self.assert_(broker.empty())
def test_reclaim(self):
broker = AccountBroker(':memory:', account='test_account')
broker.initialize(normalize_timestamp('1'))
broker.put_container('c', normalize_timestamp(time()), 0, 0, 0)
broker.initialize(Timestamp('1').internal)
broker.put_container('c', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@ -87,7 +103,7 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 0)
broker.reclaim(normalize_timestamp(time() - 999), time())
broker.reclaim(Timestamp(time() - 999).internal, time())
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@ -96,7 +112,8 @@ class TestAccountBroker(unittest.TestCase):
"SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 0)
sleep(.00001)
broker.put_container('c', 0, normalize_timestamp(time()), 0, 0)
broker.put_container('c', 0, Timestamp(time()).internal, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@ -104,7 +121,7 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 1)
broker.reclaim(normalize_timestamp(time() - 999), time())
broker.reclaim(Timestamp(time() - 999).internal, time())
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@ -113,7 +130,7 @@ class TestAccountBroker(unittest.TestCase):
"SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 1)
sleep(.00001)
broker.reclaim(normalize_timestamp(time()), time())
broker.reclaim(Timestamp(time()).internal, time())
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@ -122,18 +139,18 @@ class TestAccountBroker(unittest.TestCase):
"SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 0)
# Test reclaim after deletion. Create 3 test containers
broker.put_container('x', 0, 0, 0, 0)
broker.put_container('y', 0, 0, 0, 0)
broker.put_container('z', 0, 0, 0, 0)
broker.reclaim(normalize_timestamp(time()), time())
broker.put_container('x', 0, 0, 0, 0, POLICIES.default.idx)
broker.put_container('y', 0, 0, 0, 0, POLICIES.default.idx)
broker.put_container('z', 0, 0, 0, 0, POLICIES.default.idx)
broker.reclaim(Timestamp(time()).internal, time())
# self.assertEqual(len(res), 2)
# self.assert_(isinstance(res, tuple))
# containers, account_name = res
# self.assert_(containers is None)
# self.assert_(account_name is None)
# Now delete the account
broker.delete_db(normalize_timestamp(time()))
broker.reclaim(normalize_timestamp(time()), time())
broker.delete_db(Timestamp(time()).internal)
broker.reclaim(Timestamp(time()).internal, time())
# self.assertEqual(len(res), 2)
# self.assert_(isinstance(res, tuple))
# containers, account_name = res
@ -144,11 +161,36 @@ class TestAccountBroker(unittest.TestCase):
# self.assert_('z' in containers)
# self.assert_('a' not in containers)
def test_delete_db_status(self):
ts = (Timestamp(t).internal for t in itertools.count(int(time())))
start = ts.next()
broker = AccountBroker(':memory:', account='a')
broker.initialize(start)
info = broker.get_info()
self.assertEqual(info['put_timestamp'], Timestamp(start).internal)
self.assert_(Timestamp(info['created_at']) >= start)
self.assertEqual(info['delete_timestamp'], '0')
if self.__class__ == TestAccountBrokerBeforeMetadata:
self.assertEqual(info['status_changed_at'], '0')
else:
self.assertEqual(info['status_changed_at'],
Timestamp(start).internal)
# delete it
delete_timestamp = ts.next()
broker.delete_db(delete_timestamp)
info = broker.get_info()
self.assertEqual(info['put_timestamp'], Timestamp(start).internal)
self.assert_(Timestamp(info['created_at']) >= start)
self.assertEqual(info['delete_timestamp'], delete_timestamp)
self.assertEqual(info['status_changed_at'], delete_timestamp)
def test_delete_container(self):
# Test AccountBroker.delete_container
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
broker.put_container('o', normalize_timestamp(time()), 0, 0, 0)
broker.initialize(Timestamp('1').internal)
broker.put_container('o', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@ -157,7 +199,8 @@ class TestAccountBroker(unittest.TestCase):
"SELECT count(*) FROM container "
"WHERE deleted = 1").fetchone()[0], 0)
sleep(.00001)
broker.put_container('o', 0, normalize_timestamp(time()), 0, 0)
broker.put_container('o', 0, Timestamp(time()).internal, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT count(*) FROM container "
@ -169,11 +212,12 @@ class TestAccountBroker(unittest.TestCase):
def test_put_container(self):
# Test AccountBroker.put_container
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
broker.initialize(Timestamp('1').internal)
# Create initial container
timestamp = normalize_timestamp(time())
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0)
timestamp = Timestamp(time()).internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@ -185,7 +229,8 @@ class TestAccountBroker(unittest.TestCase):
"SELECT deleted FROM container").fetchone()[0], 0)
# Reput same event
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0)
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@ -198,8 +243,9 @@ class TestAccountBroker(unittest.TestCase):
# Put new event
sleep(.00001)
timestamp = normalize_timestamp(time())
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0)
timestamp = Timestamp(time()).internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@ -211,8 +257,9 @@ class TestAccountBroker(unittest.TestCase):
"SELECT deleted FROM container").fetchone()[0], 0)
# Put old event
otimestamp = normalize_timestamp(float(timestamp) - 1)
broker.put_container('"{<container \'&\' name>}"', otimestamp, 0, 0, 0)
otimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_container('"{<container \'&\' name>}"', otimestamp, 0, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@ -224,8 +271,9 @@ class TestAccountBroker(unittest.TestCase):
"SELECT deleted FROM container").fetchone()[0], 0)
# Put old delete event
dtimestamp = normalize_timestamp(float(timestamp) - 1)
broker.put_container('"{<container \'&\' name>}"', 0, dtimestamp, 0, 0)
dtimestamp = Timestamp(float(Timestamp(timestamp)) - 1).internal
broker.put_container('"{<container \'&\' name>}"', 0, dtimestamp, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@ -241,8 +289,9 @@ class TestAccountBroker(unittest.TestCase):
# Put new delete event
sleep(.00001)
timestamp = normalize_timestamp(time())
broker.put_container('"{<container \'&\' name>}"', 0, timestamp, 0, 0)
timestamp = Timestamp(time()).internal
broker.put_container('"{<container \'&\' name>}"', 0, timestamp, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@ -255,8 +304,9 @@ class TestAccountBroker(unittest.TestCase):
# Put new event
sleep(.00001)
timestamp = normalize_timestamp(time())
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0)
timestamp = Timestamp(time()).internal
broker.put_container('"{<container \'&\' name>}"', timestamp, 0, 0, 0,
POLICIES.default.idx)
with broker.get() as conn:
self.assertEqual(conn.execute(
"SELECT name FROM container").fetchone()[0],
@ -270,54 +320,68 @@ class TestAccountBroker(unittest.TestCase):
def test_get_info(self):
# Test AccountBroker.get_info
broker = AccountBroker(':memory:', account='test1')
broker.initialize(normalize_timestamp('1'))
broker.initialize(Timestamp('1').internal)
info = broker.get_info()
self.assertEqual(info['account'], 'test1')
self.assertEqual(info['hash'], '00000000000000000000000000000000')
self.assertEqual(info['put_timestamp'], Timestamp(1).internal)
self.assertEqual(info['delete_timestamp'], '0')
if self.__class__ == TestAccountBrokerBeforeMetadata:
self.assertEqual(info['status_changed_at'], '0')
else:
self.assertEqual(info['status_changed_at'], Timestamp(1).internal)
info = broker.get_info()
self.assertEqual(info['container_count'], 0)
broker.put_container('c1', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('c1', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 1)
sleep(.00001)
broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('c2', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 2)
sleep(.00001)
broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('c2', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 2)
sleep(.00001)
broker.put_container('c1', 0, normalize_timestamp(time()), 0, 0)
broker.put_container('c1', 0, Timestamp(time()).internal, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 1)
sleep(.00001)
broker.put_container('c2', 0, normalize_timestamp(time()), 0, 0)
broker.put_container('c2', 0, Timestamp(time()).internal, 0, 0,
POLICIES.default.idx)
info = broker.get_info()
self.assertEqual(info['container_count'], 0)
def test_list_containers_iter(self):
# Test AccountBroker.list_containers_iter
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
broker.initialize(Timestamp('1').internal)
for cont1 in xrange(4):
for cont2 in xrange(125):
broker.put_container('%d-%04d' % (cont1, cont2),
normalize_timestamp(time()), 0, 0, 0)
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
for cont in xrange(125):
broker.put_container('2-0051-%04d' % cont,
normalize_timestamp(time()), 0, 0, 0)
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
for cont in xrange(125):
broker.put_container('3-%04d-0049' % cont,
normalize_timestamp(time()), 0, 0, 0)
Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
listing = broker.list_containers_iter(100, '', None, None, '')
self.assertEqual(len(listing), 100)
@ -381,7 +445,8 @@ class TestAccountBroker(unittest.TestCase):
'3-0047-', '3-0048', '3-0048-', '3-0049',
'3-0049-', '3-0050'])
broker.put_container('3-0049-', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('3-0049-', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
listing = broker.list_containers_iter(10, '3-0048', None, None, None)
self.assertEqual(len(listing), 10)
self.assertEqual([row[0] for row in listing],
@ -405,17 +470,27 @@ class TestAccountBroker(unittest.TestCase):
# Test AccountBroker.list_containers_iter for an
# account that has an odd container with a trailing delimiter
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
broker.put_container('a', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('a-', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('a-a', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('a-a-a', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('a-a-b', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('a-b', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('b', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('b-a', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('b-b', normalize_timestamp(time()), 0, 0, 0)
broker.put_container('c', normalize_timestamp(time()), 0, 0, 0)
broker.initialize(Timestamp('1').internal)
broker.put_container('a', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('a-', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('a-a', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('a-a-a', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('a-a-b', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('a-b', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('b', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('b-a', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('b-b', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker.put_container('c', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
listing = broker.list_containers_iter(15, None, None, None, None)
self.assertEqual(len(listing), 10)
self.assertEqual([row[0] for row in listing],
@ -435,24 +510,30 @@ class TestAccountBroker(unittest.TestCase):
def test_chexor(self):
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
broker.put_container('a', normalize_timestamp(1),
normalize_timestamp(0), 0, 0)
broker.put_container('b', normalize_timestamp(2),
normalize_timestamp(0), 0, 0)
broker.initialize(Timestamp('1').internal)
broker.put_container('a', Timestamp(1).internal,
Timestamp(0).internal, 0, 0,
POLICIES.default.idx)
broker.put_container('b', Timestamp(2).internal,
Timestamp(0).internal, 0, 0,
POLICIES.default.idx)
hasha = hashlib.md5(
'%s-%s' % ('a', '0000000001.00000-0000000000.00000-0-0')
'%s-%s' % ('a', "%s-%s-%s-%s" % (
Timestamp(1).internal, Timestamp(0).internal, 0, 0))
).digest()
hashb = hashlib.md5(
'%s-%s' % ('b', '0000000002.00000-0000000000.00000-0-0')
'%s-%s' % ('b', "%s-%s-%s-%s" % (
Timestamp(2).internal, Timestamp(0).internal, 0, 0))
).digest()
hashc = \
''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
self.assertEqual(broker.get_info()['hash'], hashc)
broker.put_container('b', normalize_timestamp(3),
normalize_timestamp(0), 0, 0)
broker.put_container('b', Timestamp(3).internal,
Timestamp(0).internal, 0, 0,
POLICIES.default.idx)
hashb = hashlib.md5(
'%s-%s' % ('b', '0000000003.00000-0000000000.00000-0-0')
'%s-%s' % ('b', "%s-%s-%s-%s" % (
Timestamp(3).internal, Timestamp(0).internal, 0, 0))
).digest()
hashc = \
''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))
@ -460,18 +541,21 @@ class TestAccountBroker(unittest.TestCase):
def test_merge_items(self):
broker1 = AccountBroker(':memory:', account='a')
broker1.initialize(normalize_timestamp('1'))
broker1.initialize(Timestamp('1').internal)
broker2 = AccountBroker(':memory:', account='a')
broker2.initialize(normalize_timestamp('1'))
broker1.put_container('a', normalize_timestamp(1), 0, 0, 0)
broker1.put_container('b', normalize_timestamp(2), 0, 0, 0)
broker2.initialize(Timestamp('1').internal)
broker1.put_container('a', Timestamp(1).internal, 0, 0, 0,
POLICIES.default.idx)
broker1.put_container('b', Timestamp(2).internal, 0, 0, 0,
POLICIES.default.idx)
id = broker1.get_info()['id']
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
self.assertEqual(len(items), 2)
self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items]))
broker1.put_container('c', normalize_timestamp(3), 0, 0, 0)
broker1.put_container('c', Timestamp(3).internal, 0, 0, 0,
POLICIES.default.idx)
broker2.merge_items(broker1.get_items_since(
broker2.get_sync(id), 1000), id)
items = broker2.get_items_since(-1, 1000)
@ -479,6 +563,145 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(['a', 'b', 'c'],
sorted([rec['name'] for rec in items]))
def test_load_old_pending_puts(self):
# pending puts from pre-storage-policy account brokers won't contain
# the storage policy index
tempdir = mkdtemp()
broker_path = os.path.join(tempdir, 'test-load-old.db')
try:
broker = AccountBroker(broker_path, account='real')
broker.initialize(Timestamp(1).internal)
with open(broker_path + '.pending', 'a+b') as pending:
pending.write(':')
pending.write(pickle.dumps(
# name, put_timestamp, delete_timestamp, object_count,
# bytes_used, deleted
('oldcon', Timestamp(200).internal,
Timestamp(0).internal,
896, 9216695, 0)).encode('base64'))
broker._commit_puts()
with broker.get() as conn:
results = list(conn.execute('''
SELECT name, storage_policy_index FROM container
'''))
self.assertEqual(len(results), 1)
self.assertEqual(dict(results[0]),
{'name': 'oldcon', 'storage_policy_index': 0})
finally:
rmtree(tempdir)
@patch_policies([StoragePolicy(0, 'zero', False),
StoragePolicy(1, 'one', True),
StoragePolicy(2, 'two', False),
StoragePolicy(3, 'three', False)])
def test_get_policy_stats(self):
ts = (Timestamp(t).internal for t in itertools.count(int(time())))
broker = AccountBroker(':memory:', account='a')
broker.initialize(ts.next())
# check empty policy_stats
self.assertTrue(broker.empty())
policy_stats = broker.get_policy_stats()
self.assertEqual(policy_stats, {})
# add some empty containers
for policy in POLICIES:
container_name = 'c-%s' % policy.name
put_timestamp = ts.next()
broker.put_container(container_name,
put_timestamp, 0,
0, 0,
policy.idx)
policy_stats = broker.get_policy_stats()
stats = policy_stats[policy.idx]
self.assertEqual(stats['object_count'], 0)
self.assertEqual(stats['bytes_used'], 0)
# update the containers object & byte count
for policy in POLICIES:
container_name = 'c-%s' % policy.name
put_timestamp = ts.next()
count = policy.idx * 100 # good as any integer
broker.put_container(container_name,
put_timestamp, 0,
count, count,
policy.idx)
policy_stats = broker.get_policy_stats()
stats = policy_stats[policy.idx]
self.assertEqual(stats['object_count'], count)
self.assertEqual(stats['bytes_used'], count)
# check all the policy_stats at once
for policy_index, stats in policy_stats.items():
policy = POLICIES[policy_index]
count = policy.idx * 100 # coupled with policy for test
self.assertEqual(stats['object_count'], count)
self.assertEqual(stats['bytes_used'], count)
# now delete the containers one by one
for policy in POLICIES:
container_name = 'c-%s' % policy.name
delete_timestamp = ts.next()
broker.put_container(container_name,
0, delete_timestamp,
0, 0,
policy.idx)
policy_stats = broker.get_policy_stats()
stats = policy_stats[policy.idx]
self.assertEqual(stats['object_count'], 0)
self.assertEqual(stats['bytes_used'], 0)
@patch_policies([StoragePolicy(0, 'zero', False),
StoragePolicy(1, 'one', True)])
def test_policy_stats_tracking(self):
ts = (Timestamp(t).internal for t in itertools.count(int(time())))
broker = AccountBroker(':memory:', account='a')
broker.initialize(ts.next())
# policy 0
broker.put_container('con1', ts.next(), 0, 12, 2798641, 0)
broker.put_container('con1', ts.next(), 0, 13, 8156441, 0)
# policy 1
broker.put_container('con2', ts.next(), 0, 7, 5751991, 1)
broker.put_container('con2', ts.next(), 0, 8, 6085379, 1)
stats = broker.get_policy_stats()
self.assertEqual(len(stats), 2)
self.assertEqual(stats[0]['object_count'], 13)
self.assertEqual(stats[0]['bytes_used'], 8156441)
self.assertEqual(stats[1]['object_count'], 8)
self.assertEqual(stats[1]['bytes_used'], 6085379)
# Break encapsulation here to make sure that there's only 2 rows in
# the stats table. It's possible that there could be 4 rows (one per
# put_container) but that they came out in the right order so that
# get_policy_stats() collapsed them down to the right number. To prove
# that's not so, we have to go peek at the broker's internals.
with broker.get() as conn:
nrows = conn.execute(
"SELECT COUNT(*) FROM policy_stat").fetchall()[0][0]
self.assertEqual(nrows, 2)
def prespi_AccountBroker_initialize(self, conn, put_timestamp, **kwargs):
"""
The AccountBroker initialze() function before we added the
policy stat table. Used by test_policy_table_creation() to
make sure that the AccountBroker will correctly add the table
for cases where the DB existed before the policy suport was added.
:param conn: DB connection object
:param put_timestamp: put timestamp
"""
if not self.account:
raise ValueError(
'Attempting to create a new database with no account set')
self.create_container_table(conn)
self.create_account_stat_table(conn, put_timestamp)
def premetadata_create_account_stat_table(self, conn, put_timestamp):
"""
@ -511,10 +734,27 @@ def premetadata_create_account_stat_table(self, conn, put_timestamp):
conn.execute('''
UPDATE account_stat SET account = ?, created_at = ?, id = ?,
put_timestamp = ?
''', (self.account, normalize_timestamp(time()), str(uuid4()),
''', (self.account, Timestamp(time()).internal, str(uuid4()),
put_timestamp))
class TestCommonAccountBroker(TestExampleBroker):
broker_class = AccountBroker
def setUp(self):
super(TestCommonAccountBroker, self).setUp()
self.policy = random.choice(list(POLICIES))
def put_item(self, broker, timestamp):
broker.put_container('test', timestamp, 0, 0, 0,
int(self.policy))
def delete_item(self, broker, timestamp):
broker.put_container('test', 0, timestamp, 0, 0,
int(self.policy))
class TestAccountBrokerBeforeMetadata(TestAccountBroker):
"""
Tests for AccountBroker against databases created before
@ -527,7 +767,7 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker):
AccountBroker.create_account_stat_table = \
premetadata_create_account_stat_table
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
broker.initialize(Timestamp('1').internal)
exc = None
with broker.get() as conn:
try:
@ -540,6 +780,278 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker):
AccountBroker.create_account_stat_table = \
self._imported_create_account_stat_table
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
conn.execute('SELECT metadata FROM account_stat')
def prespi_create_container_table(self, conn):
"""
Copied from AccountBroker before the sstoage_policy_index column was
added; used for testing with TestAccountBrokerBeforeSPI.
Create container table which is specific to the account DB.
:param conn: DB connection object
"""
conn.executescript("""
CREATE TABLE container (
ROWID INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT,
put_timestamp TEXT,
delete_timestamp TEXT,
object_count INTEGER,
bytes_used INTEGER,
deleted INTEGER DEFAULT 0
);
CREATE INDEX ix_container_deleted_name ON
container (deleted, name);
CREATE TRIGGER container_insert AFTER INSERT ON container
BEGIN
UPDATE account_stat
SET container_count = container_count + (1 - new.deleted),
object_count = object_count + new.object_count,
bytes_used = bytes_used + new.bytes_used,
hash = chexor(hash, new.name,
new.put_timestamp || '-' ||
new.delete_timestamp || '-' ||
new.object_count || '-' || new.bytes_used);
END;
CREATE TRIGGER container_update BEFORE UPDATE ON container
BEGIN
SELECT RAISE(FAIL, 'UPDATE not allowed; DELETE and INSERT');
END;
CREATE TRIGGER container_delete AFTER DELETE ON container
BEGIN
UPDATE account_stat
SET container_count = container_count - (1 - old.deleted),
object_count = object_count - old.object_count,
bytes_used = bytes_used - old.bytes_used,
hash = chexor(hash, old.name,
old.put_timestamp || '-' ||
old.delete_timestamp || '-' ||
old.object_count || '-' || old.bytes_used);
END;
""")
class TestAccountBrokerBeforeSPI(TestAccountBroker):
"""
Tests for AccountBroker against databases created before
the storage_policy_index column was added.
"""
def setUp(self):
self._imported_create_container_table = \
AccountBroker.create_container_table
AccountBroker.create_container_table = \
prespi_create_container_table
self._imported_initialize = AccountBroker._initialize
AccountBroker._initialize = prespi_AccountBroker_initialize
broker = AccountBroker(':memory:', account='a')
broker.initialize(Timestamp('1').internal)
exc = None
with broker.get() as conn:
try:
conn.execute('SELECT storage_policy_index FROM container')
except BaseException as err:
exc = err
self.assert_('no such column: storage_policy_index' in str(exc))
with broker.get() as conn:
try:
conn.execute('SELECT * FROM policy_stat')
except sqlite3.OperationalError as err:
self.assert_('no such table: policy_stat' in str(err))
else:
self.fail('database created with policy_stat table')
def tearDown(self):
AccountBroker.create_container_table = \
self._imported_create_container_table
AccountBroker._initialize = self._imported_initialize
broker = AccountBroker(':memory:', account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
conn.execute('SELECT storage_policy_index FROM container')
@with_tempdir
def test_policy_table_migration(self, tempdir):
db_path = os.path.join(tempdir, 'account.db')
# first init an acct DB without the policy_stat table present
broker = AccountBroker(db_path, account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
try:
conn.execute('''
SELECT * FROM policy_stat
''').fetchone()[0]
except sqlite3.OperationalError as err:
# confirm that the table really isn't there
self.assert_('no such table: policy_stat' in str(err))
else:
self.fail('broker did not raise sqlite3.OperationalError '
'trying to select from policy_stat table!')
# make sure we can HEAD this thing w/o the table
stats = broker.get_policy_stats()
self.assertEqual(len(stats), 0)
# now do a PUT to create the table
broker.put_container('o', Timestamp(time()).internal, 0, 0, 0,
POLICIES.default.idx)
broker._commit_puts_stale_ok()
# now confirm that the table was created
with broker.get() as conn:
conn.execute('SELECT * FROM policy_stat')
stats = broker.get_policy_stats()
self.assertEqual(len(stats), 1)
@patch_policies
@with_tempdir
def test_container_table_migration(self, tempdir):
db_path = os.path.join(tempdir, 'account.db')
# first init an acct DB without the policy_stat table present
broker = AccountBroker(db_path, account='a')
broker.initialize(Timestamp('1').internal)
with broker.get() as conn:
try:
conn.execute('''
SELECT storage_policy_index FROM container
''').fetchone()[0]
except sqlite3.OperationalError as err:
# confirm that the table doesn't have this column
self.assert_('no such column: storage_policy_index' in
str(err))
else:
self.fail('broker did not raise sqlite3.OperationalError '
'trying to select from storage_policy_index '
'from container table!')
# manually insert an existing row to avoid migration
with broker.get() as conn:
conn.execute('''
INSERT INTO container (name, put_timestamp,
delete_timestamp, object_count, bytes_used,
deleted)
VALUES (?, ?, ?, ?, ?, ?)
''', ('test_name', Timestamp(time()).internal, 0, 1, 2, 0))
conn.commit()
# make sure we can iter containers without the migration
for c in broker.list_containers_iter(1, None, None, None, None):
self.assertEqual(c, ('test_name', 1, 2, 0))
# stats table is mysteriously empty...
stats = broker.get_policy_stats()
self.assertEqual(len(stats), 0)
# now do a PUT with a different value for storage_policy_index
# which will update the DB schema as well as update policy_stats
# for legacy containers in the DB (those without an SPI)
other_policy = [p for p in POLICIES if p.idx != 0][0]
broker.put_container('test_second', Timestamp(time()).internal,
0, 3, 4, other_policy.idx)
broker._commit_puts_stale_ok()
with broker.get() as conn:
rows = conn.execute('''
SELECT name, storage_policy_index FROM container
''').fetchall()
for row in rows:
if row[0] == 'test_name':
self.assertEqual(row[1], 0)
else:
self.assertEqual(row[1], other_policy.idx)
# we should have stats for both containers
stats = broker.get_policy_stats()
self.assertEqual(len(stats), 2)
self.assertEqual(stats[0]['object_count'], 1)
self.assertEqual(stats[0]['bytes_used'], 2)
self.assertEqual(stats[1]['object_count'], 3)
self.assertEqual(stats[1]['bytes_used'], 4)
# now lets delete a container and make sure policy_stats is OK
with broker.get() as conn:
conn.execute('''
DELETE FROM container WHERE name = ?
''', ('test_name',))
conn.commit()
stats = broker.get_policy_stats()
self.assertEqual(len(stats), 2)
self.assertEqual(stats[0]['object_count'], 0)
self.assertEqual(stats[0]['bytes_used'], 0)
self.assertEqual(stats[1]['object_count'], 3)
self.assertEqual(stats[1]['bytes_used'], 4)
@with_tempdir
def test_half_upgraded_database(self, tempdir):
db_path = os.path.join(tempdir, 'account.db')
ts = itertools.count()
ts = (Timestamp(t).internal for t in itertools.count(int(time())))
broker = AccountBroker(db_path, account='a')
broker.initialize(ts.next())
self.assertTrue(broker.empty())
# add a container (to pending file)
broker.put_container('c', ts.next(), 0, 0, 0,
POLICIES.default.idx)
real_get = broker.get
called = []
@contextmanager
def mock_get():
with real_get() as conn:
def mock_executescript(script):
if called:
raise Exception('kaboom!')
called.append(script)
conn.executescript = mock_executescript
yield conn
broker.get = mock_get
try:
broker._commit_puts()
except Exception:
pass
else:
self.fail('mock exception was not raised')
self.assertEqual(len(called), 1)
self.assert_('CREATE TABLE policy_stat' in called[0])
# nothing was commited
broker = AccountBroker(db_path, account='a')
with broker.get() as conn:
try:
conn.execute('SELECT * FROM policy_stat')
except sqlite3.OperationalError as err:
self.assert_('no such table: policy_stat' in str(err))
else:
self.fail('half upgraded database!')
container_count = conn.execute(
'SELECT count(*) FROM container').fetchone()[0]
self.assertEqual(container_count, 0)
# try again to commit puts
self.assertFalse(broker.empty())
# full migration successful
with broker.get() as conn:
conn.execute('SELECT * FROM policy_stat')
conn.execute('SELECT storage_policy_index FROM container')

95
test/unit/account/test_reaper.py
View File

@ -15,12 +15,13 @@
import os
import time
import random
import shutil
import tempfile
import unittest
from logging import DEBUG
from mock import patch
from mock import patch, call, DEFAULT
from contextlib import nested
from swift.account import reaper
@ -28,6 +29,9 @@ from swift.account.backend import DATADIR
from swift.common.exceptions import ClientException
from swift.common.utils import normalize_timestamp
from test import unit
from swift.common.storage_policy import StoragePolicy, POLICIES, POLICY_INDEX
class FakeLogger(object):
def __init__(self, *args, **kwargs):
@ -109,6 +113,7 @@ class FakeRing(object):
def get_part_nodes(self, *args, **kwargs):
return self.nodes
acc_nodes = [{'device': 'sda1',
'ip': '',
'port': ''},
@ -130,6 +135,10 @@ cont_nodes = [{'device': 'sda1',
'port': ''}]
@unit.patch_policies([StoragePolicy(0, 'zero', False,
object_ring=unit.FakeRing()),
StoragePolicy(1, 'one', True,
object_ring=unit.FakeRing())])
class TestReaper(unittest.TestCase):
def setUp(self):
@ -150,9 +159,6 @@ class TestReaper(unittest.TestCase):
self.amount_fail += 1
raise self.myexp
def fake_object_ring(self):
return FakeRing()
def fake_direct_delete_container(self, *args, **kwargs):
if self.amount_delete_fail < self.max_delete_fail:
self.amount_delete_fail += 1
@ -265,30 +271,81 @@ class TestReaper(unittest.TestCase):
reaper.time = time_orig
def test_reap_object(self):
r = self.init_reaper({}, fakelogger=True)
self.amount_fail = 0
self.max_fail = 0
with patch('swift.account.reaper.AccountReaper.get_object_ring',
self.fake_object_ring):
with patch('swift.account.reaper.direct_delete_object',
self.fake_direct_delete_object):
r.reap_object('a', 'c', 'partition', cont_nodes, 'o')
self.assertEqual(r.stats_objects_deleted, 3)
conf = {
'mount_check': 'false',
}
r = reaper.AccountReaper(conf, logger=unit.debug_logger())
ring = unit.FakeRing()
mock_path = 'swift.account.reaper.direct_delete_object'
for policy in POLICIES:
r.reset_stats()
with patch(mock_path) as fake_direct_delete:
r.reap_object('a', 'c', 'partition', cont_nodes, 'o',
policy.idx)
for i, call_args in enumerate(
fake_direct_delete.call_args_list):
cnode = cont_nodes[i]
host = '%(ip)s:%(port)s' % cnode
device = cnode['device']
headers = {
'X-Container-Host': host,
'X-Container-Partition': 'partition',
'X-Container-Device': device,
POLICY_INDEX: policy.idx
}
ring = r.get_object_ring(policy.idx)
expected = call(ring.devs[i], 1, 'a', 'c', 'o',
headers=headers, conn_timeout=0.5,
response_timeout=10)
self.assertEqual(call_args, expected)
self.assertEqual(r.stats_objects_deleted, 3)
def test_reap_object_fail(self):
r = self.init_reaper({}, fakelogger=True)
self.amount_fail = 0
self.max_fail = 1
ctx = [patch('swift.account.reaper.AccountReaper.get_object_ring',
self.fake_object_ring),
patch('swift.account.reaper.direct_delete_object',
self.fake_direct_delete_object)]
with nested(*ctx):
r.reap_object('a', 'c', 'partition', cont_nodes, 'o')
policy = random.choice(list(POLICIES))
with patch('swift.account.reaper.direct_delete_object',
self.fake_direct_delete_object):
r.reap_object('a', 'c', 'partition', cont_nodes, 'o',
policy.idx)
self.assertEqual(r.stats_objects_deleted, 1)
self.assertEqual(r.stats_objects_remaining, 1)
self.assertEqual(r.stats_objects_possibly_remaining, 1)
@patch('swift.account.reaper.Ring',
lambda *args, **kwargs: unit.FakeRing())
def test_reap_container(self):
policy = random.choice(list(POLICIES))
r = self.init_reaper({}, fakelogger=True)
with patch.multiple('swift.account.reaper',
direct_get_container=DEFAULT,
direct_delete_object=DEFAULT,
direct_delete_container=DEFAULT) as mocks:
headers = {POLICY_INDEX: policy.idx}
obj_listing = [{'name': 'o'}]
def fake_get_container(*args, **kwargs):
try:
obj = obj_listing.pop(0)
except IndexError:
obj_list = []
else:
obj_list = [obj]
return headers, obj_list
mocks['direct_get_container'].side_effect = fake_get_container
r.reap_container('a', 'partition', acc_nodes, 'c')
mock_calls = mocks['direct_delete_object'].call_args_list
self.assertEqual(3, len(mock_calls))
for call_args in mock_calls:
_args, kwargs = call_args
self.assertEqual(kwargs['headers'][POLICY_INDEX],
policy.idx)
self.assertEquals(mocks['direct_delete_container'].call_count, 3)
self.assertEqual(r.stats_objects_deleted, 3)
def test_reap_container_get_object_fail(self):
r = self.init_reaper({}, fakelogger=True)
self.get_fail = True

130
test/unit/account/test_replicator.py
View File

@ -13,18 +13,132 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import time
import unittest
import shutil
from swift.account import replicator, backend, server
from swift.common.utils import normalize_timestamp
from swift.common.storage_policy import POLICIES
from test.unit.common import test_db_replicator
class TestReplicator(unittest.TestCase):
"""
swift.account.replicator is currently just a subclass with some class
variables overridden, but at least this test stub will ensure proper Python
syntax.
"""
class TestReplicatorSync(test_db_replicator.TestReplicatorSync):
def test_placeholder(self):
pass
backend = backend.AccountBroker
datadir = server.DATADIR
replicator_daemon = replicator.AccountReplicator
def test_sync(self):
broker = self._get_broker('a', node_index=0)
put_timestamp = normalize_timestamp(time.time())
broker.initialize(put_timestamp)
# "replicate" to same database
daemon = replicator.AccountReplicator({})
part, node = self._get_broker_part_node(broker)
info = broker.get_replication_info()
success = daemon._repl_to_node(node, broker, part, info)
# nothing to do
self.assertTrue(success)
self.assertEqual(1, daemon.stats['no_change'])
def test_sync_remote_missing(self):
broker = self._get_broker('a', node_index=0)
put_timestamp = time.time()
broker.initialize(put_timestamp)
# "replicate" to all other nodes
part, node = self._get_broker_part_node(broker)
daemon = self._run_once(node)
# complete rsync
self.assertEqual(2, daemon.stats['rsync'])
local_info = self._get_broker(
'a', node_index=0).get_info()
for i in range(1, 3):
remote_broker = self._get_broker('a', node_index=i)
self.assertTrue(os.path.exists(remote_broker.db_file))
remote_info = remote_broker.get_info()
for k, v in local_info.items():
if k == 'id':
continue
self.assertEqual(remote_info[k], v,
"mismatch remote %s %r != %r" % (
k, remote_info[k], v))
def test_sync_remote_missing_most_rows(self):
put_timestamp = time.time()
# create "local" broker
broker = self._get_broker('a', node_index=0)
broker.initialize(put_timestamp)
# create "remote" broker
remote_broker = self._get_broker('a', node_index=1)
remote_broker.initialize(put_timestamp)
# add a row to "local" db
broker.put_container('/a/c', time.time(), 0, 0, 0,
POLICIES.default.idx)
#replicate
daemon = replicator.AccountReplicator({})
def _rsync_file(db_file, remote_file, **kwargs):
remote_server, remote_path = remote_file.split('/', 1)
dest_path = os.path.join(self.root, remote_path)
shutil.copy(db_file, dest_path)
return True
daemon._rsync_file = _rsync_file
part, node = self._get_broker_part_node(remote_broker)
info = broker.get_replication_info()
success = daemon._repl_to_node(node, broker, part, info)
self.assertTrue(success)
# row merge
self.assertEqual(1, daemon.stats['remote_merge'])
local_info = self._get_broker(
'a', node_index=0).get_info()
remote_info = self._get_broker(
'a', node_index=1).get_info()
for k, v in local_info.items():
if k == 'id':
continue
self.assertEqual(remote_info[k], v,
"mismatch remote %s %r != %r" % (
k, remote_info[k], v))
def test_sync_remote_missing_one_rows(self):
put_timestamp = time.time()
# create "local" broker
broker = self._get_broker('a', node_index=0)
broker.initialize(put_timestamp)
# create "remote" broker
remote_broker = self._get_broker('a', node_index=1)
remote_broker.initialize(put_timestamp)
# add some rows to both db
for i in range(10):
put_timestamp = time.time()
for db in (broker, remote_broker):
path = '/a/c_%s' % i
db.put_container(path, put_timestamp, 0, 0, 0,
POLICIES.default.idx)
# now a row to the "local" broker only
broker.put_container('/a/c_missing', time.time(), 0, 0, 0,
POLICIES.default.idx)
# replicate
daemon = replicator.AccountReplicator({})
part, node = self._get_broker_part_node(remote_broker)
info = broker.get_replication_info()
success = daemon._repl_to_node(node, broker, part, info)
self.assertTrue(success)
# row merge
self.assertEqual(1, daemon.stats['diff'])
local_info = self._get_broker(
'a', node_index=0).get_info()
remote_info = self._get_broker(
'a', node_index=1).get_info()
for k, v in local_info.items():
if k == 'id':
continue
self.assertEqual(remote_info[k], v,
"mismatch remote %s %r != %r" % (
k, remote_info[k], v))
if __name__ == '__main__':

181
test/unit/account/test_server.py
View File

@ -22,6 +22,8 @@ from shutil import rmtree
from StringIO import StringIO
from time import gmtime
from test.unit import FakeLogger
import itertools
import random
import simplejson
import xml.dom.minidom
@ -31,8 +33,11 @@ from swift.common import constraints
from swift.account.server import AccountController
from swift.common.utils import normalize_timestamp, replication, public
from swift.common.request_helpers import get_sys_meta_prefix
from test.unit import patch_policies
from swift.common.storage_policy import StoragePolicy, POLICIES, POLICY_INDEX
@patch_policies
class TestAccountController(unittest.TestCase):
"""Test swift.account.server.AccountController"""
def setUp(self):
@ -1670,6 +1675,182 @@ class TestAccountController(unittest.TestCase):
[(('1.2.3.4 - - [01/Jan/1970:02:46:41 +0000] "HEAD /sda1/p/a" 404 '
'- "-" "-" "-" 2.0000 "-"',), {})])
def test_policy_stats_with_legacy(self):
ts = itertools.count()
# create the account
req = Request.blank('/sda1/p/a', method='PUT', headers={
'X-Timestamp': normalize_timestamp(ts.next())})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201) # sanity
# add a container
req = Request.blank('/sda1/p/a/c1', method='PUT', headers={
'X-Put-Timestamp': normalize_timestamp(ts.next()),
'X-Delete-Timestamp': '0',
'X-Object-Count': '2',
'X-Bytes-Used': '4',
})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
# read back rollup
for method in ('GET', 'HEAD'):
req = Request.blank('/sda1/p/a', method=method)
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int // 100, 2)
self.assertEquals(resp.headers['X-Account-Object-Count'], '2')
self.assertEquals(resp.headers['X-Account-Bytes-Used'], '4')
self.assertEquals(
resp.headers['X-Account-Storage-Policy-%s-Object-Count' %
POLICIES[0].name], '2')
self.assertEquals(
resp.headers['X-Account-Storage-Policy-%s-Bytes-Used' %
POLICIES[0].name], '4')
def test_policy_stats_non_default(self):
ts = itertools.count()
# create the account
req = Request.blank('/sda1/p/a', method='PUT', headers={
'X-Timestamp': normalize_timestamp(ts.next())})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201) # sanity
# add a container
non_default_policies = [p for p in POLICIES if not p.is_default]
policy = random.choice(non_default_policies)
req = Request.blank('/sda1/p/a/c1', method='PUT', headers={
'X-Put-Timestamp': normalize_timestamp(ts.next()),
'X-Delete-Timestamp': '0',
'X-Object-Count': '2',
'X-Bytes-Used': '4',
POLICY_INDEX: policy.idx,
})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
# read back rollup
for method in ('GET', 'HEAD'):
req = Request.blank('/sda1/p/a', method=method)
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int // 100, 2)
self.assertEquals(resp.headers['X-Account-Object-Count'], '2')
self.assertEquals(resp.headers['X-Account-Bytes-Used'], '4')
self.assertEquals(
resp.headers['X-Account-Storage-Policy-%s-Object-Count' %
policy.name], '2')
self.assertEquals(
resp.headers['X-Account-Storage-Policy-%s-Bytes-Used' %
policy.name], '4')
def test_empty_policy_stats(self):
ts = itertools.count()
# create the account
req = Request.blank('/sda1/p/a', method='PUT', headers={
'X-Timestamp': normalize_timestamp(ts.next())})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201) # sanity
for method in ('GET', 'HEAD'):
req = Request.blank('/sda1/p/a', method=method)
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int // 100, 2)
for key in resp.headers:
self.assert_('storage-policy' not in key.lower())
def test_empty_except_for_used_policies(self):
ts = itertools.count()
# create the account
req = Request.blank('/sda1/p/a', method='PUT', headers={
'X-Timestamp': normalize_timestamp(ts.next())})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201) # sanity
# starts empty
for method in ('GET', 'HEAD'):
req = Request.blank('/sda1/p/a', method=method)
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int // 100, 2)
for key in resp.headers:
self.assert_('storage-policy' not in key.lower())
# add a container
policy = random.choice(POLICIES)
req = Request.blank('/sda1/p/a/c1', method='PUT', headers={
'X-Put-Timestamp': normalize_timestamp(ts.next()),
'X-Delete-Timestamp': '0',
'X-Object-Count': '2',
'X-Bytes-Used': '4',
POLICY_INDEX: policy.idx,
})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
# only policy of the created container should be in headers
for method in ('GET', 'HEAD'):
req = Request.blank('/sda1/p/a', method=method)
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int // 100, 2)
for key in resp.headers:
if 'storage-policy' in key.lower():
self.assert_(policy.name.lower() in key.lower())
def test_multiple_policies_in_use(self):
ts = itertools.count()
# create the account
req = Request.blank('/sda1/p/a', method='PUT', headers={
'X-Timestamp': normalize_timestamp(ts.next())})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201) # sanity
# add some containers
for policy in POLICIES:
count = policy.idx * 100 # good as any integer
container_path = '/sda1/p/a/c_%s' % policy.name
req = Request.blank(
container_path, method='PUT', headers={
'X-Put-Timestamp': normalize_timestamp(ts.next()),
'X-Delete-Timestamp': '0',
'X-Object-Count': count,
'X-Bytes-Used': count,
POLICY_INDEX: policy.idx,
})
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201)
req = Request.blank('/sda1/p/a', method='HEAD')
resp = req.get_response(self.controller)
self.assertEqual(resp.status_int // 100, 2)
# check container counts in roll up headers
total_object_count = 0
total_bytes_used = 0
for key in resp.headers:
if 'storage-policy' not in key.lower():
continue
for policy in POLICIES:
if policy.name.lower() not in key.lower():
continue
if key.lower().endswith('object-count'):
object_count = int(resp.headers[key])
self.assertEqual(policy.idx * 100, object_count)
total_object_count += object_count
if key.lower().endswith('bytes-used'):
bytes_used = int(resp.headers[key])
self.assertEqual(policy.idx * 100, bytes_used)
total_bytes_used += bytes_used
expected_total_count = sum([p.idx * 100 for p in POLICIES])
self.assertEqual(expected_total_count, total_object_count)
self.assertEqual(expected_total_count, total_bytes_used)
@patch_policies([StoragePolicy(0, 'zero', False),
StoragePolicy(1, 'one', True),
StoragePolicy(2, 'two', False),
StoragePolicy(3, 'three', False)])
class TestNonLegacyDefaultStoragePolicy(TestAccountController):
pass
if __name__ == '__main__':
unittest.main()

128
test/unit/account/test_utils.py Normal file
View File

@ -0,0 +1,128 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import itertools
import time
import unittest
import mock
from swift.account import utils, backend
from swift.common.storage_policy import POLICIES
from swift.common.utils import Timestamp
from swift.common.swob import HeaderKeyDict
from test.unit import patch_policies
class TestFakeAccountBroker(unittest.TestCase):
def test_fake_broker_get_info(self):
broker = utils.FakeAccountBroker()
now = time.time()
with mock.patch('time.time', new=lambda: now):
info = broker.get_info()
timestamp = Timestamp(now)
expected = {
'container_count': 0,
'object_count': 0,
'bytes_used': 0,
'created_at': timestamp.internal,
'put_timestamp': timestamp.internal,
}
self.assertEqual(info, expected)
def test_fake_broker_list_containers_iter(self):
broker = utils.FakeAccountBroker()
self.assertEqual(broker.list_containers_iter(), [])
def test_fake_broker_metadata(self):
broker = utils.FakeAccountBroker()
self.assertEqual(broker.metadata, {})
def test_fake_broker_get_policy_stats(self):
broker = utils.FakeAccountBroker()
self.assertEqual(broker.get_policy_stats(), {})
class TestAccountUtils(unittest.TestCase):
def test_get_response_headers_fake_broker(self):
broker = utils.FakeAccountBroker()
now = time.time()
expected = {
'X-Account-Container-Count': 0,
'X-Account-Object-Count': 0,
'X-Account-Bytes-Used': 0,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': Timestamp(now).normal,
}
with mock.patch('time.time', new=lambda: now):
resp_headers = utils.get_response_headers(broker)
self.assertEqual(resp_headers, expected)
def test_get_response_headers_empty_memory_broker(self):
broker = backend.AccountBroker(':memory:', account='a')
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal)
expected = {
'X-Account-Container-Count': 0,
'X-Account-Object-Count': 0,
'X-Account-Bytes-Used': 0,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': Timestamp(now).normal,
}
resp_headers = utils.get_response_headers(broker)
self.assertEqual(resp_headers, expected)
@patch_policies
def test_get_response_headers_with_data(self):
broker = backend.AccountBroker(':memory:', account='a')
now = time.time()
with mock.patch('time.time', new=lambda: now):
broker.initialize(Timestamp(now).internal)
# add some container data
ts = (Timestamp(t).internal for t in itertools.count(int(now)))
total_containers = 0
total_objects = 0
total_bytes = 0
for policy in POLICIES:
delete_timestamp = ts.next()
put_timestamp = ts.next()
object_count = int(policy)
bytes_used = int(policy) * 10
broker.put_container('c-%s' % policy.name, put_timestamp,
delete_timestamp, object_count, bytes_used,
int(policy))
total_containers += 1
total_objects += object_count
total_bytes += bytes_used
expected = HeaderKeyDict({
'X-Account-Container-Count': total_containers,
'X-Account-Object-Count': total_objects,
'X-Account-Bytes-Used': total_bytes,
'X-Timestamp': Timestamp(now).normal,
'X-PUT-Timestamp': Timestamp(now).normal,
})
for policy in POLICIES:
prefix = 'X-Account-Storage-Policy-%s-' % policy.name
expected[prefix + 'Object-Count'] = int(policy)
expected[prefix + 'Bytes-Used'] = int(policy) * 10
resp_headers = utils.get_response_headers(broker)
for key, value in resp_headers.items():
expected_value = expected.pop(key)
self.assertEqual(expected_value, str(value),
'value for %r was %r not %r' % (
key, value, expected_value))
self.assertFalse(expected)

355
test/unit/cli/test_info.py
View File

@ -14,24 +14,27 @@
import os
import unittest
import cPickle as pickle
import mock
from cStringIO import StringIO
from contextlib import closing
from gzip import GzipFile
from shutil import rmtree
from tempfile import mkdtemp
from test.unit import patch_policies, write_fake_ring
from swift.common import ring, utils
from swift.common.swob import Request
from swift.common.storage_policy import StoragePolicy, POLICIES
from swift.cli.info import print_db_info_metadata, print_ring_locations, \
print_info, InfoSystemExit
print_info, print_obj_metadata, print_obj, InfoSystemExit
from swift.account.server import AccountController
from swift.container.server import ContainerController
from swift.obj.diskfile import write_metadata
class TestCliInfo(unittest.TestCase):
@patch_policies([StoragePolicy(0, 'zero', True),
StoragePolicy(1, 'one', False),
StoragePolicy(2, 'two', False)])
class TestCliInfoBase(unittest.TestCase):
def setUp(self):
self.orig_hp = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX
utils.HASH_PATH_PREFIX = 'info'
@ -44,22 +47,30 @@ class TestCliInfo(unittest.TestCase):
utils.mkdirs(os.path.join(self.testdir, 'sdb1'))
utils.mkdirs(os.path.join(self.testdir, 'sdb1', 'tmp'))
self.account_ring_path = os.path.join(self.testdir, 'account.ring.gz')
with closing(GzipFile(self.account_ring_path, 'wb')) as f:
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1',
'ip': '127.0.0.1', 'port': 42},
{'id': 1, 'zone': 1, 'device': 'sdb1',
'ip': '127.0.0.2', 'port': 43}], 30),
f)
account_devs = [
{'ip': '127.0.0.1', 'port': 42},
{'ip': '127.0.0.2', 'port': 43},
]
write_fake_ring(self.account_ring_path, *account_devs)
self.container_ring_path = os.path.join(self.testdir,
'container.ring.gz')
with closing(GzipFile(self.container_ring_path, 'wb')) as f:
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1',
'ip': '127.0.0.3', 'port': 42},
{'id': 1, 'zone': 1, 'device': 'sdb1',
'ip': '127.0.0.4', 'port': 43}], 30),
f)
container_devs = [
{'ip': '127.0.0.3', 'port': 42},
{'ip': '127.0.0.4', 'port': 43},
]
write_fake_ring(self.container_ring_path, *container_devs)
self.object_ring_path = os.path.join(self.testdir, 'object.ring.gz')
object_devs = [
{'ip': '127.0.0.3', 'port': 42},
{'ip': '127.0.0.4', 'port': 43},
]
write_fake_ring(self.object_ring_path, *object_devs)
# another ring for policy 1
self.one_ring_path = os.path.join(self.testdir, 'object-1.ring.gz')
write_fake_ring(self.one_ring_path, *object_devs)
# ... and another for policy 2
self.two_ring_path = os.path.join(self.testdir, 'object-2.ring.gz')
write_fake_ring(self.two_ring_path, *object_devs)
def tearDown(self):
utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX = self.orig_hp
@ -69,10 +80,13 @@ class TestCliInfo(unittest.TestCase):
try:
func(*args, **kwargs)
except Exception, e:
self.assertEqual(msg, str(e))
self.assertTrue(msg in str(e),
"Expected %r in %r" % (msg, str(e)))
self.assertTrue(isinstance(e, exc),
"Expected %s, got %s" % (exc, type(e)))
class TestCliInfo(TestCliInfoBase):
def test_print_db_info_metadata(self):
self.assertRaisesMessage(ValueError, 'Wrong DB type',
print_db_info_metadata, 't', {}, {})
@ -86,6 +100,7 @@ class TestCliInfo(unittest.TestCase):
created_at=100.1,
put_timestamp=106.3,
delete_timestamp=107.9,
status_changed_at=108.3,
container_count='3',
object_count='20',
bytes_used='42')
@ -100,9 +115,10 @@ class TestCliInfo(unittest.TestCase):
Account: acct
Account Hash: dc5be2aa4347a22a0fee6bc7de505b47
Metadata:
Created at: 1970-01-01 00:01:40.100000 (100.1)
Put Timestamp: 1970-01-01 00:01:46.300000 (106.3)
Delete Timestamp: 1970-01-01 00:01:47.900000 (107.9)
Created at: 1970-01-01T00:01:40.100000 (100.1)
Put Timestamp: 1970-01-01T00:01:46.300000 (106.3)
Delete Timestamp: 1970-01-01T00:01:47.900000 (107.9)
Status Timestamp: 1970-01-01T00:01:48.300000 (108.3)
Container Count: 3
Object Count: 20
Bytes Used: 42
@ -118,9 +134,11 @@ No system metadata found in db file
info = dict(
account='acct',
container='cont',
storage_policy_index=0,
created_at='0000000100.10000',
put_timestamp='0000000106.30000',
delete_timestamp='0000000107.90000',
status_changed_at='0000000108.30000',
object_count='20',
bytes_used='42',
reported_put_timestamp='0000010106.30000',
@ -140,13 +158,15 @@ No system metadata found in db file
Container: cont
Container Hash: d49d0ecbb53be1fcc49624f2f7c7ccae
Metadata:
Created at: 1970-01-01 00:01:40.100000 (0000000100.10000)
Put Timestamp: 1970-01-01 00:01:46.300000 (0000000106.30000)
Delete Timestamp: 1970-01-01 00:01:47.900000 (0000000107.90000)
Created at: 1970-01-01T00:01:40.100000 (0000000100.10000)
Put Timestamp: 1970-01-01T00:01:46.300000 (0000000106.30000)
Delete Timestamp: 1970-01-01T00:01:47.900000 (0000000107.90000)
Status Timestamp: 1970-01-01T00:01:48.300000 (0000000108.30000)
Object Count: 20
Bytes Used: 42
Reported Put Timestamp: 1970-01-01 02:48:26.300000 (0000010106.30000)
Reported Delete Timestamp: 1970-01-01 02:48:27.900000 (0000010107.90000)
Storage Policy: %s (0)
Reported Put Timestamp: 1970-01-01T02:48:26.300000 (0000010106.30000)
Reported Delete Timestamp: 1970-01-01T02:48:27.900000 (0000010107.90000)
Reported Object Count: 20
Reported Bytes Used: 42
Chexor: abaddeadbeefcafe
@ -154,54 +174,62 @@ Metadata:
X-Container-Bar: goo
X-Container-Foo: bar
System Metadata: {'mydata': 'swift'}
No user metadata found in db file'''
No user metadata found in db file''' % POLICIES[0].name
self.assertEquals(sorted(out.getvalue().strip().split('\n')),
sorted(exp_out.split('\n')))
def test_print_ring_locations(self):
self.assertRaisesMessage(ValueError, 'None type', print_ring_locations,
None, 'dir', 'acct')
self.assertRaisesMessage(ValueError, 'None type', print_ring_locations,
[], None, 'acct')
self.assertRaisesMessage(ValueError, 'None type', print_ring_locations,
[], 'dir', None)
self.assertRaisesMessage(ValueError, 'Ring error',
print_ring_locations,
[], 'dir', 'acct', 'con')
def test_print_ring_locations_invalid_args(self):
self.assertRaises(ValueError, print_ring_locations,
None, 'dir', 'acct')
self.assertRaises(ValueError, print_ring_locations,
[], None, 'acct')
self.assertRaises(ValueError, print_ring_locations,
[], 'dir', None)
self.assertRaises(ValueError, print_ring_locations,
[], 'dir', 'acct', 'con')
self.assertRaises(ValueError, print_ring_locations,
[], 'dir', 'acct', obj='o')
def test_print_ring_locations_account(self):
out = StringIO()
with mock.patch('sys.stdout', out):
acctring = ring.Ring(self.testdir, ring_name='account')
print_ring_locations(acctring, 'dir', 'acct')
exp_db2 = os.path.join('/srv', 'node', 'sdb1', 'dir', '3', 'b47',
'dc5be2aa4347a22a0fee6bc7de505b47',
'dc5be2aa4347a22a0fee6bc7de505b47.db')
exp_db1 = os.path.join('/srv', 'node', 'sda1', 'dir', '3', 'b47',
'dc5be2aa4347a22a0fee6bc7de505b47',
'dc5be2aa4347a22a0fee6bc7de505b47.db')
exp_out = ('Ring locations:\n 127.0.0.2:43 - %s\n'
' 127.0.0.1:42 - %s\n'
'\nnote: /srv/node is used as default value of `devices`,'
' the real value is set in the account config file on'
' each storage node.' % (exp_db2, exp_db1))
self.assertEquals(out.getvalue().strip(), exp_out)
exp_db = os.path.join('${DEVICE:-/srv/node*}', 'sdb1', 'dir', '3',
'b47', 'dc5be2aa4347a22a0fee6bc7de505b47')
self.assertTrue(exp_db in out.getvalue())
self.assertTrue('127.0.0.1' in out.getvalue())
self.assertTrue('127.0.0.2' in out.getvalue())
def test_print_ring_locations_container(self):
out = StringIO()
with mock.patch('sys.stdout', out):
contring = ring.Ring(self.testdir, ring_name='container')
print_ring_locations(contring, 'dir', 'acct', 'con')
exp_db4 = os.path.join('/srv', 'node', 'sdb1', 'dir', '1', 'fe6',
'63e70955d78dfc62821edc07d6ec1fe6',
'63e70955d78dfc62821edc07d6ec1fe6.db')
exp_db3 = os.path.join('/srv', 'node', 'sda1', 'dir', '1', 'fe6',
'63e70955d78dfc62821edc07d6ec1fe6',
'63e70955d78dfc62821edc07d6ec1fe6.db')
exp_out = ('Ring locations:\n 127.0.0.4:43 - %s\n'
' 127.0.0.3:42 - %s\n'
'\nnote: /srv/node is used as default value of `devices`,'
' the real value is set in the container config file on'
' each storage node.' % (exp_db4, exp_db3))
self.assertEquals(out.getvalue().strip(), exp_out)
exp_db = os.path.join('${DEVICE:-/srv/node*}', 'sdb1', 'dir', '1',
'fe6', '63e70955d78dfc62821edc07d6ec1fe6')
self.assertTrue(exp_db in out.getvalue())
def test_print_ring_locations_obj(self):
out = StringIO()
with mock.patch('sys.stdout', out):
objring = ring.Ring(self.testdir, ring_name='object')
print_ring_locations(objring, 'dir', 'acct', 'con', 'obj')
exp_obj = os.path.join('${DEVICE:-/srv/node*}', 'sda1', 'dir', '1',
'117', '4a16154fc15c75e26ba6afadf5b1c117')
self.assertTrue(exp_obj in out.getvalue())
def test_print_ring_locations_partition_number(self):
out = StringIO()
with mock.patch('sys.stdout', out):
objring = ring.Ring(self.testdir, ring_name='object')
print_ring_locations(objring, 'objects', None, tpart='1')
exp_obj1 = os.path.join('${DEVICE:-/srv/node*}', 'sda1',
'objects', '1')
exp_obj2 = os.path.join('${DEVICE:-/srv/node*}', 'sdb1',
'objects', '1')
self.assertTrue(exp_obj1 in out.getvalue())
self.assertTrue(exp_obj2 in out.getvalue())
def test_print_info(self):
db_file = 'foo'
@ -281,3 +309,202 @@ No user metadata found in db file'''
self.assertEquals(out.getvalue().strip(), exp_out)
else:
self.fail("Expected an InfoSystemExit exception to be raised")
class TestPrintObj(TestCliInfoBase):
def setUp(self):
super(TestPrintObj, self).setUp()
self.datafile = os.path.join(self.testdir,
'1402017432.46642.data')
with open(self.datafile, 'wb') as fp:
md = {'name': '/AUTH_admin/c/obj',
'Content-Type': 'application/octet-stream'}
write_metadata(fp, md)
def test_print_obj_invalid(self):
datafile = '1402017324.68634.data'
self.assertRaises(InfoSystemExit, print_obj, datafile)
datafile = os.path.join(self.testdir, './1234.data')
self.assertRaises(InfoSystemExit, print_obj, datafile)
with open(datafile, 'wb') as fp:
fp.write('1234')
out = StringIO()
with mock.patch('sys.stdout', out):
self.assertRaises(InfoSystemExit, print_obj, datafile)
self.assertEquals(out.getvalue().strip(),
'Invalid metadata')
def test_print_obj_valid(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile, swift_dir=self.testdir)
etag_msg = 'ETag: Not found in metadata'
length_msg = 'Content-Length: Not found in metadata'
self.assertTrue(etag_msg in out.getvalue())
self.assertTrue(length_msg in out.getvalue())
def test_print_obj_with_policy(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile, swift_dir=self.testdir, policy_name='one')
etag_msg = 'ETag: Not found in metadata'
length_msg = 'Content-Length: Not found in metadata'
ring_loc_msg = 'ls -lah'
self.assertTrue(etag_msg in out.getvalue())
self.assertTrue(length_msg in out.getvalue())
self.assertTrue(ring_loc_msg in out.getvalue())
def test_missing_etag(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile)
self.assertTrue('ETag: Not found in metadata' in out.getvalue())
class TestPrintObjFullMeta(TestCliInfoBase):
def setUp(self):
super(TestPrintObjFullMeta, self).setUp()
self.datafile = os.path.join(self.testdir,
'sda', 'objects-1',
'1', 'ea8',
'db4449e025aca992307c7c804a67eea8',
'1402017884.18202.data')
utils.mkdirs(os.path.dirname(self.datafile))
with open(self.datafile, 'wb') as fp:
md = {'name': '/AUTH_admin/c/obj',
'Content-Type': 'application/octet-stream',
'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
'Content-Length': 0}
write_metadata(fp, md)
def test_print_obj(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile, swift_dir=self.testdir)
self.assertTrue('/objects-1/' in out.getvalue())
def test_print_obj_no_ring(self):
no_rings_dir = os.path.join(self.testdir, 'no_rings_here')
os.mkdir(no_rings_dir)
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile, swift_dir=no_rings_dir)
self.assertTrue('d41d8cd98f00b204e9800998ecf8427e' in out.getvalue())
self.assertTrue('Partition' not in out.getvalue())
def test_print_obj_policy_name_mismatch(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile, policy_name='two', swift_dir=self.testdir)
ring_alert_msg = 'Attention: Ring does not match policy'
self.assertTrue(ring_alert_msg in out.getvalue())
def test_valid_etag(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile)
self.assertTrue('ETag: d41d8cd98f00b204e9800998ecf8427e (valid)'
in out.getvalue())
def test_invalid_etag(self):
with open(self.datafile, 'wb') as fp:
md = {'name': '/AUTH_admin/c/obj',
'Content-Type': 'application/octet-stream',
'ETag': 'badetag',
'Content-Length': 0}
write_metadata(fp, md)
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile)
self.assertTrue('ETag: badetag doesn\'t match file hash'
in out.getvalue())
def test_unchecked_etag(self):
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj(self.datafile, check_etag=False)
self.assertTrue('ETag: d41d8cd98f00b204e9800998ecf8427e (not checked)'
in out.getvalue())
def test_print_obj_metadata(self):
self.assertRaisesMessage(ValueError, 'Metadata is None',
print_obj_metadata, [])
def reset_metadata():
md = dict(name='/AUTH_admin/c/dummy')
md['Content-Type'] = 'application/octet-stream'
md['X-Timestamp'] = 106.3
md['X-Object-Meta-Mtime'] = '107.3'
return md
metadata = reset_metadata()
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj_metadata(metadata)
exp_out = '''Path: /AUTH_admin/c/dummy
Account: AUTH_admin
Container: c
Object: dummy
Object hash: 128fdf98bddd1b1e8695f4340e67a67a
Content-Type: application/octet-stream
Timestamp: 1970-01-01T00:01:46.300000 (%s)
User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' % (
utils.Timestamp(106.3).internal)
self.assertEquals(out.getvalue().strip(), exp_out)
metadata = reset_metadata()
metadata['name'] = '/a-s'
self.assertRaisesMessage(ValueError, 'Path is invalid',
print_obj_metadata, metadata)
metadata = reset_metadata()
del metadata['name']
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj_metadata(metadata)
exp_out = '''Path: Not found in metadata
Content-Type: application/octet-stream
Timestamp: 1970-01-01T00:01:46.300000 (%s)
User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' % (
utils.Timestamp(106.3).internal)
self.assertEquals(out.getvalue().strip(), exp_out)
metadata = reset_metadata()
del metadata['Content-Type']
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj_metadata(metadata)
exp_out = '''Path: /AUTH_admin/c/dummy
Account: AUTH_admin
Container: c
Object: dummy
Object hash: 128fdf98bddd1b1e8695f4340e67a67a
Content-Type: Not found in metadata
Timestamp: 1970-01-01T00:01:46.300000 (%s)
User Metadata: {'X-Object-Meta-Mtime': '107.3'}''' % (
utils.Timestamp(106.3).internal)
self.assertEquals(out.getvalue().strip(), exp_out)
metadata = reset_metadata()
del metadata['X-Timestamp']
out = StringIO()
with mock.patch('sys.stdout', out):
print_obj_metadata(metadata)
exp_out = '''Path: /AUTH_admin/c/dummy
Account: AUTH_admin
Container: c
Object: dummy
Object hash: 128fdf98bddd1b1e8695f4340e67a67a
Content-Type: application/octet-stream
Timestamp: Not found in metadata
User Metadata: {'X-Object-Meta-Mtime': '107.3'}'''
self.assertEquals(out.getvalue().strip(), exp_out)

43
test/unit/cli/test_recon.py
View File

@ -13,11 +13,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from contextlib import nested
import json
import mock
import os
import random
import string
from StringIO import StringIO
import tempfile
import time
import unittest
@ -148,6 +150,47 @@ class TestRecon(unittest.TestCase):
1, self.swift_dir, self.ring_name)
self.assertEqual(set([('127.0.0.1', 10001)]), ips)
def test_get_ringmd5(self):
for server_type in ('account', 'container', 'object', 'object-1'):
ring_name = '%s.ring.gz' % server_type
ring_file = os.path.join(self.swift_dir, ring_name)
open(ring_file, 'w')
empty_file_hash = 'd41d8cd98f00b204e9800998ecf8427e'
hosts = [("127.0.0.1", "8080")]
with mock.patch('swift.cli.recon.Scout') as mock_scout:
scout_instance = mock.MagicMock()
url = 'http://%s:%s/recon/ringmd5' % hosts[0]
response = {
'/etc/swift/account.ring.gz': empty_file_hash,
'/etc/swift/container.ring.gz': empty_file_hash,
'/etc/swift/object.ring.gz': empty_file_hash,
'/etc/swift/object-1.ring.gz': empty_file_hash,
}
status = 200
scout_instance.scout.return_value = (url, response, status)
mock_scout.return_value = scout_instance
stdout = StringIO()
mock_hash = mock.MagicMock()
patches = [
mock.patch('sys.stdout', new=stdout),
mock.patch('swift.cli.recon.md5', new=mock_hash),
]
with nested(*patches):
mock_hash.return_value.hexdigest.return_value = \
empty_file_hash
self.recon_instance.get_ringmd5(hosts, self.swift_dir)
output = stdout.getvalue()
expected = '1/1 hosts matched'
for line in output.splitlines():
if '!!' in line:
self.fail('Unexpected Error in output: %r' % line)
if expected in line:
break
else:
self.fail('Did not find expected substring %r '
'in output:\n%s' % (expected, output))
class TestReconCommands(unittest.TestCase):
def setUp(self):

43
test/unit/common/middleware/helpers.py
View File

@ -20,6 +20,8 @@ from hashlib import md5
from swift.common import swob
from swift.common.utils import split_path
from test.unit import FakeLogger, FakeRing
class FakeSwift(object):
"""
@ -33,6 +35,21 @@ class FakeSwift(object):
self.uploaded = {}
# mapping of (method, path) --> (response class, headers, body)
self._responses = {}
self.logger = FakeLogger('fake-swift')
self.account_ring = FakeRing()
self.container_ring = FakeRing()
self.get_object_ring = lambda policy_index: FakeRing()
def _get_response(self, method, path):
resp = self._responses[(method, path)]
if isinstance(resp, list):
try:
resp = resp.pop(0)
except IndexError:
raise IndexError("Didn't find any more %r "
"in allowed responses" % (
(method, path),))
return resp
def __call__(self, env, start_response):
method = env['REQUEST_METHOD']
@ -47,29 +64,30 @@ class FakeSwift(object):
if resp:
return resp(env, start_response)
headers = swob.Request(env).headers
self._calls.append((method, path, headers))
req_headers = swob.Request(env).headers
self.swift_sources.append(env.get('swift.source'))
try:
resp_class, raw_headers, body = self._responses[(method, path)]
resp_class, raw_headers, body = self._get_response(method, path)
headers = swob.HeaderKeyDict(raw_headers)
except KeyError:
if (env.get('QUERY_STRING')
and (method, env['PATH_INFO']) in self._responses):
resp_class, raw_headers, body = self._responses[
(method, env['PATH_INFO'])]
resp_class, raw_headers, body = self._get_response(
method, env['PATH_INFO'])
headers = swob.HeaderKeyDict(raw_headers)
elif method == 'HEAD' and ('GET', path) in self._responses:
resp_class, raw_headers, _ = self._responses[('GET', path)]
resp_class, raw_headers, body = self._get_response('GET', path)
body = None
headers = swob.HeaderKeyDict(raw_headers)
elif method == 'GET' and obj and path in self.uploaded:
resp_class = swob.HTTPOk
headers, body = self.uploaded[path]
else:
print "Didn't find %r in allowed responses" % ((method, path),)
raise
raise KeyError("Didn't find %r in allowed responses" % (
(method, path),))
self._calls.append((method, path, req_headers))
# simulate object PUT
if method == 'PUT' and obj:
@ -93,6 +111,10 @@ class FakeSwift(object):
def calls(self):
return [(method, path) for method, path, headers in self._calls]
@property
def headers(self):
return [headers for method, path, headers in self._calls]
@property
def calls_with_headers(self):
return self._calls
@ -101,5 +123,8 @@ class FakeSwift(object):
def call_count(self):
return len(self._calls)
def register(self, method, path, response_class, headers, body):
def register(self, method, path, response_class, headers, body=''):
self._responses[(method, path)] = (response_class, headers, body)
def register_responses(self, method, path, responses):
self._responses[(method, path)] = list(responses)

91
test/unit/common/middleware/test_container_sync.py
View File

@ -19,12 +19,15 @@ import shutil
import tempfile
import unittest
import uuid
import mock
from swift.common import swob
from swift.common.middleware import container_sync
from swift.proxy.controllers.base import _get_cache_key
from swift.proxy.controllers.info import InfoController
from test.unit import FakeLogger
class FakeApp(object):
@ -63,6 +66,94 @@ cluster_dfw1 = http://dfw1.host/v1/
def tearDown(self):
shutil.rmtree(self.tempdir, ignore_errors=1)
def test_current_not_set(self):
# no 'current' option set by default
self.assertEqual(None, self.sync.realm)
self.assertEqual(None, self.sync.cluster)
info = {}
def capture_swift_info(key, **options):
info[key] = options
with mock.patch(
'swift.common.middleware.container_sync.register_swift_info',
new=capture_swift_info):
self.sync.register_info()
for realm, realm_info in info['container_sync']['realms'].items():
for cluster, options in realm_info['clusters'].items():
self.assertEqual(options.get('current', False), False)
def test_current_invalid(self):
self.conf = {'swift_dir': self.tempdir, 'current': 'foo'}
self.sync = container_sync.ContainerSync(self.app, self.conf,
logger=FakeLogger())
self.assertEqual(None, self.sync.realm)
self.assertEqual(None, self.sync.cluster)
info = {}
def capture_swift_info(key, **options):
info[key] = options
with mock.patch(
'swift.common.middleware.container_sync.register_swift_info',
new=capture_swift_info):
self.sync.register_info()
for realm, realm_info in info['container_sync']['realms'].items():
for cluster, options in realm_info['clusters'].items():
self.assertEqual(options.get('current', False), False)
error_lines = self.sync.logger.get_lines_for_level('error')
self.assertEqual(error_lines, ['Invalid current '
'//REALM/CLUSTER (foo)'])
def test_current_in_realms_conf(self):
self.conf = {'swift_dir': self.tempdir, 'current': '//us/dfw1'}
self.sync = container_sync.ContainerSync(self.app, self.conf)
self.assertEqual('US', self.sync.realm)
self.assertEqual('DFW1', self.sync.cluster)
info = {}
def capture_swift_info(key, **options):
info[key] = options
with mock.patch(
'swift.common.middleware.container_sync.register_swift_info',
new=capture_swift_info):
self.sync.register_info()
for realm, realm_info in info['container_sync']['realms'].items():
for cluster, options in realm_info['clusters'].items():
if options.get('current'):
break
self.assertEqual(realm, self.sync.realm)
self.assertEqual(cluster, self.sync.cluster)
def test_missing_from_realms_conf(self):
self.conf = {'swift_dir': self.tempdir, 'current': 'foo/bar'}
self.sync = container_sync.ContainerSync(self.app, self.conf,
logger=FakeLogger())
self.assertEqual('FOO', self.sync.realm)
self.assertEqual('BAR', self.sync.cluster)
info = {}
def capture_swift_info(key, **options):
info[key] = options
with mock.patch(
'swift.common.middleware.container_sync.register_swift_info',
new=capture_swift_info):
self.sync.register_info()
for realm, realm_info in info['container_sync']['realms'].items():
for cluster, options in realm_info['clusters'].items():
self.assertEqual(options.get('current', False), False)
for line in self.sync.logger.get_lines_for_level('error'):
self.assertEqual(line, 'Unknown current '
'//REALM/CLUSTER (//FOO/BAR)')
def test_pass_through(self):
req = swob.Request.blank('/v1/a/c')
resp = req.get_response(self.sync)

105
test/unit/common/middleware/test_list_endpoints.py
View File

@ -19,10 +19,13 @@ from tempfile import mkdtemp
from shutil import rmtree
import os
import mock
from swift.common import ring, utils
from swift.common.utils import json
from swift.common.utils import json, split_path
from swift.common.swob import Request, Response
from swift.common.middleware import list_endpoints
from swift.common.storage_policy import StoragePolicy, POLICIES
from test.unit import patch_policies
class FakeApp(object):
@ -34,6 +37,8 @@ def start_response(*args):
pass
@patch_policies([StoragePolicy(0, 'zero', False),
StoragePolicy(1, 'one', True)])
class TestListEndpoints(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
@ -43,6 +48,9 @@ class TestListEndpoints(unittest.TestCase):
accountgz = os.path.join(self.testdir, 'account.ring.gz')
containergz = os.path.join(self.testdir, 'container.ring.gz')
objectgz = os.path.join(self.testdir, 'object.ring.gz')
objectgz_1 = os.path.join(self.testdir, 'object-1.ring.gz')
self.policy_to_test = 0
self.expected_path = ('v1', 'a', 'c', 'o1')
# Let's make the rings slightly different so we can test
# that the correct ring is consulted (e.g. we don't consult
@ -59,6 +67,10 @@ class TestListEndpoints(unittest.TestCase):
array.array('H', [0, 1, 0, 1]),
array.array('H', [0, 1, 0, 1]),
array.array('H', [3, 4, 3, 4])]
intended_replica2part2dev_id_o_1 = [
array.array('H', [1, 0, 1, 0]),
array.array('H', [1, 0, 1, 0]),
array.array('H', [4, 3, 4, 3])]
intended_devs = [{'id': 0, 'zone': 0, 'weight': 1.0,
'ip': '10.1.1.1', 'port': 6000,
'device': 'sda1'},
@ -79,6 +91,8 @@ class TestListEndpoints(unittest.TestCase):
intended_devs, intended_part_shift).save(containergz)
ring.RingData(intended_replica2part2dev_id_o,
intended_devs, intended_part_shift).save(objectgz)
ring.RingData(intended_replica2part2dev_id_o_1,
intended_devs, intended_part_shift).save(objectgz_1)
self.app = FakeApp()
self.list_endpoints = list_endpoints.filter_factory(
@ -87,6 +101,26 @@ class TestListEndpoints(unittest.TestCase):
def tearDown(self):
rmtree(self.testdir, ignore_errors=1)
def FakeGetInfo(self, env, app, swift_source=None):
info = {'status': 0, 'sync_key': None, 'meta': {},
'cors': {'allow_origin': None, 'expose_headers': None,
'max_age': None}, 'sysmeta': {}, 'read_acl': None,
'object_count': None, 'write_acl': None, 'versions': None,
'bytes': None}
info['storage_policy'] = self.policy_to_test
(version, account, container, unused) = \
split_path(env['PATH_INFO'], 3, 4, True)
self.assertEquals((version, account, container, unused),
self.expected_path)
return info
def test_get_object_ring(self):
self.assertEquals(isinstance(self.list_endpoints.get_object_ring(0),
ring.Ring), True)
self.assertEquals(isinstance(self.list_endpoints.get_object_ring(1),
ring.Ring), True)
self.assertRaises(ValueError, self.list_endpoints.get_object_ring, 99)
def test_get_endpoint(self):
# Expected results for objects taken from test_ring
# Expected results for others computed by manually invoking
@ -100,6 +134,23 @@ class TestListEndpoints(unittest.TestCase):
"http://10.1.2.2:6000/sdd1/1/a/c/o1"
])
# test policies with default endpoint name
expected = [[
"http://10.1.1.1:6000/sdb1/1/a/c/o1",
"http://10.1.2.2:6000/sdd1/1/a/c/o1"], [
"http://10.1.1.1:6000/sda1/1/a/c/o1",
"http://10.1.2.1:6000/sdc1/1/a/c/o1"
]]
PATCHGI = 'swift.common.middleware.list_endpoints.get_container_info'
for pol in POLICIES:
self.policy_to_test = pol.idx
with mock.patch(PATCHGI, self.FakeGetInfo):
resp = Request.blank('/endpoints/a/c/o1').get_response(
self.list_endpoints)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.content_type, 'application/json')
self.assertEquals(json.loads(resp.body), expected[pol.idx])
# Here, 'o1/' is the object name.
resp = Request.blank('/endpoints/a/c/o1/').get_response(
self.list_endpoints)
@ -166,33 +217,33 @@ class TestListEndpoints(unittest.TestCase):
self.assertEquals(resp.status, '200 OK')
self.assertEquals(resp.body, 'FakeApp')
# test custom path with trailing slash
custom_path_le = list_endpoints.filter_factory({
'swift_dir': self.testdir,
'list_endpoints_path': '/some/another/path/'
})(self.app)
resp = Request.blank('/some/another/path/a/c/o1') \
.get_response(custom_path_le)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.content_type, 'application/json')
self.assertEquals(json.loads(resp.body), [
"http://10.1.1.1:6000/sdb1/1/a/c/o1",
"http://10.1.2.2:6000/sdd1/1/a/c/o1"
])
# test policies with custom endpoint name
for pol in POLICIES:
# test custom path with trailing slash
custom_path_le = list_endpoints.filter_factory({
'swift_dir': self.testdir,
'list_endpoints_path': '/some/another/path/'
})(self.app)
self.policy_to_test = pol.idx
with mock.patch(PATCHGI, self.FakeGetInfo):
resp = Request.blank('/some/another/path/a/c/o1') \
.get_response(custom_path_le)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.content_type, 'application/json')
self.assertEquals(json.loads(resp.body), expected[pol.idx])
# test ustom path without trailing slash
custom_path_le = list_endpoints.filter_factory({
'swift_dir': self.testdir,
'list_endpoints_path': '/some/another/path'
})(self.app)
resp = Request.blank('/some/another/path/a/c/o1') \
.get_response(custom_path_le)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.content_type, 'application/json')
self.assertEquals(json.loads(resp.body), [
"http://10.1.1.1:6000/sdb1/1/a/c/o1",
"http://10.1.2.2:6000/sdd1/1/a/c/o1"
])
# test custom path without trailing slash
custom_path_le = list_endpoints.filter_factory({
'swift_dir': self.testdir,
'list_endpoints_path': '/some/another/path'
})(self.app)
self.policy_to_test = pol.idx
with mock.patch(PATCHGI, self.FakeGetInfo):
resp = Request.blank('/some/another/path/a/c/o1') \
.get_response(custom_path_le)
self.assertEquals(resp.status_int, 200)
self.assertEquals(resp.content_type, 'application/json')
self.assertEquals(json.loads(resp.body), expected[pol.idx])
if __name__ == '__main__':

108
test/unit/common/middleware/test_recon.py
View File

@ -17,13 +17,15 @@ import unittest
from unittest import TestCase
from contextlib import contextmanager
from posix import stat_result, statvfs_result
import array
from swift.common import ring, utils
from shutil import rmtree
import os
import mock
from swift import __version__ as swiftver
from swift.common.swob import Request
from swift.common.middleware import recon
from swift.common import utils
def fake_check_mount(a, b):
@ -186,7 +188,13 @@ class FakeRecon(object):
class TestReconSuccess(TestCase):
def setUp(self):
self.app = recon.ReconMiddleware(FakeApp(), {})
# can't use mkdtemp here as 2.6 gzip puts the filename in the header
# which will cause ring md5 checks to fail
self.tempdir = '/tmp/swift_recon_md5_test'
utils.mkdirs(self.tempdir)
self._create_rings()
self.app = recon.ReconMiddleware(FakeApp(),
{'swift_dir': self.tempdir})
self.mockos = MockOS()
self.fakecache = FakeFromCache()
self.real_listdir = os.listdir
@ -206,6 +214,96 @@ class TestReconSuccess(TestCase):
del self.mockos
self.app._from_recon_cache = self.real_from_cache
del self.fakecache
rmtree(self.tempdir)
def _create_rings(self):
def fake_time():
return 0
def fake_base(fname):
# least common denominator with gzip versions is to
# not use the .gz extension in the gzip header
return fname[:-3]
accountgz = os.path.join(self.tempdir, 'account.ring.gz')
containergz = os.path.join(self.tempdir, 'container.ring.gz')
objectgz = os.path.join(self.tempdir, 'object.ring.gz')
objectgz_1 = os.path.join(self.tempdir, 'object-1.ring.gz')
objectgz_2 = os.path.join(self.tempdir, 'object-2.ring.gz')
# make the rings unique so they have different md5 sums
intended_replica2part2dev_id_a = [
array.array('H', [3, 1, 3, 1]),
array.array('H', [0, 3, 1, 4]),
array.array('H', [1, 4, 0, 3])]
intended_replica2part2dev_id_c = [
array.array('H', [4, 3, 0, 1]),
array.array('H', [0, 1, 3, 4]),
array.array('H', [3, 4, 0, 1])]
intended_replica2part2dev_id_o = [
array.array('H', [0, 1, 0, 1]),
array.array('H', [0, 1, 0, 1]),
array.array('H', [3, 4, 3, 4])]
intended_replica2part2dev_id_o_1 = [
array.array('H', [1, 0, 1, 0]),
array.array('H', [1, 0, 1, 0]),
array.array('H', [4, 3, 4, 3])]
intended_replica2part2dev_id_o_2 = [
array.array('H', [1, 1, 1, 0]),
array.array('H', [1, 0, 1, 3]),
array.array('H', [4, 2, 4, 3])]
intended_devs = [{'id': 0, 'zone': 0, 'weight': 1.0,
'ip': '10.1.1.1', 'port': 6000,
'device': 'sda1'},
{'id': 1, 'zone': 0, 'weight': 1.0,
'ip': '10.1.1.1', 'port': 6000,
'device': 'sdb1'},
None,
{'id': 3, 'zone': 2, 'weight': 1.0,
'ip': '10.1.2.1', 'port': 6000,
'device': 'sdc1'},
{'id': 4, 'zone': 2, 'weight': 1.0,
'ip': '10.1.2.2', 'port': 6000,
'device': 'sdd1'}]
# eliminate time from the equation as gzip 2.6 includes
# it in the header resulting in md5 file mismatch, also
# have to mock basename as one version uses it, one doesn't
with mock.patch("time.time", fake_time):
with mock.patch("os.path.basename", fake_base):
ring.RingData(intended_replica2part2dev_id_a,
intended_devs, 5).save(accountgz, mtime=None)
ring.RingData(intended_replica2part2dev_id_c,
intended_devs, 5).save(containergz, mtime=None)
ring.RingData(intended_replica2part2dev_id_o,
intended_devs, 5).save(objectgz, mtime=None)
ring.RingData(intended_replica2part2dev_id_o_1,
intended_devs, 5).save(objectgz_1, mtime=None)
ring.RingData(intended_replica2part2dev_id_o_2,
intended_devs, 5).save(objectgz_2, mtime=None)
def test_get_ring_md5(self):
def fake_open(self, f):
raise IOError
expt_out = {'%s/account.ring.gz' % self.tempdir:
'd288bdf39610e90d4f0b67fa00eeec4f',
'%s/container.ring.gz' % self.tempdir:
'9a5a05a8a4fbbc61123de792dbe4592d',
'%s/object-1.ring.gz' % self.tempdir:
'3f1899b27abf5f2efcc67d6fae1e1c64',
'%s/object-2.ring.gz' % self.tempdir:
'8f0e57079b3c245d9b3d5a428e9312ee',
'%s/object.ring.gz' % self.tempdir:
'da02bfbd0bf1e7d56faea15b6fe5ab1e'}
self.assertEquals(sorted(self.app.get_ring_md5().items()),
sorted(expt_out.items()))
# cover error path
self.app.get_ring_md5(openr=fake_open)
def test_from_recon_cache(self):
oart = OpenAndReadTester(['{"notneeded": 5, "testkey1": "canhazio"}'])
@ -712,9 +810,15 @@ class TestReconSuccess(TestCase):
class TestReconMiddleware(unittest.TestCase):
def fake_list(self, path):
return ['a', 'b']
def setUp(self):
self.frecon = FakeRecon()
self.real_listdir = os.listdir
os.listdir = self.fake_list
self.app = recon.ReconMiddleware(FakeApp(), {'object_recon': "true"})
os.listdir = self.real_listdir
#self.app.object_recon = True
self.app.get_mem = self.frecon.fake_mem
self.app.get_load = self.frecon.fake_load

19
test/unit/common/ring/test_ring.py
View File

@ -28,6 +28,19 @@ from time import sleep, time
from swift.common import ring, utils
class TestRingBase(unittest.TestCase):
def setUp(self):
self._orig_hash_suffix = utils.HASH_PATH_SUFFIX
self._orig_hash_prefix = utils.HASH_PATH_PREFIX
utils.HASH_PATH_SUFFIX = 'endcap'
utils.HASH_PATH_PREFIX = ''
def tearDown(self):
utils.HASH_PATH_SUFFIX = self._orig_hash_suffix
utils.HASH_PATH_PREFIX = self._orig_hash_prefix
class TestRingData(unittest.TestCase):
def setUp(self):
@ -109,11 +122,10 @@ class TestRingData(unittest.TestCase):
'0644')
class TestRing(unittest.TestCase):
class TestRing(TestRingBase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
utils.HASH_PATH_PREFIX = ''
super(TestRing, self).setUp()
self.testdir = mkdtemp()
self.testgz = os.path.join(self.testdir, 'whatever.ring.gz')
self.intended_replica2part2dev_id = [
@ -147,6 +159,7 @@ class TestRing(unittest.TestCase):
reload_time=self.intended_reload_time, ring_name='whatever')
def tearDown(self):
super(TestRing, self).tearDown()
rmtree(self.testdir, ignore_errors=1)
def test_creation(self):

15
test/unit/common/test_constraints.py
View File

@ -16,6 +16,7 @@
import unittest
import mock
import tempfile
import time
from test import safe_repr
from test.unit import MockTrue
@ -182,6 +183,20 @@ class TestConstraints(unittest.TestCase):
self.assertFalse(constraints.check_float(''))
self.assertTrue(constraints.check_float('0'))
def test_valid_timestamp(self):
self.assertRaises(HTTPException,
constraints.valid_timestamp,
Request.blank('/'))
self.assertRaises(HTTPException,
constraints.valid_timestamp,
Request.blank('/', headers={
'X-Timestamp': 'asdf'}))
timestamp = utils.Timestamp(time.time())
req = Request.blank('/', headers={'X-Timestamp': timestamp.internal})
self.assertEqual(timestamp, constraints.valid_timestamp(req))
req = Request.blank('/', headers={'X-Timestamp': timestamp.normal})
self.assertEqual(timestamp, constraints.valid_timestamp(req))
def test_check_utf8(self):
unicode_sample = u'\uc77c\uc601'
valid_utf8_str = unicode_sample.encode('utf-8')

464
test/unit/common/test_db.py
View File

@ -20,9 +20,13 @@ import unittest
from tempfile import mkdtemp
from shutil import rmtree, copy
from uuid import uuid4
import cPickle as pickle
import simplejson
import sqlite3
import itertools
import time
import random
from mock import patch, MagicMock
from eventlet.timeout import Timeout
@ -30,10 +34,12 @@ from eventlet.timeout import Timeout
import swift.common.db
from swift.common.db import chexor, dict_factory, get_db_connection, \
DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists, \
GreenDBConnection
from swift.common.utils import normalize_timestamp, mkdirs
GreenDBConnection, PICKLE_PROTOCOL
from swift.common.utils import normalize_timestamp, mkdirs, json, Timestamp
from swift.common.exceptions import LockTimeout
from test.unit import with_tempdir
class TestDatabaseConnectionError(unittest.TestCase):
@ -82,6 +88,30 @@ class TestChexor(unittest.TestCase):
'd41d8cd98f00b204e9800998ecf8427e', None,
normalize_timestamp(1))
def test_chexor(self):
ts = (normalize_timestamp(ts) for ts in
itertools.count(int(time.time())))
objects = [
('frank', ts.next()),
('bob', ts.next()),
('tom', ts.next()),
('frank', ts.next()),
('tom', ts.next()),
('bob', ts.next()),
]
hash_ = '0'
random.shuffle(objects)
for obj in objects:
hash_ = chexor(hash_, *obj)
other_hash = '0'
random.shuffle(objects)
for obj in objects:
other_hash = chexor(other_hash, *obj)
self.assertEqual(hash_, other_hash)
class TestGreenDBConnection(unittest.TestCase):
@ -147,6 +177,401 @@ class TestGetDBConnection(unittest.TestCase):
mock_db_cmd.call_count))
class ExampleBroker(DatabaseBroker):
"""
Concrete enough implementation of a DatabaseBroker.
"""
db_type = 'test'
db_contains_type = 'test'
def _initialize(self, conn, put_timestamp, **kwargs):
conn.executescript('''
CREATE TABLE test_stat (
test_count INTEGER DEFAULT 0,
created_at TEXT,
put_timestamp TEXT DEFAULT '0',
delete_timestamp TEXT DEFAULT '0',
status_changed_at TEXT DEFAULT '0',
metadata TEXT DEFAULT ''
);
CREATE TABLE test (
ROWID INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT,
created_at TEXT,
deleted INTEGER DEFAULT 0
);
CREATE TRIGGER test_insert AFTER INSERT ON test
BEGIN
UPDATE test_stat
SET test_count = test_count + (1 - new.deleted);
END;
CREATE TRIGGER test_delete AFTER DELETE ON test
BEGIN
UPDATE test_stat
SET test_count = test_count - (1 - old.deleted);
END;
''')
conn.execute("""
INSERT INTO test_stat (
created_at, put_timestamp, status_changed_at)
VALUES (?, ?, ?);
""", (Timestamp(time.time()).internal, put_timestamp,
put_timestamp))
def merge_items(self, item_list):
with self.get() as conn:
for rec in item_list:
conn.execute(
'DELETE FROM test WHERE name = ? and created_at < ?', (
rec['name'], rec['created_at']))
if not conn.execute(
'SELECT 1 FROM test WHERE name = ?',
(rec['name'],)).fetchall():
conn.execute('''
INSERT INTO test (name, created_at, deleted)
VALUES (?, ?, ?)''', (
rec['name'], rec['created_at'], rec['deleted']))
conn.commit()
def _commit_puts_load(self, item_list, entry):
(name, timestamp, deleted) = pickle.loads(entry.decode('base64'))
item_list.append({
'name': name,
'created_at': timestamp,
'deleted': deleted,
})
def _load_item(self, name, timestamp, deleted):
if self.db_file == ':memory:':
record = {
'name': name,
'created_at': timestamp,
'deleted': deleted,
}
self.merge_items([record])
return
with open(self.pending_file, 'a+b') as fp:
fp.write(':')
fp.write(pickle.dumps(
(name, timestamp, deleted),
protocol=PICKLE_PROTOCOL).encode('base64'))
fp.flush()
def put_test(self, name, timestamp):
self._load_item(name, timestamp, 0)
def delete_test(self, name, timestamp):
self._load_item(name, timestamp, 1)
def _is_deleted(self, conn):
info = conn.execute('SELECT * FROM test_stat').fetchone()
return (info['test_count'] in (None, '', 0, '0')) and \
(Timestamp(info['delete_timestamp']) >
Timestamp(info['put_timestamp']))
class TestExampleBroker(unittest.TestCase):
"""
Tests that use the mostly Concrete enough ExampleBroker to exercise some
of the abstract methods on DatabaseBroker.
"""
broker_class = ExampleBroker
policy = 0
def test_merge_timestamps_simple_delete(self):
ts = (Timestamp(t).internal for t in
itertools.count(int(time.time())))
put_timestamp = ts.next()
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(put_timestamp)
created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp, '0')
info = broker.get_info()
self.assertEqual(info['created_at'], created_at)
self.assertEqual(info['put_timestamp'], put_timestamp)
self.assertEqual(info['delete_timestamp'], '0')
self.assertEqual(info['status_changed_at'], put_timestamp)
# delete
delete_timestamp = ts.next()
broker.merge_timestamps(created_at, put_timestamp, delete_timestamp)
self.assert_(broker.is_deleted())
info = broker.get_info()
self.assertEqual(info['created_at'], created_at)
self.assertEqual(info['put_timestamp'], put_timestamp)
self.assertEqual(info['delete_timestamp'], delete_timestamp)
self.assert_(info['status_changed_at'] > Timestamp(put_timestamp))
def put_item(self, broker, timestamp):
broker.put_test('test', timestamp)
def delete_item(self, broker, timestamp):
broker.delete_test('test', timestamp)
def test_merge_timestamps_delete_with_objects(self):
ts = (Timestamp(t).internal for t in
itertools.count(int(time.time())))
put_timestamp = ts.next()
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp, '0')
info = broker.get_info()
self.assertEqual(info['created_at'], created_at)
self.assertEqual(info['put_timestamp'], put_timestamp)
self.assertEqual(info['delete_timestamp'], '0')
self.assertEqual(info['status_changed_at'], put_timestamp)
# add object
self.put_item(broker, ts.next())
self.assertEqual(broker.get_info()[
'%s_count' % broker.db_contains_type], 1)
# delete
delete_timestamp = ts.next()
broker.merge_timestamps(created_at, put_timestamp, delete_timestamp)
self.assertFalse(broker.is_deleted())
info = broker.get_info()
self.assertEqual(info['created_at'], created_at)
self.assertEqual(info['put_timestamp'], put_timestamp)
self.assertEqual(info['delete_timestamp'], delete_timestamp)
# status is unchanged
self.assertEqual(info['status_changed_at'], put_timestamp)
# count is causing status to hold on
self.delete_item(broker, ts.next())
self.assertEqual(broker.get_info()[
'%s_count' % broker.db_contains_type], 0)
self.assert_(broker.is_deleted())
def test_merge_timestamps_simple_recreate(self):
ts = (Timestamp(t).internal for t in
itertools.count(int(time.time())))
put_timestamp = ts.next()
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
virgin_status_changed_at = broker.get_info()['status_changed_at']
created_at = broker.get_info()['created_at']
delete_timestamp = ts.next()
broker.merge_timestamps(created_at, put_timestamp, delete_timestamp)
self.assert_(broker.is_deleted())
info = broker.get_info()
self.assertEqual(info['created_at'], created_at)
self.assertEqual(info['put_timestamp'], put_timestamp)
self.assertEqual(info['delete_timestamp'], delete_timestamp)
orig_status_changed_at = info['status_changed_at']
self.assert_(orig_status_changed_at >
Timestamp(virgin_status_changed_at))
# recreate
recreate_timestamp = ts.next()
status_changed_at = time.time()
with patch('swift.common.db.time.time', new=lambda: status_changed_at):
broker.merge_timestamps(created_at, recreate_timestamp, '0')
self.assertFalse(broker.is_deleted())
info = broker.get_info()
self.assertEqual(info['created_at'], created_at)
self.assertEqual(info['put_timestamp'], recreate_timestamp)
self.assertEqual(info['delete_timestamp'], delete_timestamp)
self.assert_(info['status_changed_at'], status_changed_at)
def test_merge_timestamps_recreate_with_objects(self):
ts = (Timestamp(t).internal for t in
itertools.count(int(time.time())))
put_timestamp = ts.next()
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at']
# delete
delete_timestamp = ts.next()
broker.merge_timestamps(created_at, put_timestamp, delete_timestamp)
self.assert_(broker.is_deleted())
info = broker.get_info()
self.assertEqual(info['created_at'], created_at)
self.assertEqual(info['put_timestamp'], put_timestamp)
self.assertEqual(info['delete_timestamp'], delete_timestamp)
orig_status_changed_at = info['status_changed_at']
self.assert_(Timestamp(orig_status_changed_at) >=
Timestamp(put_timestamp))
# add object
self.put_item(broker, ts.next())
count_key = '%s_count' % broker.db_contains_type
self.assertEqual(broker.get_info()[count_key], 1)
self.assertFalse(broker.is_deleted())
# recreate
recreate_timestamp = ts.next()
broker.merge_timestamps(created_at, recreate_timestamp, '0')
self.assertFalse(broker.is_deleted())
info = broker.get_info()
self.assertEqual(info['created_at'], created_at)
self.assertEqual(info['put_timestamp'], recreate_timestamp)
self.assertEqual(info['delete_timestamp'], delete_timestamp)
self.assertEqual(info['status_changed_at'], orig_status_changed_at)
# count is not causing status to hold on
self.delete_item(broker, ts.next())
self.assertFalse(broker.is_deleted())
def test_merge_timestamps_update_put_no_status_change(self):
ts = (Timestamp(t).internal for t in
itertools.count(int(time.time())))
put_timestamp = ts.next()
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
info = broker.get_info()
orig_status_changed_at = info['status_changed_at']
created_at = info['created_at']
new_put_timestamp = ts.next()
broker.merge_timestamps(created_at, new_put_timestamp, '0')
info = broker.get_info()
self.assertEqual(new_put_timestamp, info['put_timestamp'])
self.assertEqual(orig_status_changed_at, info['status_changed_at'])
def test_merge_timestamps_update_delete_no_status_change(self):
ts = (Timestamp(t).internal for t in
itertools.count(int(time.time())))
put_timestamp = ts.next()
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp, ts.next())
orig_status_changed_at = broker.get_info()['status_changed_at']
new_delete_timestamp = ts.next()
broker.merge_timestamps(created_at, put_timestamp,
new_delete_timestamp)
info = broker.get_info()
self.assertEqual(new_delete_timestamp, info['delete_timestamp'])
self.assertEqual(orig_status_changed_at, info['status_changed_at'])
def test_get_max_row(self):
ts = (normalize_timestamp(t) for t in
itertools.count(int(time.time())))
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(ts.next(), storage_policy_index=int(self.policy))
self.assertEquals(-1, broker.get_max_row())
self.put_item(broker, ts.next())
self.assertEquals(1, broker.get_max_row())
self.delete_item(broker, ts.next())
self.assertEquals(2, broker.get_max_row())
self.put_item(broker, ts.next())
self.assertEquals(3, broker.get_max_row())
def test_get_info(self):
broker = self.broker_class(':memory:', account='test', container='c')
created_at = time.time()
with patch('swift.common.db.time.time', new=lambda: created_at):
broker.initialize(Timestamp(1).internal,
storage_policy_index=int(self.policy))
info = broker.get_info()
count_key = '%s_count' % broker.db_contains_type
expected = {
count_key: 0,
'created_at': Timestamp(created_at).internal,
'put_timestamp': Timestamp(1).internal,
'status_changed_at': Timestamp(1).internal,
'delete_timestamp': '0',
}
for k, v in expected.items():
self.assertEqual(info[k], v,
'mismatch for %s, %s != %s' % (
k, info[k], v))
def test_get_raw_metadata(self):
broker = self.broker_class(':memory:', account='test', container='c')
broker.initialize(Timestamp(0).internal,
storage_policy_index=int(self.policy))
self.assertEqual(broker.metadata, {})
self.assertEqual(broker.get_raw_metadata(), '')
key = u'test\u062a'.encode('utf-8')
value = u'value\u062a'
metadata = {
key: [value, Timestamp(1).internal]
}
broker.update_metadata(metadata)
self.assertEqual(broker.metadata, metadata)
self.assertEqual(broker.get_raw_metadata(),
json.dumps(metadata))
def test_put_timestamp(self):
ts = (Timestamp(t).internal for t in
itertools.count(int(time.time())))
broker = self.broker_class(':memory:', account='a', container='c')
orig_put_timestamp = ts.next()
broker.initialize(orig_put_timestamp,
storage_policy_index=int(self.policy))
self.assertEqual(broker.get_info()['put_timestamp'],
orig_put_timestamp)
# put_timestamp equal - no change
broker.update_put_timestamp(orig_put_timestamp)
self.assertEqual(broker.get_info()['put_timestamp'],
orig_put_timestamp)
# put_timestamp newer - gets newer
newer_put_timestamp = ts.next()
broker.update_put_timestamp(newer_put_timestamp)
self.assertEqual(broker.get_info()['put_timestamp'],
newer_put_timestamp)
# put_timestamp older - no change
broker.update_put_timestamp(orig_put_timestamp)
self.assertEqual(broker.get_info()['put_timestamp'],
newer_put_timestamp)
def test_status_changed_at(self):
ts = (Timestamp(t).internal for t in
itertools.count(int(time.time())))
broker = self.broker_class(':memory:', account='test', container='c')
put_timestamp = ts.next()
created_at = time.time()
with patch('swift.common.db.time.time', new=lambda: created_at):
broker.initialize(put_timestamp,
storage_policy_index=int(self.policy))
self.assertEquals(broker.get_info()['status_changed_at'],
put_timestamp)
self.assertEquals(broker.get_info()['created_at'],
Timestamp(created_at).internal)
status_changed_at = ts.next()
broker.update_status_changed_at(status_changed_at)
self.assertEqual(broker.get_info()['status_changed_at'],
status_changed_at)
# save the old and get a new status_changed_at
old_status_changed_at, status_changed_at = \
status_changed_at, ts.next()
broker.update_status_changed_at(status_changed_at)
self.assertEqual(broker.get_info()['status_changed_at'],
status_changed_at)
# status changed at won't go backwards...
broker.update_status_changed_at(old_status_changed_at)
self.assertEqual(broker.get_info()['status_changed_at'],
status_changed_at)
def test_get_syncs(self):
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(Timestamp(time.time()).internal,
storage_policy_index=int(self.policy))
self.assertEqual([], broker.get_syncs())
broker.merge_syncs([{'sync_point': 1, 'remote_id': 'remote1'}])
self.assertEqual([{'sync_point': 1, 'remote_id': 'remote1'}],
broker.get_syncs())
self.assertEqual([], broker.get_syncs(incoming=False))
broker.merge_syncs([{'sync_point': 2, 'remote_id': 'remote2'}],
incoming=False)
self.assertEqual([{'sync_point': 2, 'remote_id': 'remote2'}],
broker.get_syncs(incoming=False))
@with_tempdir
def test_commit_pending(self, tempdir):
ts = (Timestamp(t).internal for t in
itertools.count(int(time.time())))
broker = self.broker_class(os.path.join(tempdir, 'test.db'),
account='a', container='c')
broker.initialize(ts.next(), storage_policy_index=int(self.policy))
self.put_item(broker, ts.next())
qry = 'select * from %s_stat' % broker.db_type
with broker.get() as conn:
rows = [dict(x) for x in conn.execute(qry)]
info = rows[0]
count_key = '%s_count' % broker.db_contains_type
self.assertEqual(0, info[count_key])
broker.get_info()
self.assertEqual(1, broker.get_info()[count_key])
class TestDatabaseBroker(unittest.TestCase):
def setUp(self):
@ -226,7 +651,7 @@ class TestDatabaseBroker(unittest.TestCase):
broker.initialize, normalize_timestamp('1'))
def test_delete_db(self):
def init_stub(conn, put_timestamp):
def init_stub(conn, put_timestamp, **kwargs):
conn.execute('CREATE TABLE test (one TEXT)')
conn.execute('CREATE TABLE test_stat (id TEXT)')
conn.execute('INSERT INTO test_stat (id) VALUES (?)',
@ -383,7 +808,7 @@ class TestDatabaseBroker(unittest.TestCase):
broker.db_contains_type = 'test'
uuid1 = str(uuid4())
def _initialize(conn, timestamp):
def _initialize(conn, timestamp, **kwargs):
conn.execute('CREATE TABLE test (one TEXT)')
conn.execute('CREATE TABLE test_stat (id TEXT)')
conn.execute('INSERT INTO test_stat (id) VALUES (?)', (uuid1,))
@ -433,7 +858,7 @@ class TestDatabaseBroker(unittest.TestCase):
broker.db_type = 'test'
broker.db_contains_type = 'test'
def _initialize(conn, timestamp):
def _initialize(conn, timestamp, **kwargs):
conn.execute('CREATE TABLE test (one TEXT)')
conn.execute('INSERT INTO test (one) VALUES ("1")')
conn.execute('INSERT INTO test (one) VALUES ("2")')
@ -456,7 +881,7 @@ class TestDatabaseBroker(unittest.TestCase):
broker.db_contains_type = 'test'
uuid1 = str(uuid4())
def _initialize(conn, timestamp):
def _initialize(conn, timestamp, **kwargs):
conn.execute('CREATE TABLE test (one TEXT)')
conn.execute('CREATE TABLE test_stat (id TEXT)')
conn.execute('INSERT INTO test_stat (id) VALUES (?)', (uuid1,))
@ -531,7 +956,7 @@ class TestDatabaseBroker(unittest.TestCase):
broker_metadata = metadata and simplejson.dumps(
{'Test': ('Value', normalize_timestamp(1))}) or ''
def _initialize(conn, put_timestamp):
def _initialize(conn, put_timestamp, **kwargs):
if put_timestamp is None:
put_timestamp = normalize_timestamp(0)
conn.executescript('''
@ -562,6 +987,7 @@ class TestDatabaseBroker(unittest.TestCase):
created_at TEXT,
put_timestamp TEXT DEFAULT '0',
delete_timestamp TEXT DEFAULT '0',
status_changed_at TEXT DEFAULT '0',
test_count INTEGER,
hash TEXT default '00000000000000000000000000000000',
id TEXT
@ -571,8 +997,10 @@ class TestDatabaseBroker(unittest.TestCase):
''' % (metadata and ", metadata TEXT DEFAULT ''" or ""))
conn.execute('''
UPDATE test_stat
SET account = ?, created_at = ?, id = ?, put_timestamp = ?
''', (broker.account, broker_creation, broker_uuid, put_timestamp))
SET account = ?, created_at = ?, id = ?, put_timestamp = ?,
status_changed_at = ?
''', (broker.account, broker_creation, broker_uuid, put_timestamp,
put_timestamp))
if metadata:
conn.execute('UPDATE test_stat SET metadata = ?',
(broker_metadata,))
@ -582,11 +1010,11 @@ class TestDatabaseBroker(unittest.TestCase):
broker.initialize(put_timestamp)
info = broker.get_replication_info()
self.assertEquals(info, {
'count': 0,
'account': broker.account, 'count': 0,
'hash': '00000000000000000000000000000000',
'created_at': broker_creation, 'put_timestamp': put_timestamp,
'delete_timestamp': '0', 'max_row': -1, 'id': broker_uuid,
'metadata': broker_metadata})
'delete_timestamp': '0', 'status_changed_at': put_timestamp,
'max_row': -1, 'id': broker_uuid, 'metadata': broker_metadata})
insert_timestamp = normalize_timestamp(3)
with broker.get() as conn:
conn.execute('''
@ -595,21 +1023,21 @@ class TestDatabaseBroker(unittest.TestCase):
conn.commit()
info = broker.get_replication_info()
self.assertEquals(info, {
'count': 1,
'account': broker.account, 'count': 1,
'hash': 'bdc4c93f574b0d8c2911a27ce9dd38ba',
'created_at': broker_creation, 'put_timestamp': put_timestamp,
'delete_timestamp': '0', 'max_row': 1, 'id': broker_uuid,
'metadata': broker_metadata})
'delete_timestamp': '0', 'status_changed_at': put_timestamp,
'max_row': 1, 'id': broker_uuid, 'metadata': broker_metadata})
with broker.get() as conn:
conn.execute('DELETE FROM test')
conn.commit()
info = broker.get_replication_info()
self.assertEquals(info, {
'count': 0,
'account': broker.account, 'count': 0,
'hash': '00000000000000000000000000000000',
'created_at': broker_creation, 'put_timestamp': put_timestamp,
'delete_timestamp': '0', 'max_row': 1, 'id': broker_uuid,
'metadata': broker_metadata})
'delete_timestamp': '0', 'status_changed_at': put_timestamp,
'max_row': 1, 'id': broker_uuid, 'metadata': broker_metadata})
return broker
def test_metadata(self):

200
test/unit/common/test_db_replicator.py
View File

@ -21,18 +21,19 @@ import errno
import math
import time
from mock import patch, call
from shutil import rmtree
from shutil import rmtree, copy
from tempfile import mkdtemp, NamedTemporaryFile
import mock
import simplejson
from swift.container.backend import DATADIR
from swift.common import db_replicator
from swift.common.utils import normalize_timestamp
from swift.common.utils import (normalize_timestamp, hash_path,
storage_directory)
from swift.common.exceptions import DriveNotMounted
from swift.common.swob import HTTPException
from test.unit import FakeLogger
from test import unit
TEST_ACCOUNT_NAME = 'a c t'
@ -181,6 +182,7 @@ class FakeBroker(object):
get_repl_missing_table = False
stub_replication_info = None
db_type = 'container'
db_contains_type = 'object'
info = {'account': TEST_ACCOUNT_NAME, 'container': TEST_CONTAINER_NAME}
def __init__(self, *args, **kwargs):
@ -215,17 +217,21 @@ class FakeBroker(object):
def get_replication_info(self):
if self.get_repl_missing_table:
raise Exception('no such table')
info = dict(self.info)
info.update({
'hash': 12345,
'delete_timestamp': 0,
'put_timestamp': 1,
'created_at': 1,
'count': 0,
})
if self.stub_replication_info:
return self.stub_replication_info
return {'delete_timestamp': 0, 'put_timestamp': 1, 'count': 0,
'hash': 12345}
info.update(self.stub_replication_info)
return info
def reclaim(self, item_timestamp, sync_timestamp):
pass
def get_info(self):
return self.info
def newid(self, remote_d):
pass
@ -240,6 +246,7 @@ class FakeBroker(object):
class FakeAccountBroker(FakeBroker):
db_type = 'account'
db_contains_type = 'container'
info = {'account': TEST_ACCOUNT_NAME}
@ -267,8 +274,8 @@ class TestDBReplicator(unittest.TestCase):
self._patchers.append(patcher)
return patched_thing
def stub_delete_db(self, object_file):
self.delete_db_calls.append(object_file)
def stub_delete_db(self, broker):
self.delete_db_calls.append('/path/to/file')
def test_repl_connection(self):
node = {'replication_ip': '127.0.0.1', 'replication_port': 80,
@ -447,8 +454,7 @@ class TestDBReplicator(unittest.TestCase):
replicator.run_once()
def test_run_once_no_ips(self):
replicator = TestReplicator({})
replicator.logger = FakeLogger()
replicator = TestReplicator({}, logger=unit.FakeLogger())
self._patch(patch.object, db_replicator, 'whataremyips',
lambda *args: [])
@ -460,10 +466,10 @@ class TestDBReplicator(unittest.TestCase):
def test_run_once_node_is_not_mounted(self):
db_replicator.ring = FakeRingWithSingleNode()
replicator = TestReplicator({})
replicator.logger = FakeLogger()
replicator.mount_check = True
replicator.port = 6000
conf = {'mount_check': 'true', 'bind_port': 6000}
replicator = TestReplicator(conf, logger=unit.FakeLogger())
self.assertEqual(replicator.mount_check, True)
self.assertEqual(replicator.port, 6000)
def mock_ismount(path):
self.assertEquals(path,
@ -483,10 +489,10 @@ class TestDBReplicator(unittest.TestCase):
def test_run_once_node_is_mounted(self):
db_replicator.ring = FakeRingWithSingleNode()
replicator = TestReplicator({})
replicator.logger = FakeLogger()
replicator.mount_check = True
replicator.port = 6000
conf = {'mount_check': 'true', 'bind_port': 6000}
replicator = TestReplicator(conf, logger=unit.FakeLogger())
self.assertEqual(replicator.mount_check, True)
self.assertEqual(replicator.port, 6000)
def mock_unlink_older_than(path, mtime):
self.assertEquals(path,
@ -579,7 +585,7 @@ class TestDBReplicator(unittest.TestCase):
try:
replicator.delete_db = self.stub_delete_db
replicator.brokerclass.stub_replication_info = {
'delete_timestamp': 2, 'put_timestamp': 1, 'count': 0}
'delete_timestamp': 2, 'put_timestamp': 1}
replicator._replicate_object('0', '/path/to/file', 'node_id')
finally:
replicator.brokerclass.stub_replication_info = None
@ -592,28 +598,26 @@ class TestDBReplicator(unittest.TestCase):
self.assertEquals(['/path/to/file'], self.delete_db_calls)
def test_replicate_account_out_of_place(self):
replicator = TestReplicator({})
replicator = TestReplicator({}, logger=unit.FakeLogger())
replicator.ring = FakeRingWithNodes().Ring('path')
replicator.brokerclass = FakeAccountBroker
replicator._repl_to_node = lambda *args: True
replicator.delete_db = self.stub_delete_db
replicator.logger = FakeLogger()
# Correct node_id, wrong part
part = replicator.ring.get_part(TEST_ACCOUNT_NAME) + 1
node_id = replicator.ring.get_part_nodes(part)[0]['id']
replicator._replicate_object(str(part), '/path/to/file', node_id)
self.assertEqual(['/path/to/file'], self.delete_db_calls)
self.assertEqual(
replicator.logger.log_dict['error'],
[(('Found /path/to/file for /a%20c%20t when it should be on '
'partition 0; will replicate out and remove.',), {})])
error_msgs = replicator.logger.get_lines_for_level('error')
expected = 'Found /path/to/file for /a%20c%20t when it should be ' \
'on partition 0; will replicate out and remove.'
self.assertEqual(error_msgs, [expected])
def test_replicate_container_out_of_place(self):
replicator = TestReplicator({})
replicator = TestReplicator({}, logger=unit.FakeLogger())
replicator.ring = FakeRingWithNodes().Ring('path')
replicator._repl_to_node = lambda *args: True
replicator.delete_db = self.stub_delete_db
replicator.logger = FakeLogger()
# Correct node_id, wrong part
part = replicator.ring.get_part(
TEST_ACCOUNT_NAME, TEST_CONTAINER_NAME) + 1
@ -627,10 +631,9 @@ class TestDBReplicator(unittest.TestCase):
def test_delete_db(self):
db_replicator.lock_parent_directory = lock_parent_directory
replicator = TestReplicator({})
replicator = TestReplicator({}, logger=unit.FakeLogger())
replicator._zero_stats()
replicator.extract_device = lambda _: 'some_device'
replicator.logger = FakeLogger()
temp_dir = mkdtemp()
try:
@ -654,7 +657,8 @@ class TestDBReplicator(unittest.TestCase):
self.assertTrue(os.path.exists(temp_file2.name))
self.assertEqual(0, replicator.stats['remove'])
replicator.delete_db(temp_file.name)
temp_file.db_file = temp_file.name
replicator.delete_db(temp_file)
self.assertTrue(os.path.exists(temp_dir))
self.assertTrue(os.path.exists(temp_suf_dir))
@ -666,7 +670,8 @@ class TestDBReplicator(unittest.TestCase):
replicator.logger.log_dict['increment'])
self.assertEqual(1, replicator.stats['remove'])
replicator.delete_db(temp_file2.name)
temp_file2.db_file = temp_file2.name
replicator.delete_db(temp_file2)
self.assertTrue(os.path.exists(temp_dir))
self.assertFalse(os.path.exists(temp_suf_dir))
@ -889,10 +894,14 @@ class TestDBReplicator(unittest.TestCase):
def test_replicator_sync_with_broker_replication_missing_table(self):
rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False)
rpc.logger = unit.debug_logger()
broker = FakeBroker()
broker.get_repl_missing_table = True
called = []
def mock_quarantine_db(object_file, server_type):
called.append(True)
self.assertEquals(broker.db_file, object_file)
self.assertEquals(broker.db_type, server_type)
@ -905,6 +914,11 @@ class TestDBReplicator(unittest.TestCase):
self.assertEquals('404 Not Found', response.status)
self.assertEquals(404, response.status_int)
self.assertEqual(called, [True])
errors = rpc.logger.get_lines_for_level('error')
self.assertEqual(errors,
["Unable to decode remote metadata 'metadata'",
"Quarantining DB %s" % broker])
def test_replicator_sync(self):
rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False)
@ -1227,5 +1241,123 @@ class TestReplToNode(unittest.TestCase):
self.fake_node, FakeBroker(), '0', self.fake_info), False)
class FakeHTTPResponse(object):
def __init__(self, resp):
self.resp = resp
@property
def status(self):
return self.resp.status_int
@property
def data(self):
return self.resp.body
def attach_fake_replication_rpc(rpc, replicate_hook=None):
class FakeReplConnection(object):
def __init__(self, node, partition, hash_, logger):
self.logger = logger
self.node = node
self.partition = partition
self.path = '/%s/%s/%s' % (node['device'], partition, hash_)
self.host = node['replication_ip']
def replicate(self, op, *sync_args):
print 'REPLICATE: %s, %s, %r' % (self.path, op, sync_args)
replicate_args = self.path.lstrip('/').split('/')
args = [op] + list(sync_args)
swob_response = rpc.dispatch(replicate_args, args)
resp = FakeHTTPResponse(swob_response)
if replicate_hook:
replicate_hook(op, *sync_args)
return resp
return FakeReplConnection
class TestReplicatorSync(unittest.TestCase):
backend = None # override in subclass
datadir = None
replicator_daemon = db_replicator.Replicator
replicator_rpc = db_replicator.ReplicatorRpc
def setUp(self):
self.root = mkdtemp()
self.rpc = self.replicator_rpc(
self.root, self.datadir, self.backend, False,
logger=unit.debug_logger())
FakeReplConnection = attach_fake_replication_rpc(self.rpc)
self._orig_ReplConnection = db_replicator.ReplConnection
db_replicator.ReplConnection = FakeReplConnection
self._orig_Ring = db_replicator.ring.Ring
self._ring = unit.FakeRing()
db_replicator.ring.Ring = lambda *args, **kwargs: self._get_ring()
self.logger = unit.debug_logger()
def tearDown(self):
db_replicator.ReplConnection = self._orig_ReplConnection
db_replicator.ring.Ring = self._orig_Ring
rmtree(self.root)
def _get_ring(self):
return self._ring
def _get_broker(self, account, container=None, node_index=0):
hash_ = hash_path(account, container)
part, nodes = self._ring.get_nodes(account, container)
drive = nodes[node_index]['device']
db_path = os.path.join(self.root, drive,
storage_directory(self.datadir, part, hash_),
hash_ + '.db')
return self.backend(db_path, account=account, container=container)
def _get_broker_part_node(self, broker):
part, nodes = self._ring.get_nodes(broker.account, broker.container)
storage_dir = broker.db_file[len(self.root):].lstrip(os.path.sep)
broker_device = storage_dir.split(os.path.sep, 1)[0]
for node in nodes:
if node['device'] == broker_device:
return part, node
def _run_once(self, node, conf_updates=None, daemon=None):
conf = {
'devices': self.root,
'recon_cache_path': self.root,
'mount_check': 'false',
'bind_port': node['replication_port'],
}
if conf_updates:
conf.update(conf_updates)
daemon = daemon or self.replicator_daemon(conf, logger=self.logger)
def _rsync_file(db_file, remote_file, **kwargs):
remote_server, remote_path = remote_file.split('/', 1)
dest_path = os.path.join(self.root, remote_path)
copy(db_file, dest_path)
return True
daemon._rsync_file = _rsync_file
with mock.patch('swift.common.db_replicator.whataremyips',
new=lambda: [node['replication_ip']]):
daemon.run_once()
return daemon
def test_local_ids(self):
if self.datadir is None:
# base test case
return
for drive in ('sda', 'sdb', 'sdd'):
os.makedirs(os.path.join(self.root, drive, self.datadir))
for node in self._ring.devs:
daemon = self._run_once(node)
if node['device'] == 'sdc':
self.assertEqual(daemon._local_device_ids, set())
else:
self.assertEqual(daemon._local_device_ids,
set([node['id']]))
if __name__ == '__main__':
unittest.main()

705
test/unit/common/test_direct_client.py
View File

@ -15,325 +15,578 @@
import unittest
import os
import urllib
from contextlib import contextmanager
import StringIO
from hashlib import md5
import time
import mock
from swift.common import direct_client
from swift.common.exceptions import ClientException
from swift.common.utils import json
from swift.common.utils import json, Timestamp
from swift.common.swob import HeaderKeyDict, RESPONSE_REASONS
from swift.common.storage_policy import POLICY_INDEX, POLICIES
from test.unit import patch_policies
def mock_http_connect(status, fake_headers=None, body=None):
class FakeConn(object):
class FakeConn(object):
def __init__(self, status, fake_headers, body, *args, **kwargs):
self.status = status
def __init__(self, status, headers=None, body='', **kwargs):
self.status = status
try:
self.reason = RESPONSE_REASONS[self.status][0]
except Exception:
self.reason = 'Fake'
self.body = body
self.host = args[0]
self.port = args[1]
self.method = args[4]
self.path = args[5]
self.with_exc = False
self.headers = kwargs.get('headers', {})
self.fake_headers = fake_headers
self.body = body
self.resp_headers = HeaderKeyDict()
if headers:
self.resp_headers.update(headers)
self.with_exc = False
self.etag = None
def _update_raw_call_args(self, *args, **kwargs):
capture_attrs = ('host', 'port', 'method', 'path', 'req_headers',
'query_string')
for attr, value in zip(capture_attrs, args[:len(capture_attrs)]):
setattr(self, attr, value)
return self
def getresponse(self):
if self.etag:
self.resp_headers['etag'] = str(self.etag.hexdigest())
if self.with_exc:
raise Exception('test')
return self
def getheader(self, header, default=None):
return self.resp_headers.get(header, default)
def getheaders(self):
return self.resp_headers.items()
def read(self):
return self.body
def send(self, data):
if not self.etag:
self.etag = md5()
def getresponse(self):
if self.with_exc:
raise Exception('test')
if self.fake_headers is not None and self.method == 'POST':
self.fake_headers.append(self.headers)
return self
def getheader(self, header, default=None):
return self.headers.get(header.lower(), default)
def getheaders(self):
if self.fake_headers is not None:
for key in self.fake_headers:
self.headers.update({key: self.fake_headers[key]})
return self.headers.items()
def read(self):
return self.body
def send(self, data):
self.etag.update(data)
self.headers['etag'] = str(self.etag.hexdigest())
def close(self):
return
return lambda *args, **kwargs: FakeConn(status, fake_headers, body,
*args, **kwargs)
self.etag.update(data)
@contextmanager
def mocked_http_conn(*args, **kwargs):
fake_conn = FakeConn(*args, **kwargs)
mock_http_conn = lambda *args, **kwargs: \
fake_conn._update_raw_call_args(*args, **kwargs)
with mock.patch('swift.common.bufferedhttp.http_connect_raw',
new=mock_http_conn):
yield fake_conn
@patch_policies
class TestDirectClient(unittest.TestCase):
def setUp(self):
self.node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
self.part = '0'
self.account = u'\u062a account'
self.container = u'\u062a container'
self.obj = u'\u062a obj/name'
self.account_path = '/sda/0/%s' % urllib.quote(
self.account.encode('utf-8'))
self.container_path = '/sda/0/%s/%s' % tuple(
urllib.quote(p.encode('utf-8')) for p in (
self.account, self.container))
self.obj_path = '/sda/0/%s/%s/%s' % tuple(
urllib.quote(p.encode('utf-8')) for p in (
self.account, self.container, self.obj))
self.user_agent = 'direct-client %s' % os.getpid()
def test_gen_headers(self):
hdrs = direct_client.gen_headers()
assert 'user-agent' in hdrs
assert hdrs['user-agent'] == 'direct-client %s' % os.getpid()
assert len(hdrs.keys()) == 1
stub_user_agent = 'direct-client %s' % os.getpid()
hdrs = direct_client.gen_headers(add_ts=True)
assert 'user-agent' in hdrs
assert 'x-timestamp' in hdrs
assert len(hdrs.keys()) == 2
headers = direct_client.gen_headers()
self.assertEqual(headers['user-agent'], stub_user_agent)
self.assertEqual(1, len(headers))
hdrs = direct_client.gen_headers(hdrs_in={'foo-bar': '47'})
assert 'user-agent' in hdrs
assert 'foo-bar' in hdrs
assert hdrs['foo-bar'] == '47'
assert len(hdrs.keys()) == 2
now = time.time()
headers = direct_client.gen_headers(add_ts=True)
self.assertEqual(headers['user-agent'], stub_user_agent)
self.assert_(now - 1 < Timestamp(headers['x-timestamp']) < now + 1)
self.assertEqual(headers['x-timestamp'],
Timestamp(headers['x-timestamp']).internal)
self.assertEqual(2, len(headers))
hdrs = direct_client.gen_headers(hdrs_in={'user-agent': '47'})
assert 'user-agent' in hdrs
assert hdrs['user-agent'] == 'direct-client %s' % os.getpid()
assert len(hdrs.keys()) == 1
headers = direct_client.gen_headers(hdrs_in={'foo-bar': '47'})
self.assertEqual(headers['user-agent'], stub_user_agent)
self.assertEqual(headers['foo-bar'], '47')
self.assertEqual(2, len(headers))
headers = direct_client.gen_headers(hdrs_in={'user-agent': '47'})
self.assertEqual(headers['user-agent'], stub_user_agent)
self.assertEqual(1, len(headers))
for policy in POLICIES:
for add_ts in (True, False):
now = time.time()
headers = direct_client.gen_headers(
{POLICY_INDEX: policy.idx}, add_ts=add_ts)
self.assertEqual(headers['user-agent'], stub_user_agent)
self.assertEqual(headers[POLICY_INDEX], str(policy.idx))
expected_header_count = 2
if add_ts:
expected_header_count += 1
self.assertEqual(
headers['x-timestamp'],
Timestamp(headers['x-timestamp']).internal)
self.assert_(
now - 1 < Timestamp(headers['x-timestamp']) < now + 1)
self.assertEqual(expected_header_count, len(headers))
def test_direct_get_account(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
stub_headers = HeaderKeyDict({
'X-Account-Container-Count': '1',
'X-Account-Object-Count': '1',
'X-Account-Bytes-Used': '1',
'X-Timestamp': '1234567890',
'X-PUT-Timestamp': '1234567890'})
body = '[{"count": 1, "bytes": 20971520, "name": "c1"}]'
with mocked_http_conn(200, stub_headers, body) as conn:
resp_headers, resp = direct_client.direct_get_account(
self.node, self.part, self.account)
self.assertEqual(conn.method, 'GET')
self.assertEqual(conn.path, self.account_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(resp_headers, stub_headers)
self.assertEqual(json.loads(body), resp)
def test_direct_client_exception(self):
stub_headers = {'X-Trans-Id': 'txb5f59485c578460f8be9e-0053478d09'}
body = 'a server error has occurred'
with mocked_http_conn(500, stub_headers, body):
try:
direct_client.direct_get_account(self.node, self.part,
self.account)
except ClientException as err:
pass
else:
self.fail('ClientException not raised')
self.assertEqual(err.http_status, 500)
expected_err_msg_parts = (
'Account server %s:%s' % (self.node['ip'], self.node['port']),
'GET %r' % self.account_path,
'status 500',
)
for item in expected_err_msg_parts:
self.assert_(item in str(err), '%r was not in "%s"' % (item, err))
self.assertEqual(err.http_host, self.node['ip'])
self.assertEqual(err.http_port, self.node['port'])
self.assertEqual(err.http_device, self.node['device'])
self.assertEqual(err.http_status, 500)
self.assertEqual(err.http_reason, 'Internal Error')
self.assertEqual(err.http_headers, stub_headers)
def test_direct_get_account_no_content_does_not_parse_body(self):
headers = {
'X-Account-Container-Count': '1',
'X-Account-Object-Count': '1',
'X-Account-Bytes-Used': '1',
'X-Timestamp': '1234567890',
'X-PUT-Timestamp': '1234567890'}
with mocked_http_conn(204, headers) as conn:
resp_headers, resp = direct_client.direct_get_account(
self.node, self.part, self.account)
self.assertEqual(conn.method, 'GET')
self.assertEqual(conn.path, self.account_path)
body = '[{"count": 1, "bytes": 20971520, "name": "c1"}]'
fake_headers = {}
for header, value in headers.items():
fake_headers[header.lower()] = value
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(200, fake_headers, body)
resp_headers, resp = direct_client.direct_get_account(node, part,
account)
fake_headers.update({'user-agent': 'direct-client %s' % os.getpid()})
self.assertEqual(fake_headers, resp_headers)
self.assertEqual(json.loads(body), resp)
direct_client.http_connect = mock_http_connect(204, fake_headers, body)
resp_headers, resp = direct_client.direct_get_account(node, part,
account)
fake_headers.update({'user-agent': 'direct-client %s' % os.getpid()})
self.assertEqual(fake_headers, resp_headers)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(resp_headers, resp_headers)
self.assertEqual([], resp)
direct_client.http_connect = was_http_connector
def test_direct_get_account_error(self):
with mocked_http_conn(500) as conn:
try:
direct_client.direct_get_account(
self.node, self.part, self.account)
except ClientException as err:
pass
else:
self.fail('ClientException not raised')
self.assertEqual(conn.method, 'GET')
self.assertEqual(conn.path, self.account_path)
self.assertEqual(err.http_status, 500)
self.assert_('GET' in str(err))
def test_direct_delete_account(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
mock_path = 'swift.common.bufferedhttp.http_connect_raw'
with mock.patch(mock_path) as fake_connect:
fake_connect.return_value.getresponse.return_value.status = 200
direct_client.direct_delete_account(node, part, account)
args, kwargs = fake_connect.call_args
method = args[2]
self.assertEqual('DELETE', method)
path = args[3]
self.assertEqual('/sda/0/a', path)
headers = args[4]
self.assert_('X-Timestamp' in headers)
def test_direct_head_container(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
container = 'c'
headers = {'key': 'value'}
headers = HeaderKeyDict(key='value')
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(200, headers)
with mocked_http_conn(200, headers) as conn:
resp = direct_client.direct_head_container(
self.node, self.part, self.account, self.container)
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
resp = direct_client.direct_head_container(node, part, account,
container)
headers.update({'user-agent': 'direct-client %s' % os.getpid()})
self.assertEqual(conn.req_headers['user-agent'],
self.user_agent)
self.assertEqual(headers, resp)
direct_client.http_connect = was_http_connector
def test_direct_head_container_error(self):
headers = HeaderKeyDict(key='value')
with mocked_http_conn(503, headers) as conn:
try:
direct_client.direct_head_container(
self.node, self.part, self.account, self.container)
except ClientException as err:
pass
else:
self.fail('ClientException not raised')
# check request
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(err.http_status, 503)
self.assertEqual(err.http_headers, headers)
self.assert_('HEAD' in str(err))
def test_direct_head_container_deleted(self):
important_timestamp = Timestamp(time.time()).internal
headers = HeaderKeyDict({'X-Backend-Important-Timestamp':
important_timestamp})
with mocked_http_conn(404, headers) as conn:
try:
direct_client.direct_head_container(
self.node, self.part, self.account, self.container)
except Exception as err:
self.assert_(isinstance(err, ClientException))
else:
self.fail('ClientException not raised')
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.container_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(err.http_status, 404)
self.assertEqual(err.http_headers, headers)
def test_direct_get_container(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
container = 'c'
headers = {'key': 'value'}
headers = HeaderKeyDict({'key': 'value'})
body = '[{"hash": "8f4e3", "last_modified": "317260", "bytes": 209}]'
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(200, headers, body)
with mocked_http_conn(200, headers, body) as conn:
resp_headers, resp = direct_client.direct_get_container(
self.node, self.part, self.account, self.container)
resp_headers, resp = (
direct_client.direct_get_container(node, part, account, container))
headers.update({'user-agent': 'direct-client %s' % os.getpid()})
self.assertEqual(conn.req_headers['user-agent'],
'direct-client %s' % os.getpid())
self.assertEqual(headers, resp_headers)
self.assertEqual(json.loads(body), resp)
direct_client.http_connect = mock_http_connect(204, headers, body)
def test_direct_get_container_no_content_does_not_decode_body(self):
headers = {}
body = ''
with mocked_http_conn(204, headers, body) as conn:
resp_headers, resp = direct_client.direct_get_container(
self.node, self.part, self.account, self.container)
resp_headers, resp = (
direct_client.direct_get_container(node, part, account, container))
headers.update({'user-agent': 'direct-client %s' % os.getpid()})
self.assertEqual(conn.req_headers['user-agent'],
'direct-client %s' % os.getpid())
self.assertEqual(headers, resp_headers)
self.assertEqual([], resp)
direct_client.http_connect = was_http_connector
def test_direct_delete_container(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
container = 'c'
with mocked_http_conn(200) as conn:
direct_client.direct_delete_container(
self.node, self.part, self.account, self.container)
self.assertEqual(conn.method, 'DELETE')
self.assertEqual(conn.path, self.container_path)
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(200)
def test_direct_delete_container_error(self):
with mocked_http_conn(500) as conn:
try:
direct_client.direct_delete_container(
self.node, self.part, self.account, self.container)
except ClientException as err:
pass
else:
self.fail('ClientException not raised')
direct_client.direct_delete_container(node, part, account, container)
self.assertEqual(conn.method, 'DELETE')
self.assertEqual(conn.path, self.container_path)
direct_client.http_connect = was_http_connector
self.assertEqual(err.http_status, 500)
self.assert_('DELETE' in str(err))
def test_direct_put_container_object(self):
headers = {'x-foo': 'bar'}
with mocked_http_conn(204) as conn:
rv = direct_client.direct_put_container_object(
self.node, self.part, self.account, self.container, self.obj,
headers=headers)
self.assertEqual(conn.method, 'PUT')
self.assertEqual(conn.path, self.obj_path)
self.assert_('x-timestamp' in conn.req_headers)
self.assertEqual('bar', conn.req_headers.get('x-foo'))
self.assertEqual(rv, None)
def test_direct_put_container_object_error(self):
with mocked_http_conn(500) as conn:
try:
direct_client.direct_put_container_object(
self.node, self.part, self.account, self.container,
self.obj)
except ClientException as err:
pass
else:
self.fail('ClientException not raised')
self.assertEqual(conn.method, 'PUT')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(err.http_status, 500)
self.assert_('PUT' in str(err))
def test_direct_delete_container_object(self):
with mocked_http_conn(204) as conn:
rv = direct_client.direct_delete_container_object(
self.node, self.part, self.account, self.container, self.obj)
self.assertEqual(conn.method, 'DELETE')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(rv, None)
def test_direct_delete_container_obj_error(self):
with mocked_http_conn(500) as conn:
try:
direct_client.direct_delete_container_object(
self.node, self.part, self.account, self.container,
self.obj)
except ClientException as err:
pass
else:
self.fail('ClientException not raised')
self.assertEqual(conn.method, 'DELETE')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(err.http_status, 500)
self.assert_('DELETE' in str(err))
def test_direct_head_object(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
container = 'c'
name = 'o'
headers = {'key': 'value'}
headers = HeaderKeyDict({'x-foo': 'bar'})
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(200, headers)
with mocked_http_conn(200, headers) as conn:
resp = direct_client.direct_head_object(
self.node, self.part, self.account, self.container,
self.obj, headers=headers)
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.obj_path)
resp = direct_client.direct_head_object(node, part, account,
container, name)
headers.update({'user-agent': 'direct-client %s' % os.getpid()})
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual('bar', conn.req_headers.get('x-foo'))
self.assert_('x-timestamp' not in conn.req_headers,
'x-timestamp was in HEAD request headers')
self.assertEqual(headers, resp)
direct_client.http_connect = was_http_connector
def test_direct_head_object_error(self):
with mocked_http_conn(500) as conn:
try:
direct_client.direct_head_object(
self.node, self.part, self.account, self.container,
self.obj)
except ClientException as err:
pass
else:
self.fail('ClientException not raised')
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(err.http_status, 500)
self.assert_('HEAD' in str(err))
def test_direct_head_object_not_found(self):
important_timestamp = Timestamp(time.time()).internal
stub_headers = {'X-Backend-Important-Timestamp': important_timestamp}
with mocked_http_conn(404, headers=stub_headers) as conn:
try:
direct_client.direct_head_object(
self.node, self.part, self.account, self.container,
self.obj)
except ClientException as err:
pass
else:
self.fail('ClientException not raised')
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(err.http_status, 404)
self.assertEqual(err.http_headers['x-backend-important-timestamp'],
important_timestamp)
def test_direct_get_object(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
container = 'c'
name = 'o'
contents = StringIO.StringIO('123456')
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(200, body=contents)
resp_header, obj_body = (
direct_client.direct_get_object(node, part, account, container,
name))
with mocked_http_conn(200, body=contents) as conn:
resp_header, obj_body = direct_client.direct_get_object(
self.node, self.part, self.account, self.container, self.obj)
self.assertEqual(conn.method, 'GET')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(obj_body, contents)
direct_client.http_connect = was_http_connector
def test_direct_get_object_error(self):
with mocked_http_conn(500) as conn:
try:
direct_client.direct_get_object(
self.node, self.part,
self.account, self.container, self.obj)
except ClientException as err:
pass
else:
self.fail('ClientException not raised')
self.assertEqual(conn.method, 'GET')
self.assertEqual(conn.path, self.obj_path)
pass
self.assertEqual(err.http_status, 500)
self.assert_('GET' in str(err))
def test_direct_post_object(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
container = 'c'
name = 'o'
headers = {'Key': 'value'}
fake_headers = []
resp_headers = []
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(200, fake_headers)
with mocked_http_conn(200, resp_headers) as conn:
direct_client.direct_post_object(
self.node, self.part, self.account, self.container, self.obj,
headers)
self.assertEqual(conn.method, 'POST')
self.assertEqual(conn.path, self.obj_path)
direct_client.direct_post_object(node, part, account,
container, name, headers)
self.assertEqual(headers['Key'], fake_headers[0].get('Key'))
for header in headers:
self.assertEqual(conn.req_headers[header], headers[header])
direct_client.http_connect = was_http_connector
def test_direct_post_object_error(self):
headers = {'Key': 'value'}
with mocked_http_conn(500) as conn:
try:
direct_client.direct_post_object(
self.node, self.part, self.account, self.container,
self.obj, headers)
except ClientException as err:
pass
else:
self.fail('ClientException not raised')
self.assertEqual(conn.method, 'POST')
self.assertEqual(conn.path, self.obj_path)
for header in headers:
self.assertEqual(conn.req_headers[header], headers[header])
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assert_('x-timestamp' in conn.req_headers)
self.assertEqual(err.http_status, 500)
self.assert_('POST' in str(err))
def test_direct_delete_object(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
container = 'c'
name = 'o'
with mocked_http_conn(200) as conn:
resp = direct_client.direct_delete_object(
self.node, self.part, self.account, self.container, self.obj)
self.assertEqual(conn.method, 'DELETE')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(resp, None)
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(200)
def test_direct_delete_object_error(self):
with mocked_http_conn(503) as conn:
try:
direct_client.direct_delete_object(
self.node, self.part, self.account, self.container,
self.obj)
except ClientException as err:
pass
else:
self.fail('ClientException not raised')
self.assertEqual(conn.method, 'DELETE')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(err.http_status, 503)
self.assert_('DELETE' in str(err))
direct_client.direct_delete_object(node, part, account, container,
name)
direct_client.http_connect = was_http_connector
def test_direct_put_object(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
container = 'c'
name = 'o'
def test_direct_put_object_with_content_length(self):
contents = StringIO.StringIO('123456')
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(200)
resp = direct_client.direct_put_object(node, part, account,
container, name, contents, 6)
with mocked_http_conn(200) as conn:
resp = direct_client.direct_put_object(
self.node, self.part, self.account, self.container, self.obj,
contents, 6)
self.assertEqual(conn.method, 'PUT')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(md5('123456').hexdigest(), resp)
direct_client.http_connect = was_http_connector
def test_direct_put_object_fail(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
container = 'c'
name = 'o'
contents = StringIO.StringIO('123456')
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(500)
self.assertRaises(ClientException, direct_client.direct_put_object,
node, part, account, container, name, contents)
direct_client.http_connect = was_http_connector
with mocked_http_conn(500) as conn:
try:
direct_client.direct_put_object(
self.node, self.part, self.account, self.container,
self.obj, contents)
except ClientException as err:
pass
else:
self.fail('ClientException not raised')
self.assertEqual(conn.method, 'PUT')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(err.http_status, 500)
def test_direct_put_object_chunked(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
container = 'c'
name = 'o'
contents = StringIO.StringIO('123456')
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(200)
resp = direct_client.direct_put_object(node, part, account,
container, name, contents)
with mocked_http_conn(200) as conn:
resp = direct_client.direct_put_object(
self.node, self.part, self.account, self.container, self.obj,
contents)
self.assertEqual(conn.method, 'PUT')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(md5('6\r\n123456\r\n0\r\n\r\n').hexdigest(), resp)
direct_client.http_connect = was_http_connector
def test_retry(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
container = 'c'
name = 'o'
headers = {'key': 'value'}
headers = HeaderKeyDict({'key': 'value'})
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(200, headers)
attempts, resp = direct_client.retry(direct_client.direct_head_object,
node, part, account, container,
name)
headers.update({'user-agent': 'direct-client %s' % os.getpid()})
with mocked_http_conn(200, headers) as conn:
attempts, resp = direct_client.retry(
direct_client.direct_head_object, self.node, self.part,
self.account, self.container, self.obj)
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(headers, resp)
self.assertEqual(attempts, 1)
direct_client.http_connect = was_http_connector
if __name__ == '__main__':
unittest.main()

Some files were not shown because too many files have changed in this diff Show More