summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKota Tsuyuzaki <tsuyuzaki.kota@lab.ntt.co.jp>2017-10-16 21:04:52 +0900
committerKota Tsuyuzaki <tsuyuzaki.kota@lab.ntt.co.jp>2017-10-16 21:28:57 +0900
commitf94d6567a7e2e8b3ca1168b4a41c42c1ee371af5 (patch)
tree5e6ffadf22a6573e220f47cf58a99ccaf2d9c06a
parent7fdb83a5abe7add5cda4ccea61e8089ea424b70f (diff)
parent98e7014aafbd52bd5e388314f3c2c1e1d2375806 (diff)
Merge 'remotes/origin/master' into s3api
Notes
Notes (review): Code-Review+2: Thiago da Silva <thiago@redhat.com> Workflow+1: Thiago da Silva <thiago@redhat.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Wed, 18 Oct 2017 18:54:22 +0000 Reviewed-on: https://review.openstack.org/512277 Project: openstack/swift Branch: refs/heads/feature/s3api
-rw-r--r--CHANGELOG40
-rw-r--r--CONTRIBUTING.rst4
-rw-r--r--README.rst20
-rw-r--r--api-ref/source/storage-container-services.inc3
-rwxr-xr-xbin/swift-account-audit14
-rw-r--r--doc/manpages/account-server.conf.56
-rw-r--r--doc/manpages/container-sync-realms.conf.5138
-rw-r--r--doc/manpages/proxy-server.conf.511
-rw-r--r--doc/manpages/swift-account-audit.16
-rw-r--r--doc/saio/swift/proxy-server.conf5
-rw-r--r--doc/source/admin/objectstorage-EC.rst10
-rw-r--r--doc/source/admin_guide.rst4
-rw-r--r--doc/source/api/object_api_v1_overview.rst6
-rw-r--r--doc/source/api/temporary_url_middleware.rst2
-rw-r--r--doc/source/deployment_guide.rst15
-rw-r--r--doc/source/development_guidelines.rst4
-rw-r--r--doc/source/install/controller-include.txt2
-rw-r--r--doc/source/install/controller-install-debian.rst2
-rw-r--r--doc/source/install/controller-install-obs.rst2
-rw-r--r--doc/source/install/controller-install-rdo.rst2
-rw-r--r--doc/source/install/controller-install-ubuntu.rst2
-rw-r--r--doc/source/install/get_started.rst4
-rw-r--r--doc/source/install/initial-rings.rst2
-rw-r--r--doc/source/install/storage-include1.txt2
-rw-r--r--doc/source/install/storage-include2.txt2
-rw-r--r--doc/source/install/storage-include3.txt2
-rw-r--r--doc/source/install/storage-install-obs.rst2
-rw-r--r--doc/source/install/storage-install-rdo.rst2
-rw-r--r--doc/source/install/storage-install-ubuntu-debian.rst2
-rw-r--r--doc/source/overview_auth.rst6
-rw-r--r--doc/source/overview_encryption.rst2
-rw-r--r--etc/account-server.conf-sample6
-rw-r--r--etc/internal-client.conf-sample1
-rw-r--r--etc/proxy-server.conf-sample22
-rw-r--r--releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po67
-rw-r--r--setup.cfg1
-rw-r--r--swift/account/server.py21
-rw-r--r--swift/account/utils.py56
-rw-r--r--[-rwxr-xr-x]swift/cli/dispersion_report.py0
-rw-r--r--swift/cli/ringbuilder.py5
-rw-r--r--swift/common/constraints.py35
-rw-r--r--swift/common/db_replicator.py4
-rw-r--r--swift/common/direct_client.py15
-rw-r--r--swift/common/internal_client.py8
-rw-r--r--swift/common/memcached.py16
-rw-r--r--swift/common/middleware/copy.py68
-rw-r--r--swift/common/middleware/crypto/decrypter.py36
-rw-r--r--swift/common/middleware/dlo.py2
-rw-r--r--swift/common/middleware/domain_remap.py117
-rw-r--r--swift/common/middleware/gatekeeper.py21
-rw-r--r--swift/common/middleware/listing_formats.py211
-rw-r--r--swift/common/middleware/recon.py4
-rw-r--r--swift/common/middleware/slo.py200
-rw-r--r--swift/common/middleware/staticweb.py6
-rw-r--r--swift/common/middleware/tempauth.py23
-rw-r--r--swift/common/middleware/versioned_writes.py6
-rw-r--r--swift/common/request_helpers.py25
-rw-r--r--swift/common/ring/builder.py35
-rw-r--r--swift/common/utils.py19
-rw-r--r--swift/common/wsgi.py7
-rw-r--r--swift/container/server.py71
-rw-r--r--swift/container/sync.py1
-rw-r--r--swift/container/updater.py10
-rw-r--r--swift/obj/diskfile.py17
-rw-r--r--swift/obj/mem_diskfile.py5
-rw-r--r--swift/obj/reconstructor.py5
-rw-r--r--swift/obj/server.py10
-rw-r--r--swift/obj/updater.py8
-rw-r--r--swift/proxy/controllers/account.py10
-rw-r--r--swift/proxy/controllers/base.py11
-rw-r--r--swift/proxy/controllers/container.py7
-rw-r--r--swift/proxy/server.py11
-rw-r--r--test/functional/__init__.py9
-rw-r--r--test/functional/swift_test_client.py31
-rw-r--r--test/functional/test_object.py4
-rw-r--r--test/functional/tests.py2
-rw-r--r--test/probe/common.py5
-rw-r--r--test/probe/test_container_sync.py14
-rw-r--r--test/probe/test_object_async_update.py6
-rw-r--r--test/probe/test_object_metadata_replication.py2
-rw-r--r--test/unit/__init__.py59
-rw-r--r--test/unit/account/test_server.py98
-rw-r--r--test/unit/cli/test_dispersion_report.py21
-rw-r--r--test/unit/cli/test_ringbuilder.py15
-rw-r--r--test/unit/common/middleware/crypto/test_decrypter.py133
-rw-r--r--test/unit/common/middleware/crypto/test_encryption.py9
-rw-r--r--test/unit/common/middleware/test_copy.py250
-rw-r--r--test/unit/common/middleware/test_dlo.py24
-rw-r--r--test/unit/common/middleware/test_domain_remap.py86
-rw-r--r--test/unit/common/middleware/test_gatekeeper.py31
-rw-r--r--test/unit/common/middleware/test_listing_formats.py345
-rw-r--r--test/unit/common/middleware/test_slo.py159
-rw-r--r--test/unit/common/middleware/test_staticweb.py37
-rw-r--r--test/unit/common/middleware/test_subrequest_logging.py51
-rw-r--r--test/unit/common/middleware/test_tempauth.py52
-rw-r--r--test/unit/common/middleware/test_versioned_writes.py91
-rw-r--r--test/unit/common/ring/test_builder.py171
-rw-r--r--test/unit/common/ring/test_composite_builder.py6
-rw-r--r--test/unit/common/ring/test_utils.py19
-rw-r--r--test/unit/common/test_constraints.py60
-rw-r--r--test/unit/common/test_daemon.py16
-rw-r--r--test/unit/common/test_db_replicator.py57
-rw-r--r--test/unit/common/test_memcached.py67
-rw-r--r--test/unit/common/test_wsgi.py125
-rw-r--r--test/unit/container/test_server.py204
-rw-r--r--test/unit/helpers.py6
-rw-r--r--test/unit/obj/test_auditor.py4
-rw-r--r--test/unit/obj/test_diskfile.py52
-rw-r--r--test/unit/obj/test_reconstructor.py41
-rw-r--r--test/unit/obj/test_server.py81
-rw-r--r--test/unit/obj/test_ssync_receiver.py23
-rw-r--r--test/unit/proxy/controllers/test_base.py22
-rw-r--r--test/unit/proxy/controllers/test_container.py20
-rw-r--r--test/unit/proxy/test_server.py93
-rw-r--r--test/unit/proxy/test_sysmeta.py10
115 files changed, 2529 insertions, 1609 deletions
diff --git a/CHANGELOG b/CHANGELOG
index a305b67..e4e9629 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -244,7 +244,7 @@ swift (2.13.0, OpenStack Ocata)
244 244
245 * PUT subrequests generated from a client-side COPY will now properly log 245 * PUT subrequests generated from a client-side COPY will now properly log
246 the SSC (server-side copy) Swift source field. See 246 the SSC (server-side copy) Swift source field. See
247 https://docs.openstack.org/developer/swift/logs.html#swift-source for 247 https://docs.openstack.org/swift/latest/logs.html#swift-source for
248 more information. 248 more information.
249 249
250 * Fixed a bug where an SLO download with a range request may have resulted 250 * Fixed a bug where an SLO download with a range request may have resulted
@@ -391,13 +391,13 @@ swift (2.10.0, OpenStack Newton)
391 * Object versioning now supports a "history" mode in addition to 391 * Object versioning now supports a "history" mode in addition to
392 the older "stack" mode. The difference is in how DELETE requests 392 the older "stack" mode. The difference is in how DELETE requests
393 are handled. For full details, please read 393 are handled. For full details, please read
394 http://docs.openstack.org/developer/swift/overview_object_versioning.html. 394 https://docs.openstack.org/swift/latest/overview_object_versioning.html.
395 395
396 * New config variables to change the schedule priority and I/O 396 * New config variables to change the schedule priority and I/O
397 scheduling class. Servers and daemons now understand 397 scheduling class. Servers and daemons now understand
398 `nice_priority`, `ionice_class`, and `ionice_priority` to 398 `nice_priority`, `ionice_class`, and `ionice_priority` to
399 schedule their relative importance. Please read 399 schedule their relative importance. Please read
400 http://docs.openstack.org/developer/swift/deployment_guide.html 400 https://docs.openstack.org/swift/latest/admin_guide.html
401 for full config details. 401 for full config details.
402 402
403 * On newer kernels (3.15+ when using xfs), Swift will use the O_TMPFILE 403 * On newer kernels (3.15+ when using xfs), Swift will use the O_TMPFILE
@@ -410,7 +410,7 @@ swift (2.10.0, OpenStack Newton)
410 improved in clusters that are not completely healthy. 410 improved in clusters that are not completely healthy.
411 411
412 * Significant improvements to the api-ref doc available at 412 * Significant improvements to the api-ref doc available at
413 http://developer.openstack.org/api-ref/object-storage/. 413 https://developer.openstack.org/api-ref/object-storage/.
414 414
415 * A PUT or POST to a container will now update the container's 415 * A PUT or POST to a container will now update the container's
416 Last-Modified time, and that value will be included in a 416 Last-Modified time, and that value will be included in a
@@ -464,7 +464,7 @@ swift (2.9.0)
464 464
465 For more information on the details of the at-rest encryption 465 For more information on the details of the at-rest encryption
466 feature, please see the docs at 466 feature, please see the docs at
467 http://docs.openstack.org/developer/swift/overview_encryption.html. 467 https://docs.openstack.org/swift/latest/overview_encryption.html.
468 468
469 * `swift-recon` can now be called with more than one server type. 469 * `swift-recon` can now be called with more than one server type.
470 470
@@ -606,7 +606,7 @@ swift (2.7.0, OpenStack Mitaka)
606 default it will stagger the firing. 606 default it will stagger the firing.
607 607
608 * Added an operational procedures guide to the docs. It can be 608 * Added an operational procedures guide to the docs. It can be
609 found at http://docs.openstack.org/developer/swift/ops_runbook/index.html and 609 found at https://docs.openstack.org/swift/latest/ops_runbook/index.html and
610 includes information on detecting and handling day-to-day 610 includes information on detecting and handling day-to-day
611 operational issues in a Swift cluster. 611 operational issues in a Swift cluster.
612 612
@@ -776,7 +776,7 @@ swift (2.6.0)
776 * Container sync has been improved to more quickly find and iterate over 776 * Container sync has been improved to more quickly find and iterate over
777 the containers to be synced. This reduced server load and lowers the 777 the containers to be synced. This reduced server load and lowers the
778 time required to see data propagate between two clusters. Please see 778 time required to see data propagate between two clusters. Please see
779 http://docs.openstack.org/developer/swift/overview_container_sync.html for more details 779 https://docs.openstack.org/swift/latest/overview_container_sync.html for more details
780 about the new on-disk structure for tracking synchronized containers. 780 about the new on-disk structure for tracking synchronized containers.
781 781
782 * A container POST will now update that container's put-timestamp value. 782 * A container POST will now update that container's put-timestamp value.
@@ -862,7 +862,7 @@ swift (2.4.0)
862 server config setting ("allow_versions"), if it is currently enabled. 862 server config setting ("allow_versions"), if it is currently enabled.
863 The existing container server config setting enables existing 863 The existing container server config setting enables existing
864 containers to continue being versioned. Please see 864 containers to continue being versioned. Please see
865 http://docs.openstack.org/developer/swift/middleware.html#how-to-enable-object-versioning-in-a-swift-cluster 865 https://docs.openstack.org/swift/latest/middleware.html#how-to-enable-object-versioning-in-a-swift-cluster
866 for further upgrade notes. 866 for further upgrade notes.
867 867
868 * Allow 1+ object-servers-per-disk deployment 868 * Allow 1+ object-servers-per-disk deployment
@@ -987,7 +987,7 @@ swift (2.3.0, OpenStack Kilo)
987 ssync for durability. Deployers are urged to do extensive testing and 987 ssync for durability. Deployers are urged to do extensive testing and
988 not deploy production data using an erasure code storage policy. 988 not deploy production data using an erasure code storage policy.
989 989
990 Full docs are at http://docs.openstack.org/developer/swift/overview_erasure_code.html 990 Full docs are at https://docs.openstack.org/swift/latest/overview_erasure_code.html
991 991
992 * Add support for container TempURL Keys. 992 * Add support for container TempURL Keys.
993 993
@@ -996,7 +996,7 @@ swift (2.3.0, OpenStack Kilo)
996 996
997 * Swift now supports composite tokens. This allows another service to 997 * Swift now supports composite tokens. This allows another service to
998 act on behalf of a user, but only with that user's consent. 998 act on behalf of a user, but only with that user's consent.
999 See http://docs.openstack.org/developer/swift/overview_auth.html for more details. 999 See https://docs.openstack.org/swift/latest/overview_auth.html for more details.
1000 1000
1001 * Multi-region replication was improved. When replicating data to a 1001 * Multi-region replication was improved. When replicating data to a
1002 different region, only one replica will be pushed per replication 1002 different region, only one replica will be pushed per replication
@@ -1004,7 +1004,7 @@ swift (2.3.0, OpenStack Kilo)
1004 locally instead of pushing more data over the inter-region network. 1004 locally instead of pushing more data over the inter-region network.
1005 1005
1006 * Internal requests from the ratelimit middleware now properly log a 1006 * Internal requests from the ratelimit middleware now properly log a
1007 swift_source. See http://docs.openstack.org/developer/swift/logs.html for details. 1007 swift_source. See https://docs.openstack.org/swift/latest/logs.html for details.
1008 1008
1009 * Improved storage policy support for quarantine stats in swift-recon. 1009 * Improved storage policy support for quarantine stats in swift-recon.
1010 1010
@@ -1052,7 +1052,7 @@ swift (2.2.2)
1052 The overload and dispersion metrics have been exposed in the 1052 The overload and dispersion metrics have been exposed in the
1053 swift-ring-build CLI tools. 1053 swift-ring-build CLI tools.
1054 1054
1055 See http://docs.openstack.org/developer/swift/overview_ring.html 1055 See https://docs.openstack.org/swift/latest/overview_ring.html
1056 for more info on how data placement works now. 1056 for more info on how data placement works now.
1057 1057
1058 * Improve replication of large out-of-sync, out-of-date containers. 1058 * Improve replication of large out-of-sync, out-of-date containers.
@@ -1140,7 +1140,7 @@ swift (2.2.0, OpenStack Juno)
1140 now requires that ACLs be set on IDs, which are unique across 1140 now requires that ACLs be set on IDs, which are unique across
1141 domains, and further restricts setting new ACLs to only use IDs. 1141 domains, and further restricts setting new ACLs to only use IDs.
1142 1142
1143 Please see http://docs.openstack.org/developer/swift/overview_auth.html for 1143 Please see https://docs.openstack.org/swift/latest/overview_auth.html for
1144 more information on configuring Swift and Keystone together. 1144 more information on configuring Swift and Keystone together.
1145 1145
1146 * Swift now supports server-side account-to-account copy. Server- 1146 * Swift now supports server-side account-to-account copy. Server-
@@ -1257,7 +1257,7 @@ swift (2.0.0)
1257 them. A policy is set on a Swift container at container creation 1257 them. A policy is set on a Swift container at container creation
1258 time and cannot be changed. 1258 time and cannot be changed.
1259 1259
1260 Full docs are at http://docs.openstack.org/developer/swift/overview_policies.html 1260 Full docs are at https://docs.openstack.org/swift/latest/overview_policies.html
1261 1261
1262 * Add profiling middleware in Swift 1262 * Add profiling middleware in Swift
1263 1263
@@ -1351,7 +1351,7 @@ swift (1.13.0)
1351 the header is a JSON dictionary string to be interpreted by the 1351 the header is a JSON dictionary string to be interpreted by the
1352 auth system. A reference implementation is given in TempAuth. 1352 auth system. A reference implementation is given in TempAuth.
1353 Please see the full docs at 1353 Please see the full docs at
1354 http://docs.openstack.org/developer/swift/overview_auth.html 1354 https://docs.openstack.org/swift/latest/overview_auth.html
1355 1355
1356 * Added a WSGI environment flag to stop swob from always using 1356 * Added a WSGI environment flag to stop swob from always using
1357 absolute location. This is useful if middleware needs to use 1357 absolute location. This is useful if middleware needs to use
@@ -1433,8 +1433,8 @@ swift (1.12.0)
1433 * New container sync configuration option, separating the end user 1433 * New container sync configuration option, separating the end user
1434 from knowing the required end point and adding more secure 1434 from knowing the required end point and adding more secure
1435 signed requests. See 1435 signed requests. See
1436 http://docs.openstack.org/developer/swift/overview_container_sync.html for full 1436 https://docs.openstack.org/swift/latest/overview_container_sync.html
1437 information. 1437 for full information.
1438 1438
1439 * bulk middleware now can be configured to retry deleting containers. 1439 * bulk middleware now can be configured to retry deleting containers.
1440 1440
@@ -1699,7 +1699,7 @@ swift (1.9.0)
1699 bugrelated to content-disposition names. 1699 bugrelated to content-disposition names.
1700 1700
1701 * Added crossdomain.xml middleware. See 1701 * Added crossdomain.xml middleware. See
1702 http://docs.openstack.org/developer/swift/crossdomain.html for details 1702 https://docs.openstack.org/swift/latest/crossdomain.html for details
1703 1703
1704 * Added rsync bandwidth limit setting for object replicator 1704 * Added rsync bandwidth limit setting for object replicator
1705 1705
@@ -1720,7 +1720,7 @@ swift (1.9.0)
1720 * Improved container-sync resiliency 1720 * Improved container-sync resiliency
1721 1721
1722 * Added example Apache config files. See 1722 * Added example Apache config files. See
1723 http://docs.openstack.org/developer/swift/apache_deployment_guide.html 1723 https://docs.openstack.org/swift/latest/apache_deployment_guide.html
1724 for more info 1724 for more info
1725 1725
1726 * If an account is marked as deleted but hasn't been reaped and is still 1726 * If an account is marked as deleted but hasn't been reaped and is still
@@ -1768,7 +1768,7 @@ swift (1.8.0, OpenStack Grizzly)
1768 1768
1769 This is a change that may require an update to your proxy server 1769 This is a change that may require an update to your proxy server
1770 config file or custom middleware that you may be using. See the full 1770 config file or custom middleware that you may be using. See the full
1771 docs at http://docs.openstack.org/developer/swift/misc.html#module-swift.common.middleware.proxy_logging. 1771 docs at https://docs.openstack.org/swift/latest/misc.html.
1772 1772
1773 * Changed the default sample rate for a few high-traffic requests. 1773 * Changed the default sample rate for a few high-traffic requests.
1774 1774
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index e4958f8..6f47ef1 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -75,7 +75,7 @@ working on.
75Getting Started 75Getting Started
76--------------- 76---------------
77 77
78http://docs.openstack.org/developer/swift/first_contribution_swift.html 78https://docs.openstack.org/swift/latest/first_contribution_swift.html
79 79
80Once those steps have been completed, changes to OpenStack 80Once those steps have been completed, changes to OpenStack
81should be submitted for review via the Gerrit tool, following 81should be submitted for review via the Gerrit tool, following
@@ -116,7 +116,7 @@ Recommended workflow
116==================== 116====================
117 117
118- Set up a `Swift All-In-One 118- Set up a `Swift All-In-One
119 VM <http://docs.openstack.org/developer/swift/development_saio.html>`__\ (SAIO). 119 VM <https://docs.openstack.org/swift/latest/development_saio.html>`__\ (SAIO).
120 120
121- Make your changes. Docs and tests for your patch must land before or 121- Make your changes. Docs and tests for your patch must land before or
122 with your patch. 122 with your patch.
diff --git a/README.rst b/README.rst
index 014cf82..6a5a144 100644
--- a/README.rst
+++ b/README.rst
@@ -31,7 +31,7 @@ To build documentation install sphinx (``pip install sphinx``), run
31``python setup.py build_sphinx``, and then browse to 31``python setup.py build_sphinx``, and then browse to
32/doc/build/html/index.html. These docs are auto-generated after every 32/doc/build/html/index.html. These docs are auto-generated after every
33commit and available online at 33commit and available online at
34http://docs.openstack.org/developer/swift/. 34https://docs.openstack.org/swift/latest/.
35 35
36For Developers 36For Developers
37-------------- 37--------------
@@ -39,13 +39,14 @@ For Developers
39Getting Started 39Getting Started
40~~~~~~~~~~~~~~~ 40~~~~~~~~~~~~~~~
41 41
42Swift is part of OpenStack and follows the code contribution, review, and testing processes common to all OpenStack projects. 42Swift is part of OpenStack and follows the code contribution, review, and
43testing processes common to all OpenStack projects.
43 44
44If you would like to start contributing, check out these 45If you would like to start contributing, check out these
45`notes <CONTRIBUTING.rst>`__ to help you get started. 46`notes <CONTRIBUTING.rst>`__ to help you get started.
46 47
47The best place to get started is the 48The best place to get started is the
48`"SAIO - Swift All In One" <http://docs.openstack.org/developer/swift/development_saio.html>`__. 49`"SAIO - Swift All In One" <https://docs.openstack.org/swift/latest/development_saio.html>`__.
49This document will walk you through setting up a development cluster of 50This document will walk you through setting up a development cluster of
50Swift in a VM. The SAIO environment is ideal for running small-scale 51Swift in a VM. The SAIO environment is ideal for running small-scale
51tests against swift and trying out new features and bug fixes. 52tests against swift and trying out new features and bug fixes.
@@ -72,7 +73,7 @@ continue to work.
72 73
73Probe tests are "white box" tests that validate the internal workings of a 74Probe tests are "white box" tests that validate the internal workings of a
74Swift cluster. They are written to work against the 75Swift cluster. They are written to work against the
75`"SAIO - Swift All In One" <http://docs.openstack.org/developer/swift/development_saio.html>`__ 76`"SAIO - Swift All In One" <https://docs.openstack.org/swift/latest/development_saio.html>`__
76dev environment. For example, a probe test may create an object, delete one 77dev environment. For example, a probe test may create an object, delete one
77replica, and ensure that the background consistency processes find and correct 78replica, and ensure that the background consistency processes find and correct
78the error. 79the error.
@@ -119,10 +120,9 @@ For Deployers
119------------- 120-------------
120 121
121Deployer docs are also available at 122Deployer docs are also available at
122http://docs.openstack.org/developer/swift/. A good starting point is at 123https://docs.openstack.org/swift/latest/. A good starting point is at
123http://docs.openstack.org/developer/swift/deployment_guide.html 124https://docs.openstack.org/swift/latest/deployment_guide.html
124 125There is an `ops runbook <https://docs.openstack.org/swift/latest/ops_runbook/index.html>`__
125There is an `ops runbook <http://docs.openstack.org/developer/swift/ops_runbook/>`__
126that gives information about how to diagnose and troubleshoot common issues 126that gives information about how to diagnose and troubleshoot common issues
127when running a Swift cluster. 127when running a Swift cluster.
128 128
@@ -138,11 +138,11 @@ For client applications, official Python language bindings are provided
138at http://github.com/openstack/python-swiftclient. 138at http://github.com/openstack/python-swiftclient.
139 139
140Complete API documentation at 140Complete API documentation at
141http://developer.openstack.org/api-ref/object-store/ 141https://developer.openstack.org/api-ref/object-store/
142 142
143There is a large ecosystem of applications and libraries that support and 143There is a large ecosystem of applications and libraries that support and
144work with OpenStack Swift. Several are listed on the 144work with OpenStack Swift. Several are listed on the
145`associated projects <http://docs.openstack.org/developer/swift/associated_projects.html>`__ 145`associated projects <https://docs.openstack.org/swift/latest/associated_projects.html>`__
146page. 146page.
147 147
148-------------- 148--------------
diff --git a/api-ref/source/storage-container-services.inc b/api-ref/source/storage-container-services.inc
index d2e32a0..a66f85f 100644
--- a/api-ref/source/storage-container-services.inc
+++ b/api-ref/source/storage-container-services.inc
@@ -186,8 +186,9 @@ Example requests and responses:
186 X-Openstack-Request-Id: tx06021f10fc8642b2901e7-0052d58f37 186 X-Openstack-Request-Id: tx06021f10fc8642b2901e7-0052d58f37
187 Date: Tue, 14 Jan 2014 19:25:43 GMT 187 Date: Tue, 14 Jan 2014 19:25:43 GMT
188 188
189Error response codes:201,204, 189Normal response codes: 201, 202
190 190
191Error response codes: 400, 404, 507
191 192
192Request 193Request
193------- 194-------
diff --git a/bin/swift-account-audit b/bin/swift-account-audit
index 7e91ae3..f0ad4e7 100755
--- a/bin/swift-account-audit
+++ b/bin/swift-account-audit
@@ -44,9 +44,9 @@ You can also feed a list of urls to the script through stdin.
44 44
45Examples! 45Examples!
46 46
47 %(cmd)s SOSO_88ad0b83-b2c5-4fa1-b2d6-60c597202076 47 %(cmd)s AUTH_88ad0b83-b2c5-4fa1-b2d6-60c597202076
48 %(cmd)s SOSO_88ad0b83-b2c5-4fa1-b2d6-60c597202076/container/object 48 %(cmd)s AUTH_88ad0b83-b2c5-4fa1-b2d6-60c597202076/container/object
49 %(cmd)s -e errors.txt SOSO_88ad0b83-b2c5-4fa1-b2d6-60c597202076/container 49 %(cmd)s -e errors.txt AUTH_88ad0b83-b2c5-4fa1-b2d6-60c597202076/container
50 %(cmd)s < errors.txt 50 %(cmd)s < errors.txt
51 %(cmd)s -c 25 -d < errors.txt 51 %(cmd)s -c 25 -d < errors.txt
52""" % {'cmd': sys.argv[0]} 52""" % {'cmd': sys.argv[0]}
@@ -108,7 +108,7 @@ class Auditor(object):
108 consistent = False 108 consistent = False
109 print(' MD5 does not match etag for "%s" on %s/%s' 109 print(' MD5 does not match etag for "%s" on %s/%s'
110 % (path, node['ip'], node['device'])) 110 % (path, node['ip'], node['device']))
111 etags.append(resp.getheader('ETag')) 111 etags.append((resp.getheader('ETag'), node))
112 else: 112 else:
113 conn = http_connect(node['ip'], node['port'], 113 conn = http_connect(node['ip'], node['port'],
114 node['device'], part, 'HEAD', 114 node['device'], part, 'HEAD',
@@ -120,7 +120,7 @@ class Auditor(object):
120 print(' Bad status HEADing object "%s" on %s/%s' 120 print(' Bad status HEADing object "%s" on %s/%s'
121 % (path, node['ip'], node['device'])) 121 % (path, node['ip'], node['device']))
122 continue 122 continue
123 etags.append(resp.getheader('ETag')) 123 etags.append((resp.getheader('ETag'), node))
124 except Exception: 124 except Exception:
125 self.object_exceptions += 1 125 self.object_exceptions += 1
126 consistent = False 126 consistent = False
@@ -131,8 +131,8 @@ class Auditor(object):
131 consistent = False 131 consistent = False
132 print(" Failed fo fetch object %s at all!" % path) 132 print(" Failed fo fetch object %s at all!" % path)
133 elif hash: 133 elif hash:
134 for etag in etags: 134 for etag, node in etags:
135 if resp.getheader('ETag').strip('"') != hash: 135 if etag.strip('"') != hash:
136 consistent = False 136 consistent = False
137 self.object_checksum_mismatch += 1 137 self.object_checksum_mismatch += 1
138 print(' ETag mismatch for "%s" on %s/%s' 138 print(' ETag mismatch for "%s" on %s/%s'
diff --git a/doc/manpages/account-server.conf.5 b/doc/manpages/account-server.conf.5
index 3a06dbd..018ea4f 100644
--- a/doc/manpages/account-server.conf.5
+++ b/doc/manpages/account-server.conf.5
@@ -386,7 +386,11 @@ Connection timeout to external services. The default is 0.5 seconds.
386.IP \fBdelay_reaping\fR 386.IP \fBdelay_reaping\fR
387Normally, the reaper begins deleting account information for deleted accounts 387Normally, the reaper begins deleting account information for deleted accounts
388immediately; you can set this to delay its work however. The value is in 388immediately; you can set this to delay its work however. The value is in
389seconds. The default is 0. 389seconds. The default is 0. The sum of this value and the
390container-updater interval should be less than the account-replicator
391reclaim_age. This ensures that once the account-reaper has deleted a
392container there is sufficient time for the container-updater to report to the
393account before the account DB is removed.
390.IP \fBreap_warn_after\fR 394.IP \fBreap_warn_after\fR
391If the account fails to be be reaped due to a persistent error, the 395If the account fails to be be reaped due to a persistent error, the
392account reaper will log a message such as: 396account reaper will log a message such as:
diff --git a/doc/manpages/container-sync-realms.conf.5 b/doc/manpages/container-sync-realms.conf.5
new file mode 100644
index 0000000..6602615
--- /dev/null
+++ b/doc/manpages/container-sync-realms.conf.5
@@ -0,0 +1,138 @@
1.\"
2.\" Author: HCLTech-SSW <hcl_ss_oss@hcl.com>
3.\" Copyright (c) 2010-2017 OpenStack Foundation.
4.\"
5.\" Licensed under the Apache License, Version 2.0 (the "License");
6.\" you may not use this file except in compliance with the License.
7.\" You may obtain a copy of the License at
8.\"
9.\" http://www.apache.org/licenses/LICENSE-2.0
10.\"
11.\" Unless required by applicable law or agreed to in writing, software
12.\" distributed under the License is distributed on an "AS IS" BASIS,
13.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
14.\" implied.
15.\" See the License for the specific language governing permissions and
16.\" limitations under the License.
17.\"
18.TH container-sync-realms.conf 5 "10/09/2017" "Linux" "OpenStack Swift"
19
20.SH NAME
21.LP
22.B container-sync-realms.conf
23\- configuration file for the OpenStack Swift container sync realms
24
25
26
27.SH SYNOPSIS
28.LP
29.B container-sync-realms.conf
30
31
32
33.SH DESCRIPTION
34.PP
35This is the configuration file used by the Object storage Swift to perform container to container
36synchronization. This configuration file is used to configure clusters to allow/accept sync
37requests to/from other clusters. Using this configuration file, the user specifies where
38to sync their container to along with a secret synchronization key.
39
40You can find more information about container to container synchronization at
41\fIhttps://docs.openstack.org/swift/latest/overview_container_sync.html\fR
42
43The configuration file follows the python-pastedeploy syntax. The file is divided
44into sections, which are enclosed by square brackets. Each section will contain a
45certain number of key/value parameters which are described later.
46
47Any line that begins with a '#' symbol is ignored.
48
49You can find more information about python-pastedeploy configuration format at
50\fIhttp://pythonpaste.org/deploy/#config-format\fR
51
52
53
54.SH GLOBAL SECTION
55.PD 1
56.RS 0
57This is indicated by section named [DEFAULT]. Below are the parameters that
58are acceptable within this section.
59
60.IP "\fBmtime_check_interval\fR"
61The number of seconds between checking the modified time of this config file for changes
62and therefore reloading it. The default value is 300.
63.RE
64.PD
65
66
67
68.SH REALM SECTIONS
69.PD 1
70.RS 0
71Each section name is the name of a sync realm, for example [realm1].
72A sync realm is a set of clusters that have agreed to allow container syncing with each other.
73Realm names will be considered case insensitive. Below are the parameters that are acceptable
74within this section.
75
76.IP "\fBcluster_clustername1\fR"
77Any values in the realm section whose name begin with cluster_ will indicate the name and
78endpoint of a cluster and will be used by external users in their container's
79X-Container-Sync-To metadata header values with the format as "realm_name/cluster_name/container_name".
80The Realm and cluster names are considered to be case insensitive.
81.IP "\fBcluster_clustername2\fR"
82Any values in the realm section whose name begin with cluster_ will indicate the name and
83endpoint of a cluster and will be used by external users in their container's
84X-Container-Sync-To metadata header values with the format as "realm_name/cluster_name/container_name".
85The Realm and cluster names are considered to be case insensitive.
86
87The endpoint is what the container sync daemon will use when sending out
88requests to that cluster. Keep in mind this endpoint must be reachable by all
89container servers, since that is where the container sync daemon runs. Note
90that the endpoint ends with /v1/ and that the container sync daemon will then
91add the account/container/obj name after that.
92
93.IP "\fBkey\fR"
94The key is the overall cluster-to-cluster key used in combination with the external
95users' key that they set on their containers' X-Container-Sync-Key metadata header
96values. These keys will be used to sign each request the container sync daemon makes
97and used to validate each incoming container sync request.
98.IP "\fBkey2\fR"
99The key2 is optional and is an additional key incoming requests will be checked
100against. This is so you can rotate keys if you wish; you move the existing
101key to key2 and make a new key value.
102.RE
103.PD
104
105.SH EXAMPLE
106.nf
107.RS 0
108[DEFAULT]
109mtime_check_interval = 300
110
111
112[realm1]
113key = realm1key
114key2 = realm1key2
115cluster_clustername1 = https://host1/v1/
116cluster_clustername2 = https://host2/v1/
117
118[realm2]
119key = realm2key
120key2 = realm2key2
121cluster_clustername3 = https://host3/v1/
122cluster_clustername4 = https://host4/v1/
123.RE
124.fi
125
126
127.SH DOCUMENTATION
128.LP
129More in depth documentation in regards to
130.BI swift-container-sync
131and also about OpenStack Swift as a whole can be found at
132.BI https://docs.openstack.org/swift/latest/overview_container_sync.html
133and
134.BI https://docs.openstack.org/swift/latest/
135
136
137.SH "SEE ALSO"
138.BR swift-container-sync(1)
diff --git a/doc/manpages/proxy-server.conf.5 b/doc/manpages/proxy-server.conf.5
index 8081b34..5f0c786 100644
--- a/doc/manpages/proxy-server.conf.5
+++ b/doc/manpages/proxy-server.conf.5
@@ -996,8 +996,6 @@ Error count to consider a node error limited. The default is 10.
996Whether account PUTs and DELETEs are even callable. If set to 'true' any authorized 996Whether account PUTs and DELETEs are even callable. If set to 'true' any authorized
997user may create and delete accounts; if 'false' no one, even authorized, can. The default 997user may create and delete accounts; if 'false' no one, even authorized, can. The default
998is false. 998is false.
999.IP \fBobject_post_as_copy\fR
1000Deprecated. The default is False.
1001.IP \fBaccount_autocreate\fR 999.IP \fBaccount_autocreate\fR
1002If set to 'true' authorized accounts that do not yet exist within the Swift cluster 1000If set to 'true' authorized accounts that do not yet exist within the Swift cluster
1003will be automatically created. The default is set to false. 1001will be automatically created. The default is set to false.
@@ -1025,6 +1023,15 @@ The valid values for sorting_method are "affinity", "shuffle", and "timing".
1025.IP \fBtiming_expiry\fR 1023.IP \fBtiming_expiry\fR
1026If the "timing" sorting_method is used, the timings will only be valid for 1024If the "timing" sorting_method is used, the timings will only be valid for
1027the number of seconds configured by timing_expiry. The default is 300. 1025the number of seconds configured by timing_expiry. The default is 300.
1026.IP \fBconcurrent_gets\fR
1027If "on" then use replica count number of threads concurrently during a GET/HEAD
1028and return with the first successful response. In the EC case, this parameter
1029only affects an EC HEAD as an EC GET behaves differently. Default is "off".
1030.IP \fBconcurrency_timeout\fR
1031This parameter controls how long to wait before firing off the next
1032concurrent_get thread. A value of 0 would we fully concurrent, any other number
1033will stagger the firing of the threads. This number should be between 0 and
1034node_timeout. The default is the value of conn_timeout (0.5).
1028.IP \fBrequest_node_count\fR 1035.IP \fBrequest_node_count\fR
1029Set to the number of nodes to contact for a normal request. You can use '* replicas' 1036Set to the number of nodes to contact for a normal request. You can use '* replicas'
1030at the end to have it use the number given times the number of 1037at the end to have it use the number given times the number of
diff --git a/doc/manpages/swift-account-audit.1 b/doc/manpages/swift-account-audit.1
index 5f22cac..64d60c5 100644
--- a/doc/manpages/swift-account-audit.1
+++ b/doc/manpages/swift-account-audit.1
@@ -46,9 +46,9 @@ Also download files and verify md5
46 46
47.SH EXAMPLES 47.SH EXAMPLES
48.nf 48.nf
49/usr/bin/swift\-account\-audit\/ SOSO_88ad0b83\-b2c5\-4fa1\-b2d6\-60c597202076 49/usr/bin/swift\-account\-audit\/ AUTH_88ad0b83\-b2c5\-4fa1\-b2d6\-60c597202076
50/usr/bin/swift\-account\-audit\/ SOSO_88ad0b83\-b2c5\-4fa1\-b2d6\-60c597202076/container/object 50/usr/bin/swift\-account\-audit\/ AUTH_88ad0b83\-b2c5\-4fa1\-b2d6\-60c597202076/container/object
51/usr/bin/swift\-account\-audit\/ \fB\-e\fR errors.txt SOSO_88ad0b83\-b2c5\-4fa1\-b2d6\-60c597202076/container 51/usr/bin/swift\-account\-audit\/ \fB\-e\fR errors.txt AUTH_88ad0b83\-b2c5\-4fa1\-b2d6\-60c597202076/container
52/usr/bin/swift\-account\-audit\/ < errors.txt 52/usr/bin/swift\-account\-audit\/ < errors.txt
53/usr/bin/swift\-account\-audit\/ \fB\-c\fR 25 \fB\-d\fR < errors.txt 53/usr/bin/swift\-account\-audit\/ \fB\-c\fR 25 \fB\-d\fR < errors.txt
54.fi 54.fi
diff --git a/doc/saio/swift/proxy-server.conf b/doc/saio/swift/proxy-server.conf
index 76b85d5..12b0386 100644
--- a/doc/saio/swift/proxy-server.conf
+++ b/doc/saio/swift/proxy-server.conf
@@ -9,7 +9,7 @@ eventlet_debug = true
9[pipeline:main] 9[pipeline:main]
10# Yes, proxy-logging appears twice. This is so that 10# Yes, proxy-logging appears twice. This is so that
11# middleware-originated requests get logged too. 11# middleware-originated requests get logged too.
12pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain container_sync tempauth staticweb copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server 12pipeline = catch_errors gatekeeper healthcheck proxy-logging cache listing_formats bulk tempurl ratelimit crossdomain container_sync tempauth staticweb copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server
13 13
14[filter:catch_errors] 14[filter:catch_errors]
15use = egg:swift#catch_errors 15use = egg:swift#catch_errors
@@ -71,6 +71,9 @@ allow_versioned_writes = true
71[filter:copy] 71[filter:copy]
72use = egg:swift#copy 72use = egg:swift#copy
73 73
74[filter:listing_formats]
75use = egg:swift#listing_formats
76
74[app:proxy-server] 77[app:proxy-server]
75use = egg:swift#proxy 78use = egg:swift#proxy
76allow_account_management = true 79allow_account_management = true
diff --git a/doc/source/admin/objectstorage-EC.rst b/doc/source/admin/objectstorage-EC.rst
index 6f1996b..2e32407 100644
--- a/doc/source/admin/objectstorage-EC.rst
+++ b/doc/source/admin/objectstorage-EC.rst
@@ -17,13 +17,3 @@ erasure coding capability. It is entirely possible to share devices between
17storage policies, but for erasure coding it may make more sense to use 17storage policies, but for erasure coding it may make more sense to use
18not only separate devices but possibly even entire nodes dedicated for erasure 18not only separate devices but possibly even entire nodes dedicated for erasure
19coding. 19coding.
20
21.. important::
22
23 The erasure code support in Object Storage is considered beta in Kilo.
24 Most major functionality is included, but it has not been tested or
25 validated at large scale. This feature relies on ``ssync`` for durability.
26 We recommend deployers do extensive testing and not deploy production
27 data using an erasure code storage policy.
28 If any bugs are found during testing, please report them to
29 https://bugs.launchpad.net/swift
diff --git a/doc/source/admin_guide.rst b/doc/source/admin_guide.rst
index 497d22d..10854a4 100644
--- a/doc/source/admin_guide.rst
+++ b/doc/source/admin_guide.rst
@@ -1493,6 +1493,6 @@ See :ref:`custom-logger-hooks-label` for sample use cases.
1493Securing OpenStack Swift 1493Securing OpenStack Swift
1494------------------------ 1494------------------------
1495 1495
1496Please refer to the security guide at http://docs.openstack.org/security-guide 1496Please refer to the security guide at https://docs.openstack.org/security-guide
1497and in particular the `Object Storage 1497and in particular the `Object Storage
1498<http://docs.openstack.org/security-guide/object-storage.html>`__ section. 1498<https://docs.openstack.org/security-guide/object-storage.html>`__ section.
diff --git a/doc/source/api/object_api_v1_overview.rst b/doc/source/api/object_api_v1_overview.rst
index 30d3f04..7f8571b 100644
--- a/doc/source/api/object_api_v1_overview.rst
+++ b/doc/source/api/object_api_v1_overview.rst
@@ -169,14 +169,14 @@ The API Reference describes the operations that you can perform with the
169Object Storage API: 169Object Storage API:
170 170
171- `Storage 171- `Storage
172 accounts <http://developer.openstack.org/api-ref/object-storage/index.html#accounts>`__: 172 accounts <https://developer.openstack.org/api-ref/object-storage/index.html#accounts>`__:
173 Use to perform account-level tasks. 173 Use to perform account-level tasks.
174 174
175 Lists containers for a specified account. Creates, updates, and 175 Lists containers for a specified account. Creates, updates, and
176 deletes account metadata. Shows account metadata. 176 deletes account metadata. Shows account metadata.
177 177
178- `Storage 178- `Storage
179 containers <http://developer.openstack.org/api-ref/object-storage/index.html#containers>`__: 179 containers <https://developer.openstack.org/api-ref/object-storage/index.html#containers>`__:
180 Use to perform container-level tasks. 180 Use to perform container-level tasks.
181 181
182 Lists objects in a specified container. Creates, shows details for, 182 Lists objects in a specified container. Creates, shows details for,
@@ -184,7 +184,7 @@ Object Storage API:
184 container metadata. 184 container metadata.
185 185
186- `Storage 186- `Storage
187 objects <http://developer.openstack.org/api-ref/object-storage/index.html#objects>`__: 187 objects <https://developer.openstack.org/api-ref/object-storage/index.html#objects>`__:
188 Use to perform object-level tasks. 188 Use to perform object-level tasks.
189 189
190 Creates, replaces, shows details for, and deletes objects. Copies 190 Creates, replaces, shows details for, and deletes objects. Copies
diff --git a/doc/source/api/temporary_url_middleware.rst b/doc/source/api/temporary_url_middleware.rst
index 9acb31c..76f5dfa 100644
--- a/doc/source/api/temporary_url_middleware.rst
+++ b/doc/source/api/temporary_url_middleware.rst
@@ -222,4 +222,4 @@ Note that if the above example is copied exactly, and used in a command
222shell, then the ampersand is interpreted as an operator and the URL 222shell, then the ampersand is interpreted as an operator and the URL
223will be truncated. Enclose the URL in quotation marks to avoid this. 223will be truncated. Enclose the URL in quotation marks to avoid this.
224 224
225.. _tempurl: http://docs.openstack.org/developer/python-swiftclient/cli.html#tempurl 225.. _tempurl: https://docs.openstack.org/python-swiftclient/latest/cli/index.html#swift-tempurl
diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst
index 554302f..28df6cd 100644
--- a/doc/source/deployment_guide.rst
+++ b/doc/source/deployment_guide.rst
@@ -1650,7 +1650,15 @@ delay_reaping 0 Normally, the reaper begins deleting
1650 account information for deleted accounts 1650 account information for deleted accounts
1651 immediately; you can set this to delay 1651 immediately; you can set this to delay
1652 its work however. The value is in seconds, 1652 its work however. The value is in seconds,
1653 2592000 = 30 days, for example. 1653 2592000 = 30 days, for example. The sum of
1654 this value and the container-updater
1655 ``interval`` should be less than the
1656 account-replicator ``reclaim_age``. This
1657 ensures that once the account-reaper has
1658 deleted a container there is sufficient
1659 time for the container-updater to report
1660 to the account before the account DB is
1661 removed.
1654reap_warn_after 2892000 If the account fails to be be reaped due 1662reap_warn_after 2892000 If the account fails to be be reaped due
1655 to a persistent error, the account reaper 1663 to a persistent error, the account reaper
1656 will log a message such as: 1664 will log a message such as:
@@ -1884,7 +1892,6 @@ error_suppression_limit 10 Error count to consider
1884 node error limited 1892 node error limited
1885allow_account_management false Whether account PUTs and DELETEs 1893allow_account_management false Whether account PUTs and DELETEs
1886 are even callable 1894 are even callable
1887object_post_as_copy false Deprecated.
1888account_autocreate false If set to 'true' authorized 1895account_autocreate false If set to 'true' authorized
1889 accounts that do not yet exist 1896 accounts that do not yet exist
1890 within the Swift cluster will 1897 within the Swift cluster will
@@ -1944,12 +1951,12 @@ concurrent_gets off Use replica count numbe
1944 GET/HEAD and return with the 1951 GET/HEAD and return with the
1945 first successful response. In 1952 first successful response. In
1946 the EC case, this parameter only 1953 the EC case, this parameter only
1947 effects an EC HEAD as an EC GET 1954 affects an EC HEAD as an EC GET
1948 behaves differently. 1955 behaves differently.
1949concurrency_timeout conn_timeout This parameter controls how long 1956concurrency_timeout conn_timeout This parameter controls how long
1950 to wait before firing off the 1957 to wait before firing off the
1951 next concurrent_get thread. A 1958 next concurrent_get thread. A
1952 value of 0 would we fully concurrent 1959 value of 0 would we fully concurrent,
1953 any other number will stagger the 1960 any other number will stagger the
1954 firing of the threads. This number 1961 firing of the threads. This number
1955 should be between 0 and node_timeout. 1962 should be between 0 and node_timeout.
diff --git a/doc/source/development_guidelines.rst b/doc/source/development_guidelines.rst
index 601db80..a8d2295 100644
--- a/doc/source/development_guidelines.rst
+++ b/doc/source/development_guidelines.rst
@@ -127,9 +127,6 @@ set using environment variables:
127 environment variable ``SWIFT_TEST_IN_PROCESS_CONF_LOADER`` to 127 environment variable ``SWIFT_TEST_IN_PROCESS_CONF_LOADER`` to
128 ``ec``. 128 ``ec``.
129 129
130- the deprecated proxy-server ``object_post_as_copy`` option may be set using
131 the environment variable ``SWIFT_TEST_IN_PROCESS_OBJECT_POST_AS_COPY``.
132
133- logging to stdout may be enabled by setting ``SWIFT_TEST_DEBUG_LOGS``. 130- logging to stdout may be enabled by setting ``SWIFT_TEST_DEBUG_LOGS``.
134 131
135For example, this command would run the in-process mode functional tests with 132For example, this command would run the in-process mode functional tests with
@@ -147,7 +144,6 @@ The ``tox.ini`` file also specifies test environments for running other
147in-process functional test configurations, e.g.:: 144in-process functional test configurations, e.g.::
148 145
149 tox -e func-ec 146 tox -e func-ec
150 tox -e func-post-as-copy
151 147
152To debug the functional tests, use the 'in-process test' mode and pass the 148To debug the functional tests, use the 'in-process test' mode and pass the
153``--pdb`` flag to ``tox``:: 149``--pdb`` flag to ``tox``::
diff --git a/doc/source/install/controller-include.txt b/doc/source/install/controller-include.txt
index 184e9cd..cf9e9d1 100644
--- a/doc/source/install/controller-include.txt
+++ b/doc/source/install/controller-include.txt
@@ -28,7 +28,7 @@ following actions:
28 .. note:: 28 .. note::
29 29
30 For more information on other modules that enable additional features, 30 For more information on other modules that enable additional features,
31 see the `Deployment Guide <http://docs.openstack.org/developer/swift/deployment_guide.html>`__. 31 see the `Deployment Guide <https://docs.openstack.org/swift/latest/deployment_guide.html>`__.
32 32
33* In the ``[app:proxy-server]`` section, enable automatic account creation: 33* In the ``[app:proxy-server]`` section, enable automatic account creation:
34 34
diff --git a/doc/source/install/controller-install-debian.rst b/doc/source/install/controller-install-debian.rst
index 2a981fb..5550890 100644
--- a/doc/source/install/controller-install-debian.rst
+++ b/doc/source/install/controller-install-debian.rst
@@ -10,7 +10,7 @@ the proxy service on the controller node. However, you can run the proxy
10service on any node with network connectivity to the storage nodes. 10service on any node with network connectivity to the storage nodes.
11Additionally, you can install and configure the proxy service on multiple 11Additionally, you can install and configure the proxy service on multiple
12nodes to increase performance and redundancy. For more information, see the 12nodes to increase performance and redundancy. For more information, see the
13`Deployment Guide <http://docs.openstack.org/developer/swift/deployment_guide.html>`__. 13`Deployment Guide <https://docs.openstack.org/swift/latest/deployment_guide.html>`__.
14 14
15This section applies to Debian. 15This section applies to Debian.
16 16
diff --git a/doc/source/install/controller-install-obs.rst b/doc/source/install/controller-install-obs.rst
index 08cf545..f4a8ffc 100644
--- a/doc/source/install/controller-install-obs.rst
+++ b/doc/source/install/controller-install-obs.rst
@@ -10,7 +10,7 @@ the proxy service on the controller node. However, you can run the proxy
10service on any node with network connectivity to the storage nodes. 10service on any node with network connectivity to the storage nodes.
11Additionally, you can install and configure the proxy service on multiple 11Additionally, you can install and configure the proxy service on multiple
12nodes to increase performance and redundancy. For more information, see the 12nodes to increase performance and redundancy. For more information, see the
13`Deployment Guide <http://docs.openstack.org/developer/swift/deployment_guide.html>`__. 13`Deployment Guide <https://docs.openstack.org/swift/latest/deployment_guide.html>`__.
14 14
15This section applies to openSUSE Leap 42.2 and SUSE Linux Enterprise Server 15This section applies to openSUSE Leap 42.2 and SUSE Linux Enterprise Server
1612 SP2. 1612 SP2.
diff --git a/doc/source/install/controller-install-rdo.rst b/doc/source/install/controller-install-rdo.rst
index d6ccaa0..44dd7cc 100644
--- a/doc/source/install/controller-install-rdo.rst
+++ b/doc/source/install/controller-install-rdo.rst
@@ -10,7 +10,7 @@ the proxy service on the controller node. However, you can run the proxy
10service on any node with network connectivity to the storage nodes. 10service on any node with network connectivity to the storage nodes.
11Additionally, you can install and configure the proxy service on multiple 11Additionally, you can install and configure the proxy service on multiple
12nodes to increase performance and redundancy. For more information, see the 12nodes to increase performance and redundancy. For more information, see the
13`Deployment Guide <http://docs.openstack.org/developer/swift/deployment_guide.html>`__. 13`Deployment Guide <https://docs.openstack.org/swift/latest/deployment_guide.html>`__.
14 14
15This section applies to Red Hat Enterprise Linux 7 and CentOS 7. 15This section applies to Red Hat Enterprise Linux 7 and CentOS 7.
16 16
diff --git a/doc/source/install/controller-install-ubuntu.rst b/doc/source/install/controller-install-ubuntu.rst
index 368e609..c1bbadf 100644
--- a/doc/source/install/controller-install-ubuntu.rst
+++ b/doc/source/install/controller-install-ubuntu.rst
@@ -10,7 +10,7 @@ the proxy service on the controller node. However, you can run the proxy
10service on any node with network connectivity to the storage nodes. 10service on any node with network connectivity to the storage nodes.
11Additionally, you can install and configure the proxy service on multiple 11Additionally, you can install and configure the proxy service on multiple
12nodes to increase performance and redundancy. For more information, see the 12nodes to increase performance and redundancy. For more information, see the
13`Deployment Guide <http://docs.openstack.org/developer/swift/deployment_guide.html>`__. 13`Deployment Guide <https://docs.openstack.org/swift/latest/deployment_guide.html>`__.
14 14
15This section applies to Ubuntu 14.04 (LTS). 15This section applies to Ubuntu 14.04 (LTS).
16 16
diff --git a/doc/source/install/get_started.rst b/doc/source/install/get_started.rst
index 301f6a2..1cf9630 100644
--- a/doc/source/install/get_started.rst
+++ b/doc/source/install/get_started.rst
@@ -40,7 +40,7 @@ swift client
40swift-init 40swift-init
41 Script that initializes the building of the ring file, takes daemon 41 Script that initializes the building of the ring file, takes daemon
42 names as parameter and offers commands. Documented in 42 names as parameter and offers commands. Documented in
43 http://docs.openstack.org/developer/swift/admin_guide.html#managing-services. 43 https://docs.openstack.org/swift/latest/admin_guide.html#managing-services.
44 44
45swift-recon 45swift-recon
46 A cli tool used to retrieve various metrics and telemetry information 46 A cli tool used to retrieve various metrics and telemetry information
@@ -48,4 +48,4 @@ swift-recon
48 48
49swift-ring-builder 49swift-ring-builder
50 Storage ring build and rebalance utility. Documented in 50 Storage ring build and rebalance utility. Documented in
51 http://docs.openstack.org/developer/swift/admin_guide.html#managing-the-rings. 51 https://docs.openstack.org/swift/latest/admin_guide.html#managing-the-rings.
diff --git a/doc/source/install/initial-rings.rst b/doc/source/install/initial-rings.rst
index d7d0378..e09dfd4 100644
--- a/doc/source/install/initial-rings.rst
+++ b/doc/source/install/initial-rings.rst
@@ -9,7 +9,7 @@ maximum partitions, 3 replicas of each object, and 1 hour minimum time between
9moving a partition more than once. For Object Storage, a partition indicates a 9moving a partition more than once. For Object Storage, a partition indicates a
10directory on a storage device rather than a conventional partition table. 10directory on a storage device rather than a conventional partition table.
11For more information, see the 11For more information, see the
12`Deployment Guide <http://docs.openstack.org/developer/swift/deployment_guide.html>`__. 12`Deployment Guide <https://docs.openstack.org/swift/latest/deployment_guide.html>`__.
13 13
14.. note:: 14.. note::
15 Perform these steps on the controller node. 15 Perform these steps on the controller node.
diff --git a/doc/source/install/storage-include1.txt b/doc/source/install/storage-include1.txt
index a98f27b..7117823 100644
--- a/doc/source/install/storage-include1.txt
+++ b/doc/source/install/storage-include1.txt
@@ -28,7 +28,7 @@ following actions:
28 .. note:: 28 .. note::
29 29
30 For more information on other modules that enable additional features, 30 For more information on other modules that enable additional features,
31 see the `Deployment Guide <http://docs.openstack.org/developer/swift/deployment_guide.html>`__. 31 see the `Deployment Guide <https://docs.openstack.org/swift/latest/deployment_guide.html>`__.
32 32
33* In the ``[filter:recon]`` section, configure the recon (meters) cache 33* In the ``[filter:recon]`` section, configure the recon (meters) cache
34 directory: 34 directory:
diff --git a/doc/source/install/storage-include2.txt b/doc/source/install/storage-include2.txt
index 04cb2f3..cb320d9 100644
--- a/doc/source/install/storage-include2.txt
+++ b/doc/source/install/storage-include2.txt
@@ -28,7 +28,7 @@ following actions:
28 .. note:: 28 .. note::
29 29
30 For more information on other modules that enable additional features, 30 For more information on other modules that enable additional features,
31 see the `Deployment Guide <http://docs.openstack.org/developer/swift/deployment_guide.html>`__. 31 see the `Deployment Guide <https://docs.openstack.org/swift/latest/deployment_guide.html>`__.
32 32
33* In the ``[filter:recon]`` section, configure the recon (meters) cache 33* In the ``[filter:recon]`` section, configure the recon (meters) cache
34 directory: 34 directory:
diff --git a/doc/source/install/storage-include3.txt b/doc/source/install/storage-include3.txt
index 6551782..2cc9e2d 100644
--- a/doc/source/install/storage-include3.txt
+++ b/doc/source/install/storage-include3.txt
@@ -28,7 +28,7 @@ following actions:
28 .. note:: 28 .. note::
29 29
30 For more information on other modules that enable additional features, 30 For more information on other modules that enable additional features,
31 see the `Deployment Guide <http://docs.openstack.org/developer/swift/deployment_guide.html>`__. 31 see the `Deployment Guide <https://docs.openstack.org/swift/latest/deployment_guide.html>`__.
32 32
33* In the ``[filter:recon]`` section, configure the recon (meters) cache 33* In the ``[filter:recon]`` section, configure the recon (meters) cache
34 and lock directories: 34 and lock directories:
diff --git a/doc/source/install/storage-install-obs.rst b/doc/source/install/storage-install-obs.rst
index 7ed11cc..3199843 100644
--- a/doc/source/install/storage-install-obs.rst
+++ b/doc/source/install/storage-install-obs.rst
@@ -14,7 +14,7 @@ Although Object Storage supports any file system with
14extended attributes (xattr), testing and benchmarking 14extended attributes (xattr), testing and benchmarking
15indicate the best performance and reliability on XFS. For 15indicate the best performance and reliability on XFS. For
16more information on horizontally scaling your environment, see the 16more information on horizontally scaling your environment, see the
17`Deployment Guide <http://docs.openstack.org/developer/swift/deployment_guide.html>`_. 17`Deployment Guide <https://docs.openstack.org/swift/latest/deployment_guide.html>`_.
18 18
19This section applies to openSUSE Leap 42.2 and SUSE Linux Enterprise Server 19This section applies to openSUSE Leap 42.2 and SUSE Linux Enterprise Server
2012 SP2. 2012 SP2.
diff --git a/doc/source/install/storage-install-rdo.rst b/doc/source/install/storage-install-rdo.rst
index 8735885..91a14bd 100644
--- a/doc/source/install/storage-install-rdo.rst
+++ b/doc/source/install/storage-install-rdo.rst
@@ -14,7 +14,7 @@ Although Object Storage supports any file system with
14extended attributes (xattr), testing and benchmarking 14extended attributes (xattr), testing and benchmarking
15indicate the best performance and reliability on XFS. For 15indicate the best performance and reliability on XFS. For
16more information on horizontally scaling your environment, see the 16more information on horizontally scaling your environment, see the
17`Deployment Guide <http://docs.openstack.org/developer/swift/deployment_guide.html>`_. 17`Deployment Guide <https://docs.openstack.org/swift/latest/deployment_guide.html>`_.
18 18
19This section applies to Red Hat Enterprise Linux 7 and CentOS 7. 19This section applies to Red Hat Enterprise Linux 7 and CentOS 7.
20 20
diff --git a/doc/source/install/storage-install-ubuntu-debian.rst b/doc/source/install/storage-install-ubuntu-debian.rst
index 12fb4da..3f62bf2 100644
--- a/doc/source/install/storage-install-ubuntu-debian.rst
+++ b/doc/source/install/storage-install-ubuntu-debian.rst
@@ -14,7 +14,7 @@ Although Object Storage supports any file system with
14extended attributes (xattr), testing and benchmarking 14extended attributes (xattr), testing and benchmarking
15indicate the best performance and reliability on XFS. For 15indicate the best performance and reliability on XFS. For
16more information on horizontally scaling your environment, see the 16more information on horizontally scaling your environment, see the
17`Deployment Guide <http://docs.openstack.org/developer/swift/deployment_guide.html>`_. 17`Deployment Guide <https://docs.openstack.org/swift/latest/deployment_guide.html>`_.
18 18
19This section applies to Ubuntu 14.04 (LTS) and Debian. 19This section applies to Ubuntu 14.04 (LTS) and Debian.
20 20
diff --git a/doc/source/overview_auth.rst b/doc/source/overview_auth.rst
index b3df7f8..ab87bea 100644
--- a/doc/source/overview_auth.rst
+++ b/doc/source/overview_auth.rst
@@ -104,8 +104,8 @@ can be found in the KeystoneMiddleware_ distribution.
104The :ref:`keystoneauth` middleware performs authorization and mapping the 104The :ref:`keystoneauth` middleware performs authorization and mapping the
105Keystone roles to Swift's ACLs. 105Keystone roles to Swift's ACLs.
106 106
107.. _KeystoneMiddleware: http://docs.openstack.org/developer/keystonemiddleware/ 107.. _KeystoneMiddleware: https://docs.openstack.org/keystonemiddleware/latest/
108.. _Keystone: http://docs.openstack.org/developer/keystone/ 108.. _Keystone: https://docs.openstack.org/keystone/latest/
109 109
110.. _configuring_keystone_auth: 110.. _configuring_keystone_auth:
111 111
@@ -167,7 +167,7 @@ your situation, but in short:
167 service. The example values shown here assume a user named 'swift' with admin 167 service. The example values shown here assume a user named 'swift' with admin
168 role on a project named 'service', both being in the Keystone domain with id 168 role on a project named 'service', both being in the Keystone domain with id
169 'default'. Refer to the `KeystoneMiddleware documentation 169 'default'. Refer to the `KeystoneMiddleware documentation
170 <http://docs.openstack.org/developer/keystonemiddleware/middlewarearchitecture.html#configuration>`_ 170 <https://docs.openstack.org/keystonemiddleware/latest/middlewarearchitecture.html#configuration>`_
171 for other examples. 171 for other examples.
172 172
173* ``cache`` is set to ``swift.cache``. This means that the middleware 173* ``cache`` is set to ``swift.cache``. This means that the middleware
diff --git a/doc/source/overview_encryption.rst b/doc/source/overview_encryption.rst
index 5ebbbc8..90ab897 100644
--- a/doc/source/overview_encryption.rst
+++ b/doc/source/overview_encryption.rst
@@ -238,7 +238,7 @@ Keys currently stored in Barbican can be listed using the
238The keymaster uses the explicitly configured username and password (and 238The keymaster uses the explicitly configured username and password (and
239project name etc.) from the `keymaster.conf` file for retrieving the encryption 239project name etc.) from the `keymaster.conf` file for retrieving the encryption
240root secret from an external key management system. The `Castellan library 240root secret from an external key management system. The `Castellan library
241<http://docs.openstack.org/developer/castellan/>`_ is used to communicate with 241<https://docs.openstack.org/castellan/latest/>`_ is used to communicate with
242Barbican. 242Barbican.
243 243
244For the proxy server, reading the encryption root secret directly from the 244For the proxy server, reading the encryption root secret directly from the
diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample
index 2be6b85..257c56b 100644
--- a/etc/account-server.conf-sample
+++ b/etc/account-server.conf-sample
@@ -203,7 +203,11 @@ use = egg:swift#recon
203# 203#
204# Normally, the reaper begins deleting account information for deleted accounts 204# Normally, the reaper begins deleting account information for deleted accounts
205# immediately; you can set this to delay its work however. The value is in 205# immediately; you can set this to delay its work however. The value is in
206# seconds; 2592000 = 30 days for example. 206# seconds; 2592000 = 30 days for example. The sum of this value and the
207# container-updater interval should be less than the account-replicator
208# reclaim_age. This ensures that once the account-reaper has deleted a
209# container there is sufficient time for the container-updater to report to the
210# account before the account DB is removed.
207# delay_reaping = 0 211# delay_reaping = 0
208# 212#
209# If the account fails to be reaped due to a persistent error, the 213# If the account fails to be reaped due to a persistent error, the
diff --git a/etc/internal-client.conf-sample b/etc/internal-client.conf-sample
index 916e424..aed3c9a 100644
--- a/etc/internal-client.conf-sample
+++ b/etc/internal-client.conf-sample
@@ -28,6 +28,7 @@ pipeline = catch_errors proxy-logging cache proxy-server
28 28
29[app:proxy-server] 29[app:proxy-server]
30use = egg:swift#proxy 30use = egg:swift#proxy
31account_autocreate = true
31# See proxy-server.conf-sample for options 32# See proxy-server.conf-sample for options
32 33
33[filter:cache] 34[filter:cache]
diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample
index c07c48f..ae53b50 100644
--- a/etc/proxy-server.conf-sample
+++ b/etc/proxy-server.conf-sample
@@ -94,7 +94,7 @@ bind_port = 8080
94[pipeline:main] 94[pipeline:main]
95# This sample pipeline uses tempauth and is used for SAIO dev work and 95# This sample pipeline uses tempauth and is used for SAIO dev work and
96# testing. See below for a pipeline using keystone. 96# testing. See below for a pipeline using keystone.
97pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server 97pipeline = catch_errors gatekeeper healthcheck proxy-logging cache listing_formats container_sync bulk tempurl ratelimit tempauth copy container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server
98 98
99# The following pipeline shows keystone integration. Comment out the one 99# The following pipeline shows keystone integration. Comment out the one
100# above and uncomment this one. Additional steps for integrating keystone are 100# above and uncomment this one. Additional steps for integrating keystone are
@@ -357,7 +357,7 @@ user_test5_tester5 = testing5 service
357# Following parameters are known to work with keystonemiddleware v2.3.0 357# Following parameters are known to work with keystonemiddleware v2.3.0
358# (above v2.0.0), but checking the latest information in the wiki page[1] 358# (above v2.0.0), but checking the latest information in the wiki page[1]
359# is recommended. 359# is recommended.
360# 1. http://docs.openstack.org/developer/keystonemiddleware/middlewarearchitecture.html#configuration 360# 1. https://docs.openstack.org/keystonemiddleware/latest/middlewarearchitecture.html#configuration
361# 361#
362# [filter:authtoken] 362# [filter:authtoken]
363# paste.filter_factory = keystonemiddleware.auth_token:filter_factory 363# paste.filter_factory = keystonemiddleware.auth_token:filter_factory
@@ -544,6 +544,8 @@ use = egg:swift#domain_remap
544# can be specified separated by a comma 544# can be specified separated by a comma
545# storage_domain = example.com 545# storage_domain = example.com
546 546
547# Specify a root path part that will be added to the start of paths if not
548# already present.
547# path_root = v1 549# path_root = v1
548 550
549# Browsers can convert a host header to lowercase, so check that reseller 551# Browsers can convert a host header to lowercase, so check that reseller
@@ -556,6 +558,14 @@ use = egg:swift#domain_remap
556# reseller_prefixes = AUTH 558# reseller_prefixes = AUTH
557# default_reseller_prefix = 559# default_reseller_prefix =
558 560
561# Enable legacy remapping behavior for versioned path requests:
562# c.a.example.com/v1/o -> /v1/AUTH_a/c/o
563# instead of
564# c.a.example.com/v1/o -> /v1/AUTH_a/c/v1/o
565# ... by default all path parts after a remapped domain are considered part of
566# the object name with no special case for the path "v1"
567# mangle_client_paths = False
568
559[filter:catch_errors] 569[filter:catch_errors]
560use = egg:swift#catch_errors 570use = egg:swift#catch_errors
561# You can override the default log routing for this filter here: 571# You can override the default log routing for this filter here:
@@ -860,8 +870,6 @@ use = egg:swift#copy
860# requests are transformed into COPY requests where source and destination are 870# requests are transformed into COPY requests where source and destination are
861# the same. All client-visible behavior (save response time) should be 871# the same. All client-visible behavior (save response time) should be
862# identical. 872# identical.
863# This option is deprecated and will be ignored in a future release.
864# object_post_as_copy = false
865 873
866# Note: To enable encryption, add the following 2 dependent pieces of crypto 874# Note: To enable encryption, add the following 2 dependent pieces of crypto
867# middleware to the proxy-server pipeline. They should be to the right of all 875# middleware to the proxy-server pipeline. They should be to the right of all
@@ -915,3 +923,9 @@ use = egg:swift#encryption
915# disable_encryption to True. However, all encryption middleware should remain 923# disable_encryption to True. However, all encryption middleware should remain
916# in the pipeline in order for existing encrypted data to be read. 924# in the pipeline in order for existing encrypted data to be read.
917# disable_encryption = False 925# disable_encryption = False
926
927# listing_formats should be just right of the first proxy-logging middleware,
928# and left of most other middlewares. If it is not already present, it will
929# be automatically inserted for you.
930[filter:listing_formats]
931use = egg:swift#listing_formats
diff --git a/releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po b/releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po
new file mode 100644
index 0000000..e1922ad
--- /dev/null
+++ b/releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po
@@ -0,0 +1,67 @@
1# Andi Chandler <andi@gowling.com>, 2017. #zanata
2msgid ""
3msgstr ""
4"Project-Id-Version: Swift Release Notes 2.15.2\n"
5"Report-Msgid-Bugs-To: \n"
6"POT-Creation-Date: 2017-10-10 22:05+0000\n"
7"MIME-Version: 1.0\n"
8"Content-Type: text/plain; charset=UTF-8\n"
9"Content-Transfer-Encoding: 8bit\n"
10"PO-Revision-Date: 2017-10-05 03:59+0000\n"
11"Last-Translator: Andi Chandler <andi@gowling.com>\n"
12"Language-Team: English (United Kingdom)\n"
13"Language: en-GB\n"
14"X-Generator: Zanata 3.9.6\n"
15"Plural-Forms: nplurals=2; plural=(n != 1)\n"
16
17msgid "2.10.0"
18msgstr "2.10.0"
19
20msgid "2.10.1"
21msgstr "2.10.1"
22
23msgid "2.10.2"
24msgstr "2.10.2"
25
26msgid "2.11.0"
27msgstr "2.11.0"
28
29msgid "2.12.0"
30msgstr "2.12.0"
31
32msgid "2.13.0"
33msgstr "2.13.0"
34
35msgid "2.13.1"
36msgstr "2.13.1"
37
38msgid "2.14.0"
39msgstr "2.14.0"
40
41msgid "2.15.0"
42msgstr "2.15.0"
43
44msgid "2.15.1"
45msgstr "2.15.1"
46
47msgid ""
48"A PUT or POST to a container will now update the container's Last-Modified "
49"time, and that value will be included in a GET/HEAD response."
50msgstr ""
51"A PUT or POST to a container will now update the container's Last-Modified "
52"time, and that value will be included in a GET/HEAD response."
53
54msgid "Current (Unreleased) Release Notes"
55msgstr "Current (Unreleased) Release Notes"
56
57msgid "Swift Release Notes"
58msgstr "Swift Release Notes"
59
60msgid "domain_remap now accepts a list of domains in \"storage_domain\"."
61msgstr "domain_remap now accepts a list of domains in \"storage_domain\"."
62
63msgid "name_check and cname_lookup keys have been added to `/info`."
64msgstr "name_check and cname_lookup keys have been added to `/info`."
65
66msgid "swift-recon now respects storage policy aliases."
67msgstr "swift-recon now respects storage policy aliases."
diff --git a/setup.cfg b/setup.cfg
index e99d858..f180ffc 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -106,6 +106,7 @@ paste.filter_factory =
106 keymaster = swift.common.middleware.crypto.keymaster:filter_factory 106 keymaster = swift.common.middleware.crypto.keymaster:filter_factory
107 encryption = swift.common.middleware.crypto:filter_factory 107 encryption = swift.common.middleware.crypto:filter_factory
108 kms_keymaster = swift.common.middleware.crypto.kms_keymaster:filter_factory 108 kms_keymaster = swift.common.middleware.crypto.kms_keymaster:filter_factory
109 listing_formats = swift.common.middleware.listing_formats:filter_factory
109 110
110[build_sphinx] 111[build_sphinx]
111all_files = 1 112all_files = 1
diff --git a/swift/account/server.py b/swift/account/server.py
index 5334b97..0fe2647 100644
--- a/swift/account/server.py
+++ b/swift/account/server.py
@@ -24,15 +24,16 @@ import swift.common.db
24from swift.account.backend import AccountBroker, DATADIR 24from swift.account.backend import AccountBroker, DATADIR
25from swift.account.utils import account_listing_response, get_response_headers 25from swift.account.utils import account_listing_response, get_response_headers
26from swift.common.db import DatabaseConnectionError, DatabaseAlreadyExists 26from swift.common.db import DatabaseConnectionError, DatabaseAlreadyExists
27from swift.common.request_helpers import get_param, get_listing_content_type, \ 27from swift.common.request_helpers import get_param, \
28 split_and_validate_path 28 split_and_validate_path
29from swift.common.utils import get_logger, hash_path, public, \ 29from swift.common.utils import get_logger, hash_path, public, \
30 Timestamp, storage_directory, config_true_value, \ 30 Timestamp, storage_directory, config_true_value, \
31 json, timing_stats, replication, get_log_line 31 json, timing_stats, replication, get_log_line
32from swift.common.constraints import check_mount, valid_timestamp, check_utf8 32from swift.common.constraints import valid_timestamp, check_utf8, check_drive
33from swift.common import constraints 33from swift.common import constraints
34from swift.common.db_replicator import ReplicatorRpc 34from swift.common.db_replicator import ReplicatorRpc
35from swift.common.base_storage_server import BaseStorageServer 35from swift.common.base_storage_server import BaseStorageServer
36from swift.common.middleware import listing_formats
36from swift.common.swob import HTTPAccepted, HTTPBadRequest, \ 37from swift.common.swob import HTTPAccepted, HTTPBadRequest, \
37 HTTPCreated, HTTPForbidden, HTTPInternalServerError, \ 38 HTTPCreated, HTTPForbidden, HTTPInternalServerError, \
38 HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, \ 39 HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, \
@@ -87,7 +88,7 @@ class AccountController(BaseStorageServer):
87 def DELETE(self, req): 88 def DELETE(self, req):
88 """Handle HTTP DELETE request.""" 89 """Handle HTTP DELETE request."""
89 drive, part, account = split_and_validate_path(req, 3) 90 drive, part, account = split_and_validate_path(req, 3)
90 if self.mount_check and not check_mount(self.root, drive): 91 if not check_drive(self.root, drive, self.mount_check):
91 return HTTPInsufficientStorage(drive=drive, request=req) 92 return HTTPInsufficientStorage(drive=drive, request=req)
92 req_timestamp = valid_timestamp(req) 93 req_timestamp = valid_timestamp(req)
93 broker = self._get_account_broker(drive, part, account) 94 broker = self._get_account_broker(drive, part, account)
@@ -101,7 +102,7 @@ class AccountController(BaseStorageServer):
101 def PUT(self, req): 102 def PUT(self, req):
102 """Handle HTTP PUT request.""" 103 """Handle HTTP PUT request."""
103 drive, part, account, container = split_and_validate_path(req, 3, 4) 104 drive, part, account, container = split_and_validate_path(req, 3, 4)
104 if self.mount_check and not check_mount(self.root, drive): 105 if not check_drive(self.root, drive, self.mount_check):
105 return HTTPInsufficientStorage(drive=drive, request=req) 106 return HTTPInsufficientStorage(drive=drive, request=req)
106 if container: # put account container 107 if container: # put account container
107 if 'x-timestamp' not in req.headers: 108 if 'x-timestamp' not in req.headers:
@@ -167,8 +168,8 @@ class AccountController(BaseStorageServer):
167 def HEAD(self, req): 168 def HEAD(self, req):
168 """Handle HTTP HEAD request.""" 169 """Handle HTTP HEAD request."""
169 drive, part, account = split_and_validate_path(req, 3) 170 drive, part, account = split_and_validate_path(req, 3)
170 out_content_type = get_listing_content_type(req) 171 out_content_type = listing_formats.get_listing_content_type(req)
171 if self.mount_check and not check_mount(self.root, drive): 172 if not check_drive(self.root, drive, self.mount_check):
172 return HTTPInsufficientStorage(drive=drive, request=req) 173 return HTTPInsufficientStorage(drive=drive, request=req)
173 broker = self._get_account_broker(drive, part, account, 174 broker = self._get_account_broker(drive, part, account,
174 pending_timeout=0.1, 175 pending_timeout=0.1,
@@ -201,9 +202,9 @@ class AccountController(BaseStorageServer):
201 constraints.ACCOUNT_LISTING_LIMIT) 202 constraints.ACCOUNT_LISTING_LIMIT)
202 marker = get_param(req, 'marker', '') 203 marker = get_param(req, 'marker', '')
203 end_marker = get_param(req, 'end_marker') 204 end_marker = get_param(req, 'end_marker')
204 out_content_type = get_listing_content_type(req) 205 out_content_type = listing_formats.get_listing_content_type(req)
205 206
206 if self.mount_check and not check_mount(self.root, drive): 207 if not check_drive(self.root, drive, self.mount_check):
207 return HTTPInsufficientStorage(drive=drive, request=req) 208 return HTTPInsufficientStorage(drive=drive, request=req)
208 broker = self._get_account_broker(drive, part, account, 209 broker = self._get_account_broker(drive, part, account,
209 pending_timeout=0.1, 210 pending_timeout=0.1,
@@ -224,7 +225,7 @@ class AccountController(BaseStorageServer):
224 """ 225 """
225 post_args = split_and_validate_path(req, 3) 226 post_args = split_and_validate_path(req, 3)
226 drive, partition, hash = post_args 227 drive, partition, hash = post_args
227 if self.mount_check and not check_mount(self.root, drive): 228 if not check_drive(self.root, drive, self.mount_check):
228 return HTTPInsufficientStorage(drive=drive, request=req) 229 return HTTPInsufficientStorage(drive=drive, request=req)
229 try: 230 try:
230 args = json.load(req.environ['wsgi.input']) 231 args = json.load(req.environ['wsgi.input'])
@@ -240,7 +241,7 @@ class AccountController(BaseStorageServer):
240 """Handle HTTP POST request.""" 241 """Handle HTTP POST request."""
241 drive, part, account = split_and_validate_path(req, 3) 242 drive, part, account = split_and_validate_path(req, 3)
242 req_timestamp = valid_timestamp(req) 243 req_timestamp = valid_timestamp(req)
243 if self.mount_check and not check_mount(self.root, drive): 244 if not check_drive(self.root, drive, self.mount_check):
244 return HTTPInsufficientStorage(drive=drive, request=req) 245 return HTTPInsufficientStorage(drive=drive, request=req)
245 broker = self._get_account_broker(drive, part, account) 246 broker = self._get_account_broker(drive, part, account)
246 if broker.is_deleted(): 247 if broker.is_deleted():
diff --git a/swift/account/utils.py b/swift/account/utils.py
index 7559d00..cf7da27 100644
--- a/swift/account/utils.py
+++ b/swift/account/utils.py
@@ -14,8 +14,8 @@
14# limitations under the License. 14# limitations under the License.
15 15
16import json 16import json
17from xml.sax import saxutils
18 17
18from swift.common.middleware import listing_formats
19from swift.common.swob import HTTPOk, HTTPNoContent 19from swift.common.swob import HTTPOk, HTTPNoContent
20from swift.common.utils import Timestamp 20from swift.common.utils import Timestamp
21from swift.common.storage_policy import POLICIES 21from swift.common.storage_policy import POLICIES
@@ -78,43 +78,27 @@ def account_listing_response(account, req, response_content_type, broker=None,
78 78
79 account_list = broker.list_containers_iter(limit, marker, end_marker, 79 account_list = broker.list_containers_iter(limit, marker, end_marker,
80 prefix, delimiter, reverse) 80 prefix, delimiter, reverse)
81 if response_content_type == 'application/json': 81 data = []
82 data = [] 82 for (name, object_count, bytes_used, put_timestamp, is_subdir) \
83 for (name, object_count, bytes_used, put_timestamp, is_subdir) \ 83 in account_list:
84 in account_list: 84 if is_subdir:
85 if is_subdir: 85 data.append({'subdir': name.decode('utf8')})
86 data.append({'subdir': name}) 86 else:
87 else: 87 data.append(
88 data.append( 88 {'name': name.decode('utf8'), 'count': object_count,
89 {'name': name, 'count': object_count, 89 'bytes': bytes_used,
90 'bytes': bytes_used, 90 'last_modified': Timestamp(put_timestamp).isoformat})
91 'last_modified': Timestamp(put_timestamp).isoformat}) 91 if response_content_type.endswith('/xml'):
92 account_list = listing_formats.account_to_xml(data, account)
93 ret = HTTPOk(body=account_list, request=req, headers=resp_headers)
94 elif response_content_type.endswith('/json'):
92 account_list = json.dumps(data) 95 account_list = json.dumps(data)
93 elif response_content_type.endswith('/xml'): 96 ret = HTTPOk(body=account_list, request=req, headers=resp_headers)
94 output_list = ['<?xml version="1.0" encoding="UTF-8"?>', 97 elif data:
95 '<account name=%s>' % saxutils.quoteattr(account)] 98 account_list = listing_formats.listing_to_text(data)
96 for (name, object_count, bytes_used, put_timestamp, is_subdir) \ 99 ret = HTTPOk(body=account_list, request=req, headers=resp_headers)
97 in account_list:
98 if is_subdir:
99 output_list.append(
100 '<subdir name=%s />' % saxutils.quoteattr(name))
101 else:
102 item = '<container><name>%s</name><count>%s</count>' \
103 '<bytes>%s</bytes><last_modified>%s</last_modified>' \
104 '</container>' % \
105 (saxutils.escape(name), object_count,
106 bytes_used, Timestamp(put_timestamp).isoformat)
107 output_list.append(item)
108 output_list.append('</account>')
109 account_list = '\n'.join(output_list)
110 else: 100 else:
111 if not account_list: 101 ret = HTTPNoContent(request=req, headers=resp_headers)
112 resp = HTTPNoContent(request=req, headers=resp_headers)
113 resp.content_type = response_content_type
114 resp.charset = 'utf-8'
115 return resp
116 account_list = '\n'.join(r[0] for r in account_list) + '\n'
117 ret = HTTPOk(body=account_list, request=req, headers=resp_headers)
118 ret.content_type = response_content_type 102 ret.content_type = response_content_type
119 ret.charset = 'utf-8' 103 ret.charset = 'utf-8'
120 return ret 104 return ret
diff --git a/swift/cli/dispersion_report.py b/swift/cli/dispersion_report.py
index 61708a3..61708a3 100755..100644
--- a/swift/cli/dispersion_report.py
+++ b/swift/cli/dispersion_report.py
diff --git a/swift/cli/ringbuilder.py b/swift/cli/ringbuilder.py
index a3a3570..1b9cb05 100644
--- a/swift/cli/ringbuilder.py
+++ b/swift/cli/ringbuilder.py
@@ -218,7 +218,6 @@ def _parse_set_weight_values(argvish):
218 # --options format, 218 # --options format,
219 # but not both. If both are specified, raise an error. 219 # but not both. If both are specified, raise an error.
220 try: 220 try:
221 devs = []
222 if not new_cmd_format: 221 if not new_cmd_format:
223 if len(args) % 2 != 0: 222 if len(args) % 2 != 0:
224 print(Commands.set_weight.__doc__.strip()) 223 print(Commands.set_weight.__doc__.strip())
@@ -227,7 +226,7 @@ def _parse_set_weight_values(argvish):
227 devs_and_weights = izip(islice(argvish, 0, len(argvish), 2), 226 devs_and_weights = izip(islice(argvish, 0, len(argvish), 2),
228 islice(argvish, 1, len(argvish), 2)) 227 islice(argvish, 1, len(argvish), 2))
229 for devstr, weightstr in devs_and_weights: 228 for devstr, weightstr in devs_and_weights:
230 devs.extend(builder.search_devs( 229 devs = (builder.search_devs(
231 parse_search_value(devstr)) or []) 230 parse_search_value(devstr)) or [])
232 weight = float(weightstr) 231 weight = float(weightstr)
233 _set_weight_values(devs, weight, opts) 232 _set_weight_values(devs, weight, opts)
@@ -236,7 +235,7 @@ def _parse_set_weight_values(argvish):
236 print(Commands.set_weight.__doc__.strip()) 235 print(Commands.set_weight.__doc__.strip())
237 exit(EXIT_ERROR) 236 exit(EXIT_ERROR)
238 237
239 devs.extend(builder.search_devs( 238 devs = (builder.search_devs(
240 parse_search_values_from_opts(opts)) or []) 239 parse_search_values_from_opts(opts)) or [])
241 weight = float(args[0]) 240 weight = float(args[0])
242 _set_weight_values(devs, weight, opts) 241 _set_weight_values(devs, weight, opts)
diff --git a/swift/common/constraints.py b/swift/common/constraints.py
index 6799263..bb8fefc 100644
--- a/swift/common/constraints.py
+++ b/swift/common/constraints.py
@@ -15,6 +15,7 @@
15 15
16import functools 16import functools
17import os 17import os
18from os.path import isdir # tighter scoped import for mocking
18import time 19import time
19 20
20import six 21import six
@@ -104,11 +105,6 @@ reload_constraints()
104MAX_BUFFERED_SLO_SEGMENTS = 10000 105MAX_BUFFERED_SLO_SEGMENTS = 10000
105 106
106 107
107#: Query string format= values to their corresponding content-type values
108FORMAT2CONTENT_TYPE = {'plain': 'text/plain', 'json': 'application/json',
109 'xml': 'application/xml'}
110
111
112# By default the maximum number of allowed headers depends on the number of max 108# By default the maximum number of allowed headers depends on the number of max
113# allowed metadata settings plus a default value of 36 for swift internally 109# allowed metadata settings plus a default value of 36 for swift internally
114# generated headers and regular http headers. If for some reason this is not 110# generated headers and regular http headers. If for some reason this is not
@@ -234,9 +230,9 @@ def check_dir(root, drive):
234 230
235 :param root: base path where the dir is 231 :param root: base path where the dir is
236 :param drive: drive name to be checked 232 :param drive: drive name to be checked
237 :returns: True if it is a valid directoy, False otherwise 233 :returns: full path to the device, or None if drive fails to validate
238 """ 234 """
239 return os.path.isdir(os.path.join(root, drive)) 235 return check_drive(root, drive, False)
240 236
241 237
242def check_mount(root, drive): 238def check_mount(root, drive):
@@ -248,12 +244,31 @@ def check_mount(root, drive):
248 244
249 :param root: base path where the devices are mounted 245 :param root: base path where the devices are mounted
250 :param drive: drive name to be checked 246 :param drive: drive name to be checked
251 :returns: True if it is a valid mounted device, False otherwise 247 :returns: full path to the device, or None if drive fails to validate
248 """
249 return check_drive(root, drive, True)
250
251
252def check_drive(root, drive, mount_check):
253 """
254 Validate the path given by root and drive is a valid existing directory.
255
256 :param root: base path where the devices are mounted
257 :param drive: drive name to be checked
258 :param mount_check: additionally require path is mounted
259
260 :returns: full path to the device, or None if drive fails to validate
252 """ 261 """
253 if not (urllib.parse.quote_plus(drive) == drive): 262 if not (urllib.parse.quote_plus(drive) == drive):
254 return False 263 return None
255 path = os.path.join(root, drive) 264 path = os.path.join(root, drive)
256 return utils.ismount(path) 265 if mount_check:
266 if utils.ismount(path):
267 return path
268 else:
269 if isdir(path):
270 return path
271 return None
257 272
258 273
259def check_float(string): 274def check_float(string):
diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py
index c8a87bf..cb93d8a 100644
--- a/swift/common/db_replicator.py
+++ b/swift/common/db_replicator.py
@@ -221,9 +221,9 @@ class Replicator(Daemon):
221 'replication_last': now}, 221 'replication_last': now},
222 self.rcache, self.logger) 222 self.rcache, self.logger)
223 self.logger.info(' '.join(['%s:%s' % item for item in 223 self.logger.info(' '.join(['%s:%s' % item for item in
224 self.stats.items() if item[0] in 224 sorted(self.stats.items()) if item[0] in
225 ('no_change', 'hashmatch', 'rsync', 'diff', 'ts_repl', 225 ('no_change', 'hashmatch', 'rsync', 'diff', 'ts_repl',
226 'empty', 'diff_capped')])) 226 'empty', 'diff_capped', 'remote_merge')]))
227 227
228 def _add_failure_stats(self, failure_devs_info): 228 def _add_failure_stats(self, failure_devs_info):
229 for node, dev in failure_devs_info: 229 for node, dev in failure_devs_info:
diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py
index 71b3d07..fad4440 100644
--- a/swift/common/direct_client.py
+++ b/swift/common/direct_client.py
@@ -88,19 +88,20 @@ def _get_direct_account_container(path, stype, node, part,
88 Do not use directly use the get_direct_account or 88 Do not use directly use the get_direct_account or
89 get_direct_container instead. 89 get_direct_container instead.
90 """ 90 """
91 qs = 'format=json' 91 params = ['format=json']
92 if marker: 92 if marker:
93 qs += '&marker=%s' % quote(marker) 93 params.append('marker=%s' % quote(marker))
94 if limit: 94 if limit:
95 qs += '&limit=%d' % limit 95 params.append('limit=%d' % limit)
96 if prefix: 96 if prefix:
97 qs += '&prefix=%s' % quote(prefix) 97 params.append('prefix=%s' % quote(prefix))
98 if delimiter: 98 if delimiter:
99 qs += '&delimiter=%s' % quote(delimiter) 99 params.append('delimiter=%s' % quote(delimiter))
100 if end_marker: 100 if end_marker:
101 qs += '&end_marker=%s' % quote(end_marker) 101 params.append('end_marker=%s' % quote(end_marker))
102 if reverse: 102 if reverse:
103 qs += '&reverse=%s' % quote(reverse) 103 params.append('reverse=%s' % quote(reverse))
104 qs = '&'.join(params)
104 with Timeout(conn_timeout): 105 with Timeout(conn_timeout):
105 conn = http_connect(node['ip'], node['port'], node['device'], part, 106 conn = http_connect(node['ip'], node['port'], node['device'], part,
106 'GET', path, query_string=qs, 107 'GET', path, query_string=qs,
diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py
index 6eda392..462491f 100644
--- a/swift/common/internal_client.py
+++ b/swift/common/internal_client.py
@@ -772,12 +772,14 @@ class SimpleClient(object):
772 if name: 772 if name:
773 url = '%s/%s' % (url.rstrip('/'), quote(name)) 773 url = '%s/%s' % (url.rstrip('/'), quote(name))
774 else: 774 else:
775 url += '?format=json' 775 params = ['format=json']
776 if prefix: 776 if prefix:
777 url += '&prefix=%s' % prefix 777 params.append('prefix=%s' % prefix)
778 778
779 if marker: 779 if marker:
780 url += '&marker=%s' % quote(marker) 780 params.append('marker=%s' % quote(marker))
781
782 url += '?' + '&'.join(params)
781 783
782 req = urllib2.Request(url, headers=headers, data=contents) 784 req = urllib2.Request(url, headers=headers, data=contents)
783 if proxy: 785 if proxy:
diff --git a/swift/common/memcached.py b/swift/common/memcached.py
index b948176..6644f2b 100644
--- a/swift/common/memcached.py
+++ b/swift/common/memcached.py
@@ -164,7 +164,7 @@ class MemcacheRing(object):
164 if isinstance(e, Timeout): 164 if isinstance(e, Timeout):
165 logging.error("Timeout %(action)s to memcached: %(server)s", 165 logging.error("Timeout %(action)s to memcached: %(server)s",
166 {'action': action, 'server': server}) 166 {'action': action, 'server': server})
167 elif isinstance(e, socket.error): 167 elif isinstance(e, (socket.error, MemcacheConnectionError)):
168 logging.error("Error %(action)s to memcached: %(server)s: %(err)s", 168 logging.error("Error %(action)s to memcached: %(server)s: %(err)s",
169 {'action': action, 'server': server, 'err': e}) 169 {'action': action, 'server': server, 'err': e})
170 else: 170 else:
@@ -283,7 +283,11 @@ class MemcacheRing(object):
283 with Timeout(self._io_timeout): 283 with Timeout(self._io_timeout):
284 sock.sendall('get %s\r\n' % key) 284 sock.sendall('get %s\r\n' % key)
285 line = fp.readline().strip().split() 285 line = fp.readline().strip().split()
286 while line[0].upper() != 'END': 286 while True:
287 if not line:
288 raise MemcacheConnectionError('incomplete read')
289 if line[0].upper() == 'END':
290 break
287 if line[0].upper() == 'VALUE' and line[1] == key: 291 if line[0].upper() == 'VALUE' and line[1] == key:
288 size = int(line[3]) 292 size = int(line[3])
289 value = fp.read(size) 293 value = fp.read(size)
@@ -329,6 +333,8 @@ class MemcacheRing(object):
329 with Timeout(self._io_timeout): 333 with Timeout(self._io_timeout):
330 sock.sendall('%s %s %s\r\n' % (command, key, delta)) 334 sock.sendall('%s %s %s\r\n' % (command, key, delta))
331 line = fp.readline().strip().split() 335 line = fp.readline().strip().split()
336 if not line:
337 raise MemcacheConnectionError('incomplete read')
332 if line[0].upper() == 'NOT_FOUND': 338 if line[0].upper() == 'NOT_FOUND':
333 add_val = delta 339 add_val = delta
334 if command == 'decr': 340 if command == 'decr':
@@ -444,7 +450,11 @@ class MemcacheRing(object):
444 sock.sendall('get %s\r\n' % ' '.join(keys)) 450 sock.sendall('get %s\r\n' % ' '.join(keys))
445 line = fp.readline().strip().split() 451 line = fp.readline().strip().split()
446 responses = {} 452 responses = {}
447 while line[0].upper() != 'END': 453 while True:
454 if not line:
455 raise MemcacheConnectionError('incomplete read')
456 if line[0].upper() == 'END':
457 break
448 if line[0].upper() == 'VALUE': 458 if line[0].upper() == 'VALUE':
449 size = int(line[3]) 459 size = int(line[3])
450 value = fp.read(size) 460 value = fp.read(size)
diff --git a/swift/common/middleware/copy.py b/swift/common/middleware/copy.py
index d371857..49e85c2 100644
--- a/swift/common/middleware/copy.py
+++ b/swift/common/middleware/copy.py
@@ -112,20 +112,6 @@ If a request is sent without the query parameter, an attempt will be made to
112copy the whole object but will fail if the object size is 112copy the whole object but will fail if the object size is
113greater than 5GB. 113greater than 5GB.
114 114
115-------------------
116Object Post as Copy
117-------------------
118Historically, this has been a feature (and a configurable option with default
119set to True) in proxy server configuration. This has been moved to server side
120copy middleware and the default changed to False.
121
122When ``object_post_as_copy`` is set to ``true``, an incoming POST request is
123morphed into a COPY request where source and destination objects are same.
124
125This feature was necessary because of a previous behavior where POSTS would
126update the metadata on the object but not on the container. As a result,
127features like container sync would not work correctly. This is no longer the
128case and this option is now deprecated. It will be removed in a future release.
129""" 115"""
130 116
131import os 117import os
@@ -137,8 +123,7 @@ from swift.common.utils import get_logger, \
137 config_true_value, FileLikeIter, read_conf_dir, close_if_possible 123 config_true_value, FileLikeIter, read_conf_dir, close_if_possible
138from swift.common.swob import Request, HTTPPreconditionFailed, \ 124from swift.common.swob import Request, HTTPPreconditionFailed, \
139 HTTPRequestEntityTooLarge, HTTPBadRequest, HTTPException 125 HTTPRequestEntityTooLarge, HTTPBadRequest, HTTPException
140from swift.common.http import HTTP_MULTIPLE_CHOICES, HTTP_CREATED, \ 126from swift.common.http import HTTP_MULTIPLE_CHOICES, is_success, HTTP_OK
141 is_success, HTTP_OK
142from swift.common.constraints import check_account_format, MAX_FILE_SIZE 127from swift.common.constraints import check_account_format, MAX_FILE_SIZE
143from swift.common.request_helpers import copy_header_subset, remove_items, \ 128from swift.common.request_helpers import copy_header_subset, remove_items, \
144 is_sys_meta, is_sys_or_user_meta, is_object_transient_sysmeta 129 is_sys_meta, is_sys_or_user_meta, is_object_transient_sysmeta
@@ -238,13 +223,7 @@ class ServerSideCopyWebContext(WSGIContext):
238 return app_resp 223 return app_resp
239 224
240 def _adjust_put_response(self, req, additional_resp_headers): 225 def _adjust_put_response(self, req, additional_resp_headers):
241 if 'swift.post_as_copy' in req.environ: 226 if is_success(self._get_status_int()):
242 # Older editions returned 202 Accepted on object POSTs, so we'll
243 # convert any 201 Created responses to that for compatibility with
244 # picky clients.
245 if self._get_status_int() == HTTP_CREATED:
246 self._response_status = '202 Accepted'
247 elif is_success(self._get_status_int()):
248 for header, value in additional_resp_headers.items(): 227 for header, value in additional_resp_headers.items():
249 self._response_headers.append((header, value)) 228 self._response_headers.append((header, value))
250 229
@@ -269,17 +248,12 @@ class ServerSideCopyMiddleware(object):
269 def __init__(self, app, conf): 248 def __init__(self, app, conf):
270 self.app = app 249 self.app = app
271 self.logger = get_logger(conf, log_route="copy") 250 self.logger = get_logger(conf, log_route="copy")
272 # Read the old object_post_as_copy option from Proxy app just in case
273 # someone has set it to false (non default). This wouldn't cause
274 # problems during upgrade.
275 self._load_object_post_as_copy_conf(conf) 251 self._load_object_post_as_copy_conf(conf)
276 self.object_post_as_copy = \ 252 self.object_post_as_copy = \
277 config_true_value(conf.get('object_post_as_copy', 'false')) 253 config_true_value(conf.get('object_post_as_copy', 'false'))
278 if self.object_post_as_copy: 254 if self.object_post_as_copy:
279 msg = ('object_post_as_copy=true is deprecated; remove all ' 255 msg = ('object_post_as_copy=true is deprecated; This '
280 'references to it from %s to disable this warning. This ' 256 'option is now ignored')
281 'option will be ignored in a future release' % conf.get(
282 '__file__', 'proxy-server.conf'))
283 self.logger.warning(msg) 257 self.logger.warning(msg)
284 258
285 def _load_object_post_as_copy_conf(self, conf): 259 def _load_object_post_as_copy_conf(self, conf):
@@ -330,9 +304,6 @@ class ServerSideCopyMiddleware(object):
330 elif req.method == 'COPY': 304 elif req.method == 'COPY':
331 req.environ['swift.orig_req_method'] = req.method 305 req.environ['swift.orig_req_method'] = req.method
332 return self.handle_COPY(req, start_response) 306 return self.handle_COPY(req, start_response)
333 elif req.method == 'POST' and self.object_post_as_copy:
334 req.environ['swift.orig_req_method'] = req.method
335 return self.handle_object_post_as_copy(req, start_response)
336 elif req.method == 'OPTIONS': 307 elif req.method == 'OPTIONS':
337 # Does not interfere with OPTIONS response from 308 # Does not interfere with OPTIONS response from
338 # (account,container) servers and /info response. 309 # (account,container) servers and /info response.
@@ -343,21 +314,6 @@ class ServerSideCopyMiddleware(object):
343 314
344 return self.app(env, start_response) 315 return self.app(env, start_response)
345 316
346 def handle_object_post_as_copy(self, req, start_response):
347 req.method = 'PUT'
348 req.path_info = '/v1/%s/%s/%s' % (
349 self.account_name, self.container_name, self.object_name)
350 req.headers['Content-Length'] = 0
351 req.headers.pop('Range', None)
352 req.headers['X-Copy-From'] = quote('/%s/%s' % (self.container_name,
353 self.object_name))
354 req.environ['swift.post_as_copy'] = True
355 params = req.params
356 # for post-as-copy always copy the manifest itself if source is *LO
357 params['multipart-manifest'] = 'get'
358 req.params = params
359 return self.handle_PUT(req, start_response)
360
361 def handle_COPY(self, req, start_response): 317 def handle_COPY(self, req, start_response):
362 if not req.headers.get('Destination'): 318 if not req.headers.get('Destination'):
363 return HTTPPreconditionFailed(request=req, 319 return HTTPPreconditionFailed(request=req,
@@ -394,11 +350,6 @@ class ServerSideCopyMiddleware(object):
394 source_req.headers.pop('X-Backend-Storage-Policy-Index', None) 350 source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
395 source_req.path_info = quote(source_path) 351 source_req.path_info = quote(source_path)
396 source_req.headers['X-Newest'] = 'true' 352 source_req.headers['X-Newest'] = 'true'
397 if 'swift.post_as_copy' in req.environ:
398 # We're COPYing one object over itself because of a POST; rely on
399 # the PUT for write authorization, don't require read authorization
400 source_req.environ['swift.authorize'] = lambda req: None
401 source_req.environ['swift.authorize_override'] = True
402 353
403 # in case we are copying an SLO manifest, set format=raw parameter 354 # in case we are copying an SLO manifest, set format=raw parameter
404 params = source_req.params 355 params = source_req.params
@@ -470,11 +421,7 @@ class ServerSideCopyMiddleware(object):
470 def is_object_sysmeta(k): 421 def is_object_sysmeta(k):
471 return is_sys_meta('object', k) 422 return is_sys_meta('object', k)
472 423
473 if 'swift.post_as_copy' in sink_req.environ: 424 if config_true_value(req.headers.get('x-fresh-metadata', 'false')):
474 # Post-as-copy: ignore new sysmeta, copy existing sysmeta
475 remove_items(sink_req.headers, is_object_sysmeta)
476 copy_header_subset(source_resp, sink_req, is_object_sysmeta)
477 elif config_true_value(req.headers.get('x-fresh-metadata', 'false')):
478 # x-fresh-metadata only applies to copy, not post-as-copy: ignore 425 # x-fresh-metadata only applies to copy, not post-as-copy: ignore
479 # existing user metadata, update existing sysmeta with new 426 # existing user metadata, update existing sysmeta with new
480 copy_header_subset(source_resp, sink_req, is_object_sysmeta) 427 copy_header_subset(source_resp, sink_req, is_object_sysmeta)
@@ -497,9 +444,8 @@ class ServerSideCopyMiddleware(object):
497 params['multipart-manifest'] = 'put' 444 params['multipart-manifest'] = 'put'
498 if 'X-Object-Manifest' in source_resp.headers: 445 if 'X-Object-Manifest' in source_resp.headers:
499 del params['multipart-manifest'] 446 del params['multipart-manifest']
500 if 'swift.post_as_copy' not in sink_req.environ: 447 sink_req.headers['X-Object-Manifest'] = \
501 sink_req.headers['X-Object-Manifest'] = \ 448 source_resp.headers['X-Object-Manifest']
502 source_resp.headers['X-Object-Manifest']
503 sink_req.params = params 449 sink_req.params = params
504 450
505 # Set swift.source, data source, content length and etag 451 # Set swift.source, data source, content length and etag
diff --git a/swift/common/middleware/crypto/decrypter.py b/swift/common/middleware/crypto/decrypter.py
index 3ae17e4..c8e78a5 100644
--- a/swift/common/middleware/crypto/decrypter.py
+++ b/swift/common/middleware/crypto/decrypter.py
@@ -15,7 +15,6 @@
15 15
16import base64 16import base64
17import json 17import json
18import xml.etree.cElementTree as ElementTree
19 18
20from swift import gettext_ as _ 19from swift import gettext_ as _
21from swift.common.http import is_success 20from swift.common.http import is_success
@@ -23,7 +22,7 @@ from swift.common.middleware.crypto.crypto_utils import CryptoWSGIContext, \
23 load_crypto_meta, extract_crypto_meta, Crypto 22 load_crypto_meta, extract_crypto_meta, Crypto
24from swift.common.exceptions import EncryptionException 23from swift.common.exceptions import EncryptionException
25from swift.common.request_helpers import get_object_transient_sysmeta, \ 24from swift.common.request_helpers import get_object_transient_sysmeta, \
26 get_listing_content_type, get_sys_meta_prefix, get_user_meta_prefix 25 get_sys_meta_prefix, get_user_meta_prefix
27from swift.common.swob import Request, HTTPException, HTTPInternalServerError 26from swift.common.swob import Request, HTTPException, HTTPInternalServerError
28from swift.common.utils import get_logger, config_true_value, \ 27from swift.common.utils import get_logger, config_true_value, \
29 parse_content_range, closing_if_possible, parse_content_type, \ 28 parse_content_range, closing_if_possible, parse_content_type, \
@@ -352,15 +351,12 @@ class DecrypterContContext(BaseDecrypterContext):
352 351
353 if is_success(self._get_status_int()): 352 if is_success(self._get_status_int()):
354 # only decrypt body of 2xx responses 353 # only decrypt body of 2xx responses
355 out_content_type = get_listing_content_type(req) 354 handler = keys = None
356 if out_content_type == 'application/json': 355 for header, value in self._response_headers:
357 handler = self.process_json_resp 356 if header.lower() == 'content-type' and \
358 keys = self.get_decryption_keys(req) 357 value.split(';', 1)[0] == 'application/json':
359 elif out_content_type.endswith('/xml'): 358 handler = self.process_json_resp
360 handler = self.process_xml_resp 359 keys = self.get_decryption_keys(req)
361 keys = self.get_decryption_keys(req)
362 else:
363 handler = keys = None
364 360
365 if handler and keys: 361 if handler and keys:
366 try: 362 try:
@@ -398,24 +394,6 @@ class DecrypterContContext(BaseDecrypterContext):
398 obj_dict['hash'] = self.decrypt_value_with_meta(ciphertext, key) 394 obj_dict['hash'] = self.decrypt_value_with_meta(ciphertext, key)
399 return obj_dict 395 return obj_dict
400 396
401 def process_xml_resp(self, key, resp_iter):
402 """
403 Parses xml body listing and decrypt encrypted entries. Updates
404 Content-Length header with new body length and return a body iter.
405 """
406 with closing_if_possible(resp_iter):
407 resp_body = ''.join(resp_iter)
408 tree = ElementTree.fromstring(resp_body)
409 for elem in tree.iter('hash'):
410 ciphertext = elem.text.encode('utf8')
411 plain = self.decrypt_value_with_meta(ciphertext, key)
412 elem.text = plain.decode('utf8')
413 new_body = ElementTree.tostring(tree, encoding='UTF-8').replace(
414 "<?xml version='1.0' encoding='UTF-8'?>",
415 '<?xml version="1.0" encoding="UTF-8"?>', 1)
416 self.update_content_length(len(new_body))
417 return [new_body]
418
419 397
420class Decrypter(object): 398class Decrypter(object):
421 """Middleware for decrypting data and user metadata.""" 399 """Middleware for decrypting data and user metadata."""
diff --git a/swift/common/middleware/dlo.py b/swift/common/middleware/dlo.py
index e7bb3fe..21752ba 100644
--- a/swift/common/middleware/dlo.py
+++ b/swift/common/middleware/dlo.py
@@ -151,7 +151,7 @@ class GetContext(WSGIContext):
151 method='GET', 151 method='GET',
152 headers={'x-auth-token': req.headers.get('x-auth-token')}, 152 headers={'x-auth-token': req.headers.get('x-auth-token')},
153 agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO') 153 agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
154 con_req.query_string = 'format=json&prefix=%s' % quote(prefix) 154 con_req.query_string = 'prefix=%s' % quote(prefix)
155 if marker: 155 if marker:
156 con_req.query_string += '&marker=%s' % quote(marker) 156 con_req.query_string += '&marker=%s' % quote(marker)
157 157
diff --git a/swift/common/middleware/domain_remap.py b/swift/common/middleware/domain_remap.py
index 13cfee9..a8f4765 100644
--- a/swift/common/middleware/domain_remap.py
+++ b/swift/common/middleware/domain_remap.py
@@ -17,42 +17,91 @@
17""" 17"""
18Domain Remap Middleware 18Domain Remap Middleware
19 19
20Middleware that translates container and account parts of a domain to 20Middleware that translates container and account parts of a domain to path
21path parameters that the proxy server understands. 21parameters that the proxy server understands.
22 22
23container.account.storageurl/object gets translated to 23Translation is only performed when the request URL's host domain matches one of
24container.account.storageurl/path_root/account/container/object 24a list of domains. This list may be configured by the option
25``storage_domain``, and defaults to the single domain ``example.com``.
25 26
26account.storageurl/path_root/container/object gets translated to 27If not already present, a configurable ``path_root``, which defaults to ``v1``,
27account.storageurl/path_root/account/container/object 28will be added to the start of the translated path.
28 29
29Browsers can convert a host header to lowercase, so check that reseller 30For example, with the default configuration::
30prefix on the account is the correct case. This is done by comparing the 31
31items in the reseller_prefixes config option to the found prefix. If they 32 container.AUTH-account.example.com/object
32match except for case, the item from reseller_prefixes will be used 33 container.AUTH-account.example.com/v1/object
33instead of the found reseller prefix. When none match, the default reseller 34
34prefix is used. When no default reseller prefix is configured, any request with 35would both be translated to::
35an account prefix not in that list will be ignored by this middleware. 36
36reseller_prefixes defaults to 'AUTH'. 37 container.AUTH-account.example.com/v1/AUTH_account/container/object
38
39and::
40
41 AUTH-account.example.com/container/object
42 AUTH-account.example.com/v1/container/object
43
44would both be translated to::
45
46 AUTH-account.example.com/v1/AUTH_account/container/object
47
48Additionally, translation is only performed when the account name in the
49translated path starts with a reseller prefix matching one of a list configured
50by the option ``reseller_prefixes``, or when no match is found but a
51``default_reseller_prefix`` has been configured.
52
53The ``reseller_prefixes`` list defaults to the single prefix ``AUTH``. The
54``default_reseller_prefix`` is not configured by default.
55
56Browsers can convert a host header to lowercase, so the middleware checks that
57the reseller prefix on the account name is the correct case. This is done by
58comparing the items in the ``reseller_prefixes`` config option to the found
59prefix. If they match except for case, the item from ``reseller_prefixes`` will
60be used instead of the found reseller prefix. The middleware will also replace
61any hyphen ('-') in the account name with an underscore ('_').
62
63For example, with the default configuration::
64
65 auth-account.example.com/container/object
66 AUTH-account.example.com/container/object
67 auth_account.example.com/container/object
68 AUTH_account.example.com/container/object
69
70would all be translated to::
71
72 <unchanged>.example.com/v1/AUTH_account/container/object
73
74When no match is found in ``reseller_prefixes``, the
75``default_reseller_prefix`` config option is used. When no
76``default_reseller_prefix`` is configured, any request with an account prefix
77not in the ``reseller_prefixes`` list will be ignored by this middleware.
78
79For example, with ``default_reseller_prefix = AUTH``::
80
81 account.example.com/container/object
82
83would be translated to::
84
85 account.example.com/v1/AUTH_account/container/object
37 86
38Note that this middleware requires that container names and account names 87Note that this middleware requires that container names and account names
39(except as described above) must be DNS-compatible. This means that the 88(except as described above) must be DNS-compatible. This means that the account
40account name created in the system and the containers created by users 89name created in the system and the containers created by users cannot exceed 63
41cannot exceed 63 characters or have UTF-8 characters. These are 90characters or have UTF-8 characters. These are restrictions over and above what
42restrictions over and above what swift requires and are not explicitly 91Swift requires and are not explicitly checked. Simply put, this middleware
43checked. Simply put, the this middleware will do a best-effort attempt to 92will do a best-effort attempt to derive account and container names from
44derive account and container names from elements in the domain name and 93elements in the domain name and put those derived values into the URL path
45put those derived values into the URL path (leaving the Host header 94(leaving the ``Host`` header unchanged).
46unchanged). 95
47 96Also note that using :doc:`overview_container_sync` with remapped domain names
48Also note that using container sync with remapped domain names is not 97is not advised. With :doc:`overview_container_sync`, you should use the true
49advised. With container sync, you should use the true storage end points as 98storage end points as sync destinations.
50sync destinations.
51""" 99"""
52 100
53from swift.common.middleware import RewriteContext 101from swift.common.middleware import RewriteContext
54from swift.common.swob import Request, HTTPBadRequest 102from swift.common.swob import Request, HTTPBadRequest
55from swift.common.utils import list_from_csv, register_swift_info 103from swift.common.utils import config_true_value, list_from_csv, \
104 register_swift_info
56 105
57 106
58class _DomainRemapContext(RewriteContext): 107class _DomainRemapContext(RewriteContext):
@@ -78,12 +127,14 @@ class DomainRemapMiddleware(object):
78 if not s.startswith('.')] 127 if not s.startswith('.')]
79 self.storage_domain += [s for s in list_from_csv(storage_domain) 128 self.storage_domain += [s for s in list_from_csv(storage_domain)
80 if s.startswith('.')] 129 if s.startswith('.')]
81 self.path_root = '/' + conf.get('path_root', 'v1').strip('/') 130 self.path_root = conf.get('path_root', 'v1').strip('/') + '/'
82 prefixes = conf.get('reseller_prefixes', 'AUTH') 131 prefixes = conf.get('reseller_prefixes', 'AUTH')
83 self.reseller_prefixes = list_from_csv(prefixes) 132 self.reseller_prefixes = list_from_csv(prefixes)
84 self.reseller_prefixes_lower = [x.lower() 133 self.reseller_prefixes_lower = [x.lower()
85 for x in self.reseller_prefixes] 134 for x in self.reseller_prefixes]
86 self.default_reseller_prefix = conf.get('default_reseller_prefix') 135 self.default_reseller_prefix = conf.get('default_reseller_prefix')
136 self.mangle_client_paths = config_true_value(
137 conf.get('mangle_client_paths'))
87 138
88 def __call__(self, env, start_response): 139 def __call__(self, env, start_response):
89 if not self.storage_domain: 140 if not self.storage_domain:
@@ -129,14 +180,14 @@ class DomainRemapMiddleware(object):
129 # account prefix is not in config list. bail. 180 # account prefix is not in config list. bail.
130 return self.app(env, start_response) 181 return self.app(env, start_response)
131 182
132 requested_path = path = env['PATH_INFO'] 183 requested_path = env['PATH_INFO']
133 new_path_parts = [self.path_root, account] 184 path = requested_path[1:]
185 new_path_parts = ['', self.path_root[:-1], account]
134 if container: 186 if container:
135 new_path_parts.append(container) 187 new_path_parts.append(container)
136 if path.startswith(self.path_root): 188 if self.mangle_client_paths and (path + '/').startswith(
189 self.path_root):
137 path = path[len(self.path_root):] 190 path = path[len(self.path_root):]
138 if path.startswith('/'):
139 path = path[1:]
140 new_path_parts.append(path) 191 new_path_parts.append(path)
141 new_path = '/'.join(new_path_parts) 192 new_path = '/'.join(new_path_parts)
142 env['PATH_INFO'] = new_path 193 env['PATH_INFO'] = new_path
diff --git a/swift/common/middleware/gatekeeper.py b/swift/common/middleware/gatekeeper.py
index e5df5bf..991ec86 100644
--- a/swift/common/middleware/gatekeeper.py
+++ b/swift/common/middleware/gatekeeper.py
@@ -36,6 +36,7 @@ from swift.common.utils import get_logger, config_true_value
36from swift.common.request_helpers import ( 36from swift.common.request_helpers import (
37 remove_items, get_sys_meta_prefix, OBJECT_TRANSIENT_SYSMETA_PREFIX 37 remove_items, get_sys_meta_prefix, OBJECT_TRANSIENT_SYSMETA_PREFIX
38) 38)
39from six.moves.urllib.parse import urlsplit
39import re 40import re
40 41
41#: A list of python regular expressions that will be used to 42#: A list of python regular expressions that will be used to
@@ -89,9 +90,29 @@ class GatekeeperMiddleware(object):
89 [('X-Timestamp', ts)]) 90 [('X-Timestamp', ts)])
90 91
91 def gatekeeper_response(status, response_headers, exc_info=None): 92 def gatekeeper_response(status, response_headers, exc_info=None):
93 def fixed_response_headers():
94 def relative_path(value):
95 parsed = urlsplit(v)
96 new_path = parsed.path
97 if parsed.query:
98 new_path += ('?%s' % parsed.query)
99 if parsed.fragment:
100 new_path += ('#%s' % parsed.fragment)
101 return new_path
102
103 if not env.get('swift.leave_relative_location'):
104 return response_headers
105 else:
106 return [
107 (k, v) if k.lower() != 'location' else
108 (k, relative_path(v)) for (k, v) in response_headers
109 ]
110
111 response_headers = fixed_response_headers()
92 removed = filter( 112 removed = filter(
93 lambda h: self.outbound_condition(h[0]), 113 lambda h: self.outbound_condition(h[0]),
94 response_headers) 114 response_headers)
115
95 if removed: 116 if removed:
96 self.logger.debug('removed response headers: %s' % removed) 117 self.logger.debug('removed response headers: %s' % removed)
97 new_headers = filter( 118 new_headers = filter(
diff --git a/swift/common/middleware/listing_formats.py b/swift/common/middleware/listing_formats.py
new file mode 100644
index 0000000..53d5070
--- /dev/null
+++ b/swift/common/middleware/listing_formats.py
@@ -0,0 +1,211 @@
1# Copyright (c) 2017 OpenStack Foundation
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12# implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import json
17import six
18from xml.etree.cElementTree import Element, SubElement, tostring
19
20from swift.common.constraints import valid_api_version
21from swift.common.http import HTTP_NO_CONTENT
22from swift.common.request_helpers import get_param
23from swift.common.swob import HTTPException, HTTPNotAcceptable, Request, \
24 RESPONSE_REASONS
25
26
27#: Mapping of query string ``format=`` values to their corresponding
28#: content-type values.
29FORMAT2CONTENT_TYPE = {'plain': 'text/plain', 'json': 'application/json',
30 'xml': 'application/xml'}
31#: Maximum size of a valid JSON container listing body. If we receive
32#: a container listing response larger than this, assume it's a staticweb
33#: response and pass it on to the client.
34# Default max object length is 1024, default container listing limit is 1e4;
35# add a fudge factor for things like hash, last_modified, etc.
36MAX_CONTAINER_LISTING_CONTENT_LENGTH = 1024 * 10000 * 2
37
38
39def get_listing_content_type(req):
40 """
41 Determine the content type to use for an account or container listing
42 response.
43
44 :param req: request object
45 :returns: content type as a string (e.g. text/plain, application/json)
46 :raises HTTPNotAcceptable: if the requested content type is not acceptable
47 :raises HTTPBadRequest: if the 'format' query param is provided and
48 not valid UTF-8
49 """
50 query_format = get_param(req, 'format')
51 if query_format:
52 req.accept = FORMAT2CONTENT_TYPE.get(
53 query_format.lower(), FORMAT2CONTENT_TYPE['plain'])
54 out_content_type = req.accept.best_match(
55 ['text/plain', 'application/json', 'application/xml', 'text/xml'])
56 if not out_content_type:
57 raise HTTPNotAcceptable(request=req)
58 return out_content_type
59
60
61def account_to_xml(listing, account_name):
62 doc = Element('account', name=account_name.decode('utf-8'))
63 doc.text = '\n'
64 for record in listing:
65 if 'subdir' in record:
66 name = record.pop('subdir')
67 sub = SubElement(doc, 'subdir', name=name)
68 else:
69 sub = SubElement(doc, 'container')
70 for field in ('name', 'count', 'bytes', 'last_modified'):
71 SubElement(sub, field).text = six.text_type(
72 record.pop(field))
73 sub.tail = '\n'
74 return tostring(doc, encoding='UTF-8').replace(
75 "<?xml version='1.0' encoding='UTF-8'?>",
76 '<?xml version="1.0" encoding="UTF-8"?>', 1)
77
78
79def container_to_xml(listing, base_name):
80 doc = Element('container', name=base_name.decode('utf-8'))
81 for record in listing:
82 if 'subdir' in record:
83 name = record.pop('subdir')
84 sub = SubElement(doc, 'subdir', name=name)
85 SubElement(sub, 'name').text = name
86 else:
87 sub = SubElement(doc, 'object')
88 for field in ('name', 'hash', 'bytes', 'content_type',
89 'last_modified'):
90 SubElement(sub, field).text = six.text_type(
91 record.pop(field))
92 return tostring(doc, encoding='UTF-8').replace(
93 "<?xml version='1.0' encoding='UTF-8'?>",
94 '<?xml version="1.0" encoding="UTF-8"?>', 1)
95
96
97def listing_to_text(listing):
98 def get_lines():
99 for item in listing:
100 if 'name' in item:
101 yield item['name'].encode('utf-8') + b'\n'
102 else:
103 yield item['subdir'].encode('utf-8') + b'\n'
104 return b''.join(get_lines())
105
106
107class ListingFilter(object):
108 def __init__(self, app):
109 self.app = app
110
111 def __call__(self, env, start_response):
112 req = Request(env)
113 try:
114 # account and container only
115 version, acct, cont = req.split_path(2, 3)
116 except ValueError:
117 return self.app(env, start_response)
118
119 if not valid_api_version(version) or req.method not in ('GET', 'HEAD'):
120 return self.app(env, start_response)
121
122 # OK, definitely have an account/container request.
123 # Get the desired content-type, then force it to a JSON request.
124 try:
125 out_content_type = get_listing_content_type(req)
126 except HTTPException as err:
127 return err(env, start_response)
128
129 params = req.params
130 params['format'] = 'json'
131 req.params = params
132
133 status, headers, resp_iter = req.call_application(self.app)
134
135 header_to_index = {}
136 resp_content_type = resp_length = None
137 for i, (header, value) in enumerate(headers):
138 header = header.lower()
139 if header == 'content-type':
140 header_to_index[header] = i
141 resp_content_type = value.partition(';')[0]
142 elif header == 'content-length':
143 header_to_index[header] = i
144 resp_length = int(value)
145
146 if not status.startswith('200 '):
147 start_response(status, headers)
148 return resp_iter
149
150 if resp_content_type != 'application/json':
151 start_response(status, headers)
152 return resp_iter
153
154 if resp_length is None or \
155 resp_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
156 start_response(status, headers)
157 return resp_iter
158
159 def set_header(header, value):
160 if value is None:
161 del headers[header_to_index[header]]
162 else:
163 headers[header_to_index[header]] = (
164 headers[header_to_index[header]][0], str(value))
165
166 if req.method == 'HEAD':
167 set_header('content-type', out_content_type + '; charset=utf-8')
168 set_header('content-length', None) # don't know, can't determine
169 start_response(status, headers)
170 return resp_iter
171
172 body = b''.join(resp_iter)
173 try:
174 listing = json.loads(body)
175 # Do a couple sanity checks
176 if not isinstance(listing, list):
177 raise ValueError
178 if not all(isinstance(item, dict) for item in listing):
179 raise ValueError
180 except ValueError:
181 # Static web listing that's returning invalid JSON?
182 # Just pass it straight through; that's about all we *can* do.
183 start_response(status, headers)
184 return [body]
185
186 try:
187 if out_content_type.endswith('/xml'):
188 if cont:
189 body = container_to_xml(listing, cont)
190 else:
191 body = account_to_xml(listing, acct)
192 elif out_content_type == 'text/plain':
193 body = listing_to_text(listing)
194 # else, json -- we continue down here to be sure we set charset
195 except KeyError:
196 # listing was in a bad format -- funky static web listing??
197 start_response(status, headers)
198 return [body]
199
200 if not body:
201 status = '%s %s' % (HTTP_NO_CONTENT,
202 RESPONSE_REASONS[HTTP_NO_CONTENT][0])
203
204 set_header('content-type', out_content_type + '; charset=utf-8')
205 set_header('content-length', len(body))
206 start_response(status, headers)
207 return [body]
208
209
210def filter_factory(global_conf, **local_conf):
211 return ListingFilter
diff --git a/swift/common/middleware/recon.py b/swift/common/middleware/recon.py
index c9c994f..1ad3624 100644
--- a/swift/common/middleware/recon.py
+++ b/swift/common/middleware/recon.py
@@ -209,7 +209,7 @@ class ReconMiddleware(object):
209 continue 209 continue
210 210
211 try: 211 try:
212 mounted = check_mount(self.devices, entry) 212 mounted = bool(check_mount(self.devices, entry))
213 except OSError as err: 213 except OSError as err:
214 mounted = str(err) 214 mounted = str(err)
215 mpoint = {'device': entry, 'mounted': mounted} 215 mpoint = {'device': entry, 'mounted': mounted}
@@ -225,7 +225,7 @@ class ReconMiddleware(object):
225 continue 225 continue
226 226
227 try: 227 try:
228 mounted = check_mount(self.devices, entry) 228 mounted = bool(check_mount(self.devices, entry))
229 except OSError as err: 229 except OSError as err:
230 devices.append({'device': entry, 'mounted': str(err), 230 devices.append({'device': entry, 'mounted': str(err),
231 'size': '', 'used': '', 'avail': ''}) 231 'size': '', 'used': '', 'avail': ''})
diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py
index c5a2a63..fdc2e9e 100644
--- a/swift/common/middleware/slo.py
+++ b/swift/common/middleware/slo.py
@@ -27,7 +27,7 @@ Uploading the Manifest
27---------------------- 27----------------------
28 28
29After the user has uploaded the objects to be concatenated, a manifest is 29After the user has uploaded the objects to be concatenated, a manifest is
30uploaded. The request must be a PUT with the query parameter:: 30uploaded. The request must be a ``PUT`` with the query parameter::
31 31
32 ?multipart-manifest=put 32 ?multipart-manifest=put
33 33
@@ -47,52 +47,49 @@ range (optional) the (inclusive) range within the object to
47 use as a segment. If omitted, the entire object is used. 47 use as a segment. If omitted, the entire object is used.
48=========== ======================================================== 48=========== ========================================================
49 49
50The format of the list will be: 50The format of the list will be::
51
52 .. code::
53 51
54 [{"path": "/cont/object", 52 [{"path": "/cont/object",
55 "etag": "etagoftheobjectsegment", 53 "etag": "etagoftheobjectsegment",
56 "size_bytes": 10485760, 54 "size_bytes": 10485760,
57 "range": "1048576-2097151"}, ...] 55 "range": "1048576-2097151"},
56 ...]
58 57
59The number of object segments is limited to a configurable amount, default 58The number of object segments is limited to a configurable amount, default
601000. Each segment must be at least 1 byte. On upload, the middleware will 591000. Each segment must be at least 1 byte. On upload, the middleware will
61head every segment passed in to verify: 60head every segment passed in to verify:
62 61
63 1. the segment exists (i.e. the HEAD was successful); 621. the segment exists (i.e. the ``HEAD`` was successful);
64 2. the segment meets minimum size requirements; 632. the segment meets minimum size requirements;
65 3. if the user provided a non-null etag, the etag matches; 643. if the user provided a non-null ``etag``, the etag matches;
66 4. if the user provided a non-null size_bytes, the size_bytes matches; and 654. if the user provided a non-null ``size_bytes``, the size_bytes matches; and
67 5. if the user provided a range, it is a singular, syntactically correct range 665. if the user provided a ``range``, it is a singular, syntactically correct
68 that is satisfiable given the size of the object. 67 range that is satisfiable given the size of the object.
69 68
70Note that the etag and size_bytes keys are optional; if omitted, the 69Note that the ``etag`` and ``size_bytes`` keys are optional; if omitted, the
71verification is not performed. If any of the objects fail to verify (not 70verification is not performed. If any of the objects fail to verify (not
72found, size/etag mismatch, below minimum size, invalid range) then the user 71found, size/etag mismatch, below minimum size, invalid range) then the user
73will receive a 4xx error response. If everything does match, the user will 72will receive a 4xx error response. If everything does match, the user will
74receive a 2xx response and the SLO object is ready for downloading. 73receive a 2xx response and the SLO object is ready for downloading.
75 74
76Behind the scenes, on success, a json manifest generated from the user input is 75Behind the scenes, on success, a JSON manifest generated from the user input is
77sent to object servers with an extra "X-Static-Large-Object: True" header 76sent to object servers with an extra ``X-Static-Large-Object: True`` header
78and a modified Content-Type. The items in this manifest will include the etag 77and a modified ``Content-Type``. The items in this manifest will include the
79and size_bytes for each segment, regardless of whether the client specified 78``etag`` and ``size_bytes`` for each segment, regardless of whether the client
80them for verification. The parameter: swift_bytes=$total_size will be 79specified them for verification. The parameter ``swift_bytes=$total_size`` will
81appended to the existing Content-Type, where total_size is the sum of all 80be appended to the existing ``Content-Type``, where ``$total_size`` is the sum
82the included segments' size_bytes. This extra parameter will be hidden from 81of all the included segments' ``size_bytes``. This extra parameter will be
83the user. 82hidden from the user.
84 83
85Manifest files can reference objects in separate containers, which will improve 84Manifest files can reference objects in separate containers, which will improve
86concurrent upload speed. Objects can be referenced by multiple manifests. The 85concurrent upload speed. Objects can be referenced by multiple manifests. The
87segments of a SLO manifest can even be other SLO manifests. Treat them as any 86segments of a SLO manifest can even be other SLO manifests. Treat them as any
88other object i.e., use the Etag and Content-Length given on the PUT of the 87other object i.e., use the ``Etag`` and ``Content-Length`` given on the ``PUT``
89sub-SLO in the manifest to the parent SLO. 88of the sub-SLO in the manifest to the parent SLO.
90 89
91While uploading a manifest, a user can send Etag for verification. It needs to 90While uploading a manifest, a user can send ``Etag`` for verification. It needs
92be md5 of the segments' etags, if there is no range specified. For example, if 91to be md5 of the segments' etags, if there is no range specified. For example,
93the manifest to be uploaded looks like this: 92if the manifest to be uploaded looks like this::
94
95 .. code::
96 93
97 [{"path": "/cont/object1", 94 [{"path": "/cont/object1",
98 "etag": "etagoftheobjectsegment1", 95 "etag": "etagoftheobjectsegment1",
@@ -101,16 +98,12 @@ the manifest to be uploaded looks like this:
101 "etag": "etagoftheobjectsegment2", 98 "etag": "etagoftheobjectsegment2",
102 "size_bytes": 10485760}] 99 "size_bytes": 10485760}]
103 100
104The Etag of the above manifest would be md5 of etagoftheobjectsegment1 and 101The Etag of the above manifest would be md5 of ``etagoftheobjectsegment1`` and
105etagoftheobjectsegment2. This could be computed in the following way: 102``etagoftheobjectsegment2``. This could be computed in the following way::
106
107 .. code::
108 103
109 echo -n 'etagoftheobjectsegment1etagoftheobjectsegment2' | md5sum 104 echo -n 'etagoftheobjectsegment1etagoftheobjectsegment2' | md5sum
110 105
111If a manifest to be uploaded with a segment range looks like this: 106If a manifest to be uploaded with a segment range looks like this::
112
113 .. code::
114 107
115 [{"path": "/cont/object1", 108 [{"path": "/cont/object1",
116 "etag": "etagoftheobjectsegmentone", 109 "etag": "etagoftheobjectsegmentone",
@@ -122,10 +115,8 @@ If a manifest to be uploaded with a segment range looks like this:
122 "range": "3-4"}] 115 "range": "3-4"}]
123 116
124While computing the Etag of the above manifest, internally each segment's etag 117While computing the Etag of the above manifest, internally each segment's etag
125will be taken in the form of 'etagvalue:rangevalue;'. Hence the Etag of the 118will be taken in the form of ``etagvalue:rangevalue;``. Hence the Etag of the
126above manifest would be: 119above manifest would be::
127
128 .. code::
129 120
130 echo -n 'etagoftheobjectsegmentone:1-2;etagoftheobjectsegmenttwo:3-4;' \ 121 echo -n 'etagoftheobjectsegmentone:1-2;etagoftheobjectsegmenttwo:3-4;' \
131 | md5sum 122 | md5sum
@@ -136,65 +127,65 @@ Range Specification
136------------------- 127-------------------
137 128
138Users now have the ability to specify ranges for SLO segments. 129Users now have the ability to specify ranges for SLO segments.
139Users can now include an optional 'range' field in segment descriptions 130Users can now include an optional ``range`` field in segment descriptions
140to specify which bytes from the underlying object should be used for the 131to specify which bytes from the underlying object should be used for the
141segment data. Only one range may be specified per segment. 132segment data. Only one range may be specified per segment.
142 133
143 .. note:: 134.. note::
144 135
145 The 'etag' and 'size_bytes' fields still describe the backing object as a 136 The ``etag`` and ``size_bytes`` fields still describe the backing object
146 whole. 137 as a whole.
147 138
148If a user uploads this manifest: 139If a user uploads this manifest::
149 140
150 .. code:: 141 [{"path": "/con/obj_seg_1", "size_bytes": 2097152, "range": "0-1048576"},
151 142 {"path": "/con/obj_seg_2", "size_bytes": 2097152,
152 [{"path": "/con/obj_seg_1", "size_bytes": 2097152, "range": "0-1048576"}, 143 "range": "512-1550000"},
153 {"path": "/con/obj_seg_2", "size_bytes": 2097152, 144 {"path": "/con/obj_seg_1", "size_bytes": 2097152, "range": "-2048"}]
154 "range": "512-1550000"},
155 {"path": "/con/obj_seg_1", "size_bytes": 2097152, "range": "-2048"}]
156 145
157The segment will consist of the first 1048576 bytes of /con/obj_seg_1, 146The segment will consist of the first 1048576 bytes of /con/obj_seg_1,
158followed by bytes 513 through 1550000 (inclusive) of /con/obj_seg_2, and 147followed by bytes 513 through 1550000 (inclusive) of /con/obj_seg_2, and
159finally bytes 2095104 through 2097152 (i.e., the last 2048 bytes) of 148finally bytes 2095104 through 2097152 (i.e., the last 2048 bytes) of
160/con/obj_seg_1. 149/con/obj_seg_1.
161 150
162 .. note:: 151.. note::
163 152
164 153
165 The minimum sized range is 1 byte. This is the same as the minimum 154 The minimum sized range is 1 byte. This is the same as the minimum
166 segment size. 155 segment size.
167 156
168 157
169------------------------- 158-------------------------
170Retrieving a Large Object 159Retrieving a Large Object
171------------------------- 160-------------------------
172 161
173A GET request to the manifest object will return the concatenation of the 162A ``GET`` request to the manifest object will return the concatenation of the
174objects from the manifest much like DLO. If any of the segments from the 163objects from the manifest much like DLO. If any of the segments from the
175manifest are not found or their Etag/Content Length have changed since upload, 164manifest are not found or their ``Etag``/``Content-Length`` have changed since
176the connection will drop. In this case a 409 Conflict will be logged in the 165upload, the connection will drop. In this case a ``409 Conflict`` will be
177proxy logs and the user will receive incomplete results. Note that this will be 166logged in the proxy logs and the user will receive incomplete results. Note
178enforced regardless of whether the user performed per-segment validation during 167that this will be enforced regardless of whether the user performed per-segment
179upload. 168validation during upload.
180 169
181The headers from this GET or HEAD request will return the metadata attached 170The headers from this ``GET`` or ``HEAD`` request will return the metadata
182to the manifest object itself with some exceptions:: 171attached to the manifest object itself with some exceptions:
183 172
184 Content-Length: the total size of the SLO (the sum of the sizes of 173===================== ==================================================
185 the segments in the manifest) 174Header Value
186 X-Static-Large-Object: True 175===================== ==================================================
187 Etag: the etag of the SLO (generated the same way as DLO) 176Content-Length the total size of the SLO (the sum of the sizes of
188 177 the segments in the manifest)
189A GET request with the query parameter:: 178X-Static-Large-Object the string "True"
179Etag the etag of the SLO (generated the same way as DLO)
180===================== ==================================================
181
182A ``GET`` request with the query parameter::
190 183
191 ?multipart-manifest=get 184 ?multipart-manifest=get
192 185
193will return a transformed version of the original manifest, containing 186will return a transformed version of the original manifest, containing
194additional fields and different key names. For example, the first manifest in 187additional fields and different key names. For example, the first manifest in
195the example above would look like this: 188the example above would look like this::
196
197 .. code::
198 189
199 [{"name": "/cont/object", 190 [{"name": "/cont/object",
200 "hash": "etagoftheobjectsegment", 191 "hash": "etagoftheobjectsegment",
@@ -222,9 +213,10 @@ left to the user to use caution in handling the segments.
222Deleting a Large Object 213Deleting a Large Object
223----------------------- 214-----------------------
224 215
225A DELETE request will just delete the manifest object itself. 216A ``DELETE`` request will just delete the manifest object itself. The segment
217data referenced by the manifest will remain unchanged.
226 218
227A DELETE with a query parameter:: 219A ``DELETE`` with a query parameter::
228 220
229 ?multipart-manifest=delete 221 ?multipart-manifest=delete
230 222
@@ -235,22 +227,22 @@ itself. The failure response will be similar to the bulk delete middleware.
235Modifying a Large Object 227Modifying a Large Object
236------------------------ 228------------------------
237 229
238PUTs / POSTs will work as expected, PUTs will just overwrite the manifest 230``PUT`` and ``POST`` requests will work as expected; ``PUT``\s will just
239object for example. 231overwrite the manifest object for example.
240 232
241------------------ 233------------------
242Container Listings 234Container Listings
243------------------ 235------------------
244 236
245In a container listing the size listed for SLO manifest objects will be the 237In a container listing the size listed for SLO manifest objects will be the
246total_size of the concatenated segments in the manifest. The overall 238``total_size`` of the concatenated segments in the manifest. The overall
247X-Container-Bytes-Used for the container (and subsequently for the account) 239``X-Container-Bytes-Used`` for the container (and subsequently for the account)
248will not reflect total_size of the manifest but the actual size of the json 240will not reflect ``total_size`` of the manifest but the actual size of the JSON
249data stored. The reason for this somewhat confusing discrepancy is we want the 241data stored. The reason for this somewhat confusing discrepancy is we want the
250container listing to reflect the size of the manifest object when it is 242container listing to reflect the size of the manifest object when it is
251downloaded. We do not, however, want to count the bytes-used twice (for both 243downloaded. We do not, however, want to count the bytes-used twice (for both
252the manifest and the segments it's referring to) in the container and account 244the manifest and the segments it's referring to) in the container and account
253metadata which can be used for stats purposes. 245metadata which can be used for stats and billing purposes.
254""" 246"""
255 247
256from collections import defaultdict 248from collections import defaultdict
@@ -296,20 +288,20 @@ def parse_and_validate_input(req_body, req_path):
296 Given a request body, parses it and returns a list of dictionaries. 288 Given a request body, parses it and returns a list of dictionaries.
297 289
298 The output structure is nearly the same as the input structure, but it 290 The output structure is nearly the same as the input structure, but it
299 is not an exact copy. Given a valid input dictionary `d_in`, its 291 is not an exact copy. Given a valid input dictionary ``d_in``, its
300 corresponding output dictionary `d_out` will be as follows: 292 corresponding output dictionary ``d_out`` will be as follows:
301 293
302 * d_out['etag'] == d_in['etag'] 294 * d_out['etag'] == d_in['etag']
303 295
304 * d_out['path'] == d_in['path'] 296 * d_out['path'] == d_in['path']
305 297
306 * d_in['size_bytes'] can be a string ("12") or an integer (12), but 298 * d_in['size_bytes'] can be a string ("12") or an integer (12), but
307 d_out['size_bytes'] is an integer. 299 d_out['size_bytes'] is an integer.
308 300
309 * (optional) d_in['range'] is a string of the form "M-N", "M-", or 301 * (optional) d_in['range'] is a string of the form "M-N", "M-", or
310 "-N", where M and N are non-negative integers. d_out['range'] is the 302 "-N", where M and N are non-negative integers. d_out['range'] is the
311 corresponding swob.Range object. If d_in does not have a key 303 corresponding swob.Range object. If d_in does not have a key
312 'range', neither will d_out. 304 'range', neither will d_out.
313 305
314 :raises HTTPException: on parse errors or semantic errors (e.g. bogus 306 :raises HTTPException: on parse errors or semantic errors (e.g. bogus
315 JSON structure, syntactically invalid ranges) 307 JSON structure, syntactically invalid ranges)
@@ -435,7 +427,7 @@ class SloGetContext(WSGIContext):
435 agent='%(orig)s SLO MultipartGET', swift_source='SLO') 427 agent='%(orig)s SLO MultipartGET', swift_source='SLO')
436 sub_resp = sub_req.get_response(self.slo.app) 428 sub_resp = sub_req.get_response(self.slo.app)
437 429
438 if not is_success(sub_resp.status_int): 430 if not sub_resp.is_success:
439 close_if_possible(sub_resp.app_iter) 431 close_if_possible(sub_resp.app_iter)
440 raise ListingIterError( 432 raise ListingIterError(
441 'ERROR: while fetching %s, GET of submanifest %s ' 433 'ERROR: while fetching %s, GET of submanifest %s '
@@ -615,8 +607,9 @@ class SloGetContext(WSGIContext):
615 thing with them. Returns an iterator suitable for sending up the WSGI 607 thing with them. Returns an iterator suitable for sending up the WSGI
616 chain. 608 chain.
617 609
618 :param req: swob.Request object; is a GET or HEAD request aimed at 610 :param req: :class:`~swift.common.swob.Request` object; is a ``GET`` or
619 what may be a static large object manifest (or may not). 611 ``HEAD`` request aimed at what may (or may not) be a static
612 large object manifest.
620 :param start_response: WSGI start_response callable 613 :param start_response: WSGI start_response callable
621 """ 614 """
622 if req.params.get('multipart-manifest') != 'get': 615 if req.params.get('multipart-manifest') != 'get':
@@ -898,7 +891,9 @@ class StaticLargeObject(object):
898 The response body (only on GET, of course) will consist of the 891 The response body (only on GET, of course) will consist of the
899 concatenation of the segments. 892 concatenation of the segments.
900 893
901 :params req: a swob.Request with a path referencing an object 894 :param req: a :class:`~swift.common.swob.Request` with a path
895 referencing an object
896 :param start_response: WSGI start_response callable
902 :raises HttpException: on errors 897 :raises HttpException: on errors
903 """ 898 """
904 return SloGetContext(self).handle_slo_get_or_head(req, start_response) 899 return SloGetContext(self).handle_slo_get_or_head(req, start_response)
@@ -910,13 +905,11 @@ class StaticLargeObject(object):
910 save a manifest generated from the user input. Uses WSGIContext to 905 save a manifest generated from the user input. Uses WSGIContext to
911 call self and start_response and returns a WSGI iterator. 906 call self and start_response and returns a WSGI iterator.
912 907
913 :params req: a swob.Request with an obj in path 908 :param req: a :class:`~swift.common.swob.Request` with an obj in path
909 :param start_response: WSGI start_response callable
914 :raises HttpException: on errors 910 :raises HttpException: on errors
915 """ 911 """
916 try: 912 vrs, account, container, obj = req.split_path(4, rest_with_last=True)
917 vrs, account, container, obj = req.split_path(1, 4, True)
918 except ValueError:
919 return self.app(req.environ, start_response)
920 if req.content_length > self.max_manifest_size: 913 if req.content_length > self.max_manifest_size:
921 raise HTTPRequestEntityTooLarge( 914 raise HTTPRequestEntityTooLarge(
922 "Manifest File > %d bytes" % self.max_manifest_size) 915 "Manifest File > %d bytes" % self.max_manifest_size)
@@ -1073,7 +1066,8 @@ class StaticLargeObject(object):
1073 A generator function to be used to delete all the segments and 1066 A generator function to be used to delete all the segments and
1074 sub-segments referenced in a manifest. 1067 sub-segments referenced in a manifest.
1075 1068
1076 :params req: a swob.Request with an SLO manifest in path 1069 :param req: a :class:`~swift.common.swob.Request` with an SLO manifest
1070 in path
1077 :raises HTTPPreconditionFailed: on invalid UTF8 in request path 1071 :raises HTTPPreconditionFailed: on invalid UTF8 in request path
1078 :raises HTTPBadRequest: on too many buffered sub segments and 1072 :raises HTTPBadRequest: on too many buffered sub segments and
1079 on invalid SLO manifest path 1073 on invalid SLO manifest path
@@ -1109,8 +1103,12 @@ class StaticLargeObject(object):
1109 1103
1110 def get_slo_segments(self, obj_name, req): 1104 def get_slo_segments(self, obj_name, req):
1111 """ 1105 """
1112 Performs a swob.Request and returns the SLO manifest's segments. 1106 Performs a :class:`~swift.common.swob.Request` and returns the SLO
1107 manifest's segments.
1113 1108
1109 :param obj_name: the name of the object being deleted,
1110 as ``/container/object``
1111 :param req: the base :class:`~swift.common.swob.Request`
1114 :raises HTTPServerError: on unable to load obj_name or 1112 :raises HTTPServerError: on unable to load obj_name or
1115 on unable to load the SLO manifest data. 1113 on unable to load the SLO manifest data.
1116 :raises HTTPBadRequest: on not an SLO manifest 1114 :raises HTTPBadRequest: on not an SLO manifest
@@ -1151,7 +1149,7 @@ class StaticLargeObject(object):
1151 Will delete all the segments in the SLO manifest and then, if 1149 Will delete all the segments in the SLO manifest and then, if
1152 successful, will delete the manifest file. 1150 successful, will delete the manifest file.
1153 1151
1154 :params req: a swob.Request with an obj in path 1152 :param req: a :class:`~swift.common.swob.Request` with an obj in path
1155 :returns: swob.Response whose app_iter set to Bulk.handle_delete_iter 1153 :returns: swob.Response whose app_iter set to Bulk.handle_delete_iter
1156 """ 1154 """
1157 req.headers['Content-Type'] = None # Ignore content-type from client 1155 req.headers['Content-Type'] = None # Ignore content-type from client
diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py
index 786aef3..d01c753 100644
--- a/swift/common/middleware/staticweb.py
+++ b/swift/common/middleware/staticweb.py
@@ -260,7 +260,7 @@ class _StaticWebContext(WSGIContext):
260 env, 'GET', '/%s/%s/%s' % ( 260 env, 'GET', '/%s/%s/%s' % (
261 self.version, self.account, self.container), 261 self.version, self.account, self.container),
262 self.agent, swift_source='SW') 262 self.agent, swift_source='SW')
263 tmp_env['QUERY_STRING'] = 'delimiter=/&format=json' 263 tmp_env['QUERY_STRING'] = 'delimiter=/'
264 if prefix: 264 if prefix:
265 tmp_env['QUERY_STRING'] += '&prefix=%s' % quote(prefix) 265 tmp_env['QUERY_STRING'] += '&prefix=%s' % quote(prefix)
266 else: 266 else:
@@ -465,8 +465,8 @@ class _StaticWebContext(WSGIContext):
465 env, 'GET', '/%s/%s/%s' % ( 465 env, 'GET', '/%s/%s/%s' % (
466 self.version, self.account, self.container), 466 self.version, self.account, self.container),
467 self.agent, swift_source='SW') 467 self.agent, swift_source='SW')
468 tmp_env['QUERY_STRING'] = 'limit=1&format=json&delimiter' \ 468 tmp_env['QUERY_STRING'] = 'limit=1&delimiter=/&prefix=%s' % (
469 '=/&limit=1&prefix=%s' % quote(self.obj + '/') 469 quote(self.obj + '/'), )
470 resp = self._app_call(tmp_env) 470 resp = self._app_call(tmp_env)
471 body = ''.join(resp) 471 body = ''.join(resp)
472 if not is_success(self._get_status_int()) or not body or \ 472 if not is_success(self._get_status_int()) or not body or \
diff --git a/swift/common/middleware/tempauth.py b/swift/common/middleware/tempauth.py
index 9e9621b..3c64090 100644
--- a/swift/common/middleware/tempauth.py
+++ b/swift/common/middleware/tempauth.py
@@ -177,8 +177,6 @@ from __future__ import print_function
177from time import time 177from time import time
178from traceback import format_exc 178from traceback import format_exc
179from uuid import uuid4 179from uuid import uuid4
180from hashlib import sha1
181import hmac
182import base64 180import base64
183 181
184from eventlet import Timeout 182from eventlet import Timeout
@@ -437,20 +435,21 @@ class TempAuth(object):
437 435
438 s3_auth_details = env.get('swift3.auth_details') 436 s3_auth_details = env.get('swift3.auth_details')
439 if s3_auth_details: 437 if s3_auth_details:
438 if 'check_signature' not in s3_auth_details:
439 self.logger.warning(
440 'Swift3 did not provide a check_signature function; '
441 'upgrade Swift3 if you want to use it with tempauth')
442 return None
440 account_user = s3_auth_details['access_key'] 443 account_user = s3_auth_details['access_key']
441 signature_from_user = s3_auth_details['signature']
442 if account_user not in self.users: 444 if account_user not in self.users:
443 return None 445 return None
444 account, user = account_user.split(':', 1) 446 user = self.users[account_user]
445 account_id = self.users[account_user]['url'].rsplit('/', 1)[-1] 447 account = account_user.split(':', 1)[0]
446 path = env['PATH_INFO'] 448 account_id = user['url'].rsplit('/', 1)[-1]
447 env['PATH_INFO'] = path.replace(account_user, account_id, 1) 449 if not s3_auth_details['check_signature'](user['key']):
448 valid_signature = base64.encodestring(hmac.new(
449 self.users[account_user]['key'],
450 s3_auth_details['string_to_sign'],
451 sha1).digest()).strip()
452 if signature_from_user != valid_signature:
453 return None 450 return None
451 env['PATH_INFO'] = env['PATH_INFO'].replace(
452 account_user, account_id, 1)
454 groups = self._get_user_groups(account, account_user, account_id) 453 groups = self._get_user_groups(account, account_user, account_id)
455 454
456 return groups 455 return groups
diff --git a/swift/common/middleware/versioned_writes.py b/swift/common/middleware/versioned_writes.py
index a21c956..ee834fc 100644
--- a/swift/common/middleware/versioned_writes.py
+++ b/swift/common/middleware/versioned_writes.py
@@ -329,8 +329,7 @@ class VersionedWritesContext(WSGIContext):
329 env, method='GET', swift_source='VW', 329 env, method='GET', swift_source='VW',
330 path='/v1/%s/%s' % (account_name, lcontainer)) 330 path='/v1/%s/%s' % (account_name, lcontainer))
331 lreq.environ['QUERY_STRING'] = \ 331 lreq.environ['QUERY_STRING'] = \
332 'format=json&prefix=%s&marker=%s' % ( 332 'prefix=%s&marker=%s' % (quote(lprefix), quote(marker))
333 quote(lprefix), quote(marker))
334 if end_marker: 333 if end_marker:
335 lreq.environ['QUERY_STRING'] += '&end_marker=%s' % ( 334 lreq.environ['QUERY_STRING'] += '&end_marker=%s' % (
336 quote(end_marker)) 335 quote(end_marker))
@@ -826,8 +825,7 @@ class VersionedWritesMiddleware(object):
826 allow_versioned_writes) 825 allow_versioned_writes)
827 except HTTPException as error_response: 826 except HTTPException as error_response:
828 return error_response(env, start_response) 827 return error_response(env, start_response)
829 elif (obj and req.method in ('PUT', 'DELETE') and 828 elif (obj and req.method in ('PUT', 'DELETE')):
830 not req.environ.get('swift.post_as_copy')):
831 try: 829 try:
832 return self.object_request( 830 return self.object_request(
833 req, api_version, account, container, obj, 831 req, api_version, account, container, obj,
diff --git a/swift/common/request_helpers.py b/swift/common/request_helpers.py
index 5fdf346..5caa73c 100644
--- a/swift/common/request_helpers.py
+++ b/swift/common/request_helpers.py
@@ -31,10 +31,9 @@ from swift.common.header_key_dict import HeaderKeyDict
31 31
32from swift import gettext_ as _ 32from swift import gettext_ as _
33from swift.common.storage_policy import POLICIES 33from swift.common.storage_policy import POLICIES
34from swift.common.constraints import FORMAT2CONTENT_TYPE
35from swift.common.exceptions import ListingIterError, SegmentError 34from swift.common.exceptions import ListingIterError, SegmentError
36from swift.common.http import is_success 35from swift.common.http import is_success
37from swift.common.swob import HTTPBadRequest, HTTPNotAcceptable, \ 36from swift.common.swob import HTTPBadRequest, \
38 HTTPServiceUnavailable, Range, is_chunked, multi_range_iterator 37 HTTPServiceUnavailable, Range, is_chunked, multi_range_iterator
39from swift.common.utils import split_path, validate_device_partition, \ 38from swift.common.utils import split_path, validate_device_partition, \
40 close_if_possible, maybe_multipart_byteranges_to_document_iters, \ 39 close_if_possible, maybe_multipart_byteranges_to_document_iters, \
@@ -70,28 +69,6 @@ def get_param(req, name, default=None):
70 return value 69 return value
71 70
72 71
73def get_listing_content_type(req):
74 """
75 Determine the content type to use for an account or container listing
76 response.
77
78 :param req: request object
79 :returns: content type as a string (e.g. text/plain, application/json)
80 :raises HTTPNotAcceptable: if the requested content type is not acceptable
81 :raises HTTPBadRequest: if the 'format' query param is provided and
82 not valid UTF-8
83 """
84 query_format = get_param(req, 'format')
85 if query_format:
86 req.accept = FORMAT2CONTENT_TYPE.get(
87 query_format.lower(), FORMAT2CONTENT_TYPE['plain'])
88 out_content_type = req.accept.best_match(
89 ['text/plain', 'application/json', 'application/xml', 'text/xml'])
90 if not out_content_type:
91 raise HTTPNotAcceptable(request=req)
92 return out_content_type
93
94
95def get_name_and_placement(request, minsegs=1, maxsegs=None, 72def get_name_and_placement(request, minsegs=1, maxsegs=None,
96 rest_with_last=False): 73 rest_with_last=False):
97 """ 74 """
diff --git a/swift/common/ring/builder.py b/swift/common/ring/builder.py
index e1b4ccd..d6902dc 100644
--- a/swift/common/ring/builder.py
+++ b/swift/common/ring/builder.py
@@ -525,7 +525,9 @@ class RingBuilder(object):
525 525
526 # we'll gather a few times, or until we archive the plan 526 # we'll gather a few times, or until we archive the plan
527 for gather_count in range(MAX_BALANCE_GATHER_COUNT): 527 for gather_count in range(MAX_BALANCE_GATHER_COUNT):
528 self._gather_parts_for_balance(assign_parts, replica_plan) 528 self._gather_parts_for_balance(assign_parts, replica_plan,
529 # firsrt attempt go for disperse
530 gather_count == 0)
529 if not assign_parts: 531 if not assign_parts:
530 # most likely min part hours 532 # most likely min part hours
531 finish_status = 'Unable to finish' 533 finish_status = 'Unable to finish'
@@ -1097,6 +1099,12 @@ class RingBuilder(object):
1097 :param start: offset into self.parts to begin search 1099 :param start: offset into self.parts to begin search
1098 :param replica_plan: replicanth targets for tiers 1100 :param replica_plan: replicanth targets for tiers
1099 """ 1101 """
1102 tier2children = self._build_tier2children()
1103 parts_wanted_in_tier = defaultdict(int)
1104 for dev in self._iter_devs():
1105 wanted = max(dev['parts_wanted'], 0)
1106 for tier in dev['tiers']:
1107 parts_wanted_in_tier[tier] += wanted
1100 # Last, we gather partitions from devices that are "overweight" because 1108 # Last, we gather partitions from devices that are "overweight" because
1101 # they have more partitions than their parts_wanted. 1109 # they have more partitions than their parts_wanted.
1102 for offset in range(self.parts): 1110 for offset in range(self.parts):
@@ -1128,8 +1136,17 @@ class RingBuilder(object):
1128 replicas_at_tier[tier] < 1136 replicas_at_tier[tier] <
1129 replica_plan[tier]['max'] 1137 replica_plan[tier]['max']
1130 for tier in dev['tiers']): 1138 for tier in dev['tiers']):
1139 # we're stuck by replica plan
1140 continue
1141 for t in reversed(dev['tiers']):
1142 if replicas_at_tier[t] - 1 < replica_plan[t]['min']:
1143 # we're stuck at tier t
1144 break
1145 if sum(parts_wanted_in_tier[c]
1146 for c in tier2children[t]
1147 if c not in dev['tiers']) <= 0:
1148 # we're stuck by weight
1131 continue 1149 continue
1132
1133 # this is the most overweight_device holding a replica 1150 # this is the most overweight_device holding a replica
1134 # of this part that can shed it according to the plan 1151 # of this part that can shed it according to the plan
1135 dev['parts_wanted'] += 1 1152 dev['parts_wanted'] += 1
@@ -1141,15 +1158,19 @@ class RingBuilder(object):
1141 self._replica2part2dev[replica][part] = NONE_DEV 1158 self._replica2part2dev[replica][part] = NONE_DEV
1142 for tier in dev['tiers']: 1159 for tier in dev['tiers']:
1143 replicas_at_tier[tier] -= 1 1160 replicas_at_tier[tier] -= 1
1161 parts_wanted_in_tier[tier] -= 1
1144 self._set_part_moved(part) 1162 self._set_part_moved(part)
1145 break 1163 break
1146 1164
1147 def _gather_parts_for_balance(self, assign_parts, replica_plan): 1165 def _gather_parts_for_balance(self, assign_parts, replica_plan,
1166 disperse_first):
1148 """ 1167 """
1149 Gather parts that look like they should move for balance reasons. 1168 Gather parts that look like they should move for balance reasons.
1150 1169
1151 A simple gathers of parts that looks dispersible normally works out, 1170 A simple gathers of parts that looks dispersible normally works out,
1152 we'll switch strategies if things don't seem to move. 1171 we'll switch strategies if things don't seem to move.
1172 :param disperse_first: boolean, avoid replicas on overweight devices
1173 that need to be there for dispersion
1153 """ 1174 """
1154 # pick a random starting point on the other side of the ring 1175 # pick a random starting point on the other side of the ring
1155 quarter_turn = (self.parts // 4) 1176 quarter_turn = (self.parts // 4)
@@ -1162,10 +1183,10 @@ class RingBuilder(object):
1162 'last_start': self._last_part_gather_start}) 1183 'last_start': self._last_part_gather_start})
1163 self._last_part_gather_start = start 1184 self._last_part_gather_start = start
1164 1185
1165 self._gather_parts_for_balance_can_disperse( 1186 if disperse_first:
1166 assign_parts, start, replica_plan) 1187 self._gather_parts_for_balance_can_disperse(
1167 if not assign_parts: 1188 assign_parts, start, replica_plan)
1168 self._gather_parts_for_balance_forced(assign_parts, start) 1189 self._gather_parts_for_balance_forced(assign_parts, start)
1169 1190
1170 def _gather_parts_for_balance_forced(self, assign_parts, start, **kwargs): 1191 def _gather_parts_for_balance_forced(self, assign_parts, start, **kwargs):
1171 """ 1192 """
diff --git a/swift/common/utils.py b/swift/common/utils.py
index 6723b99..31c9e1d 100644
--- a/swift/common/utils.py
+++ b/swift/common/utils.py
@@ -52,6 +52,7 @@ import datetime
52import eventlet 52import eventlet
53import eventlet.debug 53import eventlet.debug
54import eventlet.greenthread 54import eventlet.greenthread
55import eventlet.patcher
55import eventlet.semaphore 56import eventlet.semaphore
56from eventlet import GreenPool, sleep, Timeout, tpool 57from eventlet import GreenPool, sleep, Timeout, tpool
57from eventlet.green import socket, threading 58from eventlet.green import socket, threading
@@ -77,12 +78,6 @@ from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND, \
77from swift.common.header_key_dict import HeaderKeyDict 78from swift.common.header_key_dict import HeaderKeyDict
78from swift.common.linkat import linkat 79from swift.common.linkat import linkat
79 80
80if six.PY3:
81 stdlib_queue = eventlet.patcher.original('queue')
82else:
83 stdlib_queue = eventlet.patcher.original('Queue')
84stdlib_threading = eventlet.patcher.original('threading')
85
86# logging doesn't import patched as cleanly as one would like 81# logging doesn't import patched as cleanly as one would like
87from logging.handlers import SysLogHandler 82from logging.handlers import SysLogHandler
88import logging 83import logging
@@ -470,6 +465,18 @@ def config_read_prefixed_options(conf, prefix_name, defaults):
470 return params 465 return params
471 466
472 467
468def eventlet_monkey_patch():
469 """
470 Install the appropriate Eventlet monkey patches.
471 """
472 # NOTE(sileht):
473 # monkey-patching thread is required by python-keystoneclient;
474 # monkey-patching select is required by oslo.messaging pika driver
475 # if thread is monkey-patched.
476 eventlet.patcher.monkey_patch(all=False, socket=True, select=True,
477 thread=True)
478
479
473def noop_libc_function(*args): 480def noop_libc_function(*args):
474 return 0 481 return 0
475 482
diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py
index 72eee02..add551c 100644
--- a/swift/common/wsgi.py
+++ b/swift/common/wsgi.py
@@ -412,12 +412,7 @@ def run_server(conf, logger, sock, global_conf=None):
412 wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60) 412 wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)
413 413
414 eventlet.hubs.use_hub(get_hub()) 414 eventlet.hubs.use_hub(get_hub())
415 # NOTE(sileht): 415 utils.eventlet_monkey_patch()
416 # monkey-patching thread is required by python-keystoneclient;
417 # monkey-patching select is required by oslo.messaging pika driver
418 # if thread is monkey-patched.
419 eventlet.patcher.monkey_patch(all=False, socket=True, select=True,
420 thread=True)
421 eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no')) 416 eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
422 eventlet.debug.hub_exceptions(eventlet_debug) 417 eventlet.debug.hub_exceptions(eventlet_debug)
423 wsgi_logger = NullLogger() 418 wsgi_logger = NullLogger()
diff --git a/swift/container/server.py b/swift/container/server.py
index c71925f..974a367 100644
--- a/swift/container/server.py
+++ b/swift/container/server.py
@@ -19,7 +19,6 @@ import time
19import traceback 19import traceback
20import math 20import math
21from swift import gettext_ as _ 21from swift import gettext_ as _
22from xml.etree.cElementTree import Element, SubElement, tostring
23 22
24from eventlet import Timeout 23from eventlet import Timeout
25 24
@@ -29,17 +28,18 @@ from swift.container.backend import ContainerBroker, DATADIR
29from swift.container.replicator import ContainerReplicatorRpc 28from swift.container.replicator import ContainerReplicatorRpc
30from swift.common.db import DatabaseAlreadyExists 29from swift.common.db import DatabaseAlreadyExists
31from swift.common.container_sync_realms import ContainerSyncRealms 30from swift.common.container_sync_realms import ContainerSyncRealms
32from swift.common.request_helpers import get_param, get_listing_content_type, \ 31from swift.common.request_helpers import get_param, \
33 split_and_validate_path, is_sys_or_user_meta 32 split_and_validate_path, is_sys_or_user_meta
34from swift.common.utils import get_logger, hash_path, public, \ 33from swift.common.utils import get_logger, hash_path, public, \
35 Timestamp, storage_directory, validate_sync_to, \ 34 Timestamp, storage_directory, validate_sync_to, \
36 config_true_value, timing_stats, replication, \ 35 config_true_value, timing_stats, replication, \
37 override_bytes_from_content_type, get_log_line 36 override_bytes_from_content_type, get_log_line
38from swift.common.constraints import check_mount, valid_timestamp, check_utf8 37from swift.common.constraints import valid_timestamp, check_utf8, check_drive
39from swift.common import constraints 38from swift.common import constraints
40from swift.common.bufferedhttp import http_connect 39from swift.common.bufferedhttp import http_connect
41from swift.common.exceptions import ConnectionTimeout 40from swift.common.exceptions import ConnectionTimeout
42from swift.common.http import HTTP_NOT_FOUND, is_success 41from swift.common.http import HTTP_NOT_FOUND, is_success
42from swift.common.middleware import listing_formats
43from swift.common.storage_policy import POLICIES 43from swift.common.storage_policy import POLICIES
44from swift.common.base_storage_server import BaseStorageServer 44from swift.common.base_storage_server import BaseStorageServer
45from swift.common.header_key_dict import HeaderKeyDict 45from swift.common.header_key_dict import HeaderKeyDict
@@ -111,6 +111,11 @@ class ContainerController(BaseStorageServer):
111 conf.get('auto_create_account_prefix') or '.' 111 conf.get('auto_create_account_prefix') or '.'
112 if config_true_value(conf.get('allow_versions', 'f')): 112 if config_true_value(conf.get('allow_versions', 'f')):
113 self.save_headers.append('x-versions-location') 113 self.save_headers.append('x-versions-location')
114 if 'allow_versions' in conf:
115 self.logger.warning('Option allow_versions is deprecated. '
116 'Configure the versioned_writes middleware in '
117 'the proxy-server instead. This option will '
118 'be ignored in a future release.')
114 swift.common.db.DB_PREALLOCATION = \ 119 swift.common.db.DB_PREALLOCATION = \
115 config_true_value(conf.get('db_preallocation', 'f')) 120 config_true_value(conf.get('db_preallocation', 'f'))
116 self.sync_store = ContainerSyncStore(self.root, 121 self.sync_store = ContainerSyncStore(self.root,
@@ -263,7 +268,7 @@ class ContainerController(BaseStorageServer):
263 drive, part, account, container, obj = split_and_validate_path( 268 drive, part, account, container, obj = split_and_validate_path(
264 req, 4, 5, True) 269 req, 4, 5, True)
265 req_timestamp = valid_timestamp(req) 270 req_timestamp = valid_timestamp(req)
266 if self.mount_check and not check_mount(self.root, drive): 271 if not check_drive(self.root, drive, self.mount_check):
267 return HTTPInsufficientStorage(drive=drive, request=req) 272 return HTTPInsufficientStorage(drive=drive, request=req)
268 # policy index is only relevant for delete_obj (and transitively for 273 # policy index is only relevant for delete_obj (and transitively for
269 # auto create accounts) 274 # auto create accounts)
@@ -351,7 +356,7 @@ class ContainerController(BaseStorageServer):
351 self.realms_conf) 356 self.realms_conf)
352 if err: 357 if err:
353 return HTTPBadRequest(err) 358 return HTTPBadRequest(err)
354 if self.mount_check and not check_mount(self.root, drive): 359 if not check_drive(self.root, drive, self.mount_check):
355 return HTTPInsufficientStorage(drive=drive, request=req) 360 return HTTPInsufficientStorage(drive=drive, request=req)
356 requested_policy_index = self.get_and_validate_policy_index(req) 361 requested_policy_index = self.get_and_validate_policy_index(req)
357 broker = self._get_container_broker(drive, part, account, container) 362 broker = self._get_container_broker(drive, part, account, container)
@@ -418,8 +423,8 @@ class ContainerController(BaseStorageServer):
418 """Handle HTTP HEAD request.""" 423 """Handle HTTP HEAD request."""
419 drive, part, account, container, obj = split_and_validate_path( 424 drive, part, account, container, obj = split_and_validate_path(
420 req, 4, 5, True) 425 req, 4, 5, True)
421 out_content_type = get_listing_content_type(req) 426 out_content_type = listing_formats.get_listing_content_type(req)
422 if self.mount_check and not check_mount(self.root, drive): 427 if not check_drive(self.root, drive, self.mount_check):
423 return HTTPInsufficientStorage(drive=drive, request=req) 428 return HTTPInsufficientStorage(drive=drive, request=req)
424 broker = self._get_container_broker(drive, part, account, container, 429 broker = self._get_container_broker(drive, part, account, container,
425 pending_timeout=0.1, 430 pending_timeout=0.1,
@@ -451,8 +456,8 @@ class ContainerController(BaseStorageServer):
451 """ 456 """
452 (name, created, size, content_type, etag) = record[:5] 457 (name, created, size, content_type, etag) = record[:5]
453 if content_type is None: 458 if content_type is None:
454 return {'subdir': name} 459 return {'subdir': name.decode('utf8')}
455 response = {'bytes': size, 'hash': etag, 'name': name, 460 response = {'bytes': size, 'hash': etag, 'name': name.decode('utf8'),
456 'content_type': content_type} 461 'content_type': content_type}
457 response['last_modified'] = Timestamp(created).isoformat 462 response['last_modified'] = Timestamp(created).isoformat
458 override_bytes_from_content_type(response, logger=self.logger) 463 override_bytes_from_content_type(response, logger=self.logger)
@@ -482,8 +487,8 @@ class ContainerController(BaseStorageServer):
482 request=req, 487 request=req,
483 body='Maximum limit is %d' 488 body='Maximum limit is %d'
484 % constraints.CONTAINER_LISTING_LIMIT) 489 % constraints.CONTAINER_LISTING_LIMIT)
485 out_content_type = get_listing_content_type(req) 490 out_content_type = listing_formats.get_listing_content_type(req)
486 if self.mount_check and not check_mount(self.root, drive): 491 if not check_drive(self.root, drive, self.mount_check):
487 return HTTPInsufficientStorage(drive=drive, request=req) 492 return HTTPInsufficientStorage(drive=drive, request=req)
488 broker = self._get_container_broker(drive, part, account, container, 493 broker = self._get_container_broker(drive, part, account, container,
489 pending_timeout=0.1, 494 pending_timeout=0.1,
@@ -504,36 +509,20 @@ class ContainerController(BaseStorageServer):
504 if value and (key.lower() in self.save_headers or 509 if value and (key.lower() in self.save_headers or
505 is_sys_or_user_meta('container', key)): 510 is_sys_or_user_meta('container', key)):
506 resp_headers[key] = value 511 resp_headers[key] = value
507 ret = Response(request=req, headers=resp_headers, 512 listing = [self.update_data_record(record)
508 content_type=out_content_type, charset='utf-8') 513 for record in container_list]
509 if out_content_type == 'application/json': 514 if out_content_type.endswith('/xml'):
510 ret.body = json.dumps([self.update_data_record(record) 515 body = listing_formats.container_to_xml(listing, container)
511 for record in container_list]) 516 elif out_content_type.endswith('/json'):
512 elif out_content_type.endswith('/xml'): 517 body = json.dumps(listing)
513 doc = Element('container', name=container.decode('utf-8'))
514 for obj in container_list:
515 record = self.update_data_record(obj)
516 if 'subdir' in record:
517 name = record['subdir'].decode('utf-8')
518 sub = SubElement(doc, 'subdir', name=name)
519 SubElement(sub, 'name').text = name
520 else:
521 obj_element = SubElement(doc, 'object')
522 for field in ["name", "hash", "bytes", "content_type",
523 "last_modified"]:
524 SubElement(obj_element, field).text = str(
525 record.pop(field)).decode('utf-8')
526 for field in sorted(record):
527 SubElement(obj_element, field).text = str(
528 record[field]).decode('utf-8')
529 ret.body = tostring(doc, encoding='UTF-8').replace(
530 "<?xml version='1.0' encoding='UTF-8'?>",
531 '<?xml version="1.0" encoding="UTF-8"?>', 1)
532 else: 518 else:
533 if not container_list: 519 body = listing_formats.listing_to_text(listing)
534 return HTTPNoContent(request=req, headers=resp_headers) 520
535 ret.body = '\n'.join(rec[0] for rec in container_list) + '\n' 521 ret = Response(request=req, headers=resp_headers, body=body,
522 content_type=out_content_type, charset='utf-8')
536 ret.last_modified = math.ceil(float(resp_headers['X-PUT-Timestamp'])) 523 ret.last_modified = math.ceil(float(resp_headers['X-PUT-Timestamp']))
524 if not ret.body:
525 ret.status_int = 204
537 return ret 526 return ret
538 527
539 @public 528 @public
@@ -545,7 +534,7 @@ class ContainerController(BaseStorageServer):
545 """ 534 """
546 post_args = split_and_validate_path(req, 3) 535 post_args = split_and_validate_path(req, 3)
547 drive, partition, hash = post_args 536 drive, partition, hash = post_args
548 if self.mount_check and not check_mount(self.root, drive): 537 if not check_drive(self.root, drive, self.mount_check):
549 return HTTPInsufficientStorage(drive=drive, request=req) 538 return HTTPInsufficientStorage(drive=drive, request=req)
550 try: 539 try:
551 args = json.load(req.environ['wsgi.input']) 540 args = json.load(req.environ['wsgi.input'])
@@ -567,7 +556,7 @@ class ContainerController(BaseStorageServer):
567 self.realms_conf) 556 self.realms_conf)
568 if err: 557 if err:
569 return HTTPBadRequest(err) 558 return HTTPBadRequest(err)
570 if self.mount_check and not check_mount(self.root, drive): 559 if not check_drive(self.root, drive, self.mount_check):
571 return HTTPInsufficientStorage(drive=drive, request=req) 560 return HTTPInsufficientStorage(drive=drive, request=req)
572 broker = self._get_container_broker(drive, part, account, container) 561 broker = self._get_container_broker(drive, part, account, container)
573 if broker.is_deleted(): 562 if broker.is_deleted():
diff --git a/swift/container/sync.py b/swift/container/sync.py
index 99df2e4..3d48541 100644
--- a/swift/container/sync.py
+++ b/swift/container/sync.py
@@ -77,6 +77,7 @@ pipeline = catch_errors proxy-logging cache proxy-server
77 77
78[app:proxy-server] 78[app:proxy-server]
79use = egg:swift#proxy 79use = egg:swift#proxy
80account_autocreate = true
80# See proxy-server.conf-sample for options 81# See proxy-server.conf-sample for options
81 82
82[filter:cache] 83[filter:cache]
diff --git a/swift/container/updater.py b/swift/container/updater.py
index c72acf3..ef63997 100644
--- a/swift/container/updater.py
+++ b/swift/container/updater.py
@@ -23,7 +23,7 @@ from swift import gettext_ as _
23from random import random, shuffle 23from random import random, shuffle
24from tempfile import mkstemp 24from tempfile import mkstemp
25 25
26from eventlet import spawn, patcher, Timeout 26from eventlet import spawn, Timeout
27 27
28import swift.common.db 28import swift.common.db
29from swift.container.backend import ContainerBroker, DATADIR 29from swift.container.backend import ContainerBroker, DATADIR
@@ -31,7 +31,8 @@ from swift.common.bufferedhttp import http_connect
31from swift.common.exceptions import ConnectionTimeout 31from swift.common.exceptions import ConnectionTimeout
32from swift.common.ring import Ring 32from swift.common.ring import Ring
33from swift.common.utils import get_logger, config_true_value, ismount, \ 33from swift.common.utils import get_logger, config_true_value, ismount, \
34 dump_recon_cache, majority_size, Timestamp, ratelimit_sleep 34 dump_recon_cache, majority_size, Timestamp, ratelimit_sleep, \
35 eventlet_monkey_patch
35from swift.common.daemon import Daemon 36from swift.common.daemon import Daemon
36from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR 37from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR
37 38
@@ -155,8 +156,7 @@ class ContainerUpdater(Daemon):
155 pid2filename[pid] = tmpfilename 156 pid2filename[pid] = tmpfilename
156 else: 157 else:
157 signal.signal(signal.SIGTERM, signal.SIG_DFL) 158 signal.signal(signal.SIGTERM, signal.SIG_DFL)
158 patcher.monkey_patch(all=False, socket=True, select=True, 159 eventlet_monkey_patch()
159 thread=True)
160 self.no_changes = 0 160 self.no_changes = 0
161 self.successes = 0 161 self.successes = 0
162 self.failures = 0 162 self.failures = 0
@@ -190,7 +190,7 @@ class ContainerUpdater(Daemon):
190 """ 190 """
191 Run the updater once. 191 Run the updater once.
192 """ 192 """
193 patcher.monkey_patch(all=False, socket=True, select=True, thread=True) 193 eventlet_monkey_patch()
194 self.logger.info(_('Begin container update single threaded sweep')) 194 self.logger.info(_('Begin container update single threaded sweep'))
195 begin = time.time() 195 begin = time.time()
196 self.no_changes = 0 196 self.no_changes = 0
diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py
index 362b3a3..86e53b6 100644
--- a/swift/obj/diskfile.py
+++ b/swift/obj/diskfile.py
@@ -57,7 +57,7 @@ from pyeclib.ec_iface import ECDriverError, ECInvalidFragmentMetadata, \
57 ECBadFragmentChecksum, ECInvalidParameter 57 ECBadFragmentChecksum, ECInvalidParameter
58 58
59from swift import gettext_ as _ 59from swift import gettext_ as _
60from swift.common.constraints import check_mount, check_dir 60from swift.common.constraints import check_drive
61from swift.common.request_helpers import is_sys_meta 61from swift.common.request_helpers import is_sys_meta
62from swift.common.utils import mkdirs, Timestamp, \ 62from swift.common.utils import mkdirs, Timestamp, \
63 storage_directory, hash_path, renamer, fallocate, fsync, fdatasync, \ 63 storage_directory, hash_path, renamer, fallocate, fsync, fdatasync, \
@@ -86,7 +86,8 @@ METADATA_KEY = 'user.swift.metadata'
86DROP_CACHE_WINDOW = 1024 * 1024 86DROP_CACHE_WINDOW = 1024 * 1024
87# These are system-set metadata keys that cannot be changed with a POST. 87# These are system-set metadata keys that cannot be changed with a POST.
88# They should be lowercase. 88# They should be lowercase.
89DATAFILE_SYSTEM_META = set('content-length deleted etag'.split()) 89RESERVED_DATAFILE_META = {'content-length', 'deleted', 'etag'}
90DATAFILE_SYSTEM_META = {'x-static-large-object'}
90DATADIR_BASE = 'objects' 91DATADIR_BASE = 'objects'
91ASYNCDIR_BASE = 'async_pending' 92ASYNCDIR_BASE = 'async_pending'
92TMP_BASE = 'tmp' 93TMP_BASE = 'tmp'
@@ -1191,12 +1192,11 @@ class BaseDiskFileManager(object):
1191 # we'll do some kind of check unless explicitly forbidden 1192 # we'll do some kind of check unless explicitly forbidden
1192 if mount_check is not False: 1193 if mount_check is not False:
1193 if mount_check or self.mount_check: 1194 if mount_check or self.mount_check:
1194 check = check_mount 1195 mount_check = True
1195 else: 1196 else:
1196 check = check_dir 1197 mount_check = False
1197 if not check(self.devices, device): 1198 return check_drive(self.devices, device, mount_check)
1198 return None 1199 return join(self.devices, device)
1199 return os.path.join(self.devices, device)
1200 1200
1201 @contextmanager 1201 @contextmanager
1202 def replication_lock(self, device): 1202 def replication_lock(self, device):
@@ -2416,7 +2416,8 @@ class BaseDiskFile(object):
2416 self._merge_content_type_metadata(ctype_file) 2416 self._merge_content_type_metadata(ctype_file)
2417 sys_metadata = dict( 2417 sys_metadata = dict(
2418 [(key, val) for key, val in self._datafile_metadata.items() 2418 [(key, val) for key, val in self._datafile_metadata.items()
2419 if key.lower() in DATAFILE_SYSTEM_META 2419 if key.lower() in (RESERVED_DATAFILE_META |
2420 DATAFILE_SYSTEM_META)
2420 or is_sys_meta('object', key)]) 2421 or is_sys_meta('object', key)])
2421 self._metadata.update(self._metafile_metadata) 2422 self._metadata.update(self._metafile_metadata)
2422 self._metadata.update(sys_metadata) 2423 self._metadata.update(sys_metadata)
diff --git a/swift/obj/mem_diskfile.py b/swift/obj/mem_diskfile.py
index 1764f8a..83a7309 100644
--- a/swift/obj/mem_diskfile.py
+++ b/swift/obj/mem_diskfile.py
@@ -27,7 +27,7 @@ from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \
27 DiskFileCollision, DiskFileDeleted, DiskFileNotOpen 27 DiskFileCollision, DiskFileDeleted, DiskFileNotOpen
28from swift.common.request_helpers import is_sys_meta 28from swift.common.request_helpers import is_sys_meta
29from swift.common.swob import multi_range_iterator 29from swift.common.swob import multi_range_iterator
30from swift.obj.diskfile import DATAFILE_SYSTEM_META 30from swift.obj.diskfile import DATAFILE_SYSTEM_META, RESERVED_DATAFILE_META
31 31
32 32
33class InMemoryFileSystem(object): 33class InMemoryFileSystem(object):
@@ -433,7 +433,8 @@ class DiskFile(object):
433 # with the object data. 433 # with the object data.
434 immutable_metadata = dict( 434 immutable_metadata = dict(
435 [(key, val) for key, val in cur_mdata.items() 435 [(key, val) for key, val in cur_mdata.items()
436 if key.lower() in DATAFILE_SYSTEM_META 436 if key.lower() in (RESERVED_DATAFILE_META |
437 DATAFILE_SYSTEM_META)
437 or is_sys_meta('object', key)]) 438 or is_sys_meta('object', key)])
438 metadata.update(immutable_metadata) 439 metadata.update(immutable_metadata)
439 metadata['name'] = self._name 440 metadata['name'] = self._name
diff --git a/swift/obj/reconstructor.py b/swift/obj/reconstructor.py
index ef45380..6c2b247 100644
--- a/swift/obj/reconstructor.py
+++ b/swift/obj/reconstructor.py
@@ -1097,8 +1097,9 @@ class ObjectReconstructor(Daemon):
1097 self.part_count += len(partitions) 1097 self.part_count += len(partitions)
1098 for partition in partitions: 1098 for partition in partitions:
1099 part_path = join(obj_path, partition) 1099 part_path = join(obj_path, partition)
1100 if partition in ('auditor_status_ALL.json', 1100 if (partition.startswith('auditor_status_') and
1101 'auditor_status_ZBF.json'): 1101 partition.endswith('.json')):
1102 # ignore auditor status files
1102 continue 1103 continue
1103 if not partition.isdigit(): 1104 if not partition.isdigit():
1104 self.logger.warning( 1105 self.logger.warning(
diff --git a/swift/obj/server.py b/swift/obj/server.py
index 8885383..563ccb9 100644
--- a/swift/obj/server.py
+++ b/swift/obj/server.py
@@ -55,7 +55,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
55 HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \ 55 HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \
56 HTTPInsufficientStorage, HTTPForbidden, HTTPException, HTTPConflict, \