openstack
/
swift
Code Issues Proposed changes

Merge remote-tracking branch 'remotes/origin/master' into merge-master 

Include a trivial whitespace change in .zuul.yaml to make sure Zuul
realizes it changed on master.

Change-Id: I24069547265db9ebd39e565f966f4d0b82c0aea2
This commit is contained in:
Romain LE DISEZ 2019-03-27 22:09:23 +01:00 committed by Tim Burke
parent d435992d79 89e5927f7d
commit 6afc1130fd
53 changed files with 1526 additions and 861 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.mailmap
View File

@ -79,7 +79,7 @@ Minwoo Bae <minwoob@us.ibm.com> Minwoo B
Jaivish Kothari <jaivish.kothari@nectechnologies.in> <janonymous.codevulture@gmail.com>
Michael Matur <michael.matur@gmail.com>
Kazuhiro Miyahara <miyahara.kazuhiro@lab.ntt.co.jp>
Alexandra Settle <alexandra.settle@rackspace.com>
Alexandra Settle <asettle@suse.com> <alexandra.settle@rackspace.com>
Kenichiro Matsuda <matsuda_kenichi@jp.fujitsu.com>
Atsushi Sakai <sakaia@jp.fujitsu.com>
Takashi Natsume <natsume.takashi@lab.ntt.co.jp>

31
.zuul.yaml
View File

@ -35,6 +35,7 @@
- job:
name: swift-tox-py35
parent: swift-tox-base
nodeset: ubuntu-xenial
description: |
Run unit-tests for swift under cPython version 3.5.
@ -67,6 +68,25 @@
NOSE_COVER_HTML_DIR: '{toxinidir}/cover'
post-run: tools/playbooks/common/cover-post.yaml
- job:
name: swift-tox-py37
parent: swift-tox-base
nodeset: ubuntu-bionic
description: |
Run unit-tests for swift under cPython version 3.7.
Uses tox with the ``py37`` environment.
It sets TMPDIR to an XFS mount point created via
tools/test-setup.sh.
vars:
tox_envlist: py37
bindep_profile: test py37
python_version: 3.7
tox_environment:
NOSE_COVER_HTML: 1
NOSE_COVER_HTML_DIR: '{toxinidir}/cover'
post-run: tools/playbooks/common/cover-post.yaml
- job:
name: swift-tox-func
parent: swift-tox-base
@ -316,6 +336,11 @@
- ^(api-ref|doc|releasenotes)/.*$
- ^test/(functional|probe)/.*$
voting: false
- swift-tox-py37:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
- ^test/(functional|probe)/.*$
voting: false
- swift-tox-func:
irrelevant-files:
- ^(api-ref|doc|releasenotes)/.*$
@ -344,7 +369,10 @@
- swift-tox-func-s3api-ceph-s3tests-tempauth:
irrelevant-files:
- ^(api-ref|releasenotes)/.*$
- ^test/probe/.*$
# Keep doc/saio -- we use those sample configs in the saio playbooks
# Also keep doc/s3api -- it holds known failures for these tests
- ^doc/(requirements.txt|(manpages|source)/.*)$
- ^test/(unit|probe)/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
- swift-probetests-centos-7:
irrelevant-files:
@ -428,3 +456,4 @@
post:
jobs:
- publish-openstack-python-branch-tarball

6
AUTHORS
View File

@ -45,7 +45,7 @@ Alex Holden (alex@alexjonasholden.com)
Alex Pecoraro (alex.pecoraro@emc.com)
Alex Szarka (szarka@inf.u-szeged.hu)
Alex Yang (alex890714@gmail.com)
Alexandra Settle (alexandra.settle@rackspace.com)
Alexandra Settle (asettle@suse.com)
Alexandre Lécuyer (alexandre.lecuyer@corp.ovh.com)
Alfredo Moralejo (amoralej@redhat.com)
Alistair Coles (alistairncoles@gmail.com)
@ -166,6 +166,7 @@ Fujita Tomonori (fujita.tomonori@lab.ntt.co.jp)
Félix Cantournet (felix.cantournet@cloudwatt.com)
Gage Hugo (gh159m@att.com)
Ganesh Maharaj Mahalingam (ganesh.mahalingam@intel.com)
gaobin (gaobin@inspur.com)
gaofei (gao.fei@inspur.com)
Gaurav B. Gangalwar (gaurav@gluster.com)
gecong1973 (ge.cong@zte.com.cn)
@ -173,6 +174,7 @@ gengchc2 (geng.changcai2@zte.com.cn)
Gerard Gine (ggine@swiftstack.com)
Gerry Drudy (gerry.drudy@hpe.com)
Gil Vernik (gilv@il.ibm.com)
Gleb Samsonov (sams-gleb@yandex.ru)
Gonéri Le Bouder (goneri.lebouder@enovance.com)
Graham Hayes (graham.hayes@hpe.com)
Gregory Haynes (greg@greghaynes.net)
@ -276,6 +278,7 @@ Mehdi Abaakouk (sileht@redhat.com)
melissaml (ma.lei@99cloud.net)
Michael Matur (michael.matur@gmail.com)
Michael Shuler (mshuler@gmail.com)
Michele Valsecchi (mvalsecc@redhat.com)
Mike Fedosin (mfedosin@mirantis.com)
Mingyu Li (li.mingyu@99cloud.net)
Minwoo Bae (minwoob@us.ibm.com)
@ -286,6 +289,7 @@ Monty Taylor (mordred@inaugust.com)
Morgan Fainberg (morgan.fainberg@gmail.com)
Morita Kazutaka (morita.kazutaka@gmail.com)
Motonobu Ichimura (motonobu@gmail.com)
Nadeem Syed (snadeem.hameed@gmail.com)
Nakagawa Masaaki (nakagawamsa@nttdata.co.jp)
Nakul Dahiwade (nakul.dahiwade@intel.com)
Nam Nguyen Hoai (namnh@vn.fujitsu.com)

58
CHANGELOG
View File

@ -1,3 +1,61 @@
swift (2.21.0, OpenStack Stein release)
* Change the behavior of the EC reconstructor to perform a
fragment rebuild to a handoff node when a primary peer responds
with 507 to the REPLICATE request. This changes EC to match the
existing behavior of replication when drives fail. After a
rebalance of EC rings (potentially removing unmounted/failed
devices), it's most IO efficient to run in handoffs_only mode to
avoid unnecessary rebuilds.
* O_TMPFILE support is now detected by attempting to use it
instead of looking at the kernel version. This allows older
kernels with backported patches to take advantage of the
O_TMPFILE functionality.
* Add slo_manifest_hook callback to allow other middlewares to
impose additional constraints on or make edits to SLO manifests
before being written. For example, a middleware could enforce
minimum segment size or insert data segments.
* Fixed an issue with multi-region EC policies that caused the EC
reconstructor to constantly attempt cross-region rebuild
traffic.
* Fixed an issue where S3 API v4 signatures would not be validated
against the body of the request, allowing a replay attack if
request headers were captured by a malicious third party.
* Display crypto data/metadata details in swift-object-info.
* formpost can now accept a content-encoding parameter.
* Fixed an issue where multipart uploads with the S3 API would
sometimes report an error despite all segments being upload
successfully.
* Multipart object segments are now actually deleted when the
multipart object is deleted via the S3 API.
* Swift now returns a 503 (instead of a 500) when an account
auto-create fails.
* Fixed a bug where encryption would store the incorrect key
metadata if the object name starts with a slash.
* Fixed an issue where an object server failure during a client
download could leave an open socket between the proxy and
client.
* Fixed an issue where deleted EC objects didn't have their
on-disk directories cleaned up. This would cause extra resource
usage on the object servers.
* Fixed issue where bulk requests using xml and expect
100-continue would return a malformed HTTP response.
* Various other minor bug fixes and improvements.
swift (2.20.0)
* S3 API compatibility updates

2
doc/saio/swift/object-expirer.conf
View File

@ -27,7 +27,7 @@ log_level = INFO
interval = 300
# auto_create_account_prefix = .
# report_interval = 300
# concurrency is the level of concurrency o use to do the work, this value
# concurrency is the level of concurrency to use to do the work, this value
# must be set to at least 1
# concurrency = 1
# processes is how many parts to divide the work into, one part per process

613
doc/source/development_saio.rst
View File

@ -76,9 +76,10 @@ Installing dependencies
python2-netifaces python2-pip python2-dnspython \
python2-mock
Note: This installs necessary system dependencies and *most* of the python
dependencies. Later in the process setuptools/distribute or pip will install
and/or upgrade packages.
.. note::
This installs necessary system dependencies and *most* of the python
dependencies. Later in the process setuptools/distribute or pip will install
and/or upgrade packages.
Next, choose either :ref:`partition-section` or :ref:`loopback-section`.
@ -90,50 +91,52 @@ Using a partition for storage
If you are going to use a separate partition for Swift data, be sure to add
another device when creating the VM, and follow these instructions:
#. Set up a single partition::
#. Set up a single partition::
sudo fdisk /dev/sdb
sudo mkfs.xfs /dev/sdb1
sudo fdisk /dev/sdb
sudo mkfs.xfs /dev/sdb1
#. Edit ``/etc/fstab`` and add::
#. Edit ``/etc/fstab`` and add::
/dev/sdb1 /mnt/sdb1 xfs noatime,nodiratime,nobarrier,logbufs=8 0 0
/dev/sdb1 /mnt/sdb1 xfs noatime,nodiratime,nobarrier,logbufs=8 0 0
#. Create the mount point and the individualized links::
#. Create the mount point and the individualized links::
sudo mkdir /mnt/sdb1
sudo mount /mnt/sdb1
sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4
sudo chown ${USER}:${USER} /mnt/sdb1/*
sudo mkdir /srv
for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done
sudo mkdir -p /srv/1/node/sdb1 /srv/1/node/sdb5 \
/srv/2/node/sdb2 /srv/2/node/sdb6 \
/srv/3/node/sdb3 /srv/3/node/sdb7 \
/srv/4/node/sdb4 /srv/4/node/sdb8 \
/var/run/swift
sudo chown -R ${USER}:${USER} /var/run/swift
# **Make sure to include the trailing slash after /srv/$x/**
for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done
sudo mkdir /mnt/sdb1
sudo mount /mnt/sdb1
sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4
sudo chown ${USER}:${USER} /mnt/sdb1/*
sudo mkdir /srv
for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done
sudo mkdir -p /srv/1/node/sdb1 /srv/1/node/sdb5 \
/srv/2/node/sdb2 /srv/2/node/sdb6 \
/srv/3/node/sdb3 /srv/3/node/sdb7 \
/srv/4/node/sdb4 /srv/4/node/sdb8 \
/var/run/swift
sudo chown -R ${USER}:${USER} /var/run/swift
# **Make sure to include the trailing slash after /srv/$x/**
for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done
Note: For OpenSuse users, a user's primary group is `users`, so you have 2 options:
.. note::
For OpenSuse users, a user's primary group is ``users``, so you have 2 options:
* Change `${USER}:${USER}` to `${USER}:users` in all references of this guide; or
* Create a group for your username and add yourself to it::
* Change ``${USER}:${USER}`` to ``${USER}:users`` in all references of this guide; or
* Create a group for your username and add yourself to it::
sudo groupadd ${USER} && sudo gpasswd -a ${USER} ${USER}
sudo groupadd ${USER} && sudo gpasswd -a ${USER} ${USER}
Note: We create the mount points and mount the storage disk under
/mnt/sdb1. This disk will contain one directory per simulated swift node,
each owned by the current swift user.
.. note::
We create the mount points and mount the storage disk under
/mnt/sdb1. This disk will contain one directory per simulated swift node,
each owned by the current swift user.
We then create symlinks to these directories under /srv.
If the disk sdb is unmounted, files will not be written under
/srv/\*, because the symbolic link destination /mnt/sdb1/* will not
exist. This prevents disk sync operations from writing to the root
partition in the event a drive is unmounted.
We then create symlinks to these directories under /srv.
If the disk sdb is unmounted, files will not be written under
/srv/\*, because the symbolic link destination /mnt/sdb1/* will not
exist. This prevents disk sync operations from writing to the root
partition in the event a drive is unmounted.
#. Next, skip to :ref:`common-dev-section`.
#. Next, skip to :ref:`common-dev-section`.
.. _loopback-section:
@ -144,51 +147,53 @@ Using a loopback device for storage
If you want to use a loopback device instead of another partition, follow
these instructions:
#. Create the file for the loopback device::
#. Create the file for the loopback device::
sudo mkdir /srv
sudo truncate -s 1GB /srv/swift-disk
sudo mkfs.xfs /srv/swift-disk
sudo mkdir /srv
sudo truncate -s 1GB /srv/swift-disk
sudo mkfs.xfs /srv/swift-disk
Modify size specified in the ``truncate`` command to make a larger or
smaller partition as needed.
Modify size specified in the ``truncate`` command to make a larger or
smaller partition as needed.
#. Edit `/etc/fstab` and add::
#. Edit `/etc/fstab` and add::
/srv/swift-disk /mnt/sdb1 xfs loop,noatime,nodiratime,nobarrier,logbufs=8 0 0
/srv/swift-disk /mnt/sdb1 xfs loop,noatime,nodiratime,nobarrier,logbufs=8 0 0
#. Create the mount point and the individualized links::
#. Create the mount point and the individualized links::
sudo mkdir /mnt/sdb1
sudo mount /mnt/sdb1
sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4
sudo chown ${USER}:${USER} /mnt/sdb1/*
for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done
sudo mkdir -p /srv/1/node/sdb1 /srv/1/node/sdb5 \
/srv/2/node/sdb2 /srv/2/node/sdb6 \
/srv/3/node/sdb3 /srv/3/node/sdb7 \
/srv/4/node/sdb4 /srv/4/node/sdb8 \
/var/run/swift
sudo chown -R ${USER}:${USER} /var/run/swift
# **Make sure to include the trailing slash after /srv/$x/**
for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done
sudo mkdir /mnt/sdb1
sudo mount /mnt/sdb1
sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4
sudo chown ${USER}:${USER} /mnt/sdb1/*
for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done
sudo mkdir -p /srv/1/node/sdb1 /srv/1/node/sdb5 \
/srv/2/node/sdb2 /srv/2/node/sdb6 \
/srv/3/node/sdb3 /srv/3/node/sdb7 \
/srv/4/node/sdb4 /srv/4/node/sdb8 \
/var/run/swift
sudo chown -R ${USER}:${USER} /var/run/swift
# **Make sure to include the trailing slash after /srv/$x/**
for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done
Note: For OpenSuse users, a user's primary group is `users`, so you have 2 options:
.. note::
For OpenSuse users, a user's primary group is ``users``, so you have 2 options:
* Change `${USER}:${USER}` to `${USER}:users` in all references of this guide; or
* Create a group for your username and add yourself to it::
* Change ``${USER}:${USER}`` to ``${USER}:users`` in all references of this guide; or
* Create a group for your username and add yourself to it::
sudo groupadd ${USER} && sudo gpasswd -a ${USER} ${USER}
sudo groupadd ${USER} && sudo gpasswd -a ${USER} ${USER}
Note: We create the mount points and mount the loopback file under
/mnt/sdb1. This file will contain one directory per simulated swift node,
each owned by the current swift user.
.. note::
We create the mount points and mount the loopback file under
/mnt/sdb1. This file will contain one directory per simulated swift node,
each owned by the current swift user.
We then create symlinks to these directories under /srv.
If the loopback file is unmounted, files will not be written under
/srv/\*, because the symbolic link destination /mnt/sdb1/* will not
exist. This prevents disk sync operations from writing to the root
partition in the event a drive is unmounted.
We then create symlinks to these directories under /srv.
If the loopback file is unmounted, files will not be written under
/srv/\*, because the symbolic link destination /mnt/sdb1/* will not
exist. This prevents disk sync operations from writing to the root
partition in the event a drive is unmounted.
.. _common-dev-section:
@ -229,117 +234,120 @@ To persist this, edit and add the following to ``/etc/fstab``::
Getting the code
----------------
#. Check out the python-swiftclient repo::
#. Check out the python-swiftclient repo::
cd $HOME; git clone https://github.com/openstack/python-swiftclient.git
cd $HOME; git clone https://github.com/openstack/python-swiftclient.git
#. Build a development installation of python-swiftclient::
#. Build a development installation of python-swiftclient::
cd $HOME/python-swiftclient; sudo python setup.py develop; cd -
cd $HOME/python-swiftclient; sudo python setup.py develop; cd -
Ubuntu 12.04 users need to install python-swiftclient's dependencies before the installation of
python-swiftclient. This is due to a bug in an older version of setup tools::
Ubuntu 12.04 users need to install python-swiftclient's dependencies before the installation of
python-swiftclient. This is due to a bug in an older version of setup tools::
cd $HOME/python-swiftclient; sudo pip install -r requirements.txt; sudo python setup.py develop; cd -
cd $HOME/python-swiftclient; sudo pip install -r requirements.txt; sudo python setup.py develop; cd -
#. Check out the swift repo::
#. Check out the swift repo::
git clone https://github.com/openstack/swift.git
git clone https://github.com/openstack/swift.git
#. Build a development installation of swift::
#. Build a development installation of swift::
cd $HOME/swift; sudo pip install --no-binary cryptography -r requirements.txt; sudo python setup.py develop; cd -
cd $HOME/swift; sudo pip install --no-binary cryptography -r requirements.txt; sudo python setup.py develop; cd -
Note: Due to a difference in libssl.so naming in OpenSuse to other Linux distros the wheel/binary wont work so the
cryptography must be built, thus the ``--no-binary cryptography``.
.. note::
Due to a difference in how ``libssl.so`` is named in OpenSuse vs. other Linux distros the
wheel/binary won't work; thus we use ``--no-binary cryptography`` to build ``cryptography``
locally.
Fedora 19 or later users might have to perform the following if development
installation of swift fails::
Fedora 19 or later users might have to perform the following if development
installation of swift fails::
sudo pip install -U xattr
sudo pip install -U xattr
#. Install swift's test dependencies::
#. Install swift's test dependencies::
cd $HOME/swift; sudo pip install -r test-requirements.txt
cd $HOME/swift; sudo pip install -r test-requirements.txt
----------------
Setting up rsync
----------------
#. Create ``/etc/rsyncd.conf``::
#. Create ``/etc/rsyncd.conf``::
sudo cp $HOME/swift/doc/saio/rsyncd.conf /etc/
sudo sed -i "s/<your-user-name>/${USER}/" /etc/rsyncd.conf
sudo cp $HOME/swift/doc/saio/rsyncd.conf /etc/
sudo sed -i "s/<your-user-name>/${USER}/" /etc/rsyncd.conf
Here is the default ``rsyncd.conf`` file contents maintained in the repo
that is copied and fixed up above:
Here is the default ``rsyncd.conf`` file contents maintained in the repo
that is copied and fixed up above:
.. literalinclude:: /../saio/rsyncd.conf
.. literalinclude:: /../saio/rsyncd.conf
:language: ini
#. On Ubuntu, edit the following line in ``/etc/default/rsync``::
#. On Ubuntu, edit the following line in ``/etc/default/rsync``::
RSYNC_ENABLE=true
RSYNC_ENABLE=true
On Fedora, edit the following line in ``/etc/xinetd.d/rsync``::
On Fedora, edit the following line in ``/etc/xinetd.d/rsync``::
disable = no
disable = no
One might have to create the above files to perform the edits.
One might have to create the above files to perform the edits.
On OpenSuse, nothing needs to happen here.
On OpenSuse, nothing needs to happen here.
#. On platforms with SELinux in ``Enforcing`` mode, either set to ``Permissive``::
#. On platforms with SELinux in ``Enforcing`` mode, either set to ``Permissive``::
sudo setenforce Permissive
sudo setenforce Permissive
Or just allow rsync full access::
Or just allow rsync full access::
sudo setsebool -P rsync_full_access 1
sudo setsebool -P rsync_full_access 1
#. Start the rsync daemon
#. Start the rsync daemon
* On Ubuntu 14.04, run::
* On Ubuntu 14.04, run::
sudo service rsync restart
sudo service rsync restart
* On Ubuntu 16.04, run::
* On Ubuntu 16.04, run::
sudo systemctl enable rsync
sudo systemctl start rsync
sudo systemctl enable rsync
sudo systemctl start rsync
* On Fedora, run::
* On Fedora, run::
sudo systemctl restart xinetd.service
sudo systemctl enable rsyncd.service
sudo systemctl start rsyncd.service
sudo systemctl restart xinetd.service
sudo systemctl enable rsyncd.service
sudo systemctl start rsyncd.service
* On OpenSuse, run::
* On OpenSuse, run::
sudo systemctl enable rsyncd.service
sudo systemctl start rsyncd.service
sudo systemctl enable rsyncd.service
sudo systemctl start rsyncd.service
* On other xinetd based systems simply run::
* On other xinetd based systems simply run::
sudo service xinetd restart
sudo service xinetd restart
#. Verify rsync is accepting connections for all servers::
#. Verify rsync is accepting connections for all servers::
rsync rsync://pub@localhost/
rsync rsync://pub@localhost/
You should see the following output from the above command::
You should see the following output from the above command::
account6012
account6022
account6032
account6042
container6011
container6021
container6031
container6041
object6010
object6020
object6030
object6040
account6012
account6022
account6032
account6042
container6011
container6021
container6031
container6041
object6010
object6020
object6030
object6040
------------------
Starting memcached
@ -362,50 +370,51 @@ running, tokens cannot be validated, and accessing Swift becomes impossible.
Optional: Setting up rsyslog for individual logging
---------------------------------------------------
#. Install the swift rsyslogd configuration::
#. Install the swift rsyslogd configuration::
sudo cp $HOME/swift/doc/saio/rsyslog.d/10-swift.conf /etc/rsyslog.d/
sudo cp $HOME/swift/doc/saio/rsyslog.d/10-swift.conf /etc/rsyslog.d/
Note: OpenSuse may have the systemd logger installed, so if you want this
to work, you need to install rsyslog::
Note: OpenSuse may have the systemd logger installed, so if you want this
to work, you need to install rsyslog::
sudo zypper install rsyslog
sudo systemctl start rsyslog.service
sudo systemctl enable rsyslog.service
sudo zypper install rsyslog
sudo systemctl start rsyslog.service
sudo systemctl enable rsyslog.service
Be sure to review that conf file to determine if you want all the logs
in one file vs. all the logs separated out, and if you want hourly logs
for stats processing. For convenience, we provide its default contents
below:
Be sure to review that conf file to determine if you want all the logs
in one file vs. all the logs separated out, and if you want hourly logs
for stats processing. For convenience, we provide its default contents
below:
.. literalinclude:: /../saio/rsyslog.d/10-swift.conf
.. literalinclude:: /../saio/rsyslog.d/10-swift.conf
:language: ini
#. Edit ``/etc/rsyslog.conf`` and make the following change (usually in the
"GLOBAL DIRECTIVES" section)::
#. Edit ``/etc/rsyslog.conf`` and make the following change (usually in the
"GLOBAL DIRECTIVES" section)::
$PrivDropToGroup adm
$PrivDropToGroup adm
#. If using hourly logs (see above) perform::
#. If using hourly logs (see above) perform::
sudo mkdir -p /var/log/swift/hourly
sudo mkdir -p /var/log/swift/hourly
Otherwise perform::
Otherwise perform::
sudo mkdir -p /var/log/swift
sudo mkdir -p /var/log/swift
#. Setup the logging directory and start syslog:
#. Setup the logging directory and start syslog:
* On Ubuntu::
* On Ubuntu::
sudo chown -R syslog.adm /var/log/swift
sudo chmod -R g+w /var/log/swift
sudo service rsyslog restart
sudo chown -R syslog.adm /var/log/swift
sudo chmod -R g+w /var/log/swift
sudo service rsyslog restart
* On Fedora and OpenSuse::
* On Fedora and OpenSuse::
sudo chown -R root:adm /var/log/swift
sudo chmod -R g+w /var/log/swift
sudo systemctl restart rsyslog.service
sudo chown -R root:adm /var/log/swift
sudo chmod -R g+w /var/log/swift
sudo systemctl restart rsyslog.service
---------------------
Configuring each node
@ -415,89 +424,106 @@ After performing the following steps, be sure to verify that Swift has access
to resulting configuration files (sample configuration files are provided with
all defaults in line-by-line comments).
#. Optionally remove an existing swift directory::
#. Optionally remove an existing swift directory::
sudo rm -rf /etc/swift
sudo rm -rf /etc/swift
#. Populate the ``/etc/swift`` directory itself::
#. Populate the ``/etc/swift`` directory itself::
cd $HOME/swift/doc; sudo cp -r saio/swift /etc/swift; cd -
sudo chown -R ${USER}:${USER} /etc/swift
cd $HOME/swift/doc; sudo cp -r saio/swift /etc/swift; cd -
sudo chown -R ${USER}:${USER} /etc/swift
#. Update ``<your-user-name>`` references in the Swift config files::
#. Update ``<your-user-name>`` references in the Swift config files::
find /etc/swift/ -name \*.conf | xargs sudo sed -i "s/<your-user-name>/${USER}/"
find /etc/swift/ -name \*.conf | xargs sudo sed -i "s/<your-user-name>/${USER}/"
The contents of the configuration files provided by executing the above
commands are as follows:
#. ``/etc/swift/swift.conf``
#. ``/etc/swift/swift.conf``
.. literalinclude:: /../saio/swift/swift.conf
.. literalinclude:: /../saio/swift/swift.conf
:language: ini
#. ``/etc/swift/proxy-server.conf``
#. ``/etc/swift/proxy-server.conf``
.. literalinclude:: /../saio/swift/proxy-server.conf
.. literalinclude:: /../saio/swift/proxy-server.conf
:language: ini
#. ``/etc/swift/object-expirer.conf``
#. ``/etc/swift/object-expirer.conf``
.. literalinclude:: /../saio/swift/object-expirer.conf
.. literalinclude:: /../saio/swift/object-expirer.conf
:language: ini
#. ``/etc/swift/container-reconciler.conf``
#. ``/etc/swift/container-reconciler.conf``
.. literalinclude:: /../saio/swift/container-reconciler.conf
.. literalinclude:: /../saio/swift/container-reconciler.conf
:language: ini
#. ``/etc/swift/container-sync-realms.conf``
#. ``/etc/swift/container-sync-realms.conf``
.. literalinclude:: /../saio/swift/container-sync-realms.conf
.. literalinclude:: /../saio/swift/container-sync-realms.conf
:language: ini
#. ``/etc/swift/account-server/1.conf``
#. ``/etc/swift/account-server/1.conf``
.. literalinclude:: /../saio/swift/account-server/1.conf
.. literalinclude:: /../saio/swift/account-server/1.conf
:language: ini
#. ``/etc/swift/container-server/1.conf``
#. ``/etc/swift/container-server/1.conf``
.. literalinclude:: /../saio/swift/container-server/1.conf
.. literalinclude:: /../saio/swift/container-server/1.conf
:language: ini
#. ``/etc/swift/object-server/1.conf``
#. ``/etc/swift/object-server/1.conf``
.. literalinclude:: /../saio/swift/object-server/1.conf
.. literalinclude:: /../saio/swift/object-server/1.conf
:language: ini
#. ``/etc/swift/account-server/2.conf``
#. ``/etc/swift/account-server/2.conf``
.. literalinclude:: /../saio/swift/account-server/2.conf
.. literalinclude:: /../saio/swift/account-server/2.conf
:language: ini
#. ``/etc/swift/container-server/2.conf``
#. ``/etc/swift/container-server/2.conf``
.. literalinclude:: /../saio/swift/container-server/2.conf
.. literalinclude:: /../saio/swift/container-server/2.conf
:language: ini
#. ``/etc/swift/object-server/2.conf``
#. ``/etc/swift/object-server/2.conf``
.. literalinclude:: /../saio/swift/object-server/2.conf
.. literalinclude:: /../saio/swift/object-server/2.conf
:language: ini
#. ``/etc/swift/account-server/3.conf``
#. ``/etc/swift/account-server/3.conf``
.. literalinclude:: /../saio/swift/account-server/3.conf
.. literalinclude:: /../saio/swift/account-server/3.conf
:language: ini
#. ``/etc/swift/container-server/3.conf``
#. ``/etc/swift/container-server/3.conf``
.. literalinclude:: /../saio/swift/container-server/3.conf
.. literalinclude:: /../saio/swift/container-server/3.conf
:language: ini
#. ``/etc/swift/object-server/3.conf``
#. ``/etc/swift/object-server/3.conf``
.. literalinclude:: /../saio/swift/object-server/3.conf
.. literalinclude:: /../saio/swift/object-server/3.conf
:language: ini
#. ``/etc/swift/account-server/4.conf``
#. ``/etc/swift/account-server/4.conf``
.. literalinclude:: /../saio/swift/account-server/4.conf
.. literalinclude:: /../saio/swift/account-server/4.conf
:language: ini
#. ``/etc/swift/container-server/4.conf``
#. ``/etc/swift/container-server/4.conf``
.. literalinclude:: /../saio/swift/container-server/4.conf
.. literalinclude:: /../saio/swift/container-server/4.conf
:language: ini
#. ``/etc/swift/object-server/4.conf``
#. ``/etc/swift/object-server/4.conf``
.. literalinclude:: /../saio/swift/object-server/4.conf
.. literalinclude:: /../saio/swift/object-server/4.conf
:language: ini
.. _setup_scripts:
@ -505,139 +531,146 @@ commands are as follows:
Setting up scripts for running Swift
------------------------------------
#. Copy the SAIO scripts for resetting the environment::
#. Copy the SAIO scripts for resetting the environment::
mkdir -p $HOME/bin
cd $HOME/swift/doc; cp saio/bin/* $HOME/bin; cd -
chmod +x $HOME/bin/*
mkdir -p $HOME/bin
cd $HOME/swift/doc; cp saio/bin/* $HOME/bin; cd -
chmod +x $HOME/bin/*
#. Edit the ``$HOME/bin/resetswift`` script
#. Edit the ``$HOME/bin/resetswift`` script
The template ``resetswift`` script looks like the following:
The template ``resetswift`` script looks like the following:
.. literalinclude:: /../saio/bin/resetswift
.. literalinclude:: /../saio/bin/resetswift
:language: bash
If you are using a loopback device add an environment var to
substitute ``/dev/sdb1`` with ``/srv/swift-disk``::
If you are using a loopback device add an environment var to
substitute ``/dev/sdb1`` with ``/srv/swift-disk``::
echo "export SAIO_BLOCK_DEVICE=/srv/swift-disk" >> $HOME/.bashrc
echo "export SAIO_BLOCK_DEVICE=/srv/swift-disk" >> $HOME/.bashrc
If you did not set up rsyslog for individual logging, remove the ``find
/var/log/swift...`` line::
If you did not set up rsyslog for individual logging, remove the ``find
/var/log/swift...`` line::
sed -i "/find \/var\/log\/swift/d" $HOME/bin/resetswift
sed -i "/find \/var\/log\/swift/d" $HOME/bin/resetswift
#. Install the sample configuration file for running tests::
#. Install the sample configuration file for running tests::
cp $HOME/swift/test/sample.conf /etc/swift/test.conf
cp $HOME/swift/test/sample.conf /etc/swift/test.conf
The template ``test.conf`` looks like the following:
The template ``test.conf`` looks like the following:
.. literalinclude:: /../../test/sample.conf
.. literalinclude:: /../../test/sample.conf
:language: ini
#. Add an environment variable for running tests below::
#. Add an environment variable for running tests below::
echo "export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf" >> $HOME/.bashrc
echo "export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf" >> $HOME/.bashrc
#. Be sure that your ``PATH`` includes the ``bin`` directory::
#. Be sure that your ``PATH`` includes the ``bin`` directory::
echo "export PATH=${PATH}:$HOME/bin" >> $HOME/.bashrc
echo "export PATH=${PATH}:$HOME/bin" >> $HOME/.bashrc
#. Source the above environment variables into your current environment::
#. Source the above environment variables into your current environment::
. $HOME/.bashrc
. $HOME/.bashrc
#. Construct the initial rings using the provided script::
#. Construct the initial rings using the provided script::
remakerings
remakerings
The ``remakerings`` script looks like the following:
The ``remakerings`` script looks like the following:
.. literalinclude:: /../saio/bin/remakerings
.. literalinclude:: /../saio/bin/remakerings
:language: bash
You can expect the output from this command to produce the following. Note
that 3 object rings are created in order to test storage policies and EC in
the SAIO environment. The EC ring is the only one with all 8 devices.
There are also two replication rings, one for 3x replication and another
for 2x replication, but those rings only use 4 devices::
Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
Device d1r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 1
Device d2r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 2
Device d3r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 3
Reassigned 3072 (300.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
Device d1r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 1
Device d2r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 2
Device d3r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 3
Reassigned 2048 (200.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
Device d1r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb5_"" with 1.0 weight got id 1
Device d2r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 2
Device d3r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb6_"" with 1.0 weight got id 3
Device d4r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 4
Device d5r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb7_"" with 1.0 weight got id 5
Device d6r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 6
Device d7r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb8_"" with 1.0 weight got id 7
Reassigned 6144 (600.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
Device d0r1z1-127.0.0.1:6011R127.0.0.1:6011/sdb1_"" with 1.0 weight got id 0
Device d1r1z2-127.0.0.2:6021R127.0.0.2:6021/sdb2_"" with 1.0 weight got id 1
Device d2r1z3-127.0.0.3:6031R127.0.0.3:6031/sdb3_"" with 1.0 weight got id 2
Device d3r1z4-127.0.0.4:6041R127.0.0.4:6041/sdb4_"" with 1.0 weight got id 3
Reassigned 3072 (300.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
Device d0r1z1-127.0.0.1:6012R127.0.0.1:6012/sdb1_"" with 1.0 weight got id 0
Device d1r1z2-127.0.0.2:6022R127.0.0.2:6022/sdb2_"" with 1.0 weight got id 1
Device d2r1z3-127.0.0.3:6032R127.0.0.3:6032/sdb3_"" with 1.0 weight got id 2
Device d3r1z4-127.0.0.4:6042R127.0.0.4:6042/sdb4_"" with 1.0 weight got id 3
Reassigned 3072 (300.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
You can expect the output from this command to produce the following. Note
that 3 object rings are created in order to test storage policies and EC in
the SAIO environment. The EC ring is the only one with all 8 devices.
There are also two replication rings, one for 3x replication and another
for 2x replication, but those rings only use 4 devices:
#. Read more about Storage Policies and your SAIO :doc:`policies_saio`
.. code-block:: console
#. Verify the unit tests run::
Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
Device d1r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 1
Device d2r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 2
Device d3r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 3
Reassigned 3072 (300.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
Device d1r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 1
Device d2r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 2
Device d3r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 3
Reassigned 2048 (200.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
Device d1r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb5_"" with 1.0 weight got id 1
Device d2r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 2
Device d3r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb6_"" with 1.0 weight got id 3
Device d4r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 4
Device d5r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb7_"" with 1.0 weight got id 5
Device d6r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 6
Device d7r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb8_"" with 1.0 weight got id 7
Reassigned 6144 (600.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
Device d0r1z1-127.0.0.1:6011R127.0.0.1:6011/sdb1_"" with 1.0 weight got id 0
Device d1r1z2-127.0.0.2:6021R127.0.0.2:6021/sdb2_"" with 1.0 weight got id 1
Device d2r1z3-127.0.0.3:6031R127.0.0.3:6031/sdb3_"" with 1.0 weight got id 2
Device d3r1z4-127.0.0.4:6041R127.0.0.4:6041/sdb4_"" with 1.0 weight got id 3
Reassigned 3072 (300.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
Device d0r1z1-127.0.0.1:6012R127.0.0.1:6012/sdb1_"" with 1.0 weight got id 0
Device d1r1z2-127.0.0.2:6022R127.0.0.2:6022/sdb2_"" with 1.0 weight got id 1
Device d2r1z3-127.0.0.3:6032R127.0.0.3:6032/sdb3_"" with 1.0 weight got id 2
Device d3r1z4-127.0.0.4:6042R127.0.0.4:6042/sdb4_"" with 1.0 weight got id 3
Reassigned 3072 (300.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
$HOME/swift/.unittests
Note that the unit tests do not require any swift daemons running.
#. Read more about Storage Policies and your SAIO :doc:`policies_saio`
#. Start the "main" Swift daemon processes (proxy, account, container, and
object)::
#. Verify the unit tests run::
startmain
$HOME/swift/.unittests
(The "``Unable to increase file descriptor limit. Running as non-root?``"
warnings are expected and ok.)
Note that the unit tests do not require any swift daemons running.
The ``startmain`` script looks like the following:
#. Start the "main" Swift daemon processes (proxy, account, container, and
object)::
.. literalinclude:: /../saio/bin/startmain
startmain
#. Get an ``X-Storage-Url`` and ``X-Auth-Token``::
(The "``Unable to increase file descriptor limit. Running as non-root?``"
warnings are expected and ok.)
curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0
The ``startmain`` script looks like the following:
#. Check that you can ``GET`` account::
.. literalinclude:: /../saio/bin/startmain
:language: bash
curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>
#. Get an ``X-Storage-Url`` and ``X-Auth-Token``::
#. Check that ``swift`` command provided by the python-swiftclient package works::
curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0
swift -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat
#. Check that you can ``GET`` account::
#. Verify the functional tests run::
curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>
$HOME/swift/.functests
#. Check that ``swift`` command provided by the python-swiftclient package works::
(Note: functional tests will first delete everything in the configured
accounts.)
swift -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat
#. Verify the probe tests run::
#. Verify the functional tests run::
$HOME/swift/.probetests
$HOME/swift/.functests
(Note: probe tests will reset your environment as they call ``resetswift``
for each test.)
(Note: functional tests will first delete everything in the configured
accounts.)
#. Verify the probe tests run::
$HOME/swift/.probetests
(Note: probe tests will reset your environment as they call ``resetswift``
for each test.)
----------------
Debugging Issues

3
etc/proxy-server.conf-sample
View File

@ -589,7 +589,8 @@ reseller_prefix = AUTH_
# useful if there are multiple auth systems in the proxy pipeline.
delay_auth_decision = False
# Keystone server details
# Keystone server details. Note that this differs from how swift3 was
# configured: in particular, the Keystone API version must be included.
auth_uri = http://keystonehost:35357/v3
# Connect/read timeout to use when communicating with Keystone

69
releasenotes/notes/2_21_0_release-d8ae33ef18b7be3a.yaml Normal file
View File

@ -0,0 +1,69 @@
---
features:
- |
Change the behavior of the EC reconstructor to perform a
fragment rebuild to a handoff node when a primary peer responds
with 507 to the REPLICATE request. This changes EC to match the
existing behavior of replication when drives fail. After a
rebalance of EC rings (potentially removing unmounted/failed
devices), it's most IO efficient to run in handoffs_only mode to
avoid unnecessary rebuilds.
- |
O_TMPFILE support is now detected by attempting to use it
instead of looking at the kernel version. This allows older
kernels with backported patches to take advantage of the
O_TMPFILE functionality.
- |
Add slo_manifest_hook callback to allow other middlewares to
impose additional constraints on or make edits to SLO manifests
before being written. For example, a middleware could enforce
minimum segment size or insert data segments.
- |
Fixed an issue with multi-region EC policies that caused the EC
reconstructor to constantly attempt cross-region rebuild
traffic.
- |
Fixed an issue where S3 API v4 signatures would not be validated
against the body of the request, allowing a replay attack if
request headers were captured by a malicious third party.
- Display crypto data/metadata details in swift-object-info.
- formpost can now accept a content-encoding parameter.
- |
Fixed an issue where multipart uploads with the S3 API would
sometimes report an error despite all segments being upload
successfully.
- |
Multipart object segments are now actually deleted when the
multipart object is deleted via the S3 API.
- |
Swift now returns a 503 (instead of a 500) when an account
auto-create fails.
- |
Fixed a bug where encryption would store the incorrect key
metadata if the object name starts with a slash.
- |
Fixed an issue where an object server failure during a client
download could leave an open socket between the proxy and
client.
- |
Fixed an issue where deleted EC objects didn't have their
on-disk directories cleaned up. This would cause extra resource
usage on the object servers.
- |
Fixed issue where bulk requests using xml and expect
100-continue would return a malformed HTTP response.
- Various other minor bug fixes and improvements.

2
releasenotes/source/index.rst
View File

@ -7,6 +7,8 @@
current
stein
rocky
queens

6
releasenotes/source/stein.rst Normal file
View File

@ -0,0 +1,6 @@
===================================
Stein Series Release Notes
===================================
.. release-notes::
:branch: stable/stein

90
swift/account/reaper.py
View File

@ -16,7 +16,6 @@
import os
import random
import socket
from swift import gettext_ as _
from logging import DEBUG
from math import sqrt
from time import time
@ -142,10 +141,10 @@ class AccountReaper(Daemon):
continue
self.reap_device(device)
except (Exception, Timeout):
self.logger.exception(_("Exception in top-level account reaper "
"loop"))
self.logger.exception("Exception in top-level account reaper "
"loop")
elapsed = time() - begin
self.logger.info(_('Devices pass completed: %.02fs'), elapsed)
self.logger.info('Devices pass completed: %.02fs', elapsed)
def reap_device(self, device):
"""
@ -255,19 +254,15 @@ class AccountReaper(Daemon):
self.delay_reaping:
return False
account = info['account']
self.logger.info(_('Beginning pass on account %s'), account)
self.logger.info('Beginning pass on account %s', account)
self.reset_stats()
container_limit = 1000
if container_shard is not None:
container_limit *= len(nodes)
try:
marker = ''
while True:
containers = \
list(broker.list_containers_iter(container_limit, marker,
None, None, None))
if not containers:
break
containers = list(broker.list_containers_iter(
container_limit, '', None, None, None))
while containers:
try:
for (container, _junk, _junk, _junk, _junk) in containers:
if six.PY3:
@ -284,43 +279,44 @@ class AccountReaper(Daemon):
self.container_pool.waitall()
except (Exception, Timeout):
self.logger.exception(
_('Exception with containers for account %s'), account)
marker = containers[-1][0]
if marker == '':
break
log = 'Completed pass on account %s' % account
'Exception with containers for account %s', account)
containers = list(broker.list_containers_iter(
container_limit, containers[-1][0], None, None, None))
log_buf = ['Completed pass on account %s' % account]
except (Exception, Timeout):
self.logger.exception(
_('Exception with account %s'), account)
log = _('Incomplete pass on account %s') % account
self.logger.exception('Exception with account %s', account)
log_buf = ['Incomplete pass on account %s' % account]
if self.stats_containers_deleted:
log += _(', %s containers deleted') % self.stats_containers_deleted
log_buf.append(', %s containers deleted' %
self.stats_containers_deleted)
if self.stats_objects_deleted:
log += _(', %s objects deleted') % self.stats_objects_deleted
log_buf.append(', %s objects deleted' % self.stats_objects_deleted)
if self.stats_containers_remaining:
log += _(', %s containers remaining') % \
self.stats_containers_remaining
log_buf.append(', %s containers remaining' %
self.stats_containers_remaining)
if self.stats_objects_remaining:
log += _(', %s objects remaining') % self.stats_objects_remaining
log_buf.append(', %s objects remaining' %
self.stats_objects_remaining)
if self.stats_containers_possibly_remaining:
log += _(', %s containers possibly remaining') % \
self.stats_containers_possibly_remaining
log_buf.append(', %s containers possibly remaining' %
self.stats_containers_possibly_remaining)
if self.stats_objects_possibly_remaining:
log += _(', %s objects possibly remaining') % \
self.stats_objects_possibly_remaining
log_buf.append(', %s objects possibly remaining' %
self.stats_objects_possibly_remaining)
if self.stats_return_codes:
log += _(', return codes: ')
log_buf.append(', return codes: ')
for code in sorted(self.stats_return_codes):
log += '%s %sxxs, ' % (self.stats_return_codes[code], code)
log = log[:-2]
log += _(', elapsed: %.02fs') % (time() - begin)
self.logger.info(log)
log_buf.append('%s %sxxs, ' % (self.stats_return_codes[code],
code))
log_buf[-1] = log_buf[-1][:-2]
log_buf.append(', elapsed: %.02fs' % (time() - begin))
self.logger.info(''.join(log_buf))
self.logger.timing_since('timing', self.start_time)
delete_timestamp = Timestamp(info['delete_timestamp'])
if self.stats_containers_remaining and \
begin - float(delete_timestamp) >= self.reap_not_done_after:
self.logger.warning(
_('Account %(account)s has not been reaped since %(time)s') %
'Account %(account)s has not been reaped since %(time)s' %
{'account': account, 'time': delete_timestamp.isoformat})
return True
@ -379,14 +375,14 @@ class AccountReaper(Daemon):
except ClientException as err:
if self.logger.getEffectiveLevel() <= DEBUG:
self.logger.exception(
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
'Exception with %(ip)s:%(port)s/%(device)s', node)
self.stats_return_codes[err.http_status // 100] = \
self.stats_return_codes.get(err.http_status // 100, 0) + 1
self.logger.increment(
'return_codes.%d' % (err.http_status // 100,))
except (Timeout, socket.error) as err:
self.logger.error(
_('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
'Timeout Exception with %(ip)s:%(port)s/%(device)s',
node)
if not objects:
break
@ -397,21 +393,15 @@ class AccountReaper(Daemon):
self.logger.error('ERROR: invalid storage policy index: %r'
% policy_index)
for obj in objects:
obj_name = obj['name']
if isinstance(obj_name, six.text_type):
obj_name = obj_name.encode('utf8')
pool.spawn(self.reap_object, account, container, part,
nodes, obj_name, policy_index)
nodes, obj['name'], policy_index)
pool.waitall()
except (Exception, Timeout):
self.logger.exception(_('Exception with objects for container '
'%(container)s for account %(account)s'
),
self.logger.exception('Exception with objects for container '
'%(container)s for account %(account)s',
{'container': container,
'account': account})
marker = objects[-1]['name']
if marker == '':
break
successes = 0
failures = 0
timestamp = Timestamp.now()
@ -434,7 +424,7 @@ class AccountReaper(Daemon):
except ClientException as err:
if self.logger.getEffectiveLevel() <= DEBUG:
self.logger.exception(
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
'Exception with %(ip)s:%(port)s/%(device)s', node)
failures += 1
self.logger.increment('containers_failures')
self.stats_return_codes[err.http_status // 100] = \
@ -443,7 +433,7 @@ class AccountReaper(Daemon):
'return_codes.%d' % (err.http_status // 100,))
except (Timeout, socket.error) as err:
self.logger.error(
_('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
'Timeout Exception with %(ip)s:%(port)s/%(device)s',
node)
failures += 1
self.logger.increment('containers_failures')
@ -510,7 +500,7 @@ class AccountReaper(Daemon):
except ClientException as err:
if self.logger.getEffectiveLevel() <= DEBUG:
self.logger.exception(
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
'Exception with %(ip)s:%(port)s/%(device)s', node)
failures += 1
self.logger.increment('objects_failures')
self.stats_return_codes[err.http_status // 100] = \
@ -521,7 +511,7 @@ class AccountReaper(Daemon):
failures += 1
self.logger.increment('objects_failures')
self.logger.error(
_('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
'Timeout Exception with %(ip)s:%(port)s/%(device)s',
node)
if successes > failures:
self.stats_objects_deleted += 1

7
swift/cli/manage_shard_ranges.py
View File

@ -515,7 +515,12 @@ def main(args=None):
logger = get_logger({}, name='ContainerBroker', log_to_console=True)
broker = ContainerBroker(args.container_db, logger=logger,
skip_commits=True)
broker.get_info()
try:
broker.get_info()
except Exception as exc:
print('Error opening container DB %s: %s' % (args.container_db, exc),
file=sys.stderr)
return 2
print('Loaded db broker for %s.' % broker.path, file=sys.stderr)
return args.func(broker, args)

8
swift/common/bufferedhttp.py
View File

@ -178,6 +178,14 @@ class BufferedHTTPConnection(HTTPConnection):
return ret
def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
'''Send a request to the server.
:param method: specifies an HTTP request method, e.g. 'GET'.
:param url: specifies the object being requested, e.g. '/index.html'.
:param skip_host: if True does not add automatically a 'Host:' header
:param skip_accept_encoding: if True does not add automatically an
'Accept-Encoding:' header
'''
self._method = method
self._path = url
return HTTPConnection.putrequest(self, method, url, skip_host,

24
swift/common/middleware/bulk.py
View File

@ -380,6 +380,10 @@ class Bulk(object):
query request.
"""
last_yield = time()
if out_content_type and out_content_type.endswith('/xml'):
to_yield = '<?xml version="1.0" encoding="UTF-8"?>\n'
else:
to_yield = ' '
separator = ''
failed_files = []
resp_dict = {'Response Status': HTTPOk().status,
@ -390,8 +394,6 @@ class Bulk(object):
try:
if not out_content_type:
raise HTTPNotAcceptable(request=req)
if out_content_type.endswith('/xml'):
yield '<?xml version="1.0" encoding="UTF-8"?>\n'
try:
vrs, account, _junk = req.split_path(2, 3, True)
@ -452,9 +454,9 @@ class Bulk(object):
for resp, obj_name, retry in pile.asyncstarmap(
do_delete, names_to_delete):
if last_yield + self.yield_frequency < time():
separator = '\r\n\r\n'
last_yield = time()
yield ' '
yield to_yield
to_yield, separator = ' ', '\r\n\r\n'
self._process_delete(resp, pile, obj_name,
resp_dict, failed_files,
failed_file_response, retry)
@ -462,9 +464,9 @@ class Bulk(object):
# Abort, but drain off the in-progress deletes
for resp, obj_name, retry in pile:
if last_yield + self.yield_frequency < time():
separator = '\r\n\r\n'
last_yield = time()
yield ' '
yield to_yield
to_yield, separator = ' ', '\r\n\r\n'
# Don't pass in the pile, as we shouldn't retry
self._process_delete(
resp, None, obj_name, resp_dict,
@ -508,14 +510,16 @@ class Bulk(object):
'Response Body': '', 'Number Files Created': 0}
failed_files = []
last_yield = time()
if out_content_type and out_content_type.endswith('/xml'):
to_yield = '<?xml version="1.0" encoding="UTF-8"?>\n'
else:
to_yield = ' '
separator = ''
containers_accessed = set()
req.environ['eventlet.minimum_write_chunk_size'] = 0
try:
if not out_content_type:
raise HTTPNotAcceptable(request=req)
if out_content_type.endswith('/xml'):
yield '<?xml version="1.0" encoding="UTF-8"?>\n'
if req.content_length is None and \
req.headers.get('transfer-encoding',
@ -533,9 +537,9 @@ class Bulk(object):
containers_created = 0
while True:
if last_yield + self.yield_frequency < time():
separator = '\r\n\r\n'
last_yield = time()
yield ' '
yield to_yield
to_yield, separator = ' ', '\r\n\r\n'
tar_info = next(tar)
if tar_info is None or \
len(failed_files) >= self.max_failed_extractions:

19
swift/common/middleware/copy.py
View File

@ -114,12 +114,11 @@ greater than 5GB.
"""
from six.moves.urllib.parse import quote, unquote
from swift.common.utils import get_logger, config_true_value, FileLikeIter, \
close_if_possible
from swift.common.swob import Request, HTTPPreconditionFailed, \
HTTPRequestEntityTooLarge, HTTPBadRequest, HTTPException
HTTPRequestEntityTooLarge, HTTPBadRequest, HTTPException, \
wsgi_quote, wsgi_unquote
from swift.common.http import HTTP_MULTIPLE_CHOICES, is_success, HTTP_OK
from swift.common.constraints import check_account_format, MAX_FILE_SIZE
from swift.common.request_helpers import copy_header_subset, remove_items, \
@ -183,7 +182,7 @@ class ServerSideCopyWebContext(WSGIContext):
def get_source_resp(self, req):
sub_req = make_subrequest(
req.environ, path=quote(req.path_info), headers=req.headers,
req.environ, path=wsgi_quote(req.path_info), headers=req.headers,
swift_source='SSC')
return sub_req.get_response(self.app)
@ -257,9 +256,9 @@ class ServerSideCopyMiddleware(object):
)(req.environ, start_response)
dest_account = account
if 'Destination-Account' in req.headers:
dest_account = unquote(req.headers.get('Destination-Account'))
dest_account = wsgi_unquote(req.headers.get('Destination-Account'))
dest_account = check_account_format(req, dest_account)
req.headers['X-Copy-From-Account'] = quote(account)
req.headers['X-Copy-From-Account'] = wsgi_quote(account)
account = dest_account
del req.headers['Destination-Account']
dest_container, dest_object = _check_destination_header(req)
@ -275,7 +274,7 @@ class ServerSideCopyMiddleware(object):
req.path_info = '/%s/%s/%s/%s' % (
ver, dest_account, dest_container, dest_object)
req.headers['Content-Length'] = 0
req.headers['X-Copy-From'] = quote(source)
req.headers['X-Copy-From'] = wsgi_quote(source)
del req.headers['Destination']
return self.handle_PUT(req, start_response)
@ -312,8 +311,8 @@ class ServerSideCopyMiddleware(object):
def _create_response_headers(self, source_path, source_resp, sink_req):
resp_headers = dict()
acct, path = source_path.split('/', 3)[2:4]
resp_headers['X-Copied-From-Account'] = quote(acct)
resp_headers['X-Copied-From'] = quote(path)
resp_headers['X-Copied-From-Account'] = wsgi_quote(acct)
resp_headers['X-Copied-From'] = wsgi_quote(path)
if 'last-modified' in source_resp.headers:
resp_headers['X-Copied-From-Last-Modified'] = \
source_resp.headers['last-modified']
@ -334,7 +333,7 @@ class ServerSideCopyMiddleware(object):
src_account_name = req.headers.get('X-Copy-From-Account')
if src_account_name:
src_account_name = check_account_format(
req, unquote(src_account_name))
req, wsgi_unquote(src_account_name))
else:
src_account_name = acct
src_container_name, src_obj_name = _check_copy_from_header(req)

38
swift/common/middleware/crypto/keymaster.py
View File

@ -18,7 +18,8 @@ import hmac
from swift.common.exceptions import UnknownSecretIdError
from swift.common.middleware.crypto.crypto_utils import CRYPTO_KEY_CALLBACK
from swift.common.swob import Request, HTTPException, wsgi_to_bytes
from swift.common.utils import readconf, strict_b64decode, get_logger
from swift.common.utils import readconf, strict_b64decode, get_logger, \
split_path
from swift.common.wsgi import WSGIContext
@ -77,29 +78,54 @@ class KeyMasterContext(WSGIContext):
version = key_id['v']
if version not in ('1', '2'):
raise ValueError('Unknown key_id version: %s' % version)
if version == '1' and not key_id['path'].startswith(
'/' + self.account + '/'):
# Well shoot. This was the bug that made us notice we needed
# a v2! Hope the current account/container was the original!
key_acct, key_cont, key_obj = (
self.account, self.container, key_id['path'])
else:
key_acct, key_cont, key_obj = split_path(
key_id['path'], 1, 3, True)
check_path = (
self.account, self.container or key_cont, self.obj or key_obj)
if (key_acct, key_cont, key_obj) != check_path:
self.keymaster.logger.info(
"Path stored in meta (%r) does not match path from "
"request (%r)! Using path from meta.",
key_id['path'],
'/' + '/'.join(x for x in [
self.account, self.container, self.obj] if x))
else:
secret_id = self.keymaster.active_secret_id
# v1 had a bug where we would claim the path was just the object
# name if the object started with a slash. Bump versions to
# establish that we can trust the path.
version = '2'
key_acct, key_cont, key_obj = (
self.account, self.container, self.obj)
if (secret_id, version) in self._keys:
return self._keys[(secret_id, version)]
keys = {}
account_path = '/' + self.account
account_path = '/' + key_acct
try:
# self.account/container/obj reflect the level of the *request*,
# which may be different from the level of the key_id-path. Only
# fetch the keys that the request needs.
if self.container:
path = account_path + '/' + self.container
path = account_path + '/' + key_cont
keys['container'] = self.keymaster.create_key(
path, secret_id=secret_id)
if self.obj:
if self.obj.startswith('/') and version == '1':
path = self.obj
if key_obj.startswith('/') and version == '1':
path = key_obj
else:
path = path + '/' + self.obj
path = path + '/' + key_obj
keys['object'] = self.keymaster.create_key(
path, secret_id=secret_id)

3
swift/common/middleware/dlo.py
View File

@ -144,7 +144,8 @@ class GetContext(WSGIContext):
def _get_container_listing(self, req, version, account, container,
prefix, marker=''):
con_req = make_subrequest(
req.environ, path='/'.join(['', version, account, container]),
req.environ,
path=quote('/'.join(['', version, account, container])),
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')

21
swift/common/middleware/formpost.py
View File

@ -126,7 +126,9 @@ import hmac
from hashlib import sha1
from time import time
import six
from six.moves.urllib.parse import quote
from swift.common.exceptions import MimeInvalid
from swift.common.middleware.tempurl import get_tempurl_keys_from_metadata
from swift.common.utils import streq_const_time, register_swift_info, \
@ -229,7 +231,7 @@ class FormPost(object):
start_response(status, headers)
return [body]
except MimeInvalid:
body = 'FormPost: invalid starting boundary'
body = b'FormPost: invalid starting boundary'
start_response(
'400 Bad Request',
(('Content-Type', 'text/plain'),
@ -237,6 +239,8 @@ class FormPost(object):
return [body]
except (FormInvalid, EOFError) as err:
body = 'FormPost: %s' % err
if six.PY3:
body = body.encode('utf-8')
start_response(
'400 Bad Request',
(('Content-Type', 'text/plain'),
@ -258,6 +262,8 @@ class FormPost(object):
:returns: status_line, headers_list, body
"""
keys = self._get_keys(env)
if six.PY3:
boundary = boundary.encode('utf-8')
status = message = ''
attributes = {}
subheaders = []
@ -282,14 +288,13 @@ class FormPost(object):
hdrs['Content-Type'] or 'application/octet-stream'
if 'content-encoding' not in attributes and \
'content-encoding' in hdrs:
attributes['content-encoding'] = \
hdrs['Content-Encoding']
attributes['content-encoding'] = hdrs['Content-Encoding']
status, subheaders = \
self._perform_subrequest(env, attributes, fp, keys)
if not status.startswith('2'):
break
else:
data = ''
data = b''
mxln = MAX_VALUE_LENGTH
while mxln:
chunk = fp.read(mxln)
@ -299,6 +304,8 @@ class FormPost(object):
data += chunk
while fp.read(READ_CHUNK_SIZE):
pass
if six.PY3:
data = data.decode('utf-8')
if 'name' in attrs:
attributes[attrs['name'].lower()] = data.rstrip('\r\n--')
if not status:
@ -315,6 +322,8 @@ class FormPost(object):
body = status + '\r\nFormPost: ' + message.title()
headers.extend([('Content-Type', 'text/plain'),
('Content-Length', len(body))])
if six.PY3:
body = body.encode('utf-8')
return status, headers, body
status = status.split(' ', 1)[0]
if '?' in redirect:
@ -324,6 +333,8 @@ class FormPost(object):
redirect += 'status=%s&message=%s' % (quote(status), quote(message))
body = '<html><body><p><a href="%s">' \
'Click to continue...</a></p></body></html>' % redirect
if six.PY3:
body = body.encode('utf-8')
headers.extend(
[('Location', redirect), ('Content-Length', str(len(body)))])
return '303 See Other', headers, body
@ -385,6 +396,8 @@ class FormPost(object):
attributes.get('max_file_size') or '0',
attributes.get('max_file_count') or '0',
attributes.get('expires') or '0')
if six.PY3:
hmac_body = hmac_body.encode('utf-8')
has_valid_sig = False
for key in keys:

2
swift/common/middleware/s3api/controllers/s3_acl.py
View File

@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from urllib import quote
from six.moves.urllib.parse import quote
from swift.common.utils import public
from swift.common.middleware.s3api.controllers.base import Controller

60
swift/common/middleware/s3api/s3request.py
View File

@ -24,7 +24,8 @@ import six
from six.moves.urllib.parse import quote, unquote, parse_qsl
import string
from swift.common.utils import split_path, json, get_swift_info
from swift.common.utils import split_path, json, get_swift_info, \
close_if_possible
from swift.common import swob
from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \
HTTP_NO_CONTENT, HTTP_UNAUTHORIZED, HTTP_FORBIDDEN, HTTP_NOT_FOUND, \
@ -110,6 +111,34 @@ def _header_acl_property(resource):
doc='Get and set the %s acl property' % resource)
class HashingInput(object):
"""
wsgi.input wrapper to verify the hash of the input as it's read.
"""
def __init__(self, reader, content_length, hasher, expected_hex_hash):
self._input = reader
self._to_read = content_length
self._hasher = hasher()
self._expected = expected_hex_hash
def read(self, size=None):
chunk = self._input.read(size)
self._hasher.update(chunk)
self._to_read -= len(chunk)
if self._to_read < 0 or (size > len(chunk) and self._to_read) or (
self._to_read == 0 and
self._hasher.hexdigest() != self._expected):
self.close()
# Since we don't return the last chunk, the PUT never completes
raise swob.HTTPUnprocessableEntity(
'The X-Amz-Content-SHA56 you specified did not match '
'what we received.')
return chunk
def close(self):
close_if_possible(self._input)
class SigV4Mixin(object):
"""
A request class mixin to provide S3 signature v4 functionality
@ -401,6 +430,20 @@ class SigV4Mixin(object):
raise InvalidRequest(msg)
else:
hashed_payload = self.headers['X-Amz-Content-SHA256']
if self.content_length == 0:
if hashed_payload != sha256().hexdigest():
raise BadDigest(
'The X-Amz-Content-SHA56 you specified did not match '
'what we received.')
elif self.content_length:
self.environ['wsgi.input'] = HashingInput(
self.environ['wsgi.input'],
self.content_length,
sha256,
hashed_payload)
# else, not provided -- Swift will kick out a 411 Length Required
# which will get translated back to a S3-style response in
# S3Request._swift_error_codes
cr.append(hashed_payload)
return '\n'.join(cr).encode('utf-8')
@ -1267,12 +1310,15 @@ class S3Request(swob.Request):
sw_req = self.to_swift_req(method, container, obj, headers=headers,
body=body, query=query)
sw_resp = sw_req.get_response(app)
# reuse account and tokens
_, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
2, 3, True)
self.account = utf8encode(self.account)
try:
sw_resp = sw_req.get_response(app)
except swob.HTTPException as err:
sw_resp = err
else:
# reuse account and tokens
_, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
2, 3, True)
self.account = utf8encode(self.account)
resp = S3Response.from_swift_resp(sw_resp)
status = resp.status_int # pylint: disable-msg=E1101

19
swift/common/middleware/s3api/s3token.py
View File

@ -33,6 +33,25 @@ This middleware:
* Optionally can retrieve and cache secret from keystone
to validate signature locally
.. note::
If upgrading from swift3, the ``auth_version`` config option has been
removed, and the ``auth_uri`` option now includes the Keystone API
version. If you previously had a configuration like
.. code-block:: ini
[filter:s3token]
use = egg:swift3#s3token
auth_uri = https://keystonehost:35357
auth_version = 3
you should now use
.. code-block:: ini
[filter:s3token]
use = egg:swift#s3token
auth_uri = https://keystonehost:35357/v3
"""
import base64

10
swift/common/middleware/slo.py
View File

@ -457,8 +457,8 @@ def parse_and_validate_input(req_body, req_path):
continue
obj_path = '/'.join(['', vrs, account,
seg_dict['path'].lstrip('/')])
if req_path == quote(obj_path):
quote(seg_dict['path'].lstrip('/'))])
if req_path == obj_path:
errors.append(
b"Index %d: manifest must not include itself as a segment"
% seg_index)
@ -526,7 +526,7 @@ class SloGetContext(WSGIContext):
Raise exception on failures.
"""
sub_req = make_subrequest(
req.environ, path='/'.join(['', version, acc, con, obj]),
req.environ, path=quote('/'.join(['', version, acc, con, obj])),
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent='%(orig)s SLO MultipartGET', swift_source='SLO')
@ -1109,8 +1109,8 @@ class StaticLargeObject(object):
path2indices[seg_dict['path']].append(index)
def do_head(obj_name):
obj_path = '/'.join(['', vrs, account,
get_valid_utf8_str(obj_name).lstrip('/')])
obj_path = quote('/'.join([
'', vrs, account, get_valid_utf8_str(obj_name).lstrip('/')]))
sub_req = make_subrequest(
req.environ, path=obj_path + '?', # kill the query string

5
swift/common/middleware/tempauth.py
View File

@ -184,7 +184,7 @@ from eventlet import Timeout
import six
from swift.common.swob import Response, Request, wsgi_to_str
from swift.common.swob import HTTPBadRequest, HTTPForbidden, HTTPNotFound, \
HTTPUnauthorized
HTTPUnauthorized, HTTPMethodNotAllowed
from swift.common.request_helpers import get_sys_meta_prefix
from swift.common.middleware.acl import (
@ -688,6 +688,9 @@ class TempAuth(object):
"""
req.start_time = time()
handler = None
if req.method != 'GET':
req.response = HTTPMethodNotAllowed(request=req)
return req.response
try:
version, account, user, _junk = split_path(req.path_info,
1, 4, True)

16
swift/common/request_helpers.py
View File

@ -26,7 +26,6 @@ import sys
import time
import six
from six.moves.urllib.parse import unquote
from swift.common.header_key_dict import HeaderKeyDict
from swift import gettext_ as _
@ -35,11 +34,11 @@ from swift.common.exceptions import ListingIterError, SegmentError
from swift.common.http import is_success
from swift.common.swob import HTTPBadRequest, \
HTTPServiceUnavailable, Range, is_chunked, multi_range_iterator, \
HTTPPreconditionFailed, wsgi_to_bytes
HTTPPreconditionFailed, wsgi_to_bytes, wsgi_unquote, wsgi_to_str
from swift.common.utils import split_path, validate_device_partition, \
close_if_possible, maybe_multipart_byteranges_to_document_iters, \
multipart_byteranges_to_document_iters, parse_content_type, \
parse_content_range, csv_append, list_from_csv, Spliterator
parse_content_range, csv_append, list_from_csv, Spliterator, quote
from swift.common.wsgi import make_subrequest
@ -115,14 +114,13 @@ def split_and_validate_path(request, minsegs=1, maxsegs=None,
Utility function to split and validate the request path.
:returns: result of :meth:`~swift.common.utils.split_path` if
everything's okay
everything's okay, as native strings
:raises HTTPBadRequest: if something's not okay
"""
try:
segs = split_path(unquote(request.path),
minsegs, maxsegs, rest_with_last)
segs = request.split_path(minsegs, maxsegs, rest_with_last)
validate_device_partition(segs[0], segs[1])
return segs
return [wsgi_to_str(seg) for seg in segs]
except ValueError as err:
raise HTTPBadRequest(body=str(err), request=request,
content_type='text/plain')
@ -308,7 +306,7 @@ def check_path_header(req, name, length, error_msg):
:raise: HTTPPreconditionFailed if header value
is not well formatted.
"""
hdr = unquote(req.headers.get(name))
hdr = wsgi_unquote(req.headers.get(name))
if not hdr.startswith('/'):
hdr = '/' + hdr
try:
@ -391,7 +389,7 @@ class SegmentedIterable(object):
# segment is a plain old object, not some flavor of large
# object; therefore, its etag is its MD5sum and hence we can
# check it.
path = seg_path + '?multipart-manifest=get'
path = quote(seg_path) + '?multipart-manifest=get'
seg_req = make_subrequest(
self.req.environ, path=path, method='GET',
headers={'x-auth-token': self.req.headers.get(

44
swift/common/swob.py
View File

@ -309,6 +309,50 @@ def str_to_wsgi(native_str):
return bytes_to_wsgi(native_str.encode('utf8', errors='surrogateescape'))
def wsgi_quote(wsgi_str):
if six.PY2:
if not isinstance(wsgi_str, bytes):
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
return urllib.parse.quote(wsgi_str)
if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
return urllib.parse.quote(wsgi_str, encoding='latin-1')
def wsgi_unquote(wsgi_str):
if six.PY2:
if not isinstance(wsgi_str, bytes):
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
return urllib.parse.unquote(wsgi_str)
if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
return urllib.parse.unquote(wsgi_str, encoding='latin-1')
def wsgi_quote_plus(wsgi_str):
if six.PY2:
if not isinstance(wsgi_str, bytes):
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
return urllib.parse.quote_plus(wsgi_str)
if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
return urllib.parse.quote_plus(wsgi_str, encoding='latin-1')
def wsgi_unquote_plus(wsgi_str):
if six.PY2:
if not isinstance(wsgi_str, bytes):
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
return urllib.parse.unquote_plus(wsgi_str)
if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
return urllib.parse.unquote_plus(wsgi_str, encoding='latin-1')
def _resp_status_property():
"""
Set and retrieve the value of Response.status

3
swift/common/utils.py
View File

@ -4273,8 +4273,7 @@ def iter_multipart_mime_documents(wsgi_input, boundary, read_chunk_size=4096):
for doing that if necessary.
:param wsgi_input: The file-like object to read from.
:param boundary: The mime boundary to separate new file-like
objects on.
:param boundary: The mime boundary to separate new file-like objects on.
:returns: A generator of file-like objects for each part.
:raises MimeInvalid: if the document is malformed
"""

45
swift/common/wsgi.py
View File

@ -32,13 +32,10 @@ from eventlet.green import socket, ssl, os as green_os
import six
from six import BytesIO
from six import StringIO
from six.moves.urllib.parse import unquote
if six.PY2:
import mimetools
from swift.common import utils, constraints
from swift.common.storage_policy import BindPortsCache
from swift.common.swob import Request
from swift.common.swob import Request, wsgi_unquote
from swift.common.utils import capture_stdio, disable_fallocate, \
drop_privileges, get_logger, NullLogger, config_true_value, \
validate_configuration, get_hub, config_auto_int_value, \
@ -148,31 +145,6 @@ def wrap_conf_type(f):
appconfig = wrap_conf_type(loadwsgi.appconfig)
def monkey_patch_mimetools():
"""
mimetools.Message defaults content-type to "text/plain"
This changes it to default to None, so we can detect missing headers.
"""
if six.PY3:
# The mimetools has been removed from Python 3
return
orig_parsetype = mimetools.Message.parsetype
def parsetype(self):
if not self.typeheader:
self.type = None
self.maintype = None
self.subtype = None
self.plisttext = ''
else:
orig_parsetype(self)
parsetype.patched = True
if not getattr(mimetools.Message.parsetype, 'patched', None):
mimetools.Message.parsetype = parsetype
def get_socket(conf):
"""Bind socket to bind ip:port in conf
@ -448,6 +420,18 @@ class SwiftHttpProtocol(wsgi.HttpProtocol):
# versions the output from error is same as info anyway
self.server.log.info('ERROR WSGI: ' + f, *a)
class MessageClass(wsgi.HttpProtocol.MessageClass):
'''Subclass to see when the client didn't provide a Content-Type'''
# for py2:
def parsetype(self):
if self.typeheader is None:
self.typeheader = ''
wsgi.HttpProtocol.MessageClass.parsetype(self)
# for py3:
def get_default_type(self):
return ''
class SwiftHttpProxiedProtocol(SwiftHttpProtocol):
"""
@ -1156,7 +1140,6 @@ def _initrp(conf_path, app_section, *args, **kwargs):
if config_true_value(conf.get('disable_fallocate', 'no')):
disable_fallocate()
monkey_patch_mimetools()
return (conf, logger, log_name)
@ -1319,7 +1302,7 @@ def make_subrequest(env, method=None, path=None, body=None, headers=None,
path = path or ''
if path and '?' in path:
path, query_string = path.split('?', 1)
newenv = make_env(env, method, path=unquote(path), agent=agent,
newenv = make_env(env, method, path=wsgi_unquote(path), agent=agent,
query_string=query_string, swift_source=swift_source)
if not headers:
headers = {}

2
swift/container/backend.py
View File

@ -338,7 +338,7 @@ class ContainerBroker(DatabaseBroker):
self._db_files = None
@classmethod
def create_broker(self, device_path, part, account, container, logger=None,
def create_broker(cls, device_path, part, account, container, logger=None,
epoch=None, put_timestamp=None,
storage_policy_index=None):
"""

55
swift/obj/diskfile.py
View File

@ -131,14 +131,6 @@ def get_tmp_dir(policy_or_index):
return get_policy_string(TMP_BASE, policy_or_index)
def _unlink_if_present(filename):
try:
os.unlink(filename)
except OSError as err:
if err.errno != errno.ENOENT:
raise
def _get_filename(fd):
"""
Helper function to get to file name from a file descriptor or filename.
@ -1623,12 +1615,10 @@ class BaseDiskFileManager(object):
partition_path = get_part_path(dev_path, policy, partition)
if suffixes is None:
suffixes = self.yield_suffixes(device, partition, policy)
considering_all_suffixes = True
else:
suffixes = (
(os.path.join(partition_path, suffix), suffix)
for suffix in suffixes)
considering_all_suffixes = False
key_preference = (
('ts_meta', 'meta_info', 'timestamp'),
@ -1637,19 +1627,18 @@ class BaseDiskFileManager(object):
('ts_ctype', 'ctype_info', 'ctype_timestamp'),
)
# We delete as many empty directories as we can.
# cleanup_ondisk_files() takes care of the hash dirs, and we take
# care of the suffix dirs and possibly even the partition dir.
have_nonempty_suffix = False
# cleanup_ondisk_files() will remove empty hash dirs, and we'll
# invalidate any empty suffix dirs so they'll get cleaned up on
# the next rehash
for suffix_path, suffix in suffixes:
have_nonempty_hashdir = False
found_files = False
for object_hash in self._listdir(suffix_path):
object_path = os.path.join(suffix_path, object_hash)
try:
results = self.cleanup_ondisk_files(
object_path, **kwargs)
if results['files']:
have_nonempty_hashdir = True
found_files = True
timestamps = {}
for ts_key, info_key, info_ts_key in key_preference:
if info_key not in results:
@ -1669,38 +1658,8 @@ class BaseDiskFileManager(object):
'Invalid diskfile filename in %r (%s)' % (
object_path, err))
if have_nonempty_hashdir:
have_nonempty_suffix = True
else:
try:
os.rmdir(suffix_path)
except OSError as err:
if err.errno not in (errno.ENOENT, errno.ENOTEMPTY):
self.logger.debug(
'Error cleaning up empty suffix directory %s: %s',
suffix_path, err)
# cleanup_ondisk_files tries to remove empty hash dirs,
# but if it fails, so will we. An empty directory
# structure won't cause errors (just slowdown), so we
# ignore the exception.
if considering_all_suffixes and not have_nonempty_suffix:
# There's nothing of interest in the partition, so delete it
try:
# Remove hashes.pkl *then* hashes.invalid; otherwise, if we
# remove hashes.invalid but leave hashes.pkl, that makes it
# look as though the invalidations in hashes.invalid never
# occurred.
_unlink_if_present(os.path.join(partition_path, HASH_FILE))
_unlink_if_present(os.path.join(partition_path,
HASH_INVALIDATIONS_FILE))
# This lock is only held by people dealing with the hashes
# or the hash invalidations, and we've just removed those.
_unlink_if_present(os.path.join(partition_path, ".lock"))
_unlink_if_present(os.path.join(partition_path,
".lock-replication"))
os.rmdir(partition_path)
except OSError as err:
self.logger.debug("Error cleaning up empty partition: %s", err)
if not found_files:
self.invalidate_hash(suffix_path)
class BaseDiskFileWriter(object):

3
swift/obj/mem_diskfile.py
View File

@ -412,6 +412,9 @@ class DiskFile(object):
raise DiskFileNotOpen()
return self._metadata
get_datafile_metadata = get_metadata
get_metafile_metadata = get_metadata
def read_metadata(self, current_time=None):
"""
Return the metadata for an object.

57
swift/proxy/controllers/obj.py
View File

@ -25,6 +25,7 @@
# collected. We've seen objects hang around forever otherwise.
from six.moves.urllib.parse import unquote
from six.moves import zip
import collections
import itertools
@ -697,7 +698,8 @@ class BaseObjectController(Controller):
"""
raise NotImplementedError()
def _delete_object(self, req, obj_ring, partition, headers):
def _delete_object(self, req, obj_ring, partition, headers,
node_count=None, node_iterator=None):
"""Delete object considering write-affinity.
When deleting object in write affinity deployment, also take configured
@ -711,37 +713,12 @@ class BaseObjectController(Controller):
:param headers: system headers to storage nodes
:return: Response object
"""
policy_index = req.headers.get('X-Backend-Storage-Policy-Index')
policy = POLICIES.get_by_index(policy_index)
node_count = None
node_iterator = None
policy_options = self.app.get_policy_options(policy)
is_local = policy_options.write_affinity_is_local_fn
if is_local is not None:
primaries = obj_ring.get_part_nodes(partition)
node_count = len(primaries)
local_handoffs = policy_options.write_affinity_handoff_delete_count
if local_handoffs is None:
local_primaries = [node for node in primaries
if is_local(node)]
local_handoffs = len(primaries) - len(local_primaries)
node_count += local_handoffs
node_iterator = self.iter_nodes_local_first(
obj_ring, partition, policy=policy, local_handoffs_first=True
)
status_overrides = {404: 204}
resp = self.make_requests(req, obj_ring,
partition, 'DELETE', req.swift_entity_path,
headers, overrides=status_overrides,
node_count=node_count,
node_iterator=node_iterator)
return resp
def _post_object(self, req, obj_ring, partition, headers):
@ -861,6 +838,7 @@ class BaseObjectController(Controller):
# Include local handoff nodes if write-affinity is enabled.
node_count = len(nodes)
node_iterator = None
policy = POLICIES.get_by_index(policy_index)
policy_options = self.app.get_policy_options(policy)
is_local = policy_options.write_affinity_is_local_fn
@ -870,11 +848,16 @@ class BaseObjectController(Controller):
local_primaries = [node for node in nodes if is_local(node)]
local_handoffs = len(nodes) - len(local_primaries)
node_count += local_handoffs
node_iterator = self.iter_nodes_local_first(
obj_ring, partition, policy=policy, local_handoffs_first=True
)
headers = self._backend_requests(
req, node_count, container_partition, container_nodes,
container_path=container_path)
return self._delete_object(req, obj_ring, partition, headers)
return self._delete_object(req, obj_ring, partition, headers,
node_count=node_count,
node_iterator=node_iterator)
@ObjectControllerRouter.register(REPL_POLICY)
@ -1112,18 +1095,14 @@ class ECAppIter(object):
resp.content_type = self.learned_content_type
resp.content_length = self.obj_length
def _next_range(self):
def _next_ranges(self):
# Each FA part should have approximately the same headers. We really
# only care about Content-Range and Content-Type, and that'll be the
# same for all the different FAs.
frag_iters = []
headers = None
for parts_iter in self.internal_parts_iters:
part_info = next(parts_iter)
frag_iters.append(part_info['part_iter'])
headers = part_info['headers']
headers = HeaderKeyDict(headers)
return headers, frag_iters
for part_infos in zip(*self.internal_parts_iters):
frag_iters = [pi['part_iter'] for pi in part_infos]
headers = HeaderKeyDict(part_infos[0]['headers'])
yield headers, frag_iters
def _actual_range(self, req_start, req_end, entity_length):
# Takes 3 args: (requested-first-byte, requested-last-byte,
@ -1272,11 +1251,7 @@ class ECAppIter(object):
seen_first_headers = False
ranges_for_resp = {}
while True:
# this'll raise StopIteration and exit the loop
next_range = self._next_range()
headers, frag_iters = next_range
for headers, frag_iters in self._next_ranges():
content_type = headers['Content-Type']
content_range = headers.get('Content-Range')

17
test/functional/__init__.py
View File

@ -53,8 +53,7 @@ from test.unit import SkipTest
from swift.common import constraints, utils, ring, storage_policy
from swift.common.ring import Ring
from swift.common.wsgi import (
monkey_patch_mimetools, loadapp, SwiftHttpProtocol)
from swift.common.wsgi import loadapp, SwiftHttpProtocol
from swift.common.utils import config_true_value, split_path
from swift.account import server as account_server
from swift.container import server as container_server
@ -493,8 +492,6 @@ def in_process_setup(the_object_server=object_server):
swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf')
_info('Using swift config from %s' % swift_conf_src)
monkey_patch_mimetools()
global _testdir
_testdir = os.path.join(mkdtemp(), 'tmp_functional')
utils.mkdirs(_testdir)
@ -1292,3 +1289,15 @@ def requires_policies(f):
return f(self, *args, **kwargs)
return wrapper
def requires_bulk(f):
@functools.wraps(f)
def wrapper(*args, **kwargs):
if skip or not cluster_info:
raise SkipTest('Requires bulk middleware')
# Determine whether this cluster has bulk middleware; if not, skip test
if not cluster_info.get('bulk_upload', {}):
raise SkipTest('Requires bulk middleware')
return f(*args, **kwargs)
return wrapper

5
test/functional/test_dlo.py
View File

@ -49,7 +49,8 @@ class TestDloEnv(BaseEnv):
file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
file_item.write(letter * 10)
file_item = cls.container.file("%s/seg_upper%s" % (prefix, letter))
file_item = cls.container.file(
"%s/seg_upper_%%ff%s" % (prefix, letter))
file_item.write(letter.upper() * 10)
for letter in ('f', 'g', 'h', 'i', 'j'):
@ -64,7 +65,7 @@ class TestDloEnv(BaseEnv):
man2 = cls.container.file("man2")
man2.write('man2-contents',
hdrs={"X-Object-Manifest": "%s/%s/seg_upper" %
hdrs={"X-Object-Manifest": "%s/%s/seg_upper_%%25ff" %
(cls.container.name, prefix)})
manall = cls.container.file("manall")

34
test/functional/test_object.py
View File

@ -20,11 +20,12 @@ import json
import unittest2
from uuid import uuid4
import time
from xml.dom import minidom
from six.moves import range
from test.functional import check_response, retry, requires_acls, \
requires_policies, SkipTest
requires_policies, SkipTest, requires_bulk
import test.functional as tf
@ -1646,6 +1647,37 @@ class TestObject(unittest2.TestCase):
self.assertEqual(resp.status, 200)
self.assertEqual(body, resp.read())
@requires_bulk
def test_bulk_delete(self):
def bulk_delete(url, token, parsed, conn):
# try to bulk delete the object that was created during test setup
conn.request('DELETE', '%s/%s/%s?bulk-delete' % (
parsed.path, self.container, self.obj),
'%s/%s' % (self.container, self.obj),
{'X-Auth-Token': token,
'Accept': 'application/xml',
'Expect': '100-continue',
'Content-Type': 'text/plain'})
return check_response(conn)
resp = retry(bulk_delete)
self.assertEqual(resp.status, 200)
body = resp.read()
tree = minidom.parseString(body)
self.assertEqual(tree.documentElement.tagName, 'delete')
errors = tree.getElementsByTagName('errors')
self.assertEqual(len(errors), 1)
errors = [c.data if c.nodeType == c.TEXT_NODE else c.childNodes[0].data
for c in errors[0].childNodes
if c.nodeType != c.TEXT_NODE or c.data.strip()]
self.assertEqual(errors, [])
final_status = tree.getElementsByTagName('response_status')
self.assertEqual(len(final_status), 1)
self.assertEqual(len(final_status[0].childNodes), 1)
self.assertEqual(final_status[0].childNodes[0].data, '200 OK')
if __name__ == '__main__':
unittest2.main()

13
test/functional/test_slo.py
View File

@ -96,6 +96,8 @@ class TestSloEnv(BaseEnv):
seg_info['seg_e']]),
parms={'multipart-manifest': 'put'})
cls.container.file('seg_with_%ff_funky_name').write('z' * 10)
# Put the same manifest in the container2
file_item = cls.container2.file("manifest-abcde")
file_item.write(
@ -564,6 +566,17 @@ class TestSlo(Base):
parms={'multipart-manifest': 'put'})
self.assert_status(201)
def test_slo_funky_segment(self):
file_item = self.env.container.file("manifest-with-funky-segment")
file_item.write(
json.dumps([{
'path': '/%s/%s' % (self.env.container.name,
'seg_with_%ff_funky_name')}]),
parms={'multipart-manifest': 'put'})
self.assert_status(201)
self.assertEqual('z' * 10, file_item.read())
def test_slo_missing_etag(self):
file_item = self.env.container.file("manifest-a-missing-etag")
file_item.write(

15
test/functional/tests.py
View File

@ -128,11 +128,13 @@ class Base(unittest2.TestCase):
class Base2(object):
def setUp(self):
@classmethod
def setUpClass(cls):
Utils.create_name = Utils.create_utf8_name
super(Base2, self).setUp()
super(Base2, cls).setUpClass()
def tearDown(self):
@classmethod
def tearDownClass(cls):
Utils.create_name = Utils.create_ascii_name
@ -353,8 +355,10 @@ class TestAccount(Base):
self.assertEqual(len(containers), len(self.env.containers))
self.assert_status(200)
marker = (containers[-1] if format_type is None
else containers[-1]['name'])
containers = self.env.account.containers(
parms={'format': format_type, 'marker': containers[-1]})
parms={'format': format_type, 'marker': marker})
self.assertEqual(len(containers), 0)
if format_type is None:
self.assert_status(204)
@ -771,8 +775,9 @@ class TestContainer(Base):
self.assertEqual(len(files), len(self.env.files))
self.assert_status(200)
marker = files[-1] if format_type is None else files[-1]['name']
files = self.env.container.files(
parms={'format': format_type, 'marker': files[-1]})
parms={'format': format_type, 'marker': marker})
self.assertEqual(len(files), 0)
if format_type is None:

4
test/probe/test_replication_servers_working.py
View File

@ -142,8 +142,8 @@ class TestReplicatorFunctions(ReplProbeTest):
for files in test_node_files_list:
self.assertIn(files, new_files_list[0])
for dir in test_node_dir_list:
self.assertIn(dir, new_dir_list[0])
for directory in test_node_dir_list:
self.assertIn(directory, new_dir_list[0])
break
except Exception:
if time.time() - begin > 60:

21
test/unit/account/test_reaper.py
View File

@ -22,7 +22,6 @@ import unittest
from logging import DEBUG
from mock import patch, call, DEFAULT
import six
import eventlet
from swift.account import reaper
@ -85,9 +84,13 @@ class FakeAccountBroker(object):
'delete_timestamp': time.time() - 10}
return info
def list_containers_iter(self, *args):
def list_containers_iter(self, limit, marker, *args):
for cont in self.containers:
yield cont, None, None, None, None
if cont > marker:
yield cont, None, None, None, None
limit -= 1
if limit <= 0:
break
def is_status_deleted(self):
return True
@ -196,11 +199,11 @@ class TestReaper(unittest.TestCase):
raise self.myexp
if self.timeout:
raise eventlet.Timeout()
objects = [{'name': 'o1'},
{'name': 'o2'},
{'name': six.text_type('o3')},
{'name': ''}]
return None, objects
objects = [{'name': u'o1'},
{'name': u'o2'},
{'name': u'o3'},
{'name': u'o4'}]
return None, [o for o in objects if o['name'] > kwargs['marker']]
def fake_container_ring(self):
return FakeRing()
@ -589,7 +592,7 @@ class TestReaper(unittest.TestCase):
self.r.stats_return_codes.get(2, 0) + 1
def test_reap_account(self):
containers = ('c1', 'c2', 'c3', '')
containers = ('c1', 'c2', 'c3', 'c4')
broker = FakeAccountBroker(containers)
self.called_amount = 0
self.r = r = self.init_reaper({}, fakelogger=True)

72
test/unit/common/middleware/crypto/test_keymaster.py
View File

@ -482,6 +482,78 @@ class TestKeymaster(unittest.TestCase):
self.assertEqual(expected_keys, keys)
self.assertEqual([('/a/c', None), ('/a/c//o', None)], calls)
def test_v1_keys_with_weird_paths(self):
secrets = {None: os.urandom(32),
'22': os.urandom(33)}
conf = {}
for secret_id, secret in secrets.items():
opt = ('encryption_root_secret%s' %
(('_%s' % secret_id) if secret_id else ''))
conf[opt] = base64.b64encode(secret)
conf['active_root_secret_id'] = '22'
self.app = keymaster.KeyMaster(self.swift, conf)
orig_create_key = self.app.create_key
calls = []
def mock_create_key(path, secret_id=None):
calls.append((path, secret_id))
return orig_create_key(path, secret_id)
# request path doesn't match stored path -- this could happen if you
# misconfigured your proxy to have copy right of encryption
context = keymaster.KeyMasterContext(self.app, 'a', 'not-c', 'not-o')
for version in ('1', '2'):
with mock.patch.object(self.app, 'create_key', mock_create_key):
keys = context.fetch_crypto_keys(key_id={
'v': version, 'path': '/a/c/o'})
expected_keys = {
'container': hmac.new(secrets[None], b'/a/c',
digestmod=hashlib.sha256).digest(),
'object': hmac.new(secrets[None], b'/a/c/o',
digestmod=hashlib.sha256).digest(),
'id': {'path': '/a/c/o', 'v': version},
'all_ids': [
{'path': '/a/c/o', 'v': version},
{'path': '/a/c/o', 'secret_id': '22', 'v': version}]}
self.assertEqual(expected_keys, keys)
self.assertEqual([('/a/c', None), ('/a/c/o', None)], calls)
del calls[:]
context = keymaster.KeyMasterContext(
self.app, 'not-a', 'not-c', '/not-o')
with mock.patch.object(self.app, 'create_key', mock_create_key):
keys = context.fetch_crypto_keys(key_id={
'v': '1', 'path': '/o'})
expected_keys = {
'container': hmac.new(secrets[None], b'/not-a/not-c',
digestmod=hashlib.sha256).digest(),
'object': hmac.new(secrets[None], b'/o',
digestmod=hashlib.sha256).digest(),
'id': {'path': '/o', 'v': '1'},
'all_ids': [
{'path': '/o', 'v': '1'},
{'path': '/o', 'secret_id': '22', 'v': '1'}]}
self.assertEqual(expected_keys, keys)
self.assertEqual([('/not-a/not-c', None), ('/o', None)], calls)
del calls[:]
context = keymaster.KeyMasterContext(
self.app, 'not-a', 'not-c', '/not-o')
with mock.patch.object(self.app, 'create_key', mock_create_key):
keys = context.fetch_crypto_keys(key_id={
'v': '2', 'path': '/a/c//o'})
expected_keys = {
'container': hmac.new(secrets[None], b'/a/c',
digestmod=hashlib.sha256).digest(),
'object': hmac.new(secrets[None], b'/a/c//o',
digestmod=hashlib.sha256).digest(),
'id': {'path': '/a/c//o', 'v': '2'},
'all_ids': [
{'path': '/a/c//o', 'v': '2'},
{'path': '/a/c//o', 'secret_id': '22', 'v': '2'}]}
self.assertEqual(expected_keys, keys)
self.assertEqual([('/a/c', None), ('/a/c//o', None)], calls)
@mock.patch('swift.common.middleware.crypto.keymaster.readconf')
def test_keymaster_config_path(self, mock_readconf):
for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):

6
test/unit/common/middleware/s3api/test_multi_upload.py
View File

@ -717,7 +717,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
'Response Status': '201 Created',
'Errors': [],
})])
mock_time.return_value.time.side_effect = (
mock_time.time.side_effect = (
1, # start_time
12, # first whitespace
13, # second...
@ -769,7 +769,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
'Response Status': '400 Bad Request',
'Errors': [['some/object', '403 Forbidden']],
})])
mock_time.return_value.time.side_effect = (
mock_time.time.side_effect = (
1, # start_time
12, # first whitespace
13, # second...
@ -818,7 +818,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
'Response Status': '400 Bad Request',
'Errors': [['some/object', '404 Not Found']],
})])
mock_time.return_value.time.side_effect = (
mock_time.time.side_effect = (
1, # start_time
12, # first whitespace
13, # second...

54
test/unit/common/middleware/s3api/test_obj.py
View File

@ -469,6 +469,60 @@ class TestS3ApiObj(S3ApiTestCase):
# Check that s3api converts a Content-MD5 header into an etag.
self.assertEqual(headers['etag'], etag)
@s3acl
def test_object_PUT_v4(self):
body_sha = hashlib.sha256(self.object_body).hexdigest()
req = Request.blank(
'/bucket/object',
environ={'REQUEST_METHOD': 'PUT'},
headers={
'Authorization':
'AWS4-HMAC-SHA256 '
'Credential=test:tester/%s/us-east-1/s3/aws4_request, '
'SignedHeaders=host;x-amz-date, '
'Signature=hmac' % (
self.get_v4_amz_date_header().split('T', 1)[0]),
'x-amz-date': self.get_v4_amz_date_header(),
'x-amz-storage-class': 'STANDARD',
'x-amz-content-sha256': body_sha,
'Date': self.get_date_header()},
body=self.object_body)
req.date = datetime.now()
req.content_type = 'text/plain'
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '200')
# Check that s3api returns an etag header.
self.assertEqual(headers['etag'],
'"%s"' % self.response_headers['etag'])
_, _, headers = self.swift.calls_with_headers[-1]
# No way to determine ETag to send
self.assertNotIn('etag', headers)
@s3acl
def test_object_PUT_v4_bad_hash(self):
req = Request.blank(
'/bucket/object',
environ={'REQUEST_METHOD': 'PUT'},
headers={
'Authorization':
'AWS4-HMAC-SHA256 '
'Credential=test:tester/%s/us-east-1/s3/aws4_request, '
'SignedHeaders=host;x-amz-date, '
'Signature=hmac' % (
self.get_v4_amz_date_header().split('T', 1)[0]),
'x-amz-date': self.get_v4_amz_date_header(),
'x-amz-storage-class': 'STANDARD',
'x-amz-content-sha256': 'not the hash',
'Date': self.get_date_header()},
body=self.object_body)
req.date = datetime.now()
req.content_type = 'text/plain'
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '400')
print(body)
self.assertEqual(self._get_error_code(body), 'BadDigest')
def test_object_PUT_headers(self):
content_md5 = self.etag.decode('hex').encode('base64').strip()

76
test/unit/common/middleware/s3api/test_s3request.py
View File

@ -13,9 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import hashlib
from mock import patch, MagicMock
import unittest
from six import BytesIO
from swift.common import swob
from swift.common.swob import Request, HTTPNoContent
from swift.common.middleware.s3api.utils import mktime
@ -24,7 +27,7 @@ from swift.common.middleware.s3api.subresource import ACL, User, Owner, \
Grant, encode_acl
from test.unit.common.middleware.s3api.test_s3api import S3ApiTestCase
from swift.common.middleware.s3api.s3request import S3Request, \
S3AclRequest, SigV4Request, SIGV4_X_AMZ_DATE_FORMAT
S3AclRequest, SigV4Request, SIGV4_X_AMZ_DATE_FORMAT, HashingInput
from swift.common.middleware.s3api.s3response import InvalidArgument, \
NoSuchBucket, InternalError, \
AccessDenied, SignatureDoesNotMatch, RequestTimeTooSkewed
@ -802,5 +805,76 @@ class TestRequest(S3ApiTestCase):
u'\u30c9\u30e9\u30b4\u30f3'))
class TestHashingInput(S3ApiTestCase):
def test_good(self):
raw = b'123456789'
wrapped = HashingInput(BytesIO(raw), 9, hashlib.md5,
hashlib.md5(raw).hexdigest())
self.assertEqual(b'1234', wrapped.read(4))
self.assertEqual(b'56', wrapped.read(2))
# trying to read past the end gets us whatever's left
self.assertEqual(b'789', wrapped.read(4))
# can continue trying to read -- but it'll be empty
self.assertEqual(b'', wrapped.read(2))
self.assertFalse(wrapped._input.closed)
wrapped.close()
self.assertTrue(wrapped._input.closed)
def test_empty(self):
wrapped = HashingInput(BytesIO(b''), 0, hashlib.sha256,
hashlib.sha256(b'').hexdigest())
self.assertEqual(b'', wrapped.read(4))
self.assertEqual(b'', wrapped.read(2))
self.assertFalse(wrapped._input.closed)
wrapped.close()
self.assertTrue(wrapped._input.closed)
def test_too_long(self):
raw = b'123456789'
wrapped = HashingInput(BytesIO(raw), 8, hashlib.md5,
hashlib.md5(raw).hexdigest())
self.assertEqual(b'1234', wrapped.read(4))
self.assertEqual(b'56', wrapped.read(2))
# even though the hash matches, there was more data than we expected
with self.assertRaises(swob.Response) as raised:
wrapped.read(3)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
# the error causes us to close the input
self.assertTrue(wrapped._input.closed)
def test_too_short(self):
raw = b'123456789'
wrapped = HashingInput(BytesIO(raw), 10, hashlib.md5,
hashlib.md5(raw).hexdigest())
self.assertEqual(b'1234', wrapped.read(4))
self.assertEqual(b'56', wrapped.read(2))
# even though the hash matches, there was more data than we expected
with self.assertRaises(swob.Response) as raised:
wrapped.read(4)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
self.assertTrue(wrapped._input.closed)
def test_bad_hash(self):
raw = b'123456789'
wrapped = HashingInput(BytesIO(raw), 9, hashlib.sha256,
hashlib.md5(raw).hexdigest())
self.assertEqual(b'1234', wrapped.read(4))
self.assertEqual(b'5678', wrapped.read(4))
with self.assertRaises(swob.Response) as raised:
wrapped.read(4)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
self.assertTrue(wrapped._input.closed)
def test_empty_bad_hash(self):
wrapped = HashingInput(BytesIO(b''), 0, hashlib.sha256, 'nope')
with self.assertRaises(swob.Response) as raised:
wrapped.read(3)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
# the error causes us to close the input
self.assertTrue(wrapped._input.closed)
if __name__ == '__main__':
unittest.main()

23
test/unit/common/middleware/test_copy.py
View File

@ -335,6 +335,29 @@ class TestServerSideCopyMiddleware(unittest.TestCase):
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/a/c/o', self.authorized[1].path)
def test_copy_with_unicode(self):
self.app.register('GET', '/v1/a/c/\xF0\x9F\x8C\xB4', swob.HTTPOk,
{}, 'passed')
self.app.register('PUT', '/v1/a/c/\xE2\x98\x83', swob.HTTPCreated, {})
# Just for fun, let's have a mix of properly encoded and not
req = Request.blank('/v1/a/c/%F0\x9F%8C%B4',
environ={'REQUEST_METHOD': 'COPY'},
headers={'Content-Length': '0',
'Destination': 'c/%E2\x98%83'})
status, headers, body = self.call_ssc(req)
self.assertEqual(status, '201 Created')
calls = self.app.calls_with_headers
method, path, req_headers = calls[0]
self.assertEqual('GET', method)
self.assertEqual('/v1/a/c/\xF0\x9F\x8C\xB4', path)
self.assertIn(('X-Copied-From', 'c/%F0%9F%8C%B4'), headers)
self.assertEqual(len(self.authorized), 2)
self.assertEqual('GET', self.authorized[0].method)
self.assertEqual('/v1/a/c/%F0%9F%8C%B4', self.authorized[0].path)
self.assertEqual('PUT', self.authorized[1].method)
self.assertEqual('/v1/a/c/%E2%98%83', self.authorized[1].path)
def test_copy_with_spaces_in_x_copy_from_and_account(self):
self.app.register('GET', '/v1/a/c/o o2', swob.HTTPOk, {}, b'passed')
self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {})

427
test/unit/common/middleware/test_formpost.py
View File

File diff suppressed because it is too large Load Diff

30
test/unit/common/middleware/test_tempauth.py
View File

@ -1024,6 +1024,36 @@ class TestAuth(unittest.TestCase):
resp = req.get_response(ath)
self.assertEqual(204, resp.status_int)
def test_request_method_not_allowed(self):
test_auth = auth.filter_factory({'user_ac_user': 'testing'})(FakeApp())
req = self._make_request(
'/auth/v1.0',
headers={'X-Auth-User': 'ac:user', 'X-Auth-Key': 'testing'},
environ={'REQUEST_METHOD': 'PUT'})
resp = req.get_response(test_auth)
self.assertEqual(resp.status_int, 405)
req = self._make_request(
'/auth/v1.0',
headers={'X-Auth-User': 'ac:user', 'X-Auth-Key': 'testing'},
environ={'REQUEST_METHOD': 'HEAD'})
resp = req.get_response(test_auth)
self.assertEqual(resp.status_int, 405)
req = self._make_request(
'/auth/v1.0',
headers={'X-Auth-User': 'ac:user', 'X-Auth-Key': 'testing'},
environ={'REQUEST_METHOD': 'POST'})
resp = req.get_response(test_auth)
self.assertEqual(resp.status_int, 405)
req = self._make_request(
'/auth/v1.0',
headers={'X-Auth-User': 'ac:user', 'X-Auth-Key': 'testing'},
environ={'REQUEST_METHOD': 'DELETE'})
resp = req.get_response(test_auth)
self.assertEqual(resp.status_int, 405)
class TestAuthWithMultiplePrefixes(TestAuth):
"""

51
test/unit/common/test_wsgi.py
View File

@ -27,12 +27,8 @@ import types
import eventlet.wsgi
import six
from six import BytesIO
from six import StringIO
from six.moves.urllib.parse import quote
if six.PY2:
import mimetools
import mock
@ -69,53 +65,6 @@ def _fake_rings(tmpdir):
class TestWSGI(unittest.TestCase):
"""Tests for swift.common.wsgi"""
def setUp(self):
if six.PY2:
self._orig_parsetype = mimetools.Message.parsetype
def tearDown(self):
if six.PY2:
mimetools.Message.parsetype = self._orig_parsetype
@unittest.skipIf(six.PY3, "test specific to Python 2")
def test_monkey_patch_mimetools(self):
sio = StringIO('blah')
self.assertEqual(mimetools.Message(sio).type, 'text/plain')
sio = StringIO('blah')
self.assertEqual(mimetools.Message(sio).plisttext, '')
sio = StringIO('blah')
self.assertEqual(mimetools.Message(sio).maintype, 'text')
sio = StringIO('blah')
self.assertEqual(mimetools.Message(sio).subtype, 'plain')
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).type, 'text/html')
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).plisttext,
'; charset=ISO-8859-4')
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).maintype, 'text')
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).subtype, 'html')
wsgi.monkey_patch_mimetools()
sio = StringIO('blah')
self.assertIsNone(mimetools.Message(sio).type)
sio = StringIO('blah')
self.assertEqual(mimetools.Message(sio).plisttext, '')
sio = StringIO('blah')
self.assertIsNone(mimetools.Message(sio).maintype)
sio = StringIO('blah')
self.assertIsNone(mimetools.Message(sio).subtype)
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).type, 'text/html')
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).plisttext,
'; charset=ISO-8859-4')
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).maintype, 'text')
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
self.assertEqual(mimetools.Message(sio).subtype, 'html')
def test_init_request_processor(self):
config = """
[DEFAULT]

34
test/unit/helpers.py
View File

@ -40,6 +40,7 @@ from swift.common.storage_policy import StoragePolicy, ECStoragePolicy
from swift.common.middleware import listing_formats, proxy_logging
from swift.common import utils
from swift.common.utils import mkdirs, normalize_timestamp, NullLogger
from swift.common.wsgi import SwiftHttpProtocol
from swift.container import server as container_server
from swift.obj import server as object_server
from swift.proxy import server as proxy_server
@ -212,17 +213,28 @@ def setup_servers(the_object_server=object_server, extra_conf=None):
nl = NullLogger()
logging_prosv = proxy_logging.ProxyLoggingMiddleware(
listing_formats.ListingFilter(prosrv), conf, logger=prosrv.logger)
prospa = spawn(wsgi.server, prolis, logging_prosv, nl)
acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl)
acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl)
con1spa = spawn(wsgi.server, con1lis, con1srv, nl)
con2spa = spawn(wsgi.server, con2lis, con2srv, nl)
obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl)
obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl)
obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl)
obj4spa = spawn(wsgi.server, obj4lis, obj4srv, nl)
obj5spa = spawn(wsgi.server, obj5lis, obj5srv, nl)
obj6spa = spawn(wsgi.server, obj6lis, obj6srv, nl)
prospa = spawn(wsgi.server, prolis, logging_prosv, nl,
protocol=SwiftHttpProtocol)
acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl,
protocol=SwiftHttpProtocol)
acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl,
protocol=SwiftHttpProtocol)
con1spa = spawn(wsgi.server, con1lis, con1srv, nl,
protocol=SwiftHttpProtocol)
con2spa = spawn(wsgi.server, con2lis, con2srv, nl,
protocol=SwiftHttpProtocol)
obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl,
protocol=SwiftHttpProtocol)
obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl,
protocol=SwiftHttpProtocol)
obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl,
protocol=SwiftHttpProtocol)
obj4spa = spawn(wsgi.server, obj4lis, obj4srv, nl,
protocol=SwiftHttpProtocol)
obj5spa = spawn(wsgi.server, obj5lis, obj5srv, nl,
protocol=SwiftHttpProtocol)
obj6spa = spawn(wsgi.server, obj6lis, obj6srv, nl,
protocol=SwiftHttpProtocol)
context["test_coros"] = \
(prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa, obj3spa,
obj4spa, obj5spa, obj6spa)

33
test/unit/obj/test_diskfile.py
View File

@ -1415,7 +1415,8 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
df2_suffix, df2_hash,
"1525354556.65758.ts")))
# Expire the tombstones
# Cache the hashes and expire the tombstones
self.df_mgr.get_hashes(self.existing_device, '0', [], POLICIES[0])
the_time[0] += 2 * self.df_mgr.reclaim_age
hashes = list(self.df_mgr.yield_hashes(
@ -1440,7 +1441,30 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.testdir, self.existing_device, 'objects', '0',
df2_suffix, df2_hash)))
# The empty suffix dirs are gone
# The empty suffix dirs, and partition are still there
self.assertTrue(os.path.isdir(os.path.join(
self.testdir, self.existing_device, 'objects', '0',
df1_suffix)))
self.assertTrue(os.path.isdir(os.path.join(
self.testdir, self.existing_device, 'objects', '0',
df2_suffix)))
# but the suffixes is invalid
part_dir = os.path.join(
self.testdir, self.existing_device, 'objects', '0')
invalidations_file = os.path.join(
part_dir, diskfile.HASH_INVALIDATIONS_FILE)
with open(invalidations_file) as f:
self.assertEqual('%s\n%s' % (df1_suffix, df2_suffix),
f.read().strip('\n')) # sanity
# next time get hashes runs
with mock.patch('time.time', mock_time):
hashes = self.df_mgr.get_hashes(
self.existing_device, '0', [], POLICIES[0])
self.assertEqual(hashes, {})
# ... suffixes will get cleanup
self.assertFalse(os.path.exists(os.path.join(
self.testdir, self.existing_device, 'objects', '0',
df1_suffix)))
@ -1448,8 +1472,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
self.testdir, self.existing_device, 'objects', '0',
df2_suffix)))
# The empty partition dir is gone
self.assertFalse(os.path.exists(os.path.join(
# but really it's not diskfile's jobs to decide if a partition belongs
# on a node or not
self.assertTrue(os.path.isdir(os.path.join(
self.testdir, self.existing_device, 'objects', '0')))
def test_focused_yield_hashes_does_not_clean_up(self):

23
test/unit/obj/test_reconstructor.py
View File

@ -1327,12 +1327,31 @@ class TestGlobalSetupObjectReconstructor(unittest.TestCase):
for stat_key, expected in expected_stats.items():
stat_method, stat_prefix = stat_key
self.assertStatCount(stat_method, stat_prefix, expected)
stub_data = self.reconstructor._get_hashes(
'sda1', 2, self.policy, do_listdir=True)
stub_data.update({'7ca': {None: '8f19c38e1cf8e2390d4ca29051407ae3'}})
pickle_path = os.path.join(part_path, 'hashes.pkl')
with open(pickle_path, 'w') as f:
pickle.dump(stub_data, f)
# part 2 should be totally empty
hash_gen = self.reconstructor._df_router[self.policy].yield_hashes(
'sda1', '2', self.policy)
'sda1', '2', self.policy, suffixes=stub_data.keys())
for path, hash_, ts in hash_gen:
self.fail('found %s with %s in %s' % (hash_, ts, path))
# even the partition directory is gone
new_hashes = self.reconstructor._get_hashes(
'sda1', 2, self.policy, do_listdir=True)
self.assertFalse(new_hashes)
# N.B. the partition directory is removed next pass
ssync_calls = []
with mocked_http_conn() as request_log:
with mock.patch('swift.obj.reconstructor.ssync_sender',
self._make_fake_ssync(ssync_calls)):
self.reconstructor.reconstruct(override_partitions=[2])
self.assertEqual([], ssync_calls)
self.assertFalse(os.access(part_path, os.F_OK))
def test_process_job_all_success(self):

62
test/unit/proxy/controllers/test_obj.py
View File

@ -531,7 +531,7 @@ class CommonObjectControllerMixin(BaseObjectControllerMixin):
self.assertNotIn('X-Delete-At-Host', headers)
self.assertNotIn('X-Delete-At-Device', headers)
def test_DELETE_write_affinity_before_replication(self):
def test_DELETE_write_affinity_after_replication(self):
policy_conf = self.app.get_policy_options(self.policy)
policy_conf.write_affinity_handoff_delete_count = self.replicas() // 2
policy_conf.write_affinity_is_local_fn = (
@ -545,7 +545,7 @@ class CommonObjectControllerMixin(BaseObjectControllerMixin):
self.assertEqual(resp.status_int, 204)
def test_DELETE_write_affinity_after_replication(self):
def test_DELETE_write_affinity_before_replication(self):
policy_conf = self.app.get_policy_options(self.policy)
policy_conf.write_affinity_handoff_delete_count = self.replicas() // 2
policy_conf.write_affinity_is_local_fn = (
@ -1416,6 +1416,36 @@ class TestReplicatedObjController(CommonObjectControllerMixin,
self.assertIn(req.swift_entity_path.decode('utf-8'), log_lines[0])
self.assertIn('ERROR with Object server', log_lines[0])
def test_DELETE_with_write_affinity(self):
policy_conf = self.app.get_policy_options(self.policy)
policy_conf.write_affinity_handoff_delete_count = self.replicas() // 2
policy_conf.write_affinity_is_local_fn = (
lambda node: node['region'] == 1)
req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE')
codes = [204, 204, 404, 204]
with mocked_http_conn(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 204)
codes = [204, 404, 404, 204]
with mocked_http_conn(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 204)
policy_conf.write_affinity_handoff_delete_count = 2
codes = [204, 204, 404, 204, 404]
with mocked_http_conn(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 204)
codes = [204, 404, 404, 204, 204]
with mocked_http_conn(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 204)
def test_PUT_error_during_transfer_data(self):
class FakeReader(object):
def read(self, size):
@ -1816,15 +1846,39 @@ class TestReplicatedObjController(CommonObjectControllerMixin,
@patch_policies(
[StoragePolicy(0, '1-replica', True),
StoragePolicy(1, '5-replica', False),
StoragePolicy(1, '4-replica', False),
StoragePolicy(2, '8-replica', False),
StoragePolicy(3, '15-replica', False)],
fake_ring_args=[
{'replicas': 1}, {'replicas': 5}, {'replicas': 8}, {'replicas': 15}])
{'replicas': 1}, {'replicas': 4}, {'replicas': 8}, {'replicas': 15}])
class TestReplicatedObjControllerVariousReplicas(CommonObjectControllerMixin,
unittest.TestCase):
controller_cls = obj.ReplicatedObjectController
def test_DELETE_with_write_affinity(self):
policy_index = 1
self.policy = POLICIES[policy_index]
policy_conf = self.app.get_policy_options(self.policy)
self.app.container_info['storage_policy'] = policy_index
policy_conf.write_affinity_handoff_delete_count = \
self.replicas(self.policy) // 2
policy_conf.write_affinity_is_local_fn = (
lambda node: node['region'] == 1)
req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE')
codes = [204, 204, 404, 404, 204, 204]
with mocked_http_conn(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 204)
policy_conf.write_affinity_handoff_delete_count = 1
codes = [204, 204, 404, 404, 204]
with mocked_http_conn(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 204)
@patch_policies()
class TestReplicatedObjControllerMimePutter(BaseObjectControllerMixin,

57
test/unit/proxy/test_server.py
View File

@ -71,7 +71,7 @@ from swift.common import utils, constraints
from swift.common.utils import hash_path, storage_directory, \
parse_content_type, parse_mime_headers, \
iter_multipart_mime_documents, public, mkdirs, NullLogger
from swift.common.wsgi import monkey_patch_mimetools, loadapp, ConfigString
from swift.common.wsgi import loadapp, ConfigString
from swift.proxy.controllers import base as proxy_base
from swift.proxy.controllers.base import get_cache_key, cors_validation, \
get_account_info, get_container_info
@ -97,7 +97,6 @@ def do_setup(object_server):
# setup test context and break out some globals for convenience
global _test_context, _testdir, _test_servers, _test_sockets, \
_test_POLICIES
monkey_patch_mimetools()
_test_context = setup_servers(object_server)
_testdir = _test_context["testdir"]
_test_servers = _test_context["test_servers"]
@ -3269,37 +3268,35 @@ class TestReplicatedObjectController(
self.assertNotEqual(last_modified_put, last_modified_head)
_do_conditional_GET_checks(last_modified_head)
@unpatch_policies
def test_PUT_auto_content_type(self):
with save_globals():
controller = ReplicatedObjectController(
self.app, 'account', 'container', 'object')
prolis = _test_sockets[0]
def test_content_type(filename, expected):
# The three responses here are for account_info() (HEAD to
# account server), container_info() (HEAD to container server)
# and three calls to _connect_put_node() (PUT to three object
# servers)
set_http_connect(201, 201, 201, 201, 201,
give_content_type=lambda content_type:
self.assertEqual(content_type,
next(expected)))
# We need into include a transfer-encoding to get past
# constraints.check_object_creation()
req = Request.blank('/v1/a/c/%s' % filename, {},
headers={'transfer-encoding': 'chunked'})
self.app.update_request(req)
self.app.memcache.store = {}
res = controller.PUT(req)
# If we don't check the response here we could miss problems
# in PUT()
self.assertEqual(res.status_int, 201)
def do_test(ext, content_type):
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile('rwb')
fd.write(b'PUT /v1/a/c/o.%s HTTP/1.1\r\n'
b'Host: localhost\r\n'
b'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n' %
ext.encode())
fd.flush()
headers = readuntil2crlfs(fd)
exp = b'HTTP/1.1 201'
self.assertEqual(headers[:len(exp)], exp)
test_content_type('test.jpg', iter(['', '', 'image/jpeg',
'image/jpeg', 'image/jpeg']))
test_content_type('test.html', iter(['', '', 'text/html',
'text/html', 'text/html']))
test_content_type('test.css', iter(['', '', 'text/css',
'text/css', 'text/css']))
fd.write(b'GET /v1/a/c/o.%s HTTP/1.1\r\n'
b'Host: localhost\r\nConnection: close\r\n'
b'X-Storage-Token: t\r\n\r\n' % ext.encode())
fd.flush()
headers = readuntil2crlfs(fd)
exp = b'HTTP/1.1 200'
self.assertIn(b'Content-Type: %s' % content_type.encode(),
headers.split(b'\r\n'))
sock.close()
do_test('jpg', 'image/jpeg')
do_test('html', 'text/html')
do_test('css', 'text/css')
def test_custom_mime_types_files(self):
swift_dir = mkdtemp()

3
test/unit/proxy/test_sysmeta.py
View File

@ -24,7 +24,7 @@ from swift.common.middleware.copy import ServerSideCopyMiddleware
from swift.common.storage_policy import StoragePolicy
from swift.common.swob import Request
from swift.common.utils import mkdirs, split_path
from swift.common.wsgi import monkey_patch_mimetools, WSGIContext
from swift.common.wsgi import WSGIContext
from swift.obj import server as object_server
from swift.proxy import server as proxy
import swift.proxy.controllers
@ -138,7 +138,6 @@ class TestObjectSysmeta(unittest.TestCase):
account_ring=FakeRing(replicas=1),
container_ring=FakeRing(replicas=1))
self.copy_app = ServerSideCopyMiddleware(self.app, {})
monkey_patch_mimetools()
self.tmpdir = mkdtemp()
self.testdir = os.path.join(self.tmpdir,
'tmp_test_object_server_ObjectController')

4
tox.ini
View File

@ -45,6 +45,7 @@ commands =
test/unit/common/middleware/test_copy.py \
test/unit/common/middleware/test_crossdomain.py \
test/unit/common/middleware/test_domain_remap.py \
test/unit/common/middleware/test_formpost.py \
test/unit/common/middleware/test_gatekeeper.py \
test/unit/common/middleware/test_healthcheck.py \
test/unit/common/middleware/test_keystoneauth.py \
@ -96,6 +97,9 @@ commands =
[testenv:py36]
commands = {[testenv:py35]commands}
[testenv:py37]
commands = {[testenv:py35]commands}
[testenv:pep8]
basepython = python2.7
commands =