merged master into ec

Change-Id: I65ac27670bc51481d92ea10fef088482fc46bd0f
This commit is contained in:
John Dickinson 2014-04-17 08:14:48 -07:00
commit 282548d538
26 changed files with 496 additions and 109 deletions

View File

@ -415,10 +415,18 @@ Setting up scripts for running Swift
sed -i "s/service \(.*\) restart/systemctl restart \1.service/" $HOME/bin/resetswift
The template ``resetswift`` script looks like the following:
.. literalinclude:: /../saio/bin/resetswift
#. Install the sample configuration file for running tests::
cp $HOME/swift/test/sample.conf /etc/swift/test.conf
The template ``test.conf`` looks like the following:
.. literalinclude:: /../../test/sample.conf
#. Add an environment variable for running tests below::
echo "export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf" >> $HOME/.bashrc
@ -435,6 +443,10 @@ Setting up scripts for running Swift
remakerings
The ``remakerings`` script looks like the following:
.. literalinclude:: /../saio/bin/remakerings
You can expect the ouptut from this command to produce the following::
Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
@ -467,6 +479,10 @@ Setting up scripts for running Swift
(The "``Unable to increase file descriptor limit. Running as non-root?``"
warnings are expected and ok.)
The ``startmain`` script looks like the following:
.. literalinclude:: /../saio/bin/startmain
#. Get an ``X-Storage-Url`` and ``X-Auth-Token``::
curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0

View File

@ -100,6 +100,7 @@ these log lines is::
remote_addr - - [datetime] "request_method request_path" status_int
content_length "referer" "transaction_id" "user_agent" request_time
additional_info
=================== ==========================================================
**Log Field** **Value**
@ -117,4 +118,5 @@ user_agent The value of the HTTP User-Agent header. Swift's proxy
server sets its user-agent to
``"proxy-server <pid of the proxy>".``
request_time The duration of the request.
additional_info Additional useful information.
=================== ==========================================================

View File

@ -49,7 +49,8 @@ swift_hash_path_prefix = changeme
#default = yes
# The swift-constraints section sets the basic constraints on data
# saved in the swift cluster.
# saved in the swift cluster. These constraints are automatically
# published by the proxy server in responses to /info requests.
[swift-constraints]

View File

@ -18,5 +18,5 @@
import setuptools
setuptools.setup(
setup_requires=['pbr>=0.5.21,<1.0'],
setup_requires=['pbr'],
pbr=True)

View File

@ -28,7 +28,7 @@ from swift.common.request_helpers import get_param, get_listing_content_type, \
split_and_validate_path
from swift.common.utils import get_logger, hash_path, public, \
normalize_timestamp, storage_directory, config_true_value, \
json, timing_stats, replication
json, timing_stats, replication, get_log_line
from swift.common.constraints import check_mount, check_float, check_utf8
from swift.common import constraints
from swift.common.db_replicator import ReplicatorRpc
@ -284,21 +284,13 @@ class AccountController(object):
' %(path)s '),
{'method': req.method, 'path': req.path})
res = HTTPInternalServerError(body=traceback.format_exc())
trans_time = '%.4f' % (time.time() - start_time)
additional_info = ''
if res.headers.get('x-container-timestamp') is not None:
additional_info += 'x-container-timestamp: %s' % \
res.headers['x-container-timestamp']
if self.log_requests:
log_msg = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s "%s"' % (
req.remote_addr,
time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()),
req.method, req.path,
res.status.split()[0], res.content_length or '-',
req.headers.get('x-trans-id', '-'),
req.referer or '-', req.user_agent or '-',
trans_time,
additional_info)
trans_time = time.time() - start_time
additional_info = ''
if res.headers.get('x-container-timestamp') is not None:
additional_info += 'x-container-timestamp: %s' % \
res.headers['x-container-timestamp']
log_msg = get_log_line(req, res, trans_time, additional_info)
if req.method.upper() == 'REPLICATE':
self.logger.debug(log_msg)
else:

View File

@ -107,6 +107,8 @@ def print_db_info_metadata(db_type, info, metadata):
print (' Delete Timestamp: %s (%s)' %
(datetime.utcfromtimestamp(float(info['delete_timestamp'])),
info['delete_timestamp']))
if db_type == 'account':
print ' Container Count: %s' % info['container_count']
print ' Object Count: %s' % info['object_count']
print ' Bytes Used: %s' % info['bytes_used']
if db_type == 'container':
@ -175,7 +177,7 @@ def print_info(db_type, db_file, swift_dir='/etc/swift'):
try:
info = broker.get_info()
except sqlite3.OperationalError as err:
if 'no such table' in err.message:
if 'no such table' in str(err):
print "Does not appear to be a DB of type \"%s\": %s" % (
db_type, db_file)
raise InfoSystemExit()

View File

@ -34,6 +34,10 @@ ACCOUNT_LISTING_LIMIT = 10000
MAX_ACCOUNT_NAME_LENGTH = 256
MAX_CONTAINER_NAME_LENGTH = 256
# If adding an entry to DEFAULT_CONSTRAINTS, note that
# these constraints are automatically published by the
# proxy server in responses to /info requests, with values
# updated by reload_constraints()
DEFAULT_CONSTRAINTS = {
'max_file_size': MAX_FILE_SIZE,
'max_meta_name_length': MAX_META_NAME_LENGTH,

View File

@ -678,7 +678,8 @@ class InternalClient(object):
"""
headers = dict(headers or {})
headers['Transfer-Encoding'] = 'chunked'
if 'Content-Length' not in headers:
headers['Transfer-Encoding'] = 'chunked'
path = self.make_path(account, container, obj)
self.make_request('PUT', path, headers, (2,), fobj)

View File

@ -120,6 +120,7 @@ class RingData(object):
tempf.flush()
os.fsync(tempf.fileno())
tempf.close()
os.chmod(tempf.name, 0o644)
os.rename(tempf.name, filename)
def to_dict(self):

View File

@ -1117,6 +1117,18 @@ class Response(object):
self.content_length = 0
return ['']
if self.last_modified and self.request.if_modified_since \
and self.last_modified <= self.request.if_modified_since:
self.status = 304
self.content_length = 0
return ['']
if self.last_modified and self.request.if_unmodified_since \
and self.last_modified > self.request.if_unmodified_since:
self.status = 412
self.content_length = 0
return ['']
if self.request and self.request.method == 'HEAD':
# We explicitly do NOT want to set self.content_length to 0 here
return ['']

View File

@ -292,6 +292,28 @@ def generate_trans_id(trans_id_suffix):
uuid.uuid4().hex[:21], time.time(), trans_id_suffix)
def get_log_line(req, res, trans_time, additional_info):
"""
Make a line for logging that matches the documented log line format
for backend servers.
:param req: the request.
:param res: the response.
:param trans_time: the time the request took to complete, a float.
:param additional_info: a string to log at the end of the line
:returns: a properly formated line for logging.
"""
return '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %.4f "%s"' % (
req.remote_addr,
time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()),
req.method, req.path, res.status.split()[0],
res.content_length or '-', req.referer or '-',
req.headers.get('x-trans-id', '-'),
req.user_agent or '-', trans_time, additional_info or '-')
def get_trans_id_time(trans_id):
if len(trans_id) >= 34 and trans_id[:2] == 'tx' and trans_id[23] == '-':
try:
@ -856,8 +878,12 @@ class LoggingHandlerWeakRef(weakref.ref):
"""
def close(self):
referent = self()
if referent:
referent.close()
try:
if referent:
referent.close()
except KeyError:
# This is to catch an issue with old py2.6 versions
pass
def flush(self):
referent = self()

View File

@ -31,7 +31,7 @@ from swift.common.request_helpers import get_param, get_listing_content_type, \
from swift.common.utils import get_logger, hash_path, public, \
normalize_timestamp, storage_directory, validate_sync_to, \
config_true_value, json, timing_stats, replication, \
override_bytes_from_content_type
override_bytes_from_content_type, get_log_line
from swift.common.constraints import check_mount, check_float, check_utf8
from swift.common import constraints
from swift.common.bufferedhttp import http_connect
@ -550,17 +550,9 @@ class ContainerController(object):
'ERROR __call__ error with %(method)s %(path)s '),
{'method': req.method, 'path': req.path})
res = HTTPInternalServerError(body=traceback.format_exc())
trans_time = '%.4f' % (time.time() - start_time)
if self.log_requests:
log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s' % (
req.remote_addr,
time.strftime('%d/%b/%Y:%H:%M:%S +0000',
time.gmtime()),
req.method, req.path,
res.status.split()[0], res.content_length or '-',
req.headers.get('x-trans-id', '-'),
req.referer or '-', req.user_agent or '-',
trans_time)
trans_time = time.time() - start_time
log_message = get_log_line(req, res, trans_time, '')
if req.method.upper() == 'REPLICATE':
self.logger.debug(log_message)
else:

View File

@ -22,14 +22,14 @@ import time
import traceback
import socket
import math
from datetime import datetime
from swift import gettext_ as _
from hashlib import md5
from eventlet import sleep, Timeout
from swift.common.utils import public, get_logger, \
config_true_value, timing_stats, replication, normalize_delete_at_timestamp
config_true_value, timing_stats, replication, \
normalize_delete_at_timestamp, get_log_line
from swift.common.bufferedhttp import http_connect
from swift.common.constraints import check_object_creation, \
check_float, check_utf8
@ -40,9 +40,9 @@ from swift.obj import ssync_receiver
from swift.common.http import is_success
from swift.common.request_helpers import get_name_and_placement, is_user_meta
from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
HTTPInternalServerError, HTTPNoContent, HTTPNotFound, HTTPNotModified, \
HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \
HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, UTC, \
HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \
HTTPInsufficientStorage, HTTPForbidden, HTTPException, HeaderKeyDict, \
HTTPConflict
from swift.obj.diskfile import DATAFILE_SYSTEM_META, DiskFileManager
@ -496,16 +496,6 @@ class ObjectController(object):
obj_size = int(metadata['Content-Length'])
file_x_ts = metadata['X-Timestamp']
file_x_ts_flt = float(file_x_ts)
file_x_ts_utc = datetime.fromtimestamp(file_x_ts_flt, UTC)
if_unmodified_since = request.if_unmodified_since
if if_unmodified_since and file_x_ts_utc > if_unmodified_since:
return HTTPPreconditionFailed(request=request)
if_modified_since = request.if_modified_since
if if_modified_since and file_x_ts_utc <= if_modified_since:
return HTTPNotModified(request=request)
keep_cache = (self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers))
@ -696,15 +686,7 @@ class ObjectController(object):
res = HTTPInternalServerError(body=traceback.format_exc())
trans_time = time.time() - start_time
if self.log_requests:
log_line = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %.4f' % (
req.remote_addr,
time.strftime('%d/%b/%Y:%H:%M:%S +0000',
time.gmtime()),
req.method, req.path, res.status.split()[0],
res.content_length or '-', req.referer or '-',
req.headers.get('x-trans-id', '-'),
req.user_agent or '-',
trans_time)
log_line = get_log_line(req, res, trans_time, '')
if req.method in ('REPLICATE', 'REPLICATION') or \
'X-Backend-Replication' in req.headers:
self.logger.debug(log_line)

View File

@ -205,17 +205,9 @@ class Application(object):
self.admin_key = conf.get('admin_key', None)
register_swift_info(
version=swift_version,
max_file_size=constraints.MAX_FILE_SIZE,
max_meta_name_length=constraints.MAX_META_NAME_LENGTH,
max_meta_value_length=constraints.MAX_META_VALUE_LENGTH,
max_meta_count=constraints.MAX_META_COUNT,
account_listing_limit=constraints.ACCOUNT_LISTING_LIMIT,
container_listing_limit=constraints.CONTAINER_LISTING_LIMIT,
max_account_name_length=constraints.MAX_ACCOUNT_NAME_LENGTH,
max_container_name_length=constraints.MAX_CONTAINER_NAME_LENGTH,
max_object_name_length=constraints.MAX_OBJECT_NAME_LENGTH,
policies=POLICIES.get_policy_info(),
strict_cors_mode=self.strict_cors_mode)
strict_cors_mode=self.strict_cors_mode,
**constraints.EFFECTIVE_CONSTRAINTS)
def check_config(self):
"""

View File

@ -778,13 +778,15 @@ class File(Base):
transferred = 0
buff = data.read(block_size)
buff_len = len(buff)
try:
while len(buff) > 0:
while buff_len > 0:
self.conn.put_data(buff)
buff = data.read(block_size)
transferred += len(buff)
transferred += buff_len
if callable(callback):
callback(transferred, self.size)
buff = data.read(block_size)
buff_len = len(buff)
self.conn.put_end()
except socket.timeout as err:

View File

@ -22,52 +22,30 @@ import locale
import random
import StringIO
import time
import threading
import unittest
import urllib
import uuid
import eventlet
import eventlet.debug
from nose import SkipTest
from swift.common.utils import get_hub
from test.functional import normalized_urls, load_constraint
import test.functional as tf
from test.functional.swift_test_client import Account, Connection, File, \
ResponseError
def chunks(s, length=3):
i, j = 0, length
while i < len(s):
yield s[i:j]
i, j = j, j + length
def timeout(seconds, method, *args, **kwargs):
class TimeoutThread(threading.Thread):
def __init__(self, method, *args, **kwargs):
threading.Thread.__init__(self)
self.method = method
self.args = args
self.kwargs = kwargs
self.exception = None
def run(self):
try:
self.method(*self.args, **self.kwargs)
except Exception as e:
self.exception = e
t = TimeoutThread(method, *args, **kwargs)
t.start()
t.join(seconds)
if t.exception:
raise t.exception
if t.isAlive():
t._Thread__stop()
return True
return False
# In order to get the proper blocking behavior of sockets without using
# threads, where we can set an arbitrary timeout for some piece of code under
# test, we use eventlet with the standard socket library patched. We have to
# perform this setup at module import time, since all the socket module
# bindings in the swiftclient code will have been made by the time nose
# invokes the package or class setup methods.
eventlet.hubs.use_hub(get_hub())
eventlet.patcher.monkey_patch(all=False, socket=True)
eventlet.debug.hub_exceptions(True)
class Utils(object):
@ -1162,6 +1140,15 @@ class TestFile(Base):
limit = load_constraint('max_file_size')
tsecs = 3
def timeout(seconds, method, *args, **kwargs):
try:
with eventlet.Timeout(seconds):
method(*args, **kwargs)
except eventlet.Timeout:
return True
else:
return False
for i in (limit - 100, limit - 10, limit - 1, limit, limit + 1,
limit + 10, limit + 100):
@ -1473,6 +1460,13 @@ class TestFile(Base):
if (tf.web_front_end == 'apache2'):
raise SkipTest("Chunked PUT can only be tested with apache2 web"
" front end")
def chunks(s, length=3):
i, j = 0, length
while i < len(s):
yield s[i:j]
i, j = j, j + length
data = File.random_data(10000)
etag = File.compute_md5sum(data)
@ -1724,19 +1718,25 @@ class TestFileComparison(Base):
for file_item in self.env.files:
hdrs = {'If-Modified-Since': self.env.time_old_f1}
self.assert_(file_item.read(hdrs=hdrs))
self.assert_(file_item.info(hdrs=hdrs))
hdrs = {'If-Modified-Since': self.env.time_new}
self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
self.assert_status(304)
self.assertRaises(ResponseError, file_item.info, hdrs=hdrs)
self.assert_status(304)
def testIfUnmodifiedSince(self):
for file_item in self.env.files:
hdrs = {'If-Unmodified-Since': self.env.time_new}
self.assert_(file_item.read(hdrs=hdrs))
self.assert_(file_item.info(hdrs=hdrs))
hdrs = {'If-Unmodified-Since': self.env.time_old_f2}
self.assertRaises(ResponseError, file_item.read, hdrs=hdrs)
self.assert_status(412)
self.assertRaises(ResponseError, file_item.info, hdrs=hdrs)
self.assert_status(412)
def testIfMatchAndUnmodified(self):
for file_item in self.env.files:

View File

@ -20,6 +20,7 @@ import unittest
from tempfile import mkdtemp
from shutil import rmtree
from StringIO import StringIO
from time import gmtime
from test.unit import FakeLogger
import itertools
import random
@ -1827,6 +1828,22 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(expected_total_count, total_object_count)
self.assertEqual(expected_total_count, total_bytes_used)
def test_log_line_format(self):
req = Request.blank(
'/sda1/p/a',
environ={'REQUEST_METHOD': 'HEAD', 'REMOTE_ADDR': '1.2.3.4'})
self.controller.logger = FakeLogger()
with mock.patch(
'time.gmtime', mock.MagicMock(side_effect=[gmtime(10001.0)])):
with mock.patch(
'time.time',
mock.MagicMock(side_effect=[10000.0, 10001.0, 10002.0])):
req.get_response(self.controller)
self.assertEqual(
self.controller.logger.log_dict['info'],
[(('1.2.3.4 - - [01/Jan/1970:02:46:41 +0000] "HEAD /sda1/p/a" 404 '
'- "-" "-" "-" 2.0000 "-"',), {})])
@patch_policies([StoragePolicy(0, 'zero', False),
StoragePolicy(1, 'one', True),
@ -1836,5 +1853,6 @@ class TestNonLegacyDefaultStoragePolicy(TestAccountController):
pass
if __name__ == '__main__':
unittest.main()

View File

@ -88,6 +88,7 @@ class TestCliInfo(unittest.TestCase):
created_at=100.1,
put_timestamp=106.3,
delete_timestamp=107.9,
container_count='3',
object_count='20',
bytes_used='42')
info['hash'] = 'abaddeadbeefcafe'
@ -104,6 +105,7 @@ Metadata:
Created at: 1970-01-01 00:01:40.100000 (100.1)
Put Timestamp: 1970-01-01 00:01:46.300000 (106.3)
Delete Timestamp: 1970-01-01 00:01:47.900000 (107.9)
Container Count: 3
Object Count: 20
Bytes Used: 42
Chexor: abaddeadbeefcafe
@ -112,7 +114,8 @@ Metadata:
No system metadata found in db file
User Metadata: {'mydata': 'swift'}'''
self.assertEquals(out.getvalue().strip(), exp_out)
self.assertEquals(sorted(out.getvalue().strip().split('\n')),
sorted(exp_out.split('\n')))
info = dict(
account='acct',
@ -156,7 +159,8 @@ Metadata:
X-Container-Foo: bar
System Metadata: {'mydata': 'swift'}
No user metadata found in db file'''
self.assertEquals(out.getvalue().strip(), exp_out)
self.assertEquals(sorted(out.getvalue().strip().split('\n')),
sorted(exp_out.split('\n')))
def test_print_ring_locations(self):
self.assertRaisesMessage(ValueError, 'None type', print_ring_locations,

View File

@ -18,6 +18,7 @@ import cPickle as pickle
import os
import sys
import unittest
import stat
from contextlib import closing
from gzip import GzipFile
from tempfile import mkdtemp
@ -98,6 +99,15 @@ class TestRingData(unittest.TestCase):
with open(ring_fname2) as ring2:
self.assertEqual(ring1.read(), ring2.read())
def test_permissions(self):
ring_fname = os.path.join(self.testdir, 'stat.ring.gz')
rd = ring.RingData(
[array.array('H', [0, 1, 0, 1]), array.array('H', [0, 1, 0, 1])],
[{'id': 0, 'zone': 0}, {'id': 1, 'zone': 1}], 30)
rd.save(ring_fname)
self.assertEqual(oct(stat.S_IMODE(os.stat(ring_fname).st_mode)),
'0644')
class TestRing(unittest.TestCase):

View File

@ -922,6 +922,33 @@ class TestInternalClient(unittest.TestCase):
client.upload_object(fobj, account, container, obj, headers)
self.assertEquals(1, client.make_request_called)
def test_upload_object_not_chunked(self):
class InternalClient(internal_client.InternalClient):
def __init__(self, test, path, headers, fobj):
self.test = test
self.path = path
self.headers = headers
self.fobj = fobj
self.make_request_called = 0
def make_request(
self, method, path, headers, acceptable_statuses,
body_file=None):
self.make_request_called += 1
self.test.assertEquals(self.path, path)
exp_headers = dict(self.headers)
self.test.assertEquals(exp_headers, headers)
self.test.assertEquals(self.fobj, fobj)
fobj = 'some_fobj'
account, container, obj = path_parts()
path = make_path(account, container, obj)
headers = {'key': 'value', 'Content-Length': len(fobj)}
client = InternalClient(self, path, headers, fobj)
client.upload_object(fobj, account, container, obj, headers)
self.assertEquals(1, client.make_request_called)
class TestGetAuth(unittest.TestCase):
@mock.patch('eventlet.green.urllib2.urlopen')

View File

@ -15,8 +15,8 @@
"Tests for swift.common.swob"
import unittest
import datetime
import unittest
import re
import time
from StringIO import StringIO
@ -1451,5 +1451,141 @@ class TestConditionalIfMatch(unittest.TestCase):
self.assertEquals(body, '')
class TestConditionalIfModifiedSince(unittest.TestCase):
def fake_app(self, environ, start_response):
start_response(
'200 OK', [('Last-Modified', 'Thu, 27 Feb 2014 03:29:37 GMT')])
return ['hi']
def fake_start_response(*a, **kw):
pass
def test_absent(self):
req = swift.common.swob.Request.blank('/')
resp = req.get_response(self.fake_app)
resp.conditional_response = True
body = ''.join(resp(req.environ, self.fake_start_response))
self.assertEquals(resp.status_int, 200)
self.assertEquals(body, 'hi')
def test_before(self):
req = swift.common.swob.Request.blank(
'/',
headers={'If-Modified-Since': 'Thu, 27 Feb 2014 03:29:36 GMT'})
resp = req.get_response(self.fake_app)
resp.conditional_response = True
body = ''.join(resp(req.environ, self.fake_start_response))
self.assertEquals(resp.status_int, 200)
self.assertEquals(body, 'hi')
def test_same(self):
req = swift.common.swob.Request.blank(
'/',
headers={'If-Modified-Since': 'Thu, 27 Feb 2014 03:29:37 GMT'})
resp = req.get_response(self.fake_app)
resp.conditional_response = True
body = ''.join(resp(req.environ, self.fake_start_response))
self.assertEquals(resp.status_int, 304)
self.assertEquals(body, '')
def test_greater(self):
req = swift.common.swob.Request.blank(
'/',
headers={'If-Modified-Since': 'Thu, 27 Feb 2014 03:29:38 GMT'})
resp = req.get_response(self.fake_app)
resp.conditional_response = True
body = ''.join(resp(req.environ, self.fake_start_response))
self.assertEquals(resp.status_int, 304)
self.assertEquals(body, '')
def test_out_of_range_is_ignored(self):
# All that datetime gives us is a ValueError or OverflowError when
# something is out of range (i.e. less than datetime.datetime.min or
# greater than datetime.datetime.max). Unfortunately, we can't
# distinguish between a date being too old and a date being too new,
# so the best we can do is ignore such headers.
max_date_list = list(datetime.datetime.max.timetuple())
max_date_list[0] += 1 # bump up the year
too_big_date_header = time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", time.struct_time(max_date_list))
req = swift.common.swob.Request.blank(
'/',
headers={'If-Modified-Since': too_big_date_header})
resp = req.get_response(self.fake_app)
resp.conditional_response = True
body = ''.join(resp(req.environ, self.fake_start_response))
self.assertEquals(resp.status_int, 200)
self.assertEquals(body, 'hi')
class TestConditionalIfUnmodifiedSince(unittest.TestCase):
def fake_app(self, environ, start_response):
start_response(
'200 OK', [('Last-Modified', 'Thu, 20 Feb 2014 03:29:37 GMT')])
return ['hi']
def fake_start_response(*a, **kw):
pass
def test_absent(self):
req = swift.common.swob.Request.blank('/')
resp = req.get_response(self.fake_app)
resp.conditional_response = True
body = ''.join(resp(req.environ, self.fake_start_response))
self.assertEquals(resp.status_int, 200)
self.assertEquals(body, 'hi')
def test_before(self):
req = swift.common.swob.Request.blank(
'/',
headers={'If-Unmodified-Since': 'Thu, 20 Feb 2014 03:29:36 GMT'})
resp = req.get_response(self.fake_app)
resp.conditional_response = True
body = ''.join(resp(req.environ, self.fake_start_response))
self.assertEquals(resp.status_int, 412)
self.assertEquals(body, '')
def test_same(self):
req = swift.common.swob.Request.blank(
'/',
headers={'If-Unmodified-Since': 'Thu, 20 Feb 2014 03:29:37 GMT'})
resp = req.get_response(self.fake_app)
resp.conditional_response = True
body = ''.join(resp(req.environ, self.fake_start_response))
self.assertEquals(resp.status_int, 200)
self.assertEquals(body, 'hi')
def test_greater(self):
req = swift.common.swob.Request.blank(
'/',
headers={'If-Unmodified-Since': 'Thu, 20 Feb 2014 03:29:38 GMT'})
resp = req.get_response(self.fake_app)
resp.conditional_response = True
body = ''.join(resp(req.environ, self.fake_start_response))
self.assertEquals(resp.status_int, 200)
self.assertEquals(body, 'hi')
def test_out_of_range_is_ignored(self):
# All that datetime gives us is a ValueError or OverflowError when
# something is out of range (i.e. less than datetime.datetime.min or
# greater than datetime.datetime.max). Unfortunately, we can't
# distinguish between a date being too old and a date being too new,
# so the best we can do is ignore such headers.
max_date_list = list(datetime.datetime.max.timetuple())
max_date_list[0] += 1 # bump up the year
too_big_date_header = time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", time.struct_time(max_date_list))
req = swift.common.swob.Request.blank(
'/',
headers={'If-Unmodified-Since': too_big_date_header})
resp = req.get_response(self.fake_app)
resp.conditional_response = True
body = ''.join(resp(req.environ, self.fake_start_response))
self.assertEquals(resp.status_int, 200)
self.assertEquals(body, 'hi')
if __name__ == '__main__':
unittest.main()

View File

@ -24,6 +24,7 @@ import eventlet.event
import grp
import logging
import os
import mock
import random
import re
import socket
@ -55,7 +56,7 @@ from swift.common.exceptions import (Timeout, MessageTimeout,
ReplicationLockTimeout)
from swift.common import utils
from swift.common.container_sync_realms import ContainerSyncRealms
from swift.common.swob import Response
from swift.common.swob import Request, Response
from test.unit import FakeLogger
@ -1893,6 +1894,22 @@ cluster_dfw1 = http://dfw1.host/v1/
utils.get_hmac('GET', '/path', 1, 'abc'),
'b17f6ff8da0e251737aa9e3ee69a881e3e092e2f')
def test_get_log_line(self):
req = Request.blank(
'/sda1/p/a/c/o',
environ={'REQUEST_METHOD': 'HEAD', 'REMOTE_ADDR': '1.2.3.4'})
res = Response()
trans_time = 1.2
additional_info = 'some information'
exp_line = '1.2.3.4 - - [01/Jan/1970:02:46:41 +0000] "HEAD ' \
'/sda1/p/a/c/o" 200 - "-" "-" "-" 1.2000 "some information"'
with mock.patch(
'time.gmtime',
mock.MagicMock(side_effect=[time.gmtime(10001.0)])):
self.assertEquals(
exp_line,
utils.get_log_line(req, res, trans_time, additional_info))
class TestSwiftInfo(unittest.TestCase):

View File

@ -23,6 +23,7 @@ from shutil import rmtree
from StringIO import StringIO
from tempfile import mkdtemp
from test.unit import FakeLogger
from time import gmtime
from xml.dom import minidom
from eventlet import spawn, Timeout, listen
@ -2214,6 +2215,22 @@ class TestContainerController(unittest.TestCase):
self.assertEqual(resp.status_int, 404)
self.assertFalse(self.controller.logger.log_dict['info'])
def test_log_line_format(self):
req = Request.blank(
'/sda1/p/a/c',
environ={'REQUEST_METHOD': 'HEAD', 'REMOTE_ADDR': '1.2.3.4'})
self.controller.logger = FakeLogger()
with mock.patch(
'time.gmtime', mock.MagicMock(side_effect=[gmtime(10001.0)])):
with mock.patch(
'time.time',
mock.MagicMock(side_effect=[10000.0, 10001.0, 10002.0])):
req.get_response(self.controller)
self.assertEqual(
self.controller.logger.log_dict['info'],
[(('1.2.3.4 - - [01/Jan/1970:02:46:41 +0000] "HEAD /sda1/p/a/c" '
'404 - "-" "-" "-" 2.0000 "-"',), {})])
if __name__ == '__main__':
unittest.main()

View File

@ -1279,6 +1279,78 @@ class TestObjectController(unittest.TestCase):
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 304)
def test_HEAD_if_modified_since(self):
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={
'X-Timestamp': timestamp,
'Content-Type': 'application/octet-stream',
'Content-Length': '4'})
req.body = 'test'
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 201)
req = Request.blank('/sda1/p/a/c/o',
environ={'REQUEST_METHOD': 'HEAD'})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 200)
since = strftime('%a, %d %b %Y %H:%M:%S GMT',
gmtime(float(timestamp) + 1))
req = Request.blank('/sda1/p/a/c/o',
environ={'REQUEST_METHOD': 'HEAD'},
headers={'If-Modified-Since': since})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 304)
since = \
strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) - 1))
req = Request.blank('/sda1/p/a/c/o',
environ={'REQUEST_METHOD': 'HEAD'},
headers={'If-Modified-Since': since})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 200)
since = \
strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) + 1))
req = Request.blank('/sda1/p/a/c/o',
environ={'REQUEST_METHOD': 'HEAD'},
headers={'If-Modified-Since': since})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 304)
req = Request.blank('/sda1/p/a/c/o',
environ={'REQUEST_METHOD': 'HEAD'})
resp = req.get_response(self.object_controller)
since = resp.headers['Last-Modified']
self.assertEquals(since, strftime('%a, %d %b %Y %H:%M:%S GMT',
gmtime(math.ceil(float(timestamp)))))
req = Request.blank('/sda1/p/a/c/o',
environ={'REQUEST_METHOD': 'HEAD'},
headers={'If-Modified-Since': since})
resp = self.object_controller.GET(req)
self.assertEquals(resp.status_int, 304)
timestamp = normalize_timestamp(int(time()))
req = Request.blank('/sda1/p/a/c/o2',
environ={'REQUEST_METHOD': 'PUT'},
headers={
'X-Timestamp': timestamp,
'Content-Type': 'application/octet-stream',
'Content-Length': '4'})
req.body = 'test'
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 201)
since = strftime('%a, %d %b %Y %H:%M:%S GMT',
gmtime(float(timestamp)))
req = Request.blank('/sda1/p/a/c/o2',
environ={'REQUEST_METHOD': 'HEAD'},
headers={'If-Modified-Since': since})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 304)
def test_GET_if_unmodified_since(self):
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
@ -1327,6 +1399,42 @@ class TestObjectController(unittest.TestCase):
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 200)
def test_HEAD_if_unmodified_since(self):
timestamp = normalize_timestamp(time())
req = Request.blank(
'/sda1/p/a/c/o',
environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': timestamp,
'Content-Type': 'application/octet-stream',
'Content-Length': '4'})
req.body = 'test'
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 201)
since = strftime('%a, %d %b %Y %H:%M:%S GMT',
gmtime(math.ceil(float(timestamp)) + 1))
req = Request.blank('/sda1/p/a/c/o',
environ={'REQUEST_METHOD': 'HEAD'},
headers={'If-Unmodified-Since': since})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 200)
since = strftime('%a, %d %b %Y %H:%M:%S GMT',
gmtime(math.ceil(float(timestamp))))
req = Request.blank('/sda1/p/a/c/o',
environ={'REQUEST_METHOD': 'HEAD'},
headers={'If-Unmodified-Since': since})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 200)
since = strftime('%a, %d %b %Y %H:%M:%S GMT',
gmtime(math.ceil(float(timestamp)) - 1))
req = Request.blank('/sda1/p/a/c/o',
environ={'REQUEST_METHOD': 'HEAD'},
headers={'If-Unmodified-Since': since})
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 412)
def test_GET_quarantine(self):
# Test swift.obj.server.ObjectController.GET
timestamp = normalize_timestamp(time())
@ -3399,7 +3507,7 @@ class TestObjectController(unittest.TestCase):
self.assertEqual(
self.object_controller.logger.log_dict['info'],
[(('None - - [01/Jan/1970:02:46:41 +0000] "PUT'
' /sda1/p/a/c/o" 405 - "-" "-" "-" 1.0000',),
' /sda1/p/a/c/o" 405 - "-" "-" "-" 1.0000 "-"',),
{})])
def test_not_utf8_and_not_logging_requests(self):
@ -3569,5 +3677,22 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 201)
self.assertTrue(os.path.isdir(object_dir))
def test_log_line_format(self):
req = Request.blank(
'/sda1/p/a/c/o',
environ={'REQUEST_METHOD': 'HEAD', 'REMOTE_ADDR': '1.2.3.4'})
self.object_controller.logger = FakeLogger()
with mock.patch(
'time.gmtime', mock.MagicMock(side_effect=[gmtime(10001.0)])):
with mock.patch(
'time.time',
mock.MagicMock(side_effect=[10000.0, 10001.0, 10002.0])):
req.get_response(self.object_controller)
self.assertEqual(
self.object_controller.logger.log_dict['info'],
[(('1.2.3.4 - - [01/Jan/1970:02:46:41 +0000] "HEAD /sda1/p/a/c/o" '
'404 - "-" "-" "-" 2.0000 "-"',), {})])
if __name__ == '__main__':
unittest.main()

View File

@ -86,7 +86,7 @@ class TestObjControllerWriteAffinity(unittest.TestCase):
def test_connect_put_node_timeout(self):
controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
self.app.conn_timeout = 0.1
self.app.conn_timeout = 0.05
with set_http_connect(200, slow_connect=True):
nodes = [dict(ip='', port='', device='')]
res = controller._connect_put_node(nodes, '', '', {}, ('', ''))

View File

@ -6299,6 +6299,9 @@ class TestSwiftInfo(unittest.TestCase):
self.assertEqual(si['max_meta_value_length'],
constraints.MAX_META_VALUE_LENGTH)
self.assertEqual(si['max_meta_count'], constraints.MAX_META_COUNT)
self.assertEqual(si['max_header_size'], constraints.MAX_HEADER_SIZE)
self.assertEqual(si['max_meta_overall_size'],
constraints.MAX_META_OVERALL_SIZE)
self.assertEqual(si['account_listing_limit'],
constraints.ACCOUNT_LISTING_LIMIT)
self.assertEqual(si['container_listing_limit'],
@ -6309,6 +6312,11 @@ class TestSwiftInfo(unittest.TestCase):
constraints.MAX_CONTAINER_NAME_LENGTH)
self.assertEqual(si['max_object_name_length'],
constraints.MAX_OBJECT_NAME_LENGTH)
self.assertTrue('strict_cors_mode' in si)
self.assertTrue('policies' in si)
# this next test is deliberately brittle in order to alert if
# other items are added to swift info
self.assertEqual(len(si), 14)
self.assertTrue('policies' in si)
sorted_pols = sorted(si['policies'], key=operator.itemgetter('name'))