From 62a1e7e0593723ff19a83c1533b02f6f444f6942 Mon Sep 17 00:00:00 2001 From: Samuel Merritt Date: Tue, 4 Mar 2014 11:52:48 -0800 Subject: [PATCH 01/15] Support If-[Un]Modified-Since for object HEAD We already supported it for object GET requests, but not for HEAD. This lets clients keep metadata up-to-date without having to either fetch the whole object when it's changed or do their own date parsing. They can just treat Last-Modified as opaque and update their idea of metadata when they get a 200. Change-Id: Iff25d8989a93d651fd2c327e1e58036e79e1bde1 --- swift/common/swob.py | 12 +++ swift/obj/server.py | 15 +--- test/functional/tests.py | 6 ++ test/unit/common/test_swob.py | 138 +++++++++++++++++++++++++++++++++- test/unit/obj/test_server.py | 108 ++++++++++++++++++++++++++ 5 files changed, 265 insertions(+), 14 deletions(-) diff --git a/swift/common/swob.py b/swift/common/swob.py index 638086ea0e..68b19cf875 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -1117,6 +1117,18 @@ class Response(object): self.content_length = 0 return [''] + if self.last_modified and self.request.if_modified_since \ + and self.last_modified <= self.request.if_modified_since: + self.status = 304 + self.content_length = 0 + return [''] + + if self.last_modified and self.request.if_unmodified_since \ + and self.last_modified > self.request.if_unmodified_since: + self.status = 412 + self.content_length = 0 + return [''] + if self.request and self.request.method == 'HEAD': # We explicitly do NOT want to set self.content_length to 0 here return [''] diff --git a/swift/obj/server.py b/swift/obj/server.py index a4d8de7c93..e356bf53b4 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -22,7 +22,6 @@ import time import traceback import socket import math -from datetime import datetime from swift import gettext_ as _ from hashlib import md5 @@ -40,9 +39,9 @@ from swift.obj import ssync_receiver from swift.common.http import is_success from swift.common.request_helpers import split_and_validate_path, is_user_meta from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \ - HTTPInternalServerError, HTTPNoContent, HTTPNotFound, HTTPNotModified, \ + HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \ HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \ - HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, UTC, \ + HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \ HTTPInsufficientStorage, HTTPForbidden, HTTPException, HeaderKeyDict, \ HTTPConflict from swift.obj.diskfile import DATAFILE_SYSTEM_META, DiskFileManager @@ -481,16 +480,6 @@ class ObjectController(object): obj_size = int(metadata['Content-Length']) file_x_ts = metadata['X-Timestamp'] file_x_ts_flt = float(file_x_ts) - file_x_ts_utc = datetime.fromtimestamp(file_x_ts_flt, UTC) - - if_unmodified_since = request.if_unmodified_since - if if_unmodified_since and file_x_ts_utc > if_unmodified_since: - return HTTPPreconditionFailed(request=request) - - if_modified_since = request.if_modified_since - if if_modified_since and file_x_ts_utc <= if_modified_since: - return HTTPNotModified(request=request) - keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) diff --git a/test/functional/tests.py b/test/functional/tests.py index 97e45a6275..a95a30ab38 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -1752,19 +1752,25 @@ class TestFileComparison(Base): for file_item in self.env.files: hdrs = {'If-Modified-Since': self.env.time_old_f1} self.assert_(file_item.read(hdrs=hdrs)) + self.assert_(file_item.info(hdrs=hdrs)) hdrs = {'If-Modified-Since': self.env.time_new} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(304) + self.assertRaises(ResponseError, file_item.info, hdrs=hdrs) + self.assert_status(304) def testIfUnmodifiedSince(self): for file_item in self.env.files: hdrs = {'If-Unmodified-Since': self.env.time_new} self.assert_(file_item.read(hdrs=hdrs)) + self.assert_(file_item.info(hdrs=hdrs)) hdrs = {'If-Unmodified-Since': self.env.time_old_f2} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(412) + self.assertRaises(ResponseError, file_item.info, hdrs=hdrs) + self.assert_status(412) def testIfMatchAndUnmodified(self): for file_item in self.env.files: diff --git a/test/unit/common/test_swob.py b/test/unit/common/test_swob.py index 7cc5439e91..b7438dfead 100644 --- a/test/unit/common/test_swob.py +++ b/test/unit/common/test_swob.py @@ -15,8 +15,8 @@ "Tests for swift.common.swob" -import unittest import datetime +import unittest import re import time from StringIO import StringIO @@ -1451,5 +1451,141 @@ class TestConditionalIfMatch(unittest.TestCase): self.assertEquals(body, '') +class TestConditionalIfModifiedSince(unittest.TestCase): + def fake_app(self, environ, start_response): + start_response( + '200 OK', [('Last-Modified', 'Thu, 27 Feb 2014 03:29:37 GMT')]) + return ['hi'] + + def fake_start_response(*a, **kw): + pass + + def test_absent(self): + req = swift.common.swob.Request.blank('/') + resp = req.get_response(self.fake_app) + resp.conditional_response = True + body = ''.join(resp(req.environ, self.fake_start_response)) + self.assertEquals(resp.status_int, 200) + self.assertEquals(body, 'hi') + + def test_before(self): + req = swift.common.swob.Request.blank( + '/', + headers={'If-Modified-Since': 'Thu, 27 Feb 2014 03:29:36 GMT'}) + resp = req.get_response(self.fake_app) + resp.conditional_response = True + body = ''.join(resp(req.environ, self.fake_start_response)) + self.assertEquals(resp.status_int, 200) + self.assertEquals(body, 'hi') + + def test_same(self): + req = swift.common.swob.Request.blank( + '/', + headers={'If-Modified-Since': 'Thu, 27 Feb 2014 03:29:37 GMT'}) + resp = req.get_response(self.fake_app) + resp.conditional_response = True + body = ''.join(resp(req.environ, self.fake_start_response)) + self.assertEquals(resp.status_int, 304) + self.assertEquals(body, '') + + def test_greater(self): + req = swift.common.swob.Request.blank( + '/', + headers={'If-Modified-Since': 'Thu, 27 Feb 2014 03:29:38 GMT'}) + resp = req.get_response(self.fake_app) + resp.conditional_response = True + body = ''.join(resp(req.environ, self.fake_start_response)) + self.assertEquals(resp.status_int, 304) + self.assertEquals(body, '') + + def test_out_of_range_is_ignored(self): + # All that datetime gives us is a ValueError or OverflowError when + # something is out of range (i.e. less than datetime.datetime.min or + # greater than datetime.datetime.max). Unfortunately, we can't + # distinguish between a date being too old and a date being too new, + # so the best we can do is ignore such headers. + max_date_list = list(datetime.datetime.max.timetuple()) + max_date_list[0] += 1 # bump up the year + too_big_date_header = time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", time.struct_time(max_date_list)) + + req = swift.common.swob.Request.blank( + '/', + headers={'If-Modified-Since': too_big_date_header}) + resp = req.get_response(self.fake_app) + resp.conditional_response = True + body = ''.join(resp(req.environ, self.fake_start_response)) + self.assertEquals(resp.status_int, 200) + self.assertEquals(body, 'hi') + + +class TestConditionalIfUnmodifiedSince(unittest.TestCase): + def fake_app(self, environ, start_response): + start_response( + '200 OK', [('Last-Modified', 'Thu, 20 Feb 2014 03:29:37 GMT')]) + return ['hi'] + + def fake_start_response(*a, **kw): + pass + + def test_absent(self): + req = swift.common.swob.Request.blank('/') + resp = req.get_response(self.fake_app) + resp.conditional_response = True + body = ''.join(resp(req.environ, self.fake_start_response)) + self.assertEquals(resp.status_int, 200) + self.assertEquals(body, 'hi') + + def test_before(self): + req = swift.common.swob.Request.blank( + '/', + headers={'If-Unmodified-Since': 'Thu, 20 Feb 2014 03:29:36 GMT'}) + resp = req.get_response(self.fake_app) + resp.conditional_response = True + body = ''.join(resp(req.environ, self.fake_start_response)) + self.assertEquals(resp.status_int, 412) + self.assertEquals(body, '') + + def test_same(self): + req = swift.common.swob.Request.blank( + '/', + headers={'If-Unmodified-Since': 'Thu, 20 Feb 2014 03:29:37 GMT'}) + resp = req.get_response(self.fake_app) + resp.conditional_response = True + body = ''.join(resp(req.environ, self.fake_start_response)) + self.assertEquals(resp.status_int, 200) + self.assertEquals(body, 'hi') + + def test_greater(self): + req = swift.common.swob.Request.blank( + '/', + headers={'If-Unmodified-Since': 'Thu, 20 Feb 2014 03:29:38 GMT'}) + resp = req.get_response(self.fake_app) + resp.conditional_response = True + body = ''.join(resp(req.environ, self.fake_start_response)) + self.assertEquals(resp.status_int, 200) + self.assertEquals(body, 'hi') + + def test_out_of_range_is_ignored(self): + # All that datetime gives us is a ValueError or OverflowError when + # something is out of range (i.e. less than datetime.datetime.min or + # greater than datetime.datetime.max). Unfortunately, we can't + # distinguish between a date being too old and a date being too new, + # so the best we can do is ignore such headers. + max_date_list = list(datetime.datetime.max.timetuple()) + max_date_list[0] += 1 # bump up the year + too_big_date_header = time.strftime( + "%a, %d %b %Y %H:%M:%S GMT", time.struct_time(max_date_list)) + + req = swift.common.swob.Request.blank( + '/', + headers={'If-Unmodified-Since': too_big_date_header}) + resp = req.get_response(self.fake_app) + resp.conditional_response = True + body = ''.join(resp(req.environ, self.fake_start_response)) + self.assertEquals(resp.status_int, 200) + self.assertEquals(body, 'hi') + + if __name__ == '__main__': unittest.main() diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 749c9705ec..3b5f80ffc9 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -1226,6 +1226,78 @@ class TestObjectController(unittest.TestCase): resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 304) + def test_HEAD_if_modified_since(self): + timestamp = normalize_timestamp(time()) + req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={ + 'X-Timestamp': timestamp, + 'Content-Type': 'application/octet-stream', + 'Content-Length': '4'}) + req.body = 'test' + resp = req.get_response(self.object_controller) + self.assertEquals(resp.status_int, 201) + + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}) + resp = req.get_response(self.object_controller) + self.assertEquals(resp.status_int, 200) + + since = strftime('%a, %d %b %Y %H:%M:%S GMT', + gmtime(float(timestamp) + 1)) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'If-Modified-Since': since}) + resp = req.get_response(self.object_controller) + self.assertEquals(resp.status_int, 304) + + since = \ + strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) - 1)) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'If-Modified-Since': since}) + resp = req.get_response(self.object_controller) + self.assertEquals(resp.status_int, 200) + + since = \ + strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) + 1)) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'If-Modified-Since': since}) + resp = req.get_response(self.object_controller) + self.assertEquals(resp.status_int, 304) + + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}) + resp = req.get_response(self.object_controller) + since = resp.headers['Last-Modified'] + self.assertEquals(since, strftime('%a, %d %b %Y %H:%M:%S GMT', + gmtime(math.ceil(float(timestamp))))) + + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'If-Modified-Since': since}) + resp = self.object_controller.GET(req) + self.assertEquals(resp.status_int, 304) + + timestamp = normalize_timestamp(int(time())) + req = Request.blank('/sda1/p/a/c/o2', + environ={'REQUEST_METHOD': 'PUT'}, + headers={ + 'X-Timestamp': timestamp, + 'Content-Type': 'application/octet-stream', + 'Content-Length': '4'}) + req.body = 'test' + resp = req.get_response(self.object_controller) + self.assertEquals(resp.status_int, 201) + + since = strftime('%a, %d %b %Y %H:%M:%S GMT', + gmtime(float(timestamp))) + req = Request.blank('/sda1/p/a/c/o2', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'If-Modified-Since': since}) + resp = req.get_response(self.object_controller) + self.assertEquals(resp.status_int, 304) + def test_GET_if_unmodified_since(self): timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, @@ -1274,6 +1346,42 @@ class TestObjectController(unittest.TestCase): resp = self.object_controller.GET(req) self.assertEquals(resp.status_int, 200) + def test_HEAD_if_unmodified_since(self): + timestamp = normalize_timestamp(time()) + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': timestamp, + 'Content-Type': 'application/octet-stream', + 'Content-Length': '4'}) + req.body = 'test' + resp = req.get_response(self.object_controller) + self.assertEquals(resp.status_int, 201) + + since = strftime('%a, %d %b %Y %H:%M:%S GMT', + gmtime(math.ceil(float(timestamp)) + 1)) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'If-Unmodified-Since': since}) + resp = req.get_response(self.object_controller) + self.assertEquals(resp.status_int, 200) + + since = strftime('%a, %d %b %Y %H:%M:%S GMT', + gmtime(math.ceil(float(timestamp)))) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'If-Unmodified-Since': since}) + resp = req.get_response(self.object_controller) + self.assertEquals(resp.status_int, 200) + + since = strftime('%a, %d %b %Y %H:%M:%S GMT', + gmtime(math.ceil(float(timestamp)) - 1)) + req = Request.blank('/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD'}, + headers={'If-Unmodified-Since': since}) + resp = req.get_response(self.object_controller) + self.assertEquals(resp.status_int, 412) + def test_GET_quarantine(self): # Test swift.obj.server.ObjectController.GET timestamp = normalize_timestamp(time()) From f63b37572df9188f2fc753c06f32488a2b2f59a9 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Tue, 1 Apr 2014 22:56:43 -0400 Subject: [PATCH 02/15] Update callback with proper bytes transferred The first set of bytes transferred was being dropped. Change-Id: I8e055190d04af1718c5e88bf5e8c44fde6794c74 --- test/functional/swift_test_client.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index b4dcb56cf9..ce7508841c 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -778,13 +778,15 @@ class File(Base): transferred = 0 buff = data.read(block_size) + buff_len = len(buff) try: - while len(buff) > 0: + while buff_len > 0: self.conn.put_data(buff) - buff = data.read(block_size) - transferred += len(buff) + transferred += buff_len if callable(callback): callback(transferred, self.size) + buff = data.read(block_size) + buff_len = len(buff) self.conn.put_end() except socket.timeout as err: From 9d0067a0f546ce8a0fddb257655f2944c05e35e6 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Fri, 4 Apr 2014 15:26:32 -0400 Subject: [PATCH 03/15] Attempt to ensure connect always timesout It seems that the test_connect_put_timeout() test does not always fail when it is expected. Sometimes, not very often, the attempt to connect succeeds, resulting in a failed test. This might be because the fake-connection infrastructure uses a sleep(0.1) and the test uses a connect timeout of 0.1. There might be a case where the two values result in the exact time where the entries happen to be added in the wrong order such that the sleep() completes first before the connect timeout fires, where the connect completes successfully. Closes bug 1302781 Change-Id: Ie23e40cf294170eccdf0713e313f9a31a92f9071 --- test/unit/proxy/controllers/test_obj.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 4942691722..c31481e764 100755 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -82,7 +82,7 @@ class TestObjControllerWriteAffinity(unittest.TestCase): def test_connect_put_node_timeout(self): controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') - self.app.conn_timeout = 0.1 + self.app.conn_timeout = 0.05 with set_http_connect(200, slow_connect=True): nodes = [dict(ip='', port='', device='')] res = controller._connect_put_node(nodes, '', '', {}, ('', '')) From b9b5fef89af51c66905de33e2436c063f4b09d36 Mon Sep 17 00:00:00 2001 From: James Page Date: Sat, 5 Apr 2014 09:38:12 +0100 Subject: [PATCH 04/15] Set permissions on generated ring files The use of NamedTemporaryFile creates rings with permissions 0600; however most installs probably generate the rings as root but the swift-proxy runs as user swift. Set the permissions on the generated ring to 0644 prior to rename so that the swift user can read the rings. Change-Id: Ia511931f471c5c9840012c3a75b89c1f35b1b245 Closes-Bug: #1302700 --- swift/common/ring/ring.py | 1 + test/unit/common/ring/test_ring.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/swift/common/ring/ring.py b/swift/common/ring/ring.py index 5b315285ac..a1f9024bc4 100644 --- a/swift/common/ring/ring.py +++ b/swift/common/ring/ring.py @@ -120,6 +120,7 @@ class RingData(object): tempf.flush() os.fsync(tempf.fileno()) tempf.close() + os.chmod(tempf.name, 0o644) os.rename(tempf.name, filename) def to_dict(self): diff --git a/test/unit/common/ring/test_ring.py b/test/unit/common/ring/test_ring.py index 04eb1b7cb5..1892d19923 100644 --- a/test/unit/common/ring/test_ring.py +++ b/test/unit/common/ring/test_ring.py @@ -18,6 +18,7 @@ import cPickle as pickle import os import sys import unittest +import stat from contextlib import closing from gzip import GzipFile from tempfile import mkdtemp @@ -98,6 +99,15 @@ class TestRingData(unittest.TestCase): with open(ring_fname2) as ring2: self.assertEqual(ring1.read(), ring2.read()) + def test_permissions(self): + ring_fname = os.path.join(self.testdir, 'stat.ring.gz') + rd = ring.RingData( + [array.array('H', [0, 1, 0, 1]), array.array('H', [0, 1, 0, 1])], + [{'id': 0, 'zone': 0}, {'id': 1, 'zone': 1}], 30) + rd.save(ring_fname) + self.assertEqual(oct(stat.S_IMODE(os.stat(ring_fname).st_mode)), + '0644') + class TestRing(unittest.TestCase): From b67a4b993815f1830b59054ae0d08a468044a53e Mon Sep 17 00:00:00 2001 From: Gil Vernik Date: Mon, 7 Apr 2014 08:44:54 +0300 Subject: [PATCH 05/15] Support for content-length in the upload object method for internal client. Internal client uses 'chunked' transfer encoding in it's upload_object method. If content-length is known in advance, internal_client should allowed to use content length in the upload_object method and not the chunked method. Change-Id: I68d2ebde78e01fa16b7187a2f045ea20f4310722 --- swift/common/internal_client.py | 3 ++- test/unit/common/test_internal_client.py | 27 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py index b0fda5f2b1..c477a4960e 100644 --- a/swift/common/internal_client.py +++ b/swift/common/internal_client.py @@ -678,7 +678,8 @@ class InternalClient(object): """ headers = dict(headers or {}) - headers['Transfer-Encoding'] = 'chunked' + if 'Content-Length' not in headers: + headers['Transfer-Encoding'] = 'chunked' path = self.make_path(account, container, obj) self.make_request('PUT', path, headers, (2,), fobj) diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py index 76198de28a..8a10106737 100644 --- a/test/unit/common/test_internal_client.py +++ b/test/unit/common/test_internal_client.py @@ -922,6 +922,33 @@ class TestInternalClient(unittest.TestCase): client.upload_object(fobj, account, container, obj, headers) self.assertEquals(1, client.make_request_called) + def test_upload_object_not_chunked(self): + class InternalClient(internal_client.InternalClient): + def __init__(self, test, path, headers, fobj): + self.test = test + self.path = path + self.headers = headers + self.fobj = fobj + self.make_request_called = 0 + + def make_request( + self, method, path, headers, acceptable_statuses, + body_file=None): + self.make_request_called += 1 + self.test.assertEquals(self.path, path) + exp_headers = dict(self.headers) + self.test.assertEquals(exp_headers, headers) + self.test.assertEquals(self.fobj, fobj) + + fobj = 'some_fobj' + account, container, obj = path_parts() + path = make_path(account, container, obj) + headers = {'key': 'value', 'Content-Length': len(fobj)} + + client = InternalClient(self, path, headers, fobj) + client.upload_object(fobj, account, container, obj, headers) + self.assertEquals(1, client.make_request_called) + class TestGetAuth(unittest.TestCase): @mock.patch('eventlet.green.urllib2.urlopen') From d32dc8d49c19de8b1264cae23b82b3ffd9707454 Mon Sep 17 00:00:00 2001 From: Greg Lange Date: Wed, 26 Mar 2014 22:55:55 +0000 Subject: [PATCH 06/15] Unify backend logging Make account, object, and container servers construct log lines using the same utility function so they will produce identically formatted lines. This change reorders the fields logged for the account server. This change also adds the "additional info" field to the two servers that didn't log that field. This makes the log lines identical across all 3 servers. If people don't like that, I can take that out. I think it makes the documentation, parsing of the log lines, and the code a tad cleaner. DocImpact Change-Id: I268dc0df9dd07afa5382592a28ea37b96c6c2f44 Closes-Bug: 1280955 --- doc/source/logs.rst | 2 ++ swift/account/server.py | 22 +++++++--------------- swift/common/utils.py | 22 ++++++++++++++++++++++ swift/container/server.py | 14 +++----------- swift/obj/server.py | 13 +++---------- test/unit/account/test_server.py | 17 +++++++++++++++++ test/unit/common/test_utils.py | 19 ++++++++++++++++++- test/unit/container/test_server.py | 17 +++++++++++++++++ test/unit/obj/test_server.py | 18 +++++++++++++++++- 9 files changed, 106 insertions(+), 38 deletions(-) diff --git a/doc/source/logs.rst b/doc/source/logs.rst index 56f0588f1b..c2981aac34 100644 --- a/doc/source/logs.rst +++ b/doc/source/logs.rst @@ -100,6 +100,7 @@ these log lines is:: remote_addr - - [datetime] "request_method request_path" status_int content_length "referer" "transaction_id" "user_agent" request_time + additional_info =================== ========================================================== **Log Field** **Value** @@ -117,4 +118,5 @@ user_agent The value of the HTTP User-Agent header. Swift's proxy server sets its user-agent to ``"proxy-server ".`` request_time The duration of the request. +additional_info Additional useful information. =================== ========================================================== diff --git a/swift/account/server.py b/swift/account/server.py index 83a2bff79f..919b51bbfd 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -28,7 +28,7 @@ from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path from swift.common.utils import get_logger, hash_path, public, \ normalize_timestamp, storage_directory, config_true_value, \ - json, timing_stats, replication + json, timing_stats, replication, get_log_line from swift.common.constraints import check_mount, check_float, check_utf8 from swift.common import constraints from swift.common.db_replicator import ReplicatorRpc @@ -290,21 +290,13 @@ class AccountController(object): ' %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) - trans_time = '%.4f' % (time.time() - start_time) - additional_info = '' - if res.headers.get('x-container-timestamp') is not None: - additional_info += 'x-container-timestamp: %s' % \ - res.headers['x-container-timestamp'] if self.log_requests: - log_msg = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s "%s"' % ( - req.remote_addr, - time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), - req.method, req.path, - res.status.split()[0], res.content_length or '-', - req.headers.get('x-trans-id', '-'), - req.referer or '-', req.user_agent or '-', - trans_time, - additional_info) + trans_time = time.time() - start_time + additional_info = '' + if res.headers.get('x-container-timestamp') is not None: + additional_info += 'x-container-timestamp: %s' % \ + res.headers['x-container-timestamp'] + log_msg = get_log_line(req, res, trans_time, additional_info) if req.method.upper() == 'REPLICATE': self.logger.debug(log_msg) else: diff --git a/swift/common/utils.py b/swift/common/utils.py index 5281be9410..c1f0a1031e 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -292,6 +292,28 @@ def generate_trans_id(trans_id_suffix): uuid.uuid4().hex[:21], time.time(), trans_id_suffix) +def get_log_line(req, res, trans_time, additional_info): + """ + Make a line for logging that matches the documented log line format + for backend servers. + + :param req: the request. + :param res: the response. + :param trans_time: the time the request took to complete, a float. + :param additional_info: a string to log at the end of the line + + :returns: a properly formated line for logging. + """ + + return '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %.4f "%s"' % ( + req.remote_addr, + time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), + req.method, req.path, res.status.split()[0], + res.content_length or '-', req.referer or '-', + req.headers.get('x-trans-id', '-'), + req.user_agent or '-', trans_time, additional_info or '-') + + def get_trans_id_time(trans_id): if len(trans_id) >= 34 and trans_id[:2] == 'tx' and trans_id[23] == '-': try: diff --git a/swift/container/server.py b/swift/container/server.py index 380bb39ca1..784ee7fe42 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -31,7 +31,7 @@ from swift.common.request_helpers import get_param, get_listing_content_type, \ from swift.common.utils import get_logger, hash_path, public, \ normalize_timestamp, storage_directory, validate_sync_to, \ config_true_value, json, timing_stats, replication, \ - override_bytes_from_content_type + override_bytes_from_content_type, get_log_line from swift.common.constraints import check_mount, check_float, check_utf8 from swift.common import constraints from swift.common.bufferedhttp import http_connect @@ -504,17 +504,9 @@ class ContainerController(object): 'ERROR __call__ error with %(method)s %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) - trans_time = '%.4f' % (time.time() - start_time) if self.log_requests: - log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s' % ( - req.remote_addr, - time.strftime('%d/%b/%Y:%H:%M:%S +0000', - time.gmtime()), - req.method, req.path, - res.status.split()[0], res.content_length or '-', - req.headers.get('x-trans-id', '-'), - req.referer or '-', req.user_agent or '-', - trans_time) + trans_time = time.time() - start_time + log_message = get_log_line(req, res, trans_time, '') if req.method.upper() == 'REPLICATE': self.logger.debug(log_message) else: diff --git a/swift/obj/server.py b/swift/obj/server.py index 025c6f15ba..2c8214e07f 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -29,7 +29,8 @@ from hashlib import md5 from eventlet import sleep, Timeout from swift.common.utils import public, get_logger, \ - config_true_value, timing_stats, replication, normalize_delete_at_timestamp + config_true_value, timing_stats, replication, \ + normalize_delete_at_timestamp, get_log_line from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, \ check_float, check_utf8 @@ -680,15 +681,7 @@ class ObjectController(object): res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: - log_line = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %.4f' % ( - req.remote_addr, - time.strftime('%d/%b/%Y:%H:%M:%S +0000', - time.gmtime()), - req.method, req.path, res.status.split()[0], - res.content_length or '-', req.referer or '-', - req.headers.get('x-trans-id', '-'), - req.user_agent or '-', - trans_time) + log_line = get_log_line(req, res, trans_time, '') if req.method in ('REPLICATE', 'REPLICATION') or \ 'X-Backend-Replication' in req.headers: self.logger.debug(log_line) diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py index ca9d6f6c87..9e6cc0d7de 100644 --- a/test/unit/account/test_server.py +++ b/test/unit/account/test_server.py @@ -20,6 +20,7 @@ import unittest from tempfile import mkdtemp from shutil import rmtree from StringIO import StringIO +from time import gmtime from test.unit import FakeLogger import simplejson @@ -1653,6 +1654,22 @@ class TestAccountController(unittest.TestCase): self.assertEqual(resp.status_int, 404) self.assertFalse(self.controller.logger.log_dict['info']) + def test_log_line_format(self): + req = Request.blank( + '/sda1/p/a', + environ={'REQUEST_METHOD': 'HEAD', 'REMOTE_ADDR': '1.2.3.4'}) + self.controller.logger = FakeLogger() + with mock.patch( + 'time.gmtime', mock.MagicMock(side_effect=[gmtime(10001.0)])): + with mock.patch( + 'time.time', + mock.MagicMock(side_effect=[10000.0, 10001.0, 10002.0])): + req.get_response(self.controller) + self.assertEqual( + self.controller.logger.log_dict['info'], + [(('1.2.3.4 - - [01/Jan/1970:02:46:41 +0000] "HEAD /sda1/p/a" 404 ' + '- "-" "-" "-" 2.0000 "-"',), {})]) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 2996f35557..f8849821d3 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -24,6 +24,7 @@ import eventlet.event import grp import logging import os +import mock import random import re import socket @@ -55,7 +56,7 @@ from swift.common.exceptions import (Timeout, MessageTimeout, ReplicationLockTimeout) from swift.common import utils from swift.common.container_sync_realms import ContainerSyncRealms -from swift.common.swob import Response +from swift.common.swob import Request, Response from test.unit import FakeLogger @@ -1893,6 +1894,22 @@ cluster_dfw1 = http://dfw1.host/v1/ utils.get_hmac('GET', '/path', 1, 'abc'), 'b17f6ff8da0e251737aa9e3ee69a881e3e092e2f') + def test_get_log_line(self): + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD', 'REMOTE_ADDR': '1.2.3.4'}) + res = Response() + trans_time = 1.2 + additional_info = 'some information' + exp_line = '1.2.3.4 - - [01/Jan/1970:02:46:41 +0000] "HEAD ' \ + '/sda1/p/a/c/o" 200 - "-" "-" "-" 1.2000 "some information"' + with mock.patch( + 'time.gmtime', + mock.MagicMock(side_effect=[time.gmtime(10001.0)])): + self.assertEquals( + exp_line, + utils.get_log_line(req, res, trans_time, additional_info)) + class TestSwiftInfo(unittest.TestCase): diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 628308117a..4fd319ab90 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -22,6 +22,7 @@ from shutil import rmtree from StringIO import StringIO from tempfile import mkdtemp from test.unit import FakeLogger +from time import gmtime from xml.dom import minidom from eventlet import spawn, Timeout, listen @@ -1956,6 +1957,22 @@ class TestContainerController(unittest.TestCase): self.assertEqual(resp.status_int, 404) self.assertFalse(self.controller.logger.log_dict['info']) + def test_log_line_format(self): + req = Request.blank( + '/sda1/p/a/c', + environ={'REQUEST_METHOD': 'HEAD', 'REMOTE_ADDR': '1.2.3.4'}) + self.controller.logger = FakeLogger() + with mock.patch( + 'time.gmtime', mock.MagicMock(side_effect=[gmtime(10001.0)])): + with mock.patch( + 'time.time', + mock.MagicMock(side_effect=[10000.0, 10001.0, 10002.0])): + req.get_response(self.controller) + self.assertEqual( + self.controller.logger.log_dict['info'], + [(('1.2.3.4 - - [01/Jan/1970:02:46:41 +0000] "HEAD /sda1/p/a/c" ' + '404 - "-" "-" "-" 2.0000 "-"',), {})]) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index c21bf222fa..1614783d4c 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -3385,7 +3385,7 @@ class TestObjectController(unittest.TestCase): self.assertEqual( self.object_controller.logger.log_dict['info'], [(('None - - [01/Jan/1970:02:46:41 +0000] "PUT' - ' /sda1/p/a/c/o" 405 - "-" "-" "-" 1.0000',), + ' /sda1/p/a/c/o" 405 - "-" "-" "-" 1.0000 "-"',), {})]) def test_not_utf8_and_not_logging_requests(self): @@ -3519,6 +3519,22 @@ class TestObjectController(unittest.TestCase): self.assertEqual( self.object_controller.logger.log_dict['info'], []) + def test_log_line_format(self): + req = Request.blank( + '/sda1/p/a/c/o', + environ={'REQUEST_METHOD': 'HEAD', 'REMOTE_ADDR': '1.2.3.4'}) + self.object_controller.logger = FakeLogger() + with mock.patch( + 'time.gmtime', mock.MagicMock(side_effect=[gmtime(10001.0)])): + with mock.patch( + 'time.time', + mock.MagicMock(side_effect=[10000.0, 10001.0, 10002.0])): + req.get_response(self.object_controller) + self.assertEqual( + self.object_controller.logger.log_dict['info'], + [(('1.2.3.4 - - [01/Jan/1970:02:46:41 +0000] "HEAD /sda1/p/a/c/o" ' + '404 - "-" "-" "-" 2.0000 "-"',), {})]) + if __name__ == '__main__': unittest.main() From 677ee3c931b5d307e44416b431207495435bf402 Mon Sep 17 00:00:00 2001 From: anc Date: Mon, 7 Apr 2014 18:42:50 +0100 Subject: [PATCH 07/15] Add missing constraints to /info Two of the default constraints, max_header_size and max_meta_overall_size, don't get registered for the /info response by the proxy server. Rather than adding them individually to the proxy's register_swift_info call, this patch proposes to register all the constraints for /info at once using the constraints.EFFECTIVE_CONSTRAINTS dict. Any future additions to default constraints will then be automatically included in /info. Change-Id: I5c10d4c8eda90ba94745b6f89df85aafbb50f8ef --- swift/proxy/server.py | 12 ++---------- test/unit/proxy/test_server.py | 3 +++ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index f6af6e63c8..afa7916329 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -204,16 +204,8 @@ class Application(object): self.admin_key = conf.get('admin_key', None) register_swift_info( version=swift_version, - max_file_size=constraints.MAX_FILE_SIZE, - max_meta_name_length=constraints.MAX_META_NAME_LENGTH, - max_meta_value_length=constraints.MAX_META_VALUE_LENGTH, - max_meta_count=constraints.MAX_META_COUNT, - account_listing_limit=constraints.ACCOUNT_LISTING_LIMIT, - container_listing_limit=constraints.CONTAINER_LISTING_LIMIT, - max_account_name_length=constraints.MAX_ACCOUNT_NAME_LENGTH, - max_container_name_length=constraints.MAX_CONTAINER_NAME_LENGTH, - max_object_name_length=constraints.MAX_OBJECT_NAME_LENGTH, - strict_cors_mode=self.strict_cors_mode) + strict_cors_mode=self.strict_cors_mode, + **constraints.EFFECTIVE_CONSTRAINTS) def check_config(self): """ diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index a520a23992..abd49599a0 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -5902,6 +5902,9 @@ class TestSwiftInfo(unittest.TestCase): self.assertEqual(si['max_meta_value_length'], constraints.MAX_META_VALUE_LENGTH) self.assertEqual(si['max_meta_count'], constraints.MAX_META_COUNT) + self.assertEqual(si['max_header_size'], constraints.MAX_HEADER_SIZE) + self.assertEqual(si['max_meta_overall_size'], + constraints.MAX_META_OVERALL_SIZE) self.assertEqual(si['account_listing_limit'], constraints.ACCOUNT_LISTING_LIMIT) self.assertEqual(si['container_listing_limit'], From d47a582e749d3a61071ff249175d67e37bef7e6c Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Tue, 8 Apr 2014 10:26:40 -0700 Subject: [PATCH 08/15] update setup.py with pbr version This file is normally updated in the standard "update from global requirements" patch, but since those contain other changes we don't want, the setup.py file never gets updated. This patch updates setup.py to bring it in line with the standard global requirements way of doing things without also updating [test-]requirements.txt Change-Id: Id0e4f8e17dd4c714b4d2dd75985431a9e12e6a4f --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2a0786a8b2..70c2b3f32b 100644 --- a/setup.py +++ b/setup.py @@ -18,5 +18,5 @@ import setuptools setuptools.setup( - setup_requires=['pbr>=0.5.21,<1.0'], + setup_requires=['pbr'], pbr=True) From c2744caac43586b745c43c37a9c31483f7a126fc Mon Sep 17 00:00:00 2001 From: Samuel Merritt Date: Tue, 8 Apr 2014 11:44:58 -0700 Subject: [PATCH 09/15] Fix deprecation warning Accessing BaseException.message spews a warning; we can get the same information with str(err), which does not spew. Change-Id: I67648d53d25522be074a78c44b3ce97dc27d1f07 --- swift/cli/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/cli/info.py b/swift/cli/info.py index c5cf8daaee..179d5b16aa 100644 --- a/swift/cli/info.py +++ b/swift/cli/info.py @@ -168,7 +168,7 @@ def print_info(db_type, db_file, swift_dir='/etc/swift'): try: info = broker.get_info() except sqlite3.OperationalError as err: - if 'no such table' in err.message: + if 'no such table' in str(err): print "Does not appear to be a DB of type \"%s\": %s" % ( db_type, db_file) raise InfoSystemExit() From 58fe2f256fccc14f5078e9fde9b9f7b2219a06e5 Mon Sep 17 00:00:00 2001 From: anc Date: Tue, 8 Apr 2014 18:44:06 +0100 Subject: [PATCH 10/15] Add tests and comments re constraints in /info Add test to check that only the expected keys are reported by proxy in /info, and add comments to raise awareness that default constraints will be automatically published by proxy in response to /info requests. Change-Id: Ia5f6339b06cdc2e1dc960d1f75562a2505530202 --- etc/swift.conf-sample | 3 ++- swift/common/constraints.py | 4 ++++ test/unit/proxy/test_server.py | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/etc/swift.conf-sample b/etc/swift.conf-sample index 50c0463bd8..8bb8c4ae5e 100644 --- a/etc/swift.conf-sample +++ b/etc/swift.conf-sample @@ -10,7 +10,8 @@ swift_hash_path_prefix = changeme # The swift-constraints section sets the basic constraints on data -# saved in the swift cluster. +# saved in the swift cluster. These constraints are automatically +# published by the proxy server in responses to /info requests. [swift-constraints] diff --git a/swift/common/constraints.py b/swift/common/constraints.py index 84545fce92..d3cbea0863 100644 --- a/swift/common/constraints.py +++ b/swift/common/constraints.py @@ -34,6 +34,10 @@ ACCOUNT_LISTING_LIMIT = 10000 MAX_ACCOUNT_NAME_LENGTH = 256 MAX_CONTAINER_NAME_LENGTH = 256 +# If adding an entry to DEFAULT_CONSTRAINTS, note that +# these constraints are automatically published by the +# proxy server in responses to /info requests, with values +# updated by reload_constraints() DEFAULT_CONSTRAINTS = { 'max_file_size': MAX_FILE_SIZE, 'max_meta_name_length': MAX_META_NAME_LENGTH, diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index abd49599a0..3925910612 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -5915,6 +5915,10 @@ class TestSwiftInfo(unittest.TestCase): constraints.MAX_CONTAINER_NAME_LENGTH) self.assertEqual(si['max_object_name_length'], constraints.MAX_OBJECT_NAME_LENGTH) + self.assertTrue('strict_cors_mode' in si) + # this next test is deliberately brittle in order to alert if + # other items are added to swift info + self.assertEqual(len(si), 13) if __name__ == '__main__': From 5ff6a4d5d6cdbf39ba9f6d5d416cdd1c6c6a52ac Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Mon, 7 Apr 2014 13:01:44 -0400 Subject: [PATCH 11/15] Use eventlet instead of threading for timeout The only explicit use of Python threading is found in the testFileSizeLimit test. Using eventlet seems a bit easier to follow, accomplishing the same goal, and does not constrain us to a multi-threaded environment. The chunks() and timeout() module level functions are only used by one test each, so we just move them to those tests to indicate they are not used globally. Change-Id: I50b9fb798fbfd1d552b3c3f90309f6b86da34853 --- test/functional/tests.py | 64 ++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/test/functional/tests.py b/test/functional/tests.py index c160a3cd97..485439a029 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -22,52 +22,30 @@ import locale import random import StringIO import time -import threading import unittest import urllib import uuid +import eventlet +import eventlet.debug from nose import SkipTest +from swift.common.utils import get_hub + from test.functional import normalized_urls, load_constraint import test.functional as tf from test.functional.swift_test_client import Account, Connection, File, \ ResponseError -def chunks(s, length=3): - i, j = 0, length - while i < len(s): - yield s[i:j] - i, j = j, j + length - - -def timeout(seconds, method, *args, **kwargs): - class TimeoutThread(threading.Thread): - def __init__(self, method, *args, **kwargs): - threading.Thread.__init__(self) - - self.method = method - self.args = args - self.kwargs = kwargs - self.exception = None - - def run(self): - try: - self.method(*self.args, **self.kwargs) - except Exception as e: - self.exception = e - - t = TimeoutThread(method, *args, **kwargs) - t.start() - t.join(seconds) - - if t.exception: - raise t.exception - - if t.isAlive(): - t._Thread__stop() - return True - return False +# In order to get the proper blocking behavior of sockets without using +# threads, where we can set an arbitrary timeout for some piece of code under +# test, we use eventlet with the standard socket library patched. We have to +# perform this setup at module import time, since all the socket module +# bindings in the swiftclient code will have been made by the time nose +# invokes the package or class setup methods. +eventlet.hubs.use_hub(get_hub()) +eventlet.patcher.monkey_patch(all=False, socket=True) +eventlet.debug.hub_exceptions(True) class Utils(object): @@ -1162,6 +1140,15 @@ class TestFile(Base): limit = load_constraint('max_file_size') tsecs = 3 + def timeout(seconds, method, *args, **kwargs): + try: + with eventlet.Timeout(seconds): + method(*args, **kwargs) + except eventlet.Timeout: + return True + else: + return False + for i in (limit - 100, limit - 10, limit - 1, limit, limit + 1, limit + 10, limit + 100): @@ -1473,6 +1460,13 @@ class TestFile(Base): if (tf.web_front_end == 'apache2'): raise SkipTest("Chunked PUT can only be tested with apache2 web" " front end") + + def chunks(s, length=3): + i, j = 0, length + while i < len(s): + yield s[i:j] + i, j = j, j + length + data = File.random_data(10000) etag = File.compute_md5sum(data) From 2c2ede22338d6ca9637233076157f1d26248fbf1 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Thu, 10 Apr 2014 18:59:01 +0000 Subject: [PATCH 12/15] Fix logging issue when services stop on py26 On older versions of python 2.6, exceptions would be spewed to the error log whenever a service would stop. This gets magnified by the container-updater which seems to do it with every pass. This catches and squelches the error. Change-Id: I128c09c240e768e8195af1f6fe79b10d4e432471 Closes-Bug: #1306027 --- swift/common/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/swift/common/utils.py b/swift/common/utils.py index c1f0a1031e..5d2306c8a2 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -878,8 +878,12 @@ class LoggingHandlerWeakRef(weakref.ref): """ def close(self): referent = self() - if referent: - referent.close() + try: + if referent: + referent.close() + except KeyError: + # This is to catch an issue with old py2.6 versions + pass def flush(self): referent = self() From deb0dfd090cee2feef664bdd4449194c274d830e Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Thu, 10 Apr 2014 14:36:43 -0400 Subject: [PATCH 13/15] Add includes of referenced SAIO bin scripts Change-Id: I6810e69a757336a3aed0a38146c27f270fe2dde1 --- doc/source/development_saio.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 4fe40d6d77..d1caff7229 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -415,10 +415,18 @@ Setting up scripts for running Swift sed -i "s/service \(.*\) restart/systemctl restart \1.service/" $HOME/bin/resetswift + The template ``resetswift`` script looks like the following: + + .. literalinclude:: /../saio/bin/resetswift + #. Install the sample configuration file for running tests:: cp $HOME/swift/test/sample.conf /etc/swift/test.conf + The template ``test.conf`` looks like the following: + + .. literalinclude:: /../../test/sample.conf + #. Add an environment variable for running tests below:: echo "export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf" >> $HOME/.bashrc @@ -435,6 +443,10 @@ Setting up scripts for running Swift remakerings + The ``remakerings`` script looks like the following: + + .. literalinclude:: /../saio/bin/remakerings + You can expect the ouptut from this command to produce the following:: Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0 @@ -467,6 +479,10 @@ Setting up scripts for running Swift (The "``Unable to increase file descriptor limit. Running as non-root?``" warnings are expected and ok.) + The ``startmain`` script looks like the following: + + .. literalinclude:: /../saio/bin/startmain + #. Get an ``X-Storage-Url`` and ``X-Auth-Token``:: curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0 From 67fff5b2973f5ca12316041664eddf3e8ece45be Mon Sep 17 00:00:00 2001 From: Madhuri Kumari Date: Fri, 11 Apr 2014 11:53:48 +0530 Subject: [PATCH 14/15] Print 'Container Count' in data base info Currently, 'Container Count' was missing in data base info. So this patch will help printing 'Container Count' also. Change-Id: I1ca80ee79e71b086b30fd2d1ab024ea1cfb324f5 --- swift/cli/info.py | 2 ++ test/unit/cli/test_info.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/swift/cli/info.py b/swift/cli/info.py index 179d5b16aa..9f893f6037 100644 --- a/swift/cli/info.py +++ b/swift/cli/info.py @@ -106,6 +106,8 @@ def print_db_info_metadata(db_type, info, metadata): print (' Delete Timestamp: %s (%s)' % (datetime.utcfromtimestamp(float(info['delete_timestamp'])), info['delete_timestamp'])) + if db_type == 'account': + print ' Container Count: %s' % info['container_count'] print ' Object Count: %s' % info['object_count'] print ' Bytes Used: %s' % info['bytes_used'] if db_type == 'container': diff --git a/test/unit/cli/test_info.py b/test/unit/cli/test_info.py index 88914493a9..306bb608b6 100644 --- a/test/unit/cli/test_info.py +++ b/test/unit/cli/test_info.py @@ -86,6 +86,7 @@ class TestCliInfo(unittest.TestCase): created_at=100.1, put_timestamp=106.3, delete_timestamp=107.9, + container_count='3', object_count='20', bytes_used='42') info['hash'] = 'abaddeadbeefcafe' @@ -102,6 +103,7 @@ Metadata: Created at: 1970-01-01 00:01:40.100000 (100.1) Put Timestamp: 1970-01-01 00:01:46.300000 (106.3) Delete Timestamp: 1970-01-01 00:01:47.900000 (107.9) + Container Count: 3 Object Count: 20 Bytes Used: 42 Chexor: abaddeadbeefcafe From 856c15539a958dabe3b8a1f22d305048ca39de9a Mon Sep 17 00:00:00 2001 From: Paul Luse Date: Mon, 14 Apr 2014 15:19:22 -0700 Subject: [PATCH 15/15] Fix testcase test_print_db_info_metadata() Test compares cluster info to hardcoded expected data and wasn't sorting the two sets of things being compared leading to some sporadic unit test failures. Change-Id: I3ef98260a62c15d06ba8cc196196d4e90abca3f0 --- test/unit/cli/test_info.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/unit/cli/test_info.py b/test/unit/cli/test_info.py index 306bb608b6..55516460c6 100644 --- a/test/unit/cli/test_info.py +++ b/test/unit/cli/test_info.py @@ -112,7 +112,8 @@ Metadata: No system metadata found in db file User Metadata: {'mydata': 'swift'}''' - self.assertEquals(out.getvalue().strip(), exp_out) + self.assertEquals(sorted(out.getvalue().strip().split('\n')), + sorted(exp_out.split('\n'))) info = dict( account='acct', @@ -154,7 +155,8 @@ Metadata: X-Container-Foo: bar System Metadata: {'mydata': 'swift'} No user metadata found in db file''' - self.assertEquals(out.getvalue().strip(), exp_out) + self.assertEquals(sorted(out.getvalue().strip().split('\n')), + sorted(exp_out.split('\n'))) def test_print_ring_locations(self): self.assertRaisesMessage(ValueError, 'None type', print_ring_locations,