openstack
/
swift
Code Issues Proposed changes

Merged trunk. Enabled unittests in setup.py. w00t.

This commit is contained in:
Monty Taylor 2010-07-14 09:30:31 -05:00
parent 02622ba3ac c2e75a5aad
commit d91a16b7cf
6 changed files with 111 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
doc/source/development_saio.rst
View File

@ -20,7 +20,7 @@ good idea what to do on other environments.
from above.
#. Select `Linux` and `Ubuntu 64-bit`.
#. Fill in the *Linux Easy Install* details (you should make the user
name match your bzr repo user name).
name match your launchpad user id).
#. `Customize Settings`, name the image whatever you want
(`SAIO` for instance.)
#. When the `Settings` window comes up, select `Hard Disk`, create an
@ -45,6 +45,7 @@ good idea what to do on other environments.
#. `mount /mnt/sdb1`
#. `mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4 /mnt/sdb1/test`
#. `chown <your-user-name>:<your-group-name> /mnt/sdb1/*`
#. `mkdir /srv`
#. `for x in {1..4}; do ln -s /mnt/sdb1/$x /srv/$x; done`
#. `mkdir -p /etc/swift/object-server /etc/swift/container-server /etc/swift/account-server /srv/1/node/sdb1 /srv/2/node/sdb2 /srv/3/node/sdb3 /srv/4/node/sdb4 /var/run/swift`
#. `chown -R <your-user-name>:<your-group-name> /etc/swift /srv/[1-4] /var/run/swift`
@ -148,13 +149,14 @@ good idea what to do on other environments.
[DEFAULT]
email = Your Name <your-email-address>
#. If you are using launchpad to get the code or make changes, run
`bzr launchpad-login <launchpad_id>`
#. Check out your bzr repo of swift, for example:
`bzr branch lp:swift`
#. ``for f in `ls ~/openswift/bin/`; do sudo ln -s /home/<your-user-name>/openswift/bin/$f /usr/bin/`basename $f .py`; done``
`bzr branch lp:~swift-core/swift/trunk swift`
#. ``for f in `ls ~/swift/bin/`; do sudo ln -s /home/<your-user-name>/swift/bin/$f /usr/bin/`basename $f .py`; done``
#. Edit `~/.bashrc` and add to the end::
export PYTHONPATH=~/openswift
export PYTHONPATH=~/swift
export PATH_TO_TEST_XFS=/mnt/sdb1/test
export SWIFT_TEST_CONFIG_FILE=/etc/swift/func_test.conf
export PATH=${PATH}:~/bin
@ -422,11 +424,11 @@ good idea what to do on other environments.
#. `chmod +x ~/bin/*`
#. `remakerings`
#. `cd ~/openswift; ./.unittests`
#. `cd ~/swift; ./.unittests`
#. `startmain`
#. `swift-auth-create-account test tester testing`
#. Get an `X-Storage-Url` and `X-Auth-Token`: `curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:11000/v1.0`
#. Check that you can GET account: `curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>`
#. Get an `X-Storage-Url` and `X-Auth-Token`: ``curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:11000/v1.0``
#. Check that you can GET account: ``curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>``
#. Check that `st` works: `st -A http://127.0.0.1:11000/v1.0 -U test:tester -K testing stat`
#. Create `/etc/swift/func_test.conf`::
@ -440,6 +442,22 @@ good idea what to do on other environments.
collate = C
#. `cd ~/openswift; ./.functests`
#. `cd ~/openswift; ./.probetests`
#. `cd ~/swift; ./.functests`
#. `cd ~/swift; ./.probetests`
----------------
Debugging Issues
----------------
If all doesn't go as planned, and tests fail, or you can't auth, or something doesn't work, here are some good starting places to look for issues:
#. Everything is logged in /var/log/syslog, so that is a good first place to
look for errors (most likely python tracebacks).
#. Make sure all of the server processes are running. For the base
functionality, the Proxy, Account, Container, Object and Auth servers
should be running
#. If one of the servers are not running, and no errors are logged to syslog,
it may be useful to try to start the server manually, for example:
`swift-object server /etc/swift/object-server/1.conf` will start the
object server. If there are problems not showing up in syslog,
then you will likely see the traceback on startup.

4
setup.py
View File

@ -24,9 +24,7 @@ setup(
author='OpenStack, LLC.',
url='https://launchpad.net/swift',
packages=find_packages(exclude=['tests','bin']),
#Uncomment this once unittests work without /etc
#Also, figure out how to make this only run unit tests
#test_suite = 'nose.collector',
test_suite = 'nose.collector',
classifiers=[
'Development Status :: 4 - Beta',
'License :: OSI Approved :: Apache Software License',

19
test/unit/container/test_server.py
View File

@ -14,6 +14,7 @@
# limitations under the License.
import os
import sys
import unittest
from shutil import rmtree
from StringIO import StringIO
@ -32,13 +33,17 @@ class TestContainerController(unittest.TestCase):
""" Test swift.container_server.ContainerController """
def setUp(self):
""" Set up for testing swift.object_server.ObjectController """
path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
if not path_to_test_xfs or not os.path.exists(path_to_test_xfs):
raise Exception('PATH_TO_TEST_XFS not set or not pointing to a '
'valid directory.\nPlease set PATH_TO_TEST_XFS to '
'a directory on an XFS file system for testing.')
self.testdir = os.path.join(path_to_test_xfs,
'tmp_test_object_server_ObjectController')
self.path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
if not self.path_to_test_xfs or \
not os.path.exists(self.path_to_test_xfs):
print >>sys.stderr, 'WARNING: PATH_TO_TEST_XFS not set or not ' \
'pointing to a valid directory.\n' \
'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \
'system for testing.'
self.testdir = '/tmp/SWIFTUNITTEST'
else:
self.testdir = os.path.join(self.path_to_test_xfs,
'tmp_test_object_server_ObjectController')
mkdirs(self.testdir)
rmtree(self.testdir)
mkdirs(os.path.join(self.testdir, 'sda1'))

19
test/unit/container/test_updater.py
View File

@ -15,6 +15,7 @@
import cPickle as pickle
import os
import sys
import unittest
from gzip import GzipFile
from shutil import rmtree
@ -32,13 +33,17 @@ from swift.common.utils import normalize_timestamp
class TestContainerUpdater(unittest.TestCase):
def setUp(self):
path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
if not path_to_test_xfs or not os.path.exists(path_to_test_xfs):
raise Exception('PATH_TO_TEST_XFS not set or not pointing to a '
'valid directory.\nPlease set PATH_TO_TEST_XFS to '
'a directory on an XFS file system for testing.')
self.testdir = os.path.join(path_to_test_xfs,
'tmp_test_container_updater')
self.path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
if not self.path_to_test_xfs or \
not os.path.exists(self.path_to_test_xfs):
print >>sys.stderr, 'WARNING: PATH_TO_TEST_XFS not set or not ' \
'pointing to a valid directory.\n' \
'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \
'system for testing.'
self.testdir = '/tmp/SWIFTUNITTEST'
else:
self.testdir = os.path.join(self.path_to_test_xfs,
'tmp_test_container_updater')
rmtree(self.testdir, ignore_errors=1)
os.mkdir(self.testdir)
pickle.dump(RingData([[0, 1, 0, 1], [1, 0, 1, 0]],

60
test/unit/obj/test_server.py
View File

@ -38,13 +38,17 @@ class TestObjectController(unittest.TestCase):
def setUp(self):
""" Set up for testing swift.object_server.ObjectController """
path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
if not path_to_test_xfs or not os.path.exists(path_to_test_xfs):
raise Exception('PATH_TO_TEST_XFS not set or not pointing to a '
'valid directory.\nPlease set PATH_TO_TEST_XFS to '
'a directory on an XFS file system for testing.')
self.testdir = os.path.join(path_to_test_xfs,
'tmp_test_object_server_ObjectController')
self.path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
if not self.path_to_test_xfs or \
not os.path.exists(self.path_to_test_xfs):
print >>sys.stderr, 'WARNING: PATH_TO_TEST_XFS not set or not ' \
'pointing to a valid directory.\n' \
'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \
'system for testing.'
self.testdir = '/tmp/SWIFTUNITTEST'
else:
self.testdir = os.path.join(self.path_to_test_xfs,
'tmp_test_object_server_ObjectController')
mkdirs(self.testdir)
rmtree(self.testdir)
mkdirs(os.path.join(self.testdir, 'sda1'))
@ -59,6 +63,8 @@ class TestObjectController(unittest.TestCase):
def test_POST_update_meta(self):
""" Test swift.object_server.ObjectController.POST """
if not self.path_to_test_xfs:
return
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': timestamp,
@ -85,6 +91,8 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.headers['Content-Type'], 'application/x-test')
def test_POST_not_exist(self):
if not self.path_to_test_xfs:
return
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/fail', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': timestamp,
@ -105,6 +113,8 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 400)
def test_POST_container_connection(self):
if not self.path_to_test_xfs:
return
def mock_http_connect(response, with_exc=False):
class FakeConn(object):
def __init__(self, status, with_exc):
@ -199,6 +209,8 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 411)
def test_PUT_common(self):
if not self.path_to_test_xfs:
return
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': timestamp,
@ -221,6 +233,8 @@ class TestObjectController(unittest.TestCase):
'name': '/a/c/o'})
def test_PUT_overwrite(self):
if not self.path_to_test_xfs:
return
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(time()),
'Content-Length': '6',
@ -252,6 +266,8 @@ class TestObjectController(unittest.TestCase):
'Content-Encoding': 'gzip'})
def test_PUT_no_etag(self):
if not self.path_to_test_xfs:
return
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(time()),
'Content-Type': 'text/plain'})
@ -269,6 +285,8 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 422)
def test_PUT_user_metadata(self):
if not self.path_to_test_xfs:
return
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': timestamp,
@ -295,6 +313,8 @@ class TestObjectController(unittest.TestCase):
'X-Object-Meta-Two': 'Two'})
def test_PUT_container_connection(self):
if not self.path_to_test_xfs:
return
def mock_http_connect(response, with_exc=False):
class FakeConn(object):
def __init__(self, status, with_exc):
@ -355,6 +375,8 @@ class TestObjectController(unittest.TestCase):
def test_HEAD(self):
""" Test swift.object_server.ObjectController.HEAD """
if not self.path_to_test_xfs:
return
req = Request.blank('/sda1/p/a/c')
resp = self.object_controller.HEAD(req)
self.assertEquals(resp.status_int, 400)
@ -420,6 +442,8 @@ class TestObjectController(unittest.TestCase):
def test_GET(self):
""" Test swift.object_server.ObjectController.GET """
if not self.path_to_test_xfs:
return
req = Request.blank('/sda1/p/a/c')
resp = self.object_controller.GET(req)
self.assertEquals(resp.status_int, 400)
@ -507,6 +531,8 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 404)
def test_GET_if_match(self):
if not self.path_to_test_xfs:
return
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={
'X-Timestamp': normalize_timestamp(time()),
@ -559,6 +585,8 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 412)
def test_GET_if_none_match(self):
if not self.path_to_test_xfs:
return
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={
'X-Timestamp': normalize_timestamp(time()),
@ -608,6 +636,8 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.etag, etag)
def test_GET_if_modified_since(self):
if not self.path_to_test_xfs:
return
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={
@ -643,6 +673,8 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 304)
def test_GET_if_unmodified_since(self):
if not self.path_to_test_xfs:
return
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={
@ -680,6 +712,8 @@ class TestObjectController(unittest.TestCase):
def test_DELETE(self):
""" Test swift.object_server.ObjectController.DELETE """
if not self.path_to_test_xfs:
return
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'})
resp = self.object_controller.DELETE(req)
self.assertEquals(resp.status_int, 400)
@ -832,6 +866,8 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(outbuf.getvalue()[:4], '405 ')
def test_chunked_put(self):
if not self.path_to_test_xfs:
return
listener = listen(('localhost', 0))
port = listener.getsockname()[1]
killer = spawn(wsgi.server, listener, self.object_controller,
@ -856,6 +892,8 @@ class TestObjectController(unittest.TestCase):
killer.kill()
def test_max_object_name_length(self):
if not self.path_to_test_xfs:
return
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/' + ('1' * 1024),
environ={'REQUEST_METHOD': 'PUT'},
@ -875,6 +913,8 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 400)
def test_disk_file_app_iter_corners(self):
if not self.path_to_test_xfs:
return
df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o')
mkdirs(df.datadir)
f = open(os.path.join(df.datadir,
@ -906,6 +946,8 @@ class TestObjectController(unittest.TestCase):
self.assert_(os.path.exists(tmpdir))
def test_max_upload_time(self):
if not self.path_to_test_xfs:
return
class SlowBody():
def __init__(self):
self.sent = 0
@ -946,6 +988,8 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 499)
def test_bad_sinces(self):
if not self.path_to_test_xfs:
return
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(time()),
'Content-Length': '4', 'Content-Type': 'text/plain'},
@ -970,6 +1014,8 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(resp.status_int, 412)
def test_content_encoding(self):
if not self.path_to_test_xfs:
return
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(time()),
'Content-Length': '4', 'Content-Type': 'text/plain',

8
test/unit/proxy/test_server.py
View File

@ -1028,9 +1028,11 @@ class TestObjectController(unittest.TestCase):
# proxy_server.Application we couldn't get to easily otherwise.
path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
if not path_to_test_xfs or not os.path.exists(path_to_test_xfs):
raise Exception('PATH_TO_TEST_XFS not set or not pointing to '
'a valid directory.\nPlease set PATH_TO_TEST_XFS to a '
'directory on an XFS file system for testing.')
print >>sys.stderr, 'WARNING: PATH_TO_TEST_XFS not set or not ' \
'pointing to a valid directory.\n' \
'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \
'system for testing.'
return
testdir = \
os.path.join(path_to_test_xfs, 'tmp_test_proxy_server_chunked')
mkdirs(testdir)