Merged trunk. Enabled unittests in setup.py. w00t.

doc/source/development_saio.rst

@@ -20,7 +20,7 @@ good idea what to do on other environments.
      from above.
   #. Select `Linux` and `Ubuntu 64-bit`.
   #. Fill in the *Linux Easy Install* details (you should make the user
-     name match your bzr repo user name).
+     name match your launchpad user id).
   #. `Customize Settings`, name the image whatever you want 
      (`SAIO` for instance.)
   #. When the `Settings` window comes up, select `Hard Disk`, create an
@@ -45,6 +45,7 @@ good idea what to do on other environments.
   #. `mount /mnt/sdb1`
   #. `mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4 /mnt/sdb1/test`
   #. `chown <your-user-name>:<your-group-name> /mnt/sdb1/*`
+  #. `mkdir /srv`
   #. `for x in {1..4}; do ln -s /mnt/sdb1/$x /srv/$x; done`
   #. `mkdir -p /etc/swift/object-server /etc/swift/container-server /etc/swift/account-server /srv/1/node/sdb1 /srv/2/node/sdb2 /srv/3/node/sdb3 /srv/4/node/sdb4 /var/run/swift`
   #. `chown -R <your-user-name>:<your-group-name> /etc/swift /srv/[1-4] /var/run/swift`
@@ -148,13 +149,14 @@ good idea what to do on other environments.
 
         [DEFAULT]
                 email = Your Name <your-email-address>
-
+  #. If you are using launchpad to get the code or make changes, run
+     `bzr launchpad-login <launchpad_id>`
   #. Check out your bzr repo of swift, for example:
-     `bzr branch lp:swift`
+     `bzr branch lp:~swift-core/swift/trunk swift`
-  #. ``for f in `ls ~/openswift/bin/`; do sudo ln -s /home/<your-user-name>/openswift/bin/$f /usr/bin/`basename $f .py`; done``
+  #. ``for f in `ls ~/swift/bin/`; do sudo ln -s /home/<your-user-name>/swift/bin/$f /usr/bin/`basename $f .py`; done``
   #. Edit `~/.bashrc` and add to the end::
 
-        export PYTHONPATH=~/openswift
+        export PYTHONPATH=~/swift
         export PATH_TO_TEST_XFS=/mnt/sdb1/test
         export SWIFT_TEST_CONFIG_FILE=/etc/swift/func_test.conf
         export PATH=${PATH}:~/bin
@@ -422,11 +424,11 @@ good idea what to do on other environments.
 
   #. `chmod +x ~/bin/*`
   #. `remakerings`
-  #. `cd ~/openswift; ./.unittests`
+  #. `cd ~/swift; ./.unittests`
   #. `startmain`
   #. `swift-auth-create-account test tester testing`
-  #. Get an `X-Storage-Url` and `X-Auth-Token`: `curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:11000/v1.0`
+  #. Get an `X-Storage-Url` and `X-Auth-Token`: ``curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:11000/v1.0``
-  #. Check that you can GET account: `curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>`
+  #. Check that you can GET account: ``curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>``
   #. Check that `st` works: `st -A http://127.0.0.1:11000/v1.0 -U test:tester -K testing stat`
   #. Create `/etc/swift/func_test.conf`::
 
@@ -440,6 +442,22 @@ good idea what to do on other environments.
 
         collate = C
 
-  #. `cd ~/openswift; ./.functests`
+  #. `cd ~/swift; ./.functests`
-  #. `cd ~/openswift; ./.probetests`
+  #. `cd ~/swift; ./.probetests`
 
+----------------
+Debugging Issues
+----------------
+
+If all doesn't go as planned, and tests fail, or you can't auth, or something doesn't work, here are some good starting places to look for issues:
+
+#. Everything is logged in /var/log/syslog, so that is a good first place to
+   look for errors (most likely python tracebacks).
+#. Make sure all of the server processes are running.  For the base
+   functionality, the Proxy, Account, Container, Object and Auth servers
+   should be running
+#. If one of the servers are not running, and no errors are logged to syslog,
+   it may be useful to try to start the server manually, for example: 
+   `swift-object server /etc/swift/object-server/1.conf` will start the 
+   object server.  If there are problems not showing up in syslog, 
+   then you will likely see the traceback on startup.

setup.py

@@ -24,9 +24,7 @@ setup(
     author='OpenStack, LLC.',
     url='https://launchpad.net/swift',
     packages=find_packages(exclude=['tests','bin']),
-    #Uncomment this once unittests work without /etc
+    test_suite = 'nose.collector',
-    #Also, figure out how to make this only run unit tests
-    #test_suite = 'nose.collector',
     classifiers=[
         'Development Status :: 4 - Beta',
         'License :: OSI Approved :: Apache Software License',

test/unit/container/test_server.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import os
+import sys
 import unittest
 from shutil import rmtree
 from StringIO import StringIO
@@ -32,13 +33,17 @@ class TestContainerController(unittest.TestCase):
     """ Test swift.container_server.ContainerController """
     def setUp(self):
         """ Set up for testing swift.object_server.ObjectController """
-        path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
+        self.path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
-        if not path_to_test_xfs or not os.path.exists(path_to_test_xfs):
+        if not self.path_to_test_xfs or \
-            raise Exception('PATH_TO_TEST_XFS not set or not pointing to a '
+                not os.path.exists(self.path_to_test_xfs):
-                            'valid directory.\nPlease set PATH_TO_TEST_XFS to '
+            print >>sys.stderr, 'WARNING: PATH_TO_TEST_XFS not set or not ' \
-                            'a directory on an XFS file system for testing.')
+                'pointing to a valid directory.\n' \
-        self.testdir = os.path.join(path_to_test_xfs,
+                'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \
-                                    'tmp_test_object_server_ObjectController')
+                'system for testing.'
+            self.testdir = '/tmp/SWIFTUNITTEST'
+        else:
+            self.testdir = os.path.join(self.path_to_test_xfs,
+                            'tmp_test_object_server_ObjectController')
         mkdirs(self.testdir)
         rmtree(self.testdir)
         mkdirs(os.path.join(self.testdir, 'sda1'))

test/unit/container/test_updater.py

@@ -15,6 +15,7 @@
 
 import cPickle as pickle
 import os
+import sys
 import unittest
 from gzip import GzipFile
 from shutil import rmtree
@@ -32,13 +33,17 @@ from swift.common.utils import normalize_timestamp
 class TestContainerUpdater(unittest.TestCase):
 
     def setUp(self):
-        path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
+        self.path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
-        if not path_to_test_xfs or not os.path.exists(path_to_test_xfs):
+        if not self.path_to_test_xfs or \
-            raise Exception('PATH_TO_TEST_XFS not set or not pointing to a '
+                not os.path.exists(self.path_to_test_xfs):
-                            'valid directory.\nPlease set PATH_TO_TEST_XFS to '
+            print >>sys.stderr, 'WARNING: PATH_TO_TEST_XFS not set or not ' \
-                            'a directory on an XFS file system for testing.')
+                'pointing to a valid directory.\n' \
-        self.testdir = os.path.join(path_to_test_xfs,
+                'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \
-                                    'tmp_test_container_updater')
+                'system for testing.'
+            self.testdir = '/tmp/SWIFTUNITTEST'
+        else:
+            self.testdir = os.path.join(self.path_to_test_xfs,
+                                        'tmp_test_container_updater')
         rmtree(self.testdir, ignore_errors=1)
         os.mkdir(self.testdir)
         pickle.dump(RingData([[0, 1, 0, 1], [1, 0, 1, 0]],

test/unit/obj/test_server.py

@@ -38,13 +38,17 @@ class TestObjectController(unittest.TestCase):
 
     def setUp(self):
         """ Set up for testing swift.object_server.ObjectController """
-        path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
+        self.path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
-        if not path_to_test_xfs or not os.path.exists(path_to_test_xfs):
+        if not self.path_to_test_xfs or \
-            raise Exception('PATH_TO_TEST_XFS not set or not pointing to a '
+                not os.path.exists(self.path_to_test_xfs):
-                            'valid directory.\nPlease set PATH_TO_TEST_XFS to '
+            print >>sys.stderr, 'WARNING: PATH_TO_TEST_XFS not set or not ' \
-                            'a directory on an XFS file system for testing.')
+                'pointing to a valid directory.\n' \
-        self.testdir = os.path.join(path_to_test_xfs,
+                'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \
-                                    'tmp_test_object_server_ObjectController')
+                'system for testing.'
+            self.testdir = '/tmp/SWIFTUNITTEST'
+        else:
+            self.testdir = os.path.join(self.path_to_test_xfs,
+                            'tmp_test_object_server_ObjectController')
         mkdirs(self.testdir)
         rmtree(self.testdir)
         mkdirs(os.path.join(self.testdir, 'sda1'))
@@ -59,6 +63,8 @@ class TestObjectController(unittest.TestCase):
 
     def test_POST_update_meta(self):
         """ Test swift.object_server.ObjectController.POST """
+        if not self.path_to_test_xfs:
+            return
         timestamp = normalize_timestamp(time())
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                             headers={'X-Timestamp': timestamp,
@@ -85,6 +91,8 @@ class TestObjectController(unittest.TestCase):
         self.assertEquals(resp.headers['Content-Type'], 'application/x-test')
 
     def test_POST_not_exist(self):
+        if not self.path_to_test_xfs:
+            return
         timestamp = normalize_timestamp(time())
         req = Request.blank('/sda1/p/a/c/fail', environ={'REQUEST_METHOD': 'POST'},
                             headers={'X-Timestamp': timestamp,
@@ -105,6 +113,8 @@ class TestObjectController(unittest.TestCase):
         self.assertEquals(resp.status_int, 400)
 
     def test_POST_container_connection(self):
+        if not self.path_to_test_xfs:
+            return
         def mock_http_connect(response, with_exc=False):
             class FakeConn(object):
                 def __init__(self, status, with_exc):
@@ -199,6 +209,8 @@ class TestObjectController(unittest.TestCase):
         self.assertEquals(resp.status_int, 411)
 
     def test_PUT_common(self):
+        if not self.path_to_test_xfs:
+            return
         timestamp = normalize_timestamp(time())
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                 headers={'X-Timestamp': timestamp,
@@ -221,6 +233,8 @@ class TestObjectController(unittest.TestCase):
                            'name': '/a/c/o'})
 
     def test_PUT_overwrite(self):
+        if not self.path_to_test_xfs:
+            return
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                 headers={'X-Timestamp': normalize_timestamp(time()),
                          'Content-Length': '6',
@@ -252,6 +266,8 @@ class TestObjectController(unittest.TestCase):
                            'Content-Encoding': 'gzip'})
 
     def test_PUT_no_etag(self):
+        if not self.path_to_test_xfs:
+            return
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                             headers={'X-Timestamp': normalize_timestamp(time()),
                                      'Content-Type': 'text/plain'})
@@ -269,6 +285,8 @@ class TestObjectController(unittest.TestCase):
         self.assertEquals(resp.status_int, 422)
 
     def test_PUT_user_metadata(self):
+        if not self.path_to_test_xfs:
+            return
         timestamp = normalize_timestamp(time())
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                 headers={'X-Timestamp': timestamp,
@@ -295,6 +313,8 @@ class TestObjectController(unittest.TestCase):
                            'X-Object-Meta-Two': 'Two'})
 
     def test_PUT_container_connection(self):
+        if not self.path_to_test_xfs:
+            return
         def mock_http_connect(response, with_exc=False):
             class FakeConn(object):
                 def __init__(self, status, with_exc):
@@ -355,6 +375,8 @@ class TestObjectController(unittest.TestCase):
 
     def test_HEAD(self):
         """ Test swift.object_server.ObjectController.HEAD """
+        if not self.path_to_test_xfs:
+            return
         req = Request.blank('/sda1/p/a/c')
         resp = self.object_controller.HEAD(req)
         self.assertEquals(resp.status_int, 400)
@@ -420,6 +442,8 @@ class TestObjectController(unittest.TestCase):
 
     def test_GET(self):
         """ Test swift.object_server.ObjectController.GET """
+        if not self.path_to_test_xfs:
+            return
         req = Request.blank('/sda1/p/a/c')
         resp = self.object_controller.GET(req)
         self.assertEquals(resp.status_int, 400)
@@ -507,6 +531,8 @@ class TestObjectController(unittest.TestCase):
         self.assertEquals(resp.status_int, 404)
 
     def test_GET_if_match(self):
+        if not self.path_to_test_xfs:
+            return
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                             headers={
                                 'X-Timestamp': normalize_timestamp(time()),
@@ -559,6 +585,8 @@ class TestObjectController(unittest.TestCase):
         self.assertEquals(resp.status_int, 412)
 
     def test_GET_if_none_match(self):
+        if not self.path_to_test_xfs:
+            return
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                             headers={
                                 'X-Timestamp': normalize_timestamp(time()),
@@ -608,6 +636,8 @@ class TestObjectController(unittest.TestCase):
         self.assertEquals(resp.etag, etag)
 
     def test_GET_if_modified_since(self):
+        if not self.path_to_test_xfs:
+            return
         timestamp = normalize_timestamp(time())
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                             headers={
@@ -643,6 +673,8 @@ class TestObjectController(unittest.TestCase):
         self.assertEquals(resp.status_int, 304)
 
     def test_GET_if_unmodified_since(self):
+        if not self.path_to_test_xfs:
+            return
         timestamp = normalize_timestamp(time())
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                             headers={
@@ -680,6 +712,8 @@ class TestObjectController(unittest.TestCase):
 
     def test_DELETE(self):
         """ Test swift.object_server.ObjectController.DELETE """
+        if not self.path_to_test_xfs:
+            return
         req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'})
         resp = self.object_controller.DELETE(req)
         self.assertEquals(resp.status_int, 400)
@@ -832,6 +866,8 @@ class TestObjectController(unittest.TestCase):
         self.assertEquals(outbuf.getvalue()[:4], '405 ')
 
     def test_chunked_put(self):
+        if not self.path_to_test_xfs:
+            return
         listener = listen(('localhost', 0))
         port = listener.getsockname()[1]
         killer = spawn(wsgi.server, listener, self.object_controller,
@@ -856,6 +892,8 @@ class TestObjectController(unittest.TestCase):
         killer.kill()
 
     def test_max_object_name_length(self):
+        if not self.path_to_test_xfs:
+            return
         timestamp = normalize_timestamp(time())
         req = Request.blank('/sda1/p/a/c/' + ('1' * 1024),
                 environ={'REQUEST_METHOD': 'PUT'},
@@ -875,6 +913,8 @@ class TestObjectController(unittest.TestCase):
         self.assertEquals(resp.status_int, 400)
 
     def test_disk_file_app_iter_corners(self):
+        if not self.path_to_test_xfs:
+            return
         df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o')
         mkdirs(df.datadir)
         f = open(os.path.join(df.datadir,
@@ -906,6 +946,8 @@ class TestObjectController(unittest.TestCase):
             self.assert_(os.path.exists(tmpdir))
 
     def test_max_upload_time(self):
+        if not self.path_to_test_xfs:
+            return
         class SlowBody():
             def __init__(self):
                 self.sent = 0
@@ -946,6 +988,8 @@ class TestObjectController(unittest.TestCase):
         self.assertEquals(resp.status_int, 499)
 
     def test_bad_sinces(self):
+        if not self.path_to_test_xfs:
+            return
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
             headers={'X-Timestamp': normalize_timestamp(time()),
                      'Content-Length': '4', 'Content-Type': 'text/plain'},
@@ -970,6 +1014,8 @@ class TestObjectController(unittest.TestCase):
         self.assertEquals(resp.status_int, 412)
 
     def test_content_encoding(self):
+        if not self.path_to_test_xfs:
+            return
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
             headers={'X-Timestamp': normalize_timestamp(time()),
                      'Content-Length': '4', 'Content-Type': 'text/plain',

test/unit/proxy/test_server.py

@@ -1028,9 +1028,11 @@ class TestObjectController(unittest.TestCase):
         # proxy_server.Application we couldn't get to easily otherwise.
         path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
         if not path_to_test_xfs or not os.path.exists(path_to_test_xfs):
-            raise Exception('PATH_TO_TEST_XFS not set or not pointing to '
+            print >>sys.stderr, 'WARNING: PATH_TO_TEST_XFS not set or not ' \
-                'a valid directory.\nPlease set PATH_TO_TEST_XFS to a '
+                'pointing to a valid directory.\n' \
-                'directory on an XFS file system for testing.')
+                'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \
+                'system for testing.'
+            return
         testdir = \
             os.path.join(path_to_test_xfs, 'tmp_test_proxy_server_chunked')
         mkdirs(testdir)