Merge "Sync oslo lockutils for "fix lockutils.lock() to make it thread-safe""

nova/openstack/common/excutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # Copyright 2012, Red Hat, Inc.
 #
@@ -24,7 +22,9 @@ import sys
 import time
 import traceback
 
-from nova.openstack.common.gettextutils import _
+import six
+
+from nova.openstack.common.gettextutils import _  # noqa
 
 
 class save_and_reraise_exception(object):
@@ -65,7 +65,7 @@ class save_and_reraise_exception(object):
                                                      self.tb))
             return False
         if self.reraise:
-            raise self.type_, self.value, self.tb
+            six.reraise(self.type_, self.value, self.tb)
 
 
 def forever_retry_uncaught_exceptions(infunc):
@@ -77,7 +77,8 @@ def forever_retry_uncaught_exceptions(infunc):
             try:
                 return infunc(*args, **kwargs)
             except Exception as exc:
-                if exc.message == last_exc_message:
+                this_exc_message = six.u(str(exc))
+                if this_exc_message == last_exc_message:
                     exc_count += 1
                 else:
                     exc_count = 1
@@ -85,12 +86,12 @@ def forever_retry_uncaught_exceptions(infunc):
                 # the exception message changes
                 cur_time = int(time.time())
                 if (cur_time - last_log_time > 60 or
-                        exc.message != last_exc_message):
+                        this_exc_message != last_exc_message):
                     logging.exception(
                         _('Unexpected exception occurred %d time(s)... '
                           'retrying.') % exc_count)
                     last_log_time = cur_time
-                    last_exc_message = exc.message
+                    last_exc_message = this_exc_message
                     exc_count = 0
                 # This should be a very rare event. In case it isn't, do
                 # a sleep.

nova/openstack/common/fileutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -19,6 +17,7 @@
 import contextlib
 import errno
 import os
+import tempfile
 
 from nova.openstack.common import excutils
 from nova.openstack.common.gettextutils import _  # noqa
@@ -109,3 +108,30 @@ def file_open(*args, **kwargs):
     state at all (for unit tests)
     """
     return file(*args, **kwargs)
+
+
+def write_to_tempfile(content, path=None, suffix='', prefix='tmp'):
+    """Create temporary file or use existing file.
+
+    This util is needed for creating temporary file with
+    specified content, suffix and prefix. If path is not None,
+    it will be used for writing content. If the path doesn't
+    exist it'll be created.
+
+    :param content: content for temporary file.
+    :param path: same as parameter 'dir' for mkstemp
+    :param suffix: same as parameter 'suffix' for mkstemp
+    :param prefix: same as parameter 'prefix' for mkstemp
+
+    For example: it can be used in database tests for creating
+    configuration files.
+    """
+    if path:
+        ensure_tree(path)
+
+    (fd, path) = tempfile.mkstemp(suffix=suffix, dir=path, prefix=prefix)
+    try:
+        os.write(fd, content)
+    finally:
+        os.close(fd)
+    return path

nova/openstack/common/gettextutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 Red Hat, Inc.
 # Copyright 2013 IBM Corp.
 # All Rights Reserved.

nova/openstack/common/importutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #

nova/openstack/common/jsonutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # Copyright 2011 Justin Santa Barbara
@@ -124,7 +122,7 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
                                       level=level,
                                       max_depth=max_depth)
         if isinstance(value, dict):
-            return dict((k, recursive(v)) for k, v in value.iteritems())
+            return dict((k, recursive(v)) for k, v in six.iteritems(value))
         elif isinstance(value, (list, tuple)):
             return [recursive(lv) for lv in value]
 

nova/openstack/common/local.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #

nova/openstack/common/lockutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -20,6 +18,10 @@ import contextlib
 import errno
 import functools
 import os
+import shutil
+import subprocess
+import sys
+import tempfile
 import threading
 import time
 import weakref
@@ -39,6 +41,7 @@ util_opts = [
     cfg.BoolOpt('disable_process_locking', default=False,
                 help='Whether to disable inter-process locks'),
     cfg.StrOpt('lock_path',
+               default=os.environ.get("NOVA_LOCK_PATH"),
                help=('Directory to use for lock files.'))
 ]
 
@@ -131,6 +134,7 @@ else:
     InterProcessLock = _PosixLock
 
 _semaphores = weakref.WeakValueDictionary()
+_semaphores_lock = threading.Lock()
 
 
 @contextlib.contextmanager
@@ -153,15 +157,12 @@ def lock(name, lock_file_prefix=None, external=False, lock_path=None):
     special location for external lock files to live. If nothing is set, then
     CONF.lock_path is used as a default.
     """
-    # NOTE(soren): If we ever go natively threaded, this will be racy.
-    #              See http://stackoverflow.com/questions/5390569/dyn
-    #              amically-allocating-and-destroying-mutexes
-    sem = _semaphores.get(name, threading.Semaphore())
-    if name not in _semaphores:
-        # this check is not racy - we're already holding ref locally
-        # so GC won't remove the item and there was no IO switch
-        # (only valid in greenthreads)
-        _semaphores[name] = sem
+    with _semaphores_lock:
+        try:
+            sem = _semaphores[name]
+        except KeyError:
+            sem = threading.Semaphore()
+            _semaphores[name] = sem
 
     with sem:
         LOG.debug(_('Got semaphore "%(lock)s"'), {'lock': name})
@@ -276,3 +277,27 @@ def synchronized_with_prefix(lock_file_prefix):
     """
 
     return functools.partial(synchronized, lock_file_prefix=lock_file_prefix)
+
+
+def main(argv):
+    """Create a dir for locks and pass it to command from arguments
+
+    If you run this:
+    python -m openstack.common.lockutils python setup.py testr <etc>
+
+    a temporary directory will be created for all your locks and passed to all
+    your tests in an environment variable. The temporary dir will be deleted
+    afterwards and the return value will be preserved.
+    """
+
+    lock_dir = tempfile.mkdtemp()
+    os.environ["NOVA_LOCK_PATH"] = lock_dir
+    try:
+        ret_val = subprocess.call(argv[1:])
+    finally:
+        shutil.rmtree(lock_dir, ignore_errors=True)
+    return ret_val
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

nova/openstack/common/timeutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2011 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -178,6 +176,15 @@ def delta_seconds(before, after):
     datetime objects (as a float, to microsecond resolution).
     """
     delta = after - before
+    return total_seconds(delta)
+
+
+def total_seconds(delta):
+    """Return the total seconds of datetime.timedelta object.
+
+    Compute total seconds of datetime.timedelta, datetime.timedelta
+    doesn't have method total_seconds in Python2.6, calculate it manually.
+    """
     try:
         return delta.total_seconds()
     except AttributeError: