From 12effeeb7595d16c2bc8e1204a5d8c8cea6e6bac Mon Sep 17 00:00:00 2001 From: Matt Riedemann Date: Sun, 8 Dec 2013 23:45:13 -0800 Subject: [PATCH] Sync oslo lockutils for "fix lockutils.lock() to make it thread-safe" Pull in oslo-incubator change Ia18e6e9f "fix lockutils.lock() to make it thread-safe" along with dependencies. Note: I've intentionally excluded log.py from the sync since it pulls in Change Ic2cf3e52: Adding domain to context and log which is (1) unrelated to the lockutils change and (2) also changes context.py, which lockutils does not depend on. Changes by module (from oldest to newest) ========================================= excutils: Change Ic36fa050: Enable H302 hacking check Change Ibf3c56e4: BaseException.message is deprecated since Python 2.6 Change I87fd89ff: excutils: use six.reraise to re-raise Change If640e551: excutils: replace unicode by six.u Change Ic6f0c0ef: Remove vim header fileutils: Change Ia51d416b: Add utils for creating tempfile Change Ic6f0c0ef: Remove vim header gettextutils: Change Ic6f0c0ef: Remove vim header importutils: Change Ic6f0c0ef: Remove vim header jsonutils: Change Ic6f0c0ef: Remove vim header Change I90be8797: Use six.iteritems to make dict work on Python2/3 local: Change Ic6f0c0ef: Remove vim header lockutils: Change I64fccddc: Allow lockutils to get lock_path conf from envvar Change I9e1260e2: Add main() to lockutils that creates temp dir Change Ia18e6e9f: fix lockutils.lock() to make it thread-safe Change Ic6f0c0ef: Remove vim header timeutils: Change I397bae40: Add helper method total_seconds in timeutils.py Change Ic6f0c0ef: Remove vim header Related-Bug: #1065529 Change-Id: Ia3530c0d3e78d90fd6de8ac186b252e0fbbba85e --- nova/openstack/common/excutils.py | 15 +++++---- nova/openstack/common/fileutils.py | 30 +++++++++++++++-- nova/openstack/common/gettextutils.py | 2 -- nova/openstack/common/importutils.py | 2 -- nova/openstack/common/jsonutils.py | 4 +-- nova/openstack/common/local.py | 2 -- nova/openstack/common/lockutils.py | 47 ++++++++++++++++++++------- nova/openstack/common/timeutils.py | 11 +++++-- 8 files changed, 82 insertions(+), 31 deletions(-) diff --git a/nova/openstack/common/excutils.py b/nova/openstack/common/excutils.py index f719867e8..2f5723b69 100644 --- a/nova/openstack/common/excutils.py +++ b/nova/openstack/common/excutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # Copyright 2012, Red Hat, Inc. # @@ -24,7 +22,9 @@ import sys import time import traceback -from nova.openstack.common.gettextutils import _ +import six + +from nova.openstack.common.gettextutils import _ # noqa class save_and_reraise_exception(object): @@ -65,7 +65,7 @@ class save_and_reraise_exception(object): self.tb)) return False if self.reraise: - raise self.type_, self.value, self.tb + six.reraise(self.type_, self.value, self.tb) def forever_retry_uncaught_exceptions(infunc): @@ -77,7 +77,8 @@ def forever_retry_uncaught_exceptions(infunc): try: return infunc(*args, **kwargs) except Exception as exc: - if exc.message == last_exc_message: + this_exc_message = six.u(str(exc)) + if this_exc_message == last_exc_message: exc_count += 1 else: exc_count = 1 @@ -85,12 +86,12 @@ def forever_retry_uncaught_exceptions(infunc): # the exception message changes cur_time = int(time.time()) if (cur_time - last_log_time > 60 or - exc.message != last_exc_message): + this_exc_message != last_exc_message): logging.exception( _('Unexpected exception occurred %d time(s)... ' 'retrying.') % exc_count) last_log_time = cur_time - last_exc_message = exc.message + last_exc_message = this_exc_message exc_count = 0 # This should be a very rare event. In case it isn't, do # a sleep. diff --git a/nova/openstack/common/fileutils.py b/nova/openstack/common/fileutils.py index 735d94af6..e58c9824f 100644 --- a/nova/openstack/common/fileutils.py +++ b/nova/openstack/common/fileutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -19,6 +17,7 @@ import contextlib import errno import os +import tempfile from nova.openstack.common import excutils from nova.openstack.common.gettextutils import _ # noqa @@ -109,3 +108,30 @@ def file_open(*args, **kwargs): state at all (for unit tests) """ return file(*args, **kwargs) + + +def write_to_tempfile(content, path=None, suffix='', prefix='tmp'): + """Create temporary file or use existing file. + + This util is needed for creating temporary file with + specified content, suffix and prefix. If path is not None, + it will be used for writing content. If the path doesn't + exist it'll be created. + + :param content: content for temporary file. + :param path: same as parameter 'dir' for mkstemp + :param suffix: same as parameter 'suffix' for mkstemp + :param prefix: same as parameter 'prefix' for mkstemp + + For example: it can be used in database tests for creating + configuration files. + """ + if path: + ensure_tree(path) + + (fd, path) = tempfile.mkstemp(suffix=suffix, dir=path, prefix=prefix) + try: + os.write(fd, content) + finally: + os.close(fd) + return path diff --git a/nova/openstack/common/gettextutils.py b/nova/openstack/common/gettextutils.py index ccddc98f9..c900b227e 100644 --- a/nova/openstack/common/gettextutils.py +++ b/nova/openstack/common/gettextutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 Red Hat, Inc. # Copyright 2013 IBM Corp. # All Rights Reserved. diff --git a/nova/openstack/common/importutils.py b/nova/openstack/common/importutils.py index 7a303f93f..4fd9ae2bc 100644 --- a/nova/openstack/common/importutils.py +++ b/nova/openstack/common/importutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/nova/openstack/common/jsonutils.py b/nova/openstack/common/jsonutils.py index fe0331eb0..0af24a881 100644 --- a/nova/openstack/common/jsonutils.py +++ b/nova/openstack/common/jsonutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2011 Justin Santa Barbara @@ -124,7 +122,7 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, level=level, max_depth=max_depth) if isinstance(value, dict): - return dict((k, recursive(v)) for k, v in value.iteritems()) + return dict((k, recursive(v)) for k, v in six.iteritems(value)) elif isinstance(value, (list, tuple)): return [recursive(lv) for lv in value] diff --git a/nova/openstack/common/local.py b/nova/openstack/common/local.py index e82f17d0f..0819d5b97 100644 --- a/nova/openstack/common/local.py +++ b/nova/openstack/common/local.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/nova/openstack/common/lockutils.py b/nova/openstack/common/lockutils.py index b53a82839..6f80ef6bc 100644 --- a/nova/openstack/common/lockutils.py +++ b/nova/openstack/common/lockutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -20,6 +18,10 @@ import contextlib import errno import functools import os +import shutil +import subprocess +import sys +import tempfile import threading import time import weakref @@ -39,6 +41,7 @@ util_opts = [ cfg.BoolOpt('disable_process_locking', default=False, help='Whether to disable inter-process locks'), cfg.StrOpt('lock_path', + default=os.environ.get("NOVA_LOCK_PATH"), help=('Directory to use for lock files.')) ] @@ -131,6 +134,7 @@ else: InterProcessLock = _PosixLock _semaphores = weakref.WeakValueDictionary() +_semaphores_lock = threading.Lock() @contextlib.contextmanager @@ -153,15 +157,12 @@ def lock(name, lock_file_prefix=None, external=False, lock_path=None): special location for external lock files to live. If nothing is set, then CONF.lock_path is used as a default. """ - # NOTE(soren): If we ever go natively threaded, this will be racy. - # See http://stackoverflow.com/questions/5390569/dyn - # amically-allocating-and-destroying-mutexes - sem = _semaphores.get(name, threading.Semaphore()) - if name not in _semaphores: - # this check is not racy - we're already holding ref locally - # so GC won't remove the item and there was no IO switch - # (only valid in greenthreads) - _semaphores[name] = sem + with _semaphores_lock: + try: + sem = _semaphores[name] + except KeyError: + sem = threading.Semaphore() + _semaphores[name] = sem with sem: LOG.debug(_('Got semaphore "%(lock)s"'), {'lock': name}) @@ -276,3 +277,27 @@ def synchronized_with_prefix(lock_file_prefix): """ return functools.partial(synchronized, lock_file_prefix=lock_file_prefix) + + +def main(argv): + """Create a dir for locks and pass it to command from arguments + + If you run this: + python -m openstack.common.lockutils python setup.py testr + + a temporary directory will be created for all your locks and passed to all + your tests in an environment variable. The temporary dir will be deleted + afterwards and the return value will be preserved. + """ + + lock_dir = tempfile.mkdtemp() + os.environ["NOVA_LOCK_PATH"] = lock_dir + try: + ret_val = subprocess.call(argv[1:]) + finally: + shutil.rmtree(lock_dir, ignore_errors=True) + return ret_val + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/nova/openstack/common/timeutils.py b/nova/openstack/common/timeutils.py index b79ebf378..c8b0b1539 100644 --- a/nova/openstack/common/timeutils.py +++ b/nova/openstack/common/timeutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -178,6 +176,15 @@ def delta_seconds(before, after): datetime objects (as a float, to microsecond resolution). """ delta = after - before + return total_seconds(delta) + + +def total_seconds(delta): + """Return the total seconds of datetime.timedelta object. + + Compute total seconds of datetime.timedelta, datetime.timedelta + doesn't have method total_seconds in Python2.6, calculate it manually. + """ try: return delta.total_seconds() except AttributeError: