Sync oslo lockutils for "fix lockutils.lock() to make it thread-safe"
Pull in oslo-incubator change Ia18e6e9f "fix lockutils.lock() to make it thread-safe" along with dependencies. Note: I've intentionally excluded log.py from the sync since it pulls in Change Ic2cf3e52: Adding domain to context and log which is (1) unrelated to the lockutils change and (2) also changes context.py, which lockutils does not depend on. Changes by module (from oldest to newest) ========================================= excutils: Change Ic36fa050: Enable H302 hacking check Change Ibf3c56e4: BaseException.message is deprecated since Python 2.6 Change I87fd89ff: excutils: use six.reraise to re-raise Change If640e551: excutils: replace unicode by six.u Change Ic6f0c0ef: Remove vim header fileutils: Change Ia51d416b: Add utils for creating tempfile Change Ic6f0c0ef: Remove vim header gettextutils: Change Ic6f0c0ef: Remove vim header importutils: Change Ic6f0c0ef: Remove vim header jsonutils: Change Ic6f0c0ef: Remove vim header Change I90be8797: Use six.iteritems to make dict work on Python2/3 local: Change Ic6f0c0ef: Remove vim header lockutils: Change I64fccddc: Allow lockutils to get lock_path conf from envvar Change I9e1260e2: Add main() to lockutils that creates temp dir Change Ia18e6e9f: fix lockutils.lock() to make it thread-safe Change Ic6f0c0ef: Remove vim header timeutils: Change I397bae40: Add helper method total_seconds in timeutils.py Change Ic6f0c0ef: Remove vim header Related-Bug: #1065529 Change-Id: Ia3530c0d3e78d90fd6de8ac186b252e0fbbba85e
This commit is contained in:
Matt Riedemann
Demontiê Junior
parent
30a737ca72
commit
12effeeb75
|
|
@ -1,5 +1,3 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack Foundation.
|
||||
# Copyright 2012, Red Hat, Inc.
|
||||
#
|
||||
|
|
@ -24,7 +22,9 @@ import sys
|
|||
import time
|
||||
import traceback
|
||||
|
||||
from nova.openstack.common.gettextutils import _
|
||||
import six
|
||||
|
||||
from nova.openstack.common.gettextutils import _ # noqa
|
||||
|
||||
|
||||
class save_and_reraise_exception(object):
|
||||
|
|
@ -65,7 +65,7 @@ class save_and_reraise_exception(object):
|
|||
self.tb))
|
||||
return False
|
||||
if self.reraise:
|
||||
raise self.type_, self.value, self.tb
|
||||
six.reraise(self.type_, self.value, self.tb)
|
||||
|
||||
|
||||
def forever_retry_uncaught_exceptions(infunc):
|
||||
|
|
@ -77,7 +77,8 @@ def forever_retry_uncaught_exceptions(infunc):
|
|||
try:
|
||||
return infunc(*args, **kwargs)
|
||||
except Exception as exc:
|
||||
if exc.message == last_exc_message:
|
||||
this_exc_message = six.u(str(exc))
|
||||
if this_exc_message == last_exc_message:
|
||||
exc_count += 1
|
||||
else:
|
||||
exc_count = 1
|
||||
|
|
@ -85,12 +86,12 @@ def forever_retry_uncaught_exceptions(infunc):
|
|||
# the exception message changes
|
||||
cur_time = int(time.time())
|
||||
if (cur_time - last_log_time > 60 or
|
||||
exc.message != last_exc_message):
|
||||
this_exc_message != last_exc_message):
|
||||
logging.exception(
|
||||
_('Unexpected exception occurred %d time(s)... '
|
||||
'retrying.') % exc_count)
|
||||
last_log_time = cur_time
|
||||
last_exc_message = exc.message
|
||||
last_exc_message = this_exc_message
|
||||
exc_count = 0
|
||||
# This should be a very rare event. In case it isn't, do
|
||||
# a sleep.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
|
|
@ -19,6 +17,7 @@
|
|||
import contextlib
|
||||
import errno
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
from nova.openstack.common import excutils
|
||||
from nova.openstack.common.gettextutils import _ # noqa
|
||||
|
|
@ -109,3 +108,30 @@ def file_open(*args, **kwargs):
|
|||
state at all (for unit tests)
|
||||
"""
|
||||
return file(*args, **kwargs)
|
||||
|
||||
|
||||
def write_to_tempfile(content, path=None, suffix='', prefix='tmp'):
|
||||
"""Create temporary file or use existing file.
|
||||
|
||||
This util is needed for creating temporary file with
|
||||
specified content, suffix and prefix. If path is not None,
|
||||
it will be used for writing content. If the path doesn't
|
||||
exist it'll be created.
|
||||
|
||||
:param content: content for temporary file.
|
||||
:param path: same as parameter 'dir' for mkstemp
|
||||
:param suffix: same as parameter 'suffix' for mkstemp
|
||||
:param prefix: same as parameter 'prefix' for mkstemp
|
||||
|
||||
For example: it can be used in database tests for creating
|
||||
configuration files.
|
||||
"""
|
||||
if path:
|
||||
ensure_tree(path)
|
||||
|
||||
(fd, path) = tempfile.mkstemp(suffix=suffix, dir=path, prefix=prefix)
|
||||
try:
|
||||
os.write(fd, content)
|
||||
finally:
|
||||
os.close(fd)
|
||||
return path
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2012 Red Hat, Inc.
|
||||
# Copyright 2013 IBM Corp.
|
||||
# All Rights Reserved.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2010 United States Government as represented by the
|
||||
# Administrator of the National Aeronautics and Space Administration.
|
||||
# Copyright 2011 Justin Santa Barbara
|
||||
|
|
@ -124,7 +122,7 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
|
|||
level=level,
|
||||
max_depth=max_depth)
|
||||
if isinstance(value, dict):
|
||||
return dict((k, recursive(v)) for k, v in value.iteritems())
|
||||
return dict((k, recursive(v)) for k, v in six.iteritems(value))
|
||||
elif isinstance(value, (list, tuple)):
|
||||
return [recursive(lv) for lv in value]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
|
|
@ -20,6 +18,10 @@ import contextlib
|
|||
import errno
|
||||
import functools
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
|
|
@ -39,6 +41,7 @@ util_opts = [
|
|||
cfg.BoolOpt('disable_process_locking', default=False,
|
||||
help='Whether to disable inter-process locks'),
|
||||
cfg.StrOpt('lock_path',
|
||||
default=os.environ.get("NOVA_LOCK_PATH"),
|
||||
help=('Directory to use for lock files.'))
|
||||
]
|
||||
|
||||
|
|
@ -131,6 +134,7 @@ else:
|
|||
InterProcessLock = _PosixLock
|
||||
|
||||
_semaphores = weakref.WeakValueDictionary()
|
||||
_semaphores_lock = threading.Lock()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
|
|
@ -153,15 +157,12 @@ def lock(name, lock_file_prefix=None, external=False, lock_path=None):
|
|||
special location for external lock files to live. If nothing is set, then
|
||||
CONF.lock_path is used as a default.
|
||||
"""
|
||||
# NOTE(soren): If we ever go natively threaded, this will be racy.
|
||||
# See http://stackoverflow.com/questions/5390569/dyn
|
||||
# amically-allocating-and-destroying-mutexes
|
||||
sem = _semaphores.get(name, threading.Semaphore())
|
||||
if name not in _semaphores:
|
||||
# this check is not racy - we're already holding ref locally
|
||||
# so GC won't remove the item and there was no IO switch
|
||||
# (only valid in greenthreads)
|
||||
_semaphores[name] = sem
|
||||
with _semaphores_lock:
|
||||
try:
|
||||
sem = _semaphores[name]
|
||||
except KeyError:
|
||||
sem = threading.Semaphore()
|
||||
_semaphores[name] = sem
|
||||
|
||||
with sem:
|
||||
LOG.debug(_('Got semaphore "%(lock)s"'), {'lock': name})
|
||||
|
|
@ -276,3 +277,27 @@ def synchronized_with_prefix(lock_file_prefix):
|
|||
"""
|
||||
|
||||
return functools.partial(synchronized, lock_file_prefix=lock_file_prefix)
|
||||
|
||||
|
||||
def main(argv):
|
||||
"""Create a dir for locks and pass it to command from arguments
|
||||
|
||||
If you run this:
|
||||
python -m openstack.common.lockutils python setup.py testr <etc>
|
||||
|
||||
a temporary directory will be created for all your locks and passed to all
|
||||
your tests in an environment variable. The temporary dir will be deleted
|
||||
afterwards and the return value will be preserved.
|
||||
"""
|
||||
|
||||
lock_dir = tempfile.mkdtemp()
|
||||
os.environ["NOVA_LOCK_PATH"] = lock_dir
|
||||
try:
|
||||
ret_val = subprocess.call(argv[1:])
|
||||
finally:
|
||||
shutil.rmtree(lock_dir, ignore_errors=True)
|
||||
return ret_val
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main(sys.argv))
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
|
|
@ -178,6 +176,15 @@ def delta_seconds(before, after):
|
|||
datetime objects (as a float, to microsecond resolution).
|
||||
"""
|
||||
delta = after - before
|
||||
return total_seconds(delta)
|
||||
|
||||
|
||||
def total_seconds(delta):
|
||||
"""Return the total seconds of datetime.timedelta object.
|
||||
|
||||
Compute total seconds of datetime.timedelta, datetime.timedelta
|
||||
doesn't have method total_seconds in Python2.6, calculate it manually.
|
||||
"""
|
||||
try:
|
||||
return delta.total_seconds()
|
||||
except AttributeError:
|
||||
|
|
|
|||
Loading…
Reference in New Issue