Sync Oslo and migrate from sources to oslo libs instead
Use oslo.utils, oslo.serialization and oslo.i18n instead of sources in openstack/common Change-Id: I82b0563efefdc1e897b5c2c19cce80632c3d550e
This commit is contained in:
Ilya Shakhat
parent
8ab9919af4
commit
2d6f712ca6
|
|
@ -1,9 +1,5 @@
|
|||
[DEFAULT]
|
||||
|
||||
module=importutils
|
||||
module=jsonutils
|
||||
module=log
|
||||
module=timeutils
|
||||
|
||||
# The base module to hold the copy of oslo-incubator
|
||||
base=stackalytics
|
||||
|
|
|
|||
|
|
@ -6,7 +6,10 @@ pbr>=0.6,!=0.7,<1.0
|
|||
Babel>=1.3
|
||||
Flask>=0.10,<1.0
|
||||
iso8601>=0.1.9
|
||||
oslo.config>=1.2.1
|
||||
oslo.config>=1.4.0 # Apache-2.0
|
||||
oslo.i18n>=1.0.0 # Apache-2.0
|
||||
oslo.serialization>=1.0.0 # Apache-2.0
|
||||
oslo.utils>=1.0.0 # Apache-2.0
|
||||
paramiko>=1.13.0
|
||||
psutil>=1.1.1,<2.0.0
|
||||
python3-memcached>=1.48
|
||||
|
|
|
|||
|
|
@ -6,7 +6,10 @@ pbr>=0.6,!=0.7,<1.0
|
|||
Babel>=1.3
|
||||
Flask>=0.10,<1.0
|
||||
iso8601>=0.1.9
|
||||
oslo.config>=1.2.1
|
||||
oslo.config>=1.4.0 # Apache-2.0
|
||||
oslo.i18n>=1.0.0 # Apache-2.0
|
||||
oslo.serialization>=1.0.0 # Apache-2.0
|
||||
oslo.utils>=1.0.0 # Apache-2.0
|
||||
paramiko>=1.13.0
|
||||
psutil>=1.1.1,<2.0.0
|
||||
python-memcached>=1.48
|
||||
|
|
|
|||
|
|
@ -1,17 +0,0 @@
|
|||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import six
|
||||
|
||||
|
||||
six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""oslo.i18n integration module.
|
||||
|
||||
See http://docs.openstack.org/developer/oslo.i18n/usage.html
|
||||
|
||||
"""
|
||||
|
||||
import oslo.i18n
|
||||
|
||||
|
||||
# NOTE(dhellmann): This reference to o-s-l-o will be replaced by the
|
||||
# application name when this module is synced into the separate
|
||||
# repository. It is OK to have more than one translation function
|
||||
# using the same domain, since there will still only be one message
|
||||
# catalog.
|
||||
_translators = oslo.i18n.TranslatorFactory(domain='stackalytics')
|
||||
|
||||
# The primary translation function using the well-known name "_"
|
||||
_ = _translators.primary
|
||||
|
||||
# Translators for log levels.
|
||||
#
|
||||
# The abbreviated names are meant to reflect the usual use of a short
|
||||
# name like '_'. The "L" is for "log" and the other letter comes from
|
||||
# the level.
|
||||
_LI = _translators.log_info
|
||||
_LW = _translators.log_warning
|
||||
_LE = _translators.log_error
|
||||
_LC = _translators.log_critical
|
||||
|
|
@ -1,498 +0,0 @@
|
|||
# Copyright 2012 Red Hat, Inc.
|
||||
# Copyright 2013 IBM Corp.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
gettext for openstack-common modules.
|
||||
|
||||
Usual usage in an openstack.common module:
|
||||
|
||||
from stackalytics.openstack.common.gettextutils import _
|
||||
"""
|
||||
|
||||
import copy
|
||||
import functools
|
||||
import gettext
|
||||
import locale
|
||||
from logging import handlers
|
||||
import os
|
||||
|
||||
from babel import localedata
|
||||
import six
|
||||
|
||||
_AVAILABLE_LANGUAGES = {}
|
||||
|
||||
# FIXME(dhellmann): Remove this when moving to oslo.i18n.
|
||||
USE_LAZY = False
|
||||
|
||||
|
||||
class TranslatorFactory(object):
|
||||
"""Create translator functions
|
||||
"""
|
||||
|
||||
def __init__(self, domain, lazy=False, localedir=None):
|
||||
"""Establish a set of translation functions for the domain.
|
||||
|
||||
:param domain: Name of translation domain,
|
||||
specifying a message catalog.
|
||||
:type domain: str
|
||||
:param lazy: Delays translation until a message is emitted.
|
||||
Defaults to False.
|
||||
:type lazy: Boolean
|
||||
:param localedir: Directory with translation catalogs.
|
||||
:type localedir: str
|
||||
"""
|
||||
self.domain = domain
|
||||
self.lazy = lazy
|
||||
if localedir is None:
|
||||
localedir = os.environ.get(domain.upper() + '_LOCALEDIR')
|
||||
self.localedir = localedir
|
||||
|
||||
def _make_translation_func(self, domain=None):
|
||||
"""Return a new translation function ready for use.
|
||||
|
||||
Takes into account whether or not lazy translation is being
|
||||
done.
|
||||
|
||||
The domain can be specified to override the default from the
|
||||
factory, but the localedir from the factory is always used
|
||||
because we assume the log-level translation catalogs are
|
||||
installed in the same directory as the main application
|
||||
catalog.
|
||||
|
||||
"""
|
||||
if domain is None:
|
||||
domain = self.domain
|
||||
if self.lazy:
|
||||
return functools.partial(Message, domain=domain)
|
||||
t = gettext.translation(
|
||||
domain,
|
||||
localedir=self.localedir,
|
||||
fallback=True,
|
||||
)
|
||||
if six.PY3:
|
||||
return t.gettext
|
||||
return t.ugettext
|
||||
|
||||
@property
|
||||
def primary(self):
|
||||
"The default translation function."
|
||||
return self._make_translation_func()
|
||||
|
||||
def _make_log_translation_func(self, level):
|
||||
return self._make_translation_func(self.domain + '-log-' + level)
|
||||
|
||||
@property
|
||||
def log_info(self):
|
||||
"Translate info-level log messages."
|
||||
return self._make_log_translation_func('info')
|
||||
|
||||
@property
|
||||
def log_warning(self):
|
||||
"Translate warning-level log messages."
|
||||
return self._make_log_translation_func('warning')
|
||||
|
||||
@property
|
||||
def log_error(self):
|
||||
"Translate error-level log messages."
|
||||
return self._make_log_translation_func('error')
|
||||
|
||||
@property
|
||||
def log_critical(self):
|
||||
"Translate critical-level log messages."
|
||||
return self._make_log_translation_func('critical')
|
||||
|
||||
|
||||
# NOTE(dhellmann): When this module moves out of the incubator into
|
||||
# oslo.i18n, these global variables can be moved to an integration
|
||||
# module within each application.
|
||||
|
||||
# Create the global translation functions.
|
||||
_translators = TranslatorFactory('stackalytics')
|
||||
|
||||
# The primary translation function using the well-known name "_"
|
||||
_ = _translators.primary
|
||||
|
||||
# Translators for log levels.
|
||||
#
|
||||
# The abbreviated names are meant to reflect the usual use of a short
|
||||
# name like '_'. The "L" is for "log" and the other letter comes from
|
||||
# the level.
|
||||
_LI = _translators.log_info
|
||||
_LW = _translators.log_warning
|
||||
_LE = _translators.log_error
|
||||
_LC = _translators.log_critical
|
||||
|
||||
# NOTE(dhellmann): End of globals that will move to the application's
|
||||
# integration module.
|
||||
|
||||
|
||||
def enable_lazy():
|
||||
"""Convenience function for configuring _() to use lazy gettext
|
||||
|
||||
Call this at the start of execution to enable the gettextutils._
|
||||
function to use lazy gettext functionality. This is useful if
|
||||
your project is importing _ directly instead of using the
|
||||
gettextutils.install() way of importing the _ function.
|
||||
"""
|
||||
# FIXME(dhellmann): This function will be removed in oslo.i18n,
|
||||
# because the TranslatorFactory makes it superfluous.
|
||||
global _, _LI, _LW, _LE, _LC, USE_LAZY
|
||||
tf = TranslatorFactory('stackalytics', lazy=True)
|
||||
_ = tf.primary
|
||||
_LI = tf.log_info
|
||||
_LW = tf.log_warning
|
||||
_LE = tf.log_error
|
||||
_LC = tf.log_critical
|
||||
USE_LAZY = True
|
||||
|
||||
|
||||
def install(domain, lazy=False):
|
||||
"""Install a _() function using the given translation domain.
|
||||
|
||||
Given a translation domain, install a _() function using gettext's
|
||||
install() function.
|
||||
|
||||
The main difference from gettext.install() is that we allow
|
||||
overriding the default localedir (e.g. /usr/share/locale) using
|
||||
a translation-domain-specific environment variable (e.g.
|
||||
NOVA_LOCALEDIR).
|
||||
|
||||
:param domain: the translation domain
|
||||
:param lazy: indicates whether or not to install the lazy _() function.
|
||||
The lazy _() introduces a way to do deferred translation
|
||||
of messages by installing a _ that builds Message objects,
|
||||
instead of strings, which can then be lazily translated into
|
||||
any available locale.
|
||||
"""
|
||||
if lazy:
|
||||
from six import moves
|
||||
tf = TranslatorFactory(domain, lazy=True)
|
||||
moves.builtins.__dict__['_'] = tf.primary
|
||||
else:
|
||||
localedir = '%s_LOCALEDIR' % domain.upper()
|
||||
if six.PY3:
|
||||
gettext.install(domain,
|
||||
localedir=os.environ.get(localedir))
|
||||
else:
|
||||
gettext.install(domain,
|
||||
localedir=os.environ.get(localedir),
|
||||
unicode=True)
|
||||
|
||||
|
||||
class Message(six.text_type):
|
||||
"""A Message object is a unicode object that can be translated.
|
||||
|
||||
Translation of Message is done explicitly using the translate() method.
|
||||
For all non-translation intents and purposes, a Message is simply unicode,
|
||||
and can be treated as such.
|
||||
"""
|
||||
|
||||
def __new__(cls, msgid, msgtext=None, params=None,
|
||||
domain='stackalytics', *args):
|
||||
"""Create a new Message object.
|
||||
|
||||
In order for translation to work gettext requires a message ID, this
|
||||
msgid will be used as the base unicode text. It is also possible
|
||||
for the msgid and the base unicode text to be different by passing
|
||||
the msgtext parameter.
|
||||
"""
|
||||
# If the base msgtext is not given, we use the default translation
|
||||
# of the msgid (which is in English) just in case the system locale is
|
||||
# not English, so that the base text will be in that locale by default.
|
||||
if not msgtext:
|
||||
msgtext = Message._translate_msgid(msgid, domain)
|
||||
# We want to initialize the parent unicode with the actual object that
|
||||
# would have been plain unicode if 'Message' was not enabled.
|
||||
msg = super(Message, cls).__new__(cls, msgtext)
|
||||
msg.msgid = msgid
|
||||
msg.domain = domain
|
||||
msg.params = params
|
||||
return msg
|
||||
|
||||
def translate(self, desired_locale=None):
|
||||
"""Translate this message to the desired locale.
|
||||
|
||||
:param desired_locale: The desired locale to translate the message to,
|
||||
if no locale is provided the message will be
|
||||
translated to the system's default locale.
|
||||
|
||||
:returns: the translated message in unicode
|
||||
"""
|
||||
|
||||
translated_message = Message._translate_msgid(self.msgid,
|
||||
self.domain,
|
||||
desired_locale)
|
||||
if self.params is None:
|
||||
# No need for more translation
|
||||
return translated_message
|
||||
|
||||
# This Message object may have been formatted with one or more
|
||||
# Message objects as substitution arguments, given either as a single
|
||||
# argument, part of a tuple, or as one or more values in a dictionary.
|
||||
# When translating this Message we need to translate those Messages too
|
||||
translated_params = _translate_args(self.params, desired_locale)
|
||||
|
||||
translated_message = translated_message % translated_params
|
||||
|
||||
return translated_message
|
||||
|
||||
@staticmethod
|
||||
def _translate_msgid(msgid, domain, desired_locale=None):
|
||||
if not desired_locale:
|
||||
system_locale = locale.getdefaultlocale()
|
||||
# If the system locale is not available to the runtime use English
|
||||
if not system_locale[0]:
|
||||
desired_locale = 'en_US'
|
||||
else:
|
||||
desired_locale = system_locale[0]
|
||||
|
||||
locale_dir = os.environ.get(domain.upper() + '_LOCALEDIR')
|
||||
lang = gettext.translation(domain,
|
||||
localedir=locale_dir,
|
||||
languages=[desired_locale],
|
||||
fallback=True)
|
||||
if six.PY3:
|
||||
translator = lang.gettext
|
||||
else:
|
||||
translator = lang.ugettext
|
||||
|
||||
translated_message = translator(msgid)
|
||||
return translated_message
|
||||
|
||||
def __mod__(self, other):
|
||||
# When we mod a Message we want the actual operation to be performed
|
||||
# by the parent class (i.e. unicode()), the only thing we do here is
|
||||
# save the original msgid and the parameters in case of a translation
|
||||
params = self._sanitize_mod_params(other)
|
||||
unicode_mod = super(Message, self).__mod__(params)
|
||||
modded = Message(self.msgid,
|
||||
msgtext=unicode_mod,
|
||||
params=params,
|
||||
domain=self.domain)
|
||||
return modded
|
||||
|
||||
def _sanitize_mod_params(self, other):
|
||||
"""Sanitize the object being modded with this Message.
|
||||
|
||||
- Add support for modding 'None' so translation supports it
|
||||
- Trim the modded object, which can be a large dictionary, to only
|
||||
those keys that would actually be used in a translation
|
||||
- Snapshot the object being modded, in case the message is
|
||||
translated, it will be used as it was when the Message was created
|
||||
"""
|
||||
if other is None:
|
||||
params = (other,)
|
||||
elif isinstance(other, dict):
|
||||
# Merge the dictionaries
|
||||
# Copy each item in case one does not support deep copy.
|
||||
params = {}
|
||||
if isinstance(self.params, dict):
|
||||
for key, val in self.params.items():
|
||||
params[key] = self._copy_param(val)
|
||||
for key, val in other.items():
|
||||
params[key] = self._copy_param(val)
|
||||
else:
|
||||
params = self._copy_param(other)
|
||||
return params
|
||||
|
||||
def _copy_param(self, param):
|
||||
try:
|
||||
return copy.deepcopy(param)
|
||||
except Exception:
|
||||
# Fallback to casting to unicode this will handle the
|
||||
# python code-like objects that can't be deep-copied
|
||||
return six.text_type(param)
|
||||
|
||||
def __add__(self, other):
|
||||
msg = _('Message objects do not support addition.')
|
||||
raise TypeError(msg)
|
||||
|
||||
def __radd__(self, other):
|
||||
return self.__add__(other)
|
||||
|
||||
if six.PY2:
|
||||
def __str__(self):
|
||||
# NOTE(luisg): Logging in python 2.6 tries to str() log records,
|
||||
# and it expects specifically a UnicodeError in order to proceed.
|
||||
msg = _('Message objects do not support str() because they may '
|
||||
'contain non-ascii characters. '
|
||||
'Please use unicode() or translate() instead.')
|
||||
raise UnicodeError(msg)
|
||||
|
||||
|
||||
def get_available_languages(domain):
|
||||
"""Lists the available languages for the given translation domain.
|
||||
|
||||
:param domain: the domain to get languages for
|
||||
"""
|
||||
if domain in _AVAILABLE_LANGUAGES:
|
||||
return copy.copy(_AVAILABLE_LANGUAGES[domain])
|
||||
|
||||
localedir = '%s_LOCALEDIR' % domain.upper()
|
||||
find = lambda x: gettext.find(domain,
|
||||
localedir=os.environ.get(localedir),
|
||||
languages=[x])
|
||||
|
||||
# NOTE(mrodden): en_US should always be available (and first in case
|
||||
# order matters) since our in-line message strings are en_US
|
||||
language_list = ['en_US']
|
||||
# NOTE(luisg): Babel <1.0 used a function called list(), which was
|
||||
# renamed to locale_identifiers() in >=1.0, the requirements master list
|
||||
# requires >=0.9.6, uncapped, so defensively work with both. We can remove
|
||||
# this check when the master list updates to >=1.0, and update all projects
|
||||
list_identifiers = (getattr(localedata, 'list', None) or
|
||||
getattr(localedata, 'locale_identifiers'))
|
||||
locale_identifiers = list_identifiers()
|
||||
|
||||
for i in locale_identifiers:
|
||||
if find(i) is not None:
|
||||
language_list.append(i)
|
||||
|
||||
# NOTE(luisg): Babel>=1.0,<1.3 has a bug where some OpenStack supported
|
||||
# locales (e.g. 'zh_CN', and 'zh_TW') aren't supported even though they
|
||||
# are perfectly legitimate locales:
|
||||
# https://github.com/mitsuhiko/babel/issues/37
|
||||
# In Babel 1.3 they fixed the bug and they support these locales, but
|
||||
# they are still not explicitly "listed" by locale_identifiers().
|
||||
# That is why we add the locales here explicitly if necessary so that
|
||||
# they are listed as supported.
|
||||
aliases = {'zh': 'zh_CN',
|
||||
'zh_Hant_HK': 'zh_HK',
|
||||
'zh_Hant': 'zh_TW',
|
||||
'fil': 'tl_PH'}
|
||||
for (locale_, alias) in six.iteritems(aliases):
|
||||
if locale_ in language_list and alias not in language_list:
|
||||
language_list.append(alias)
|
||||
|
||||
_AVAILABLE_LANGUAGES[domain] = language_list
|
||||
return copy.copy(language_list)
|
||||
|
||||
|
||||
def translate(obj, desired_locale=None):
|
||||
"""Gets the translated unicode representation of the given object.
|
||||
|
||||
If the object is not translatable it is returned as-is.
|
||||
If the locale is None the object is translated to the system locale.
|
||||
|
||||
:param obj: the object to translate
|
||||
:param desired_locale: the locale to translate the message to, if None the
|
||||
default system locale will be used
|
||||
:returns: the translated object in unicode, or the original object if
|
||||
it could not be translated
|
||||
"""
|
||||
message = obj
|
||||
if not isinstance(message, Message):
|
||||
# If the object to translate is not already translatable,
|
||||
# let's first get its unicode representation
|
||||
message = six.text_type(obj)
|
||||
if isinstance(message, Message):
|
||||
# Even after unicoding() we still need to check if we are
|
||||
# running with translatable unicode before translating
|
||||
return message.translate(desired_locale)
|
||||
return obj
|
||||
|
||||
|
||||
def _translate_args(args, desired_locale=None):
|
||||
"""Translates all the translatable elements of the given arguments object.
|
||||
|
||||
This method is used for translating the translatable values in method
|
||||
arguments which include values of tuples or dictionaries.
|
||||
If the object is not a tuple or a dictionary the object itself is
|
||||
translated if it is translatable.
|
||||
|
||||
If the locale is None the object is translated to the system locale.
|
||||
|
||||
:param args: the args to translate
|
||||
:param desired_locale: the locale to translate the args to, if None the
|
||||
default system locale will be used
|
||||
:returns: a new args object with the translated contents of the original
|
||||
"""
|
||||
if isinstance(args, tuple):
|
||||
return tuple(translate(v, desired_locale) for v in args)
|
||||
if isinstance(args, dict):
|
||||
translated_dict = {}
|
||||
for (k, v) in six.iteritems(args):
|
||||
translated_v = translate(v, desired_locale)
|
||||
translated_dict[k] = translated_v
|
||||
return translated_dict
|
||||
return translate(args, desired_locale)
|
||||
|
||||
|
||||
class TranslationHandler(handlers.MemoryHandler):
|
||||
"""Handler that translates records before logging them.
|
||||
|
||||
The TranslationHandler takes a locale and a target logging.Handler object
|
||||
to forward LogRecord objects to after translating them. This handler
|
||||
depends on Message objects being logged, instead of regular strings.
|
||||
|
||||
The handler can be configured declaratively in the logging.conf as follows:
|
||||
|
||||
[handlers]
|
||||
keys = translatedlog, translator
|
||||
|
||||
[handler_translatedlog]
|
||||
class = handlers.WatchedFileHandler
|
||||
args = ('/var/log/api-localized.log',)
|
||||
formatter = context
|
||||
|
||||
[handler_translator]
|
||||
class = openstack.common.log.TranslationHandler
|
||||
target = translatedlog
|
||||
args = ('zh_CN',)
|
||||
|
||||
If the specified locale is not available in the system, the handler will
|
||||
log in the default locale.
|
||||
"""
|
||||
|
||||
def __init__(self, locale=None, target=None):
|
||||
"""Initialize a TranslationHandler
|
||||
|
||||
:param locale: locale to use for translating messages
|
||||
:param target: logging.Handler object to forward
|
||||
LogRecord objects to after translation
|
||||
"""
|
||||
# NOTE(luisg): In order to allow this handler to be a wrapper for
|
||||
# other handlers, such as a FileHandler, and still be able to
|
||||
# configure it using logging.conf, this handler has to extend
|
||||
# MemoryHandler because only the MemoryHandlers' logging.conf
|
||||
# parsing is implemented such that it accepts a target handler.
|
||||
handlers.MemoryHandler.__init__(self, capacity=0, target=target)
|
||||
self.locale = locale
|
||||
|
||||
def setFormatter(self, fmt):
|
||||
self.target.setFormatter(fmt)
|
||||
|
||||
def emit(self, record):
|
||||
# We save the message from the original record to restore it
|
||||
# after translation, so other handlers are not affected by this
|
||||
original_msg = record.msg
|
||||
original_args = record.args
|
||||
|
||||
try:
|
||||
self._translate_and_log_record(record)
|
||||
finally:
|
||||
record.msg = original_msg
|
||||
record.args = original_args
|
||||
|
||||
def _translate_and_log_record(self, record):
|
||||
record.msg = translate(record.msg, self.locale)
|
||||
|
||||
# In addition to translating the message, we also need to translate
|
||||
# arguments that were passed to the log method that were not part
|
||||
# of the main message e.g., log.info(_('Some message %s'), this_one))
|
||||
record.args = _translate_args(record.args, self.locale)
|
||||
|
||||
self.target.emit(record)
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Import related utilities and helper functions.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
|
||||
def import_class(import_str):
|
||||
"""Returns a class from a string including module and class."""
|
||||
mod_str, _sep, class_str = import_str.rpartition('.')
|
||||
__import__(mod_str)
|
||||
try:
|
||||
return getattr(sys.modules[mod_str], class_str)
|
||||
except AttributeError:
|
||||
raise ImportError('Class %s cannot be found (%s)' %
|
||||
(class_str,
|
||||
traceback.format_exception(*sys.exc_info())))
|
||||
|
||||
|
||||
def import_object(import_str, *args, **kwargs):
|
||||
"""Import a class and return an instance of it."""
|
||||
return import_class(import_str)(*args, **kwargs)
|
||||
|
||||
|
||||
def import_object_ns(name_space, import_str, *args, **kwargs):
|
||||
"""Tries to import object from default namespace.
|
||||
|
||||
Imports a class and return an instance of it, first by trying
|
||||
to find the class in a default namespace, then failing back to
|
||||
a full path if not found in the default namespace.
|
||||
"""
|
||||
import_value = "%s.%s" % (name_space, import_str)
|
||||
try:
|
||||
return import_class(import_value)(*args, **kwargs)
|
||||
except ImportError:
|
||||
return import_class(import_str)(*args, **kwargs)
|
||||
|
||||
|
||||
def import_module(import_str):
|
||||
"""Import a module."""
|
||||
__import__(import_str)
|
||||
return sys.modules[import_str]
|
||||
|
||||
|
||||
def import_versioned_module(version, submodule=None):
|
||||
module = 'stackalytics.v%s' % version
|
||||
if submodule:
|
||||
module = '.'.join((module, submodule))
|
||||
return import_module(module)
|
||||
|
||||
|
||||
def try_import(import_str, default=None):
|
||||
"""Try to import a module and if it fails return default."""
|
||||
try:
|
||||
return import_module(import_str)
|
||||
except ImportError:
|
||||
return default
|
||||
|
|
@ -1,186 +0,0 @@
|
|||
# Copyright 2010 United States Government as represented by the
|
||||
# Administrator of the National Aeronautics and Space Administration.
|
||||
# Copyright 2011 Justin Santa Barbara
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
'''
|
||||
JSON related utilities.
|
||||
|
||||
This module provides a few things:
|
||||
|
||||
1) A handy function for getting an object down to something that can be
|
||||
JSON serialized. See to_primitive().
|
||||
|
||||
2) Wrappers around loads() and dumps(). The dumps() wrapper will
|
||||
automatically use to_primitive() for you if needed.
|
||||
|
||||
3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson
|
||||
is available.
|
||||
'''
|
||||
|
||||
|
||||
import codecs
|
||||
import datetime
|
||||
import functools
|
||||
import inspect
|
||||
import itertools
|
||||
import sys
|
||||
|
||||
if sys.version_info < (2, 7):
|
||||
# On Python <= 2.6, json module is not C boosted, so try to use
|
||||
# simplejson module if available
|
||||
try:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
import json
|
||||
else:
|
||||
import json
|
||||
|
||||
import six
|
||||
import six.moves.xmlrpc_client as xmlrpclib
|
||||
|
||||
from stackalytics.openstack.common import gettextutils
|
||||
from stackalytics.openstack.common import importutils
|
||||
from stackalytics.openstack.common import strutils
|
||||
from stackalytics.openstack.common import timeutils
|
||||
|
||||
netaddr = importutils.try_import("netaddr")
|
||||
|
||||
_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
|
||||
inspect.isfunction, inspect.isgeneratorfunction,
|
||||
inspect.isgenerator, inspect.istraceback, inspect.isframe,
|
||||
inspect.iscode, inspect.isbuiltin, inspect.isroutine,
|
||||
inspect.isabstract]
|
||||
|
||||
_simple_types = (six.string_types + six.integer_types
|
||||
+ (type(None), bool, float))
|
||||
|
||||
|
||||
def to_primitive(value, convert_instances=False, convert_datetime=True,
|
||||
level=0, max_depth=3):
|
||||
"""Convert a complex object into primitives.
|
||||
|
||||
Handy for JSON serialization. We can optionally handle instances,
|
||||
but since this is a recursive function, we could have cyclical
|
||||
data structures.
|
||||
|
||||
To handle cyclical data structures we could track the actual objects
|
||||
visited in a set, but not all objects are hashable. Instead we just
|
||||
track the depth of the object inspections and don't go too deep.
|
||||
|
||||
Therefore, convert_instances=True is lossy ... be aware.
|
||||
|
||||
"""
|
||||
# handle obvious types first - order of basic types determined by running
|
||||
# full tests on nova project, resulting in the following counts:
|
||||
# 572754 <type 'NoneType'>
|
||||
# 460353 <type 'int'>
|
||||
# 379632 <type 'unicode'>
|
||||
# 274610 <type 'str'>
|
||||
# 199918 <type 'dict'>
|
||||
# 114200 <type 'datetime.datetime'>
|
||||
# 51817 <type 'bool'>
|
||||
# 26164 <type 'list'>
|
||||
# 6491 <type 'float'>
|
||||
# 283 <type 'tuple'>
|
||||
# 19 <type 'long'>
|
||||
if isinstance(value, _simple_types):
|
||||
return value
|
||||
|
||||
if isinstance(value, datetime.datetime):
|
||||
if convert_datetime:
|
||||
return timeutils.strtime(value)
|
||||
else:
|
||||
return value
|
||||
|
||||
# value of itertools.count doesn't get caught by nasty_type_tests
|
||||
# and results in infinite loop when list(value) is called.
|
||||
if type(value) == itertools.count:
|
||||
return six.text_type(value)
|
||||
|
||||
# FIXME(vish): Workaround for LP bug 852095. Without this workaround,
|
||||
# tests that raise an exception in a mocked method that
|
||||
# has a @wrap_exception with a notifier will fail. If
|
||||
# we up the dependency to 0.5.4 (when it is released) we
|
||||
# can remove this workaround.
|
||||
if getattr(value, '__module__', None) == 'mox':
|
||||
return 'mock'
|
||||
|
||||
if level > max_depth:
|
||||
return '?'
|
||||
|
||||
# The try block may not be necessary after the class check above,
|
||||
# but just in case ...
|
||||
try:
|
||||
recursive = functools.partial(to_primitive,
|
||||
convert_instances=convert_instances,
|
||||
convert_datetime=convert_datetime,
|
||||
level=level,
|
||||
max_depth=max_depth)
|
||||
if isinstance(value, dict):
|
||||
return dict((k, recursive(v)) for k, v in six.iteritems(value))
|
||||
elif isinstance(value, (list, tuple)):
|
||||
return [recursive(lv) for lv in value]
|
||||
|
||||
# It's not clear why xmlrpclib created their own DateTime type, but
|
||||
# for our purposes, make it a datetime type which is explicitly
|
||||
# handled
|
||||
if isinstance(value, xmlrpclib.DateTime):
|
||||
value = datetime.datetime(*tuple(value.timetuple())[:6])
|
||||
|
||||
if convert_datetime and isinstance(value, datetime.datetime):
|
||||
return timeutils.strtime(value)
|
||||
elif isinstance(value, gettextutils.Message):
|
||||
return value.data
|
||||
elif hasattr(value, 'iteritems'):
|
||||
return recursive(dict(value.iteritems()), level=level + 1)
|
||||
elif hasattr(value, '__iter__'):
|
||||
return recursive(list(value))
|
||||
elif convert_instances and hasattr(value, '__dict__'):
|
||||
# Likely an instance of something. Watch for cycles.
|
||||
# Ignore class member vars.
|
||||
return recursive(value.__dict__, level=level + 1)
|
||||
elif netaddr and isinstance(value, netaddr.IPAddress):
|
||||
return six.text_type(value)
|
||||
else:
|
||||
if any(test(value) for test in _nasty_type_tests):
|
||||
return six.text_type(value)
|
||||
return value
|
||||
except TypeError:
|
||||
# Class objects are tricky since they may define something like
|
||||
# __iter__ defined but it isn't callable as list().
|
||||
return six.text_type(value)
|
||||
|
||||
|
||||
def dumps(value, default=to_primitive, **kwargs):
|
||||
return json.dumps(value, default=default, **kwargs)
|
||||
|
||||
|
||||
def loads(s, encoding='utf-8', **kwargs):
|
||||
return json.loads(strutils.safe_decode(s, encoding), **kwargs)
|
||||
|
||||
|
||||
def load(fp, encoding='utf-8', **kwargs):
|
||||
return json.load(codecs.getreader(encoding)(fp), **kwargs)
|
||||
|
||||
|
||||
try:
|
||||
import anyjson
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
anyjson._modules.append((__name__, 'dumps', TypeError,
|
||||
'loads', ValueError, 'load'))
|
||||
anyjson.force_implementation(__name__)
|
||||
|
|
@ -33,42 +33,24 @@ import logging
|
|||
import logging.config
|
||||
import logging.handlers
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
from oslo.config import cfg
|
||||
from oslo.serialization import jsonutils
|
||||
from oslo.utils import importutils
|
||||
import six
|
||||
from six import moves
|
||||
|
||||
from stackalytics.openstack.common.gettextutils import _
|
||||
from stackalytics.openstack.common import importutils
|
||||
from stackalytics.openstack.common import jsonutils
|
||||
_PY26 = sys.version_info[0:2] == (2, 6)
|
||||
|
||||
from stackalytics.openstack.common._i18n import _
|
||||
from stackalytics.openstack.common import local
|
||||
|
||||
|
||||
_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
|
||||
|
||||
_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
|
||||
|
||||
# NOTE(ldbragst): Let's build a list of regex objects using the list of
|
||||
# _SANITIZE_KEYS we already have. This way, we only have to add the new key
|
||||
# to the list of _SANITIZE_KEYS and we can generate regular expressions
|
||||
# for XML and JSON automatically.
|
||||
_SANITIZE_PATTERNS = []
|
||||
_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
|
||||
r'(<%(key)s>).*?(</%(key)s>)',
|
||||
r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
|
||||
r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
|
||||
r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
|
||||
'.*?([\'"])',
|
||||
r'(%(key)s\s*--?[A-z]+\s*).*?([\s])']
|
||||
|
||||
for key in _SANITIZE_KEYS:
|
||||
for pattern in _FORMAT_PATTERNS:
|
||||
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
|
||||
_SANITIZE_PATTERNS.append(reg_ex)
|
||||
|
||||
|
||||
common_cli_opts = [
|
||||
cfg.BoolOpt('debug',
|
||||
|
|
@ -138,6 +120,14 @@ generic_log_opts = [
|
|||
help='Log output to standard error.')
|
||||
]
|
||||
|
||||
DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN',
|
||||
'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO',
|
||||
'oslo.messaging=INFO', 'iso8601=WARN',
|
||||
'requests.packages.urllib3.connectionpool=WARN',
|
||||
'urllib3.connectionpool=WARN', 'websocket=WARN',
|
||||
"keystonemiddleware=WARN", "routes.middleware=WARN",
|
||||
"stevedore=WARN"]
|
||||
|
||||
log_opts = [
|
||||
cfg.StrOpt('logging_context_format_string',
|
||||
default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
|
||||
|
|
@ -156,17 +146,7 @@ log_opts = [
|
|||
'%(instance)s',
|
||||
help='Prefix each line of exception output with this format.'),
|
||||
cfg.ListOpt('default_log_levels',
|
||||
default=[
|
||||
'amqp=WARN',
|
||||
'amqplib=WARN',
|
||||
'boto=WARN',
|
||||
'qpid=WARN',
|
||||
'sqlalchemy=WARN',
|
||||
'suds=INFO',
|
||||
'oslo.messaging=INFO',
|
||||
'iso8601=WARN',
|
||||
'requests.packages.urllib3.connectionpool=WARN'
|
||||
],
|
||||
default=DEFAULT_LOG_LEVELS,
|
||||
help='List of logger=LEVEL pairs.'),
|
||||
cfg.BoolOpt('publish_errors',
|
||||
default=False,
|
||||
|
|
@ -181,11 +161,11 @@ log_opts = [
|
|||
cfg.StrOpt('instance_format',
|
||||
default='[instance: %(uuid)s] ',
|
||||
help='The format for an instance that is passed with the log '
|
||||
'message. '),
|
||||
'message.'),
|
||||
cfg.StrOpt('instance_uuid_format',
|
||||
default='[instance: %(uuid)s] ',
|
||||
help='The format for an instance UUID that is passed with the '
|
||||
'log message. '),
|
||||
'log message.'),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
|
@ -244,45 +224,20 @@ def _get_log_file_path(binary=None):
|
|||
return None
|
||||
|
||||
|
||||
def mask_password(message, secret="***"):
|
||||
"""Replace password with 'secret' in message.
|
||||
|
||||
:param message: The string which includes security information.
|
||||
:param secret: value with which to replace passwords.
|
||||
:returns: The unicode value of message with the password fields masked.
|
||||
|
||||
For example:
|
||||
|
||||
>>> mask_password("'adminPass' : 'aaaaa'")
|
||||
"'adminPass' : '***'"
|
||||
>>> mask_password("'admin_pass' : 'aaaaa'")
|
||||
"'admin_pass' : '***'"
|
||||
>>> mask_password('"password" : "aaaaa"')
|
||||
'"password" : "***"'
|
||||
>>> mask_password("'original_password' : 'aaaaa'")
|
||||
"'original_password' : '***'"
|
||||
>>> mask_password("u'original_password' : u'aaaaa'")
|
||||
"u'original_password' : u'***'"
|
||||
"""
|
||||
message = six.text_type(message)
|
||||
|
||||
# NOTE(ldbragst): Check to see if anything in message contains any key
|
||||
# specified in _SANITIZE_KEYS, if not then just return the message since
|
||||
# we don't have to mask any passwords.
|
||||
if not any(key in message for key in _SANITIZE_KEYS):
|
||||
return message
|
||||
|
||||
secret = r'\g<1>' + secret + r'\g<2>'
|
||||
for pattern in _SANITIZE_PATTERNS:
|
||||
message = re.sub(pattern, secret, message)
|
||||
return message
|
||||
|
||||
|
||||
class BaseLoggerAdapter(logging.LoggerAdapter):
|
||||
|
||||
def audit(self, msg, *args, **kwargs):
|
||||
self.log(logging.AUDIT, msg, *args, **kwargs)
|
||||
|
||||
def isEnabledFor(self, level):
|
||||
if _PY26:
|
||||
# This method was added in python 2.7 (and it does the exact
|
||||
# same logic, so we need to do the exact same logic so that
|
||||
# python 2.6 has this capability as well).
|
||||
return self.logger.isEnabledFor(level)
|
||||
else:
|
||||
return super(BaseLoggerAdapter, self).isEnabledFor(level)
|
||||
|
||||
|
||||
class LazyAdapter(BaseLoggerAdapter):
|
||||
def __init__(self, name='unknown', version='unknown'):
|
||||
|
|
@ -295,6 +250,11 @@ class LazyAdapter(BaseLoggerAdapter):
|
|||
def logger(self):
|
||||
if not self._logger:
|
||||
self._logger = getLogger(self.name, self.version)
|
||||
if six.PY3:
|
||||
# In Python 3, the code fails because the 'manager' attribute
|
||||
# cannot be found when using a LoggerAdapter as the
|
||||
# underlying logger. Work around this issue.
|
||||
self._logger.manager = self._logger.logger.manager
|
||||
return self._logger
|
||||
|
||||
|
||||
|
|
@ -340,11 +300,10 @@ class ContextAdapter(BaseLoggerAdapter):
|
|||
self.warn(stdmsg, *args, **kwargs)
|
||||
|
||||
def process(self, msg, kwargs):
|
||||
# NOTE(mrodden): catch any Message/other object and
|
||||
# coerce to unicode before they can get
|
||||
# to the python logging and possibly
|
||||
# cause string encoding trouble
|
||||
if not isinstance(msg, six.string_types):
|
||||
# NOTE(jecarey): If msg is not unicode, coerce it into unicode
|
||||
# before it can get to the python logging and
|
||||
# possibly cause string encoding trouble
|
||||
if not isinstance(msg, six.text_type):
|
||||
msg = six.text_type(msg)
|
||||
|
||||
if 'extra' not in kwargs:
|
||||
|
|
@ -448,7 +407,7 @@ def _load_log_config(log_config_append):
|
|||
try:
|
||||
logging.config.fileConfig(log_config_append,
|
||||
disable_existing_loggers=False)
|
||||
except moves.configparser.Error as exc:
|
||||
except (moves.configparser.Error, KeyError) as exc:
|
||||
raise LogConfigError(log_config_append, six.text_type(exc))
|
||||
|
||||
|
||||
|
|
@ -461,9 +420,20 @@ def setup(product_name, version='unknown'):
|
|||
sys.excepthook = _create_logging_excepthook(product_name)
|
||||
|
||||
|
||||
def set_defaults(logging_context_format_string):
|
||||
cfg.set_defaults(
|
||||
log_opts, logging_context_format_string=logging_context_format_string)
|
||||
def set_defaults(logging_context_format_string=None,
|
||||
default_log_levels=None):
|
||||
# Just in case the caller is not setting the
|
||||
# default_log_level. This is insurance because
|
||||
# we introduced the default_log_level parameter
|
||||
# later in a backwards in-compatible change
|
||||
if default_log_levels is not None:
|
||||
cfg.set_defaults(
|
||||
log_opts,
|
||||
default_log_levels=default_log_levels)
|
||||
if logging_context_format_string is not None:
|
||||
cfg.set_defaults(
|
||||
log_opts,
|
||||
logging_context_format_string=logging_context_format_string)
|
||||
|
||||
|
||||
def _find_facility_from_conf():
|
||||
|
|
@ -512,18 +482,6 @@ def _setup_logging_from_conf(project, version):
|
|||
for handler in log_root.handlers:
|
||||
log_root.removeHandler(handler)
|
||||
|
||||
if CONF.use_syslog:
|
||||
facility = _find_facility_from_conf()
|
||||
# TODO(bogdando) use the format provided by RFCSysLogHandler
|
||||
# after existing syslog format deprecation in J
|
||||
if CONF.use_syslog_rfc_format:
|
||||
syslog = RFCSysLogHandler(address='/dev/log',
|
||||
facility=facility)
|
||||
else:
|
||||
syslog = logging.handlers.SysLogHandler(address='/dev/log',
|
||||
facility=facility)
|
||||
log_root.addHandler(syslog)
|
||||
|
||||
logpath = _get_log_file_path()
|
||||
if logpath:
|
||||
filelog = logging.handlers.WatchedFileHandler(logpath)
|
||||
|
|
@ -582,6 +540,20 @@ def _setup_logging_from_conf(project, version):
|
|||
else:
|
||||
logger.setLevel(level_name)
|
||||
|
||||
if CONF.use_syslog:
|
||||
try:
|
||||
facility = _find_facility_from_conf()
|
||||
# TODO(bogdando) use the format provided by RFCSysLogHandler
|
||||
# after existing syslog format deprecation in J
|
||||
if CONF.use_syslog_rfc_format:
|
||||
syslog = RFCSysLogHandler(facility=facility)
|
||||
else:
|
||||
syslog = logging.handlers.SysLogHandler(facility=facility)
|
||||
log_root.addHandler(syslog)
|
||||
except socket.error:
|
||||
log_root.error('Unable to add syslog handler. Verify that syslog '
|
||||
'is running.')
|
||||
|
||||
|
||||
_loggers = {}
|
||||
|
||||
|
|
@ -651,6 +623,12 @@ class ContextFormatter(logging.Formatter):
|
|||
def format(self, record):
|
||||
"""Uses contextstring if request_id is set, otherwise default."""
|
||||
|
||||
# NOTE(jecarey): If msg is not unicode, coerce it into unicode
|
||||
# before it can get to the python logging and
|
||||
# possibly cause string encoding trouble
|
||||
if not isinstance(record.msg, six.text_type):
|
||||
record.msg = six.text_type(record.msg)
|
||||
|
||||
# store project info
|
||||
record.project = self.project
|
||||
record.version = self.version
|
||||
|
|
|
|||
|
|
@ -1,239 +0,0 @@
|
|||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
System-level utilities and helper functions.
|
||||
"""
|
||||
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
import unicodedata
|
||||
|
||||
import six
|
||||
|
||||
from stackalytics.openstack.common.gettextutils import _
|
||||
|
||||
|
||||
UNIT_PREFIX_EXPONENT = {
|
||||
'k': 1,
|
||||
'K': 1,
|
||||
'Ki': 1,
|
||||
'M': 2,
|
||||
'Mi': 2,
|
||||
'G': 3,
|
||||
'Gi': 3,
|
||||
'T': 4,
|
||||
'Ti': 4,
|
||||
}
|
||||
UNIT_SYSTEM_INFO = {
|
||||
'IEC': (1024, re.compile(r'(^[-+]?\d*\.?\d+)([KMGT]i?)?(b|bit|B)$')),
|
||||
'SI': (1000, re.compile(r'(^[-+]?\d*\.?\d+)([kMGT])?(b|bit|B)$')),
|
||||
}
|
||||
|
||||
TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
|
||||
FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')
|
||||
|
||||
SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
|
||||
SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")
|
||||
|
||||
|
||||
def int_from_bool_as_string(subject):
|
||||
"""Interpret a string as a boolean and return either 1 or 0.
|
||||
|
||||
Any string value in:
|
||||
|
||||
('True', 'true', 'On', 'on', '1')
|
||||
|
||||
is interpreted as a boolean True.
|
||||
|
||||
Useful for JSON-decoded stuff and config file parsing
|
||||
"""
|
||||
return bool_from_string(subject) and 1 or 0
|
||||
|
||||
|
||||
def bool_from_string(subject, strict=False, default=False):
|
||||
"""Interpret a string as a boolean.
|
||||
|
||||
A case-insensitive match is performed such that strings matching 't',
|
||||
'true', 'on', 'y', 'yes', or '1' are considered True and, when
|
||||
`strict=False`, anything else returns the value specified by 'default'.
|
||||
|
||||
Useful for JSON-decoded stuff and config file parsing.
|
||||
|
||||
If `strict=True`, unrecognized values, including None, will raise a
|
||||
ValueError which is useful when parsing values passed in from an API call.
|
||||
Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
|
||||
"""
|
||||
if not isinstance(subject, six.string_types):
|
||||
subject = six.text_type(subject)
|
||||
|
||||
lowered = subject.strip().lower()
|
||||
|
||||
if lowered in TRUE_STRINGS:
|
||||
return True
|
||||
elif lowered in FALSE_STRINGS:
|
||||
return False
|
||||
elif strict:
|
||||
acceptable = ', '.join(
|
||||
"'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
|
||||
msg = _("Unrecognized value '%(val)s', acceptable values are:"
|
||||
" %(acceptable)s") % {'val': subject,
|
||||
'acceptable': acceptable}
|
||||
raise ValueError(msg)
|
||||
else:
|
||||
return default
|
||||
|
||||
|
||||
def safe_decode(text, incoming=None, errors='strict'):
|
||||
"""Decodes incoming text/bytes string using `incoming` if they're not
|
||||
already unicode.
|
||||
|
||||
:param incoming: Text's current encoding
|
||||
:param errors: Errors handling policy. See here for valid
|
||||
values http://docs.python.org/2/library/codecs.html
|
||||
:returns: text or a unicode `incoming` encoded
|
||||
representation of it.
|
||||
:raises TypeError: If text is not an instance of str
|
||||
"""
|
||||
if not isinstance(text, (six.string_types, six.binary_type)):
|
||||
raise TypeError("%s can't be decoded" % type(text))
|
||||
|
||||
if isinstance(text, six.text_type):
|
||||
return text
|
||||
|
||||
if not incoming:
|
||||
incoming = (sys.stdin.encoding or
|
||||
sys.getdefaultencoding())
|
||||
|
||||
try:
|
||||
return text.decode(incoming, errors)
|
||||
except UnicodeDecodeError:
|
||||
# Note(flaper87) If we get here, it means that
|
||||
# sys.stdin.encoding / sys.getdefaultencoding
|
||||
# didn't return a suitable encoding to decode
|
||||
# text. This happens mostly when global LANG
|
||||
# var is not set correctly and there's no
|
||||
# default encoding. In this case, most likely
|
||||
# python will use ASCII or ANSI encoders as
|
||||
# default encodings but they won't be capable
|
||||
# of decoding non-ASCII characters.
|
||||
#
|
||||
# Also, UTF-8 is being used since it's an ASCII
|
||||
# extension.
|
||||
return text.decode('utf-8', errors)
|
||||
|
||||
|
||||
def safe_encode(text, incoming=None,
|
||||
encoding='utf-8', errors='strict'):
|
||||
"""Encodes incoming text/bytes string using `encoding`.
|
||||
|
||||
If incoming is not specified, text is expected to be encoded with
|
||||
current python's default encoding. (`sys.getdefaultencoding`)
|
||||
|
||||
:param incoming: Text's current encoding
|
||||
:param encoding: Expected encoding for text (Default UTF-8)
|
||||
:param errors: Errors handling policy. See here for valid
|
||||
values http://docs.python.org/2/library/codecs.html
|
||||
:returns: text or a bytestring `encoding` encoded
|
||||
representation of it.
|
||||
:raises TypeError: If text is not an instance of str
|
||||
"""
|
||||
if not isinstance(text, (six.string_types, six.binary_type)):
|
||||
raise TypeError("%s can't be encoded" % type(text))
|
||||
|
||||
if not incoming:
|
||||
incoming = (sys.stdin.encoding or
|
||||
sys.getdefaultencoding())
|
||||
|
||||
if isinstance(text, six.text_type):
|
||||
return text.encode(encoding, errors)
|
||||
elif text and encoding != incoming:
|
||||
# Decode text before encoding it with `encoding`
|
||||
text = safe_decode(text, incoming, errors)
|
||||
return text.encode(encoding, errors)
|
||||
else:
|
||||
return text
|
||||
|
||||
|
||||
def string_to_bytes(text, unit_system='IEC', return_int=False):
|
||||
"""Converts a string into an float representation of bytes.
|
||||
|
||||
The units supported for IEC ::
|
||||
|
||||
Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it)
|
||||
KB, KiB, MB, MiB, GB, GiB, TB, TiB
|
||||
|
||||
The units supported for SI ::
|
||||
|
||||
kb(it), Mb(it), Gb(it), Tb(it)
|
||||
kB, MB, GB, TB
|
||||
|
||||
Note that the SI unit system does not support capital letter 'K'
|
||||
|
||||
:param text: String input for bytes size conversion.
|
||||
:param unit_system: Unit system for byte size conversion.
|
||||
:param return_int: If True, returns integer representation of text
|
||||
in bytes. (default: decimal)
|
||||
:returns: Numerical representation of text in bytes.
|
||||
:raises ValueError: If text has an invalid value.
|
||||
|
||||
"""
|
||||
try:
|
||||
base, reg_ex = UNIT_SYSTEM_INFO[unit_system]
|
||||
except KeyError:
|
||||
msg = _('Invalid unit system: "%s"') % unit_system
|
||||
raise ValueError(msg)
|
||||
match = reg_ex.match(text)
|
||||
if match:
|
||||
magnitude = float(match.group(1))
|
||||
unit_prefix = match.group(2)
|
||||
if match.group(3) in ['b', 'bit']:
|
||||
magnitude /= 8
|
||||
else:
|
||||
msg = _('Invalid string format: %s') % text
|
||||
raise ValueError(msg)
|
||||
if not unit_prefix:
|
||||
res = magnitude
|
||||
else:
|
||||
res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix])
|
||||
if return_int:
|
||||
return int(math.ceil(res))
|
||||
return res
|
||||
|
||||
|
||||
def to_slug(value, incoming=None, errors="strict"):
|
||||
"""Normalize string.
|
||||
|
||||
Convert to lowercase, remove non-word characters, and convert spaces
|
||||
to hyphens.
|
||||
|
||||
Inspired by Django's `slugify` filter.
|
||||
|
||||
:param value: Text to slugify
|
||||
:param incoming: Text's current encoding
|
||||
:param errors: Errors handling policy. See here for valid
|
||||
values http://docs.python.org/2/library/codecs.html
|
||||
:returns: slugified unicode representation of `value`
|
||||
:raises TypeError: If text is not an instance of str
|
||||
"""
|
||||
value = safe_decode(value, incoming, errors)
|
||||
# NOTE(aababilov): no need to use safe_(encode|decode) here:
|
||||
# encodings are always "ascii", error handling is always "ignore"
|
||||
# and types are always known (first: unicode; second: str)
|
||||
value = unicodedata.normalize("NFKD", value).encode(
|
||||
"ascii", "ignore").decode("ascii")
|
||||
value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
|
||||
return SLUGIFY_HYPHENATE_RE.sub("-", value)
|
||||
|
|
@ -1,210 +0,0 @@
|
|||
# Copyright 2011 OpenStack Foundation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Time related utilities and helper functions.
|
||||
"""
|
||||
|
||||
import calendar
|
||||
import datetime
|
||||
import time
|
||||
|
||||
import iso8601
|
||||
import six
|
||||
|
||||
|
||||
# ISO 8601 extended time format with microseconds
|
||||
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
|
||||
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
|
||||
PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND
|
||||
|
||||
|
||||
def isotime(at=None, subsecond=False):
|
||||
"""Stringify time in ISO 8601 format."""
|
||||
if not at:
|
||||
at = utcnow()
|
||||
st = at.strftime(_ISO8601_TIME_FORMAT
|
||||
if not subsecond
|
||||
else _ISO8601_TIME_FORMAT_SUBSECOND)
|
||||
tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
|
||||
st += ('Z' if tz == 'UTC' else tz)
|
||||
return st
|
||||
|
||||
|
||||
def parse_isotime(timestr):
|
||||
"""Parse time from ISO 8601 format."""
|
||||
try:
|
||||
return iso8601.parse_date(timestr)
|
||||
except iso8601.ParseError as e:
|
||||
raise ValueError(six.text_type(e))
|
||||
except TypeError as e:
|
||||
raise ValueError(six.text_type(e))
|
||||
|
||||
|
||||
def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
|
||||
"""Returns formatted utcnow."""
|
||||
if not at:
|
||||
at = utcnow()
|
||||
return at.strftime(fmt)
|
||||
|
||||
|
||||
def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
|
||||
"""Turn a formatted time back into a datetime."""
|
||||
return datetime.datetime.strptime(timestr, fmt)
|
||||
|
||||
|
||||
def normalize_time(timestamp):
|
||||
"""Normalize time in arbitrary timezone to UTC naive object."""
|
||||
offset = timestamp.utcoffset()
|
||||
if offset is None:
|
||||
return timestamp
|
||||
return timestamp.replace(tzinfo=None) - offset
|
||||
|
||||
|
||||
def is_older_than(before, seconds):
|
||||
"""Return True if before is older than seconds."""
|
||||
if isinstance(before, six.string_types):
|
||||
before = parse_strtime(before).replace(tzinfo=None)
|
||||
else:
|
||||
before = before.replace(tzinfo=None)
|
||||
|
||||
return utcnow() - before > datetime.timedelta(seconds=seconds)
|
||||
|
||||
|
||||
def is_newer_than(after, seconds):
|
||||
"""Return True if after is newer than seconds."""
|
||||
if isinstance(after, six.string_types):
|
||||
after = parse_strtime(after).replace(tzinfo=None)
|
||||
else:
|
||||
after = after.replace(tzinfo=None)
|
||||
|
||||
return after - utcnow() > datetime.timedelta(seconds=seconds)
|
||||
|
||||
|
||||
def utcnow_ts():
|
||||
"""Timestamp version of our utcnow function."""
|
||||
if utcnow.override_time is None:
|
||||
# NOTE(kgriffs): This is several times faster
|
||||
# than going through calendar.timegm(...)
|
||||
return int(time.time())
|
||||
|
||||
return calendar.timegm(utcnow().timetuple())
|
||||
|
||||
|
||||
def utcnow():
|
||||
"""Overridable version of utils.utcnow."""
|
||||
if utcnow.override_time:
|
||||
try:
|
||||
return utcnow.override_time.pop(0)
|
||||
except AttributeError:
|
||||
return utcnow.override_time
|
||||
return datetime.datetime.utcnow()
|
||||
|
||||
|
||||
def iso8601_from_timestamp(timestamp):
|
||||
"""Returns an iso8601 formatted date from timestamp."""
|
||||
return isotime(datetime.datetime.utcfromtimestamp(timestamp))
|
||||
|
||||
|
||||
utcnow.override_time = None
|
||||
|
||||
|
||||
def set_time_override(override_time=None):
|
||||
"""Overrides utils.utcnow.
|
||||
|
||||
Make it return a constant time or a list thereof, one at a time.
|
||||
|
||||
:param override_time: datetime instance or list thereof. If not
|
||||
given, defaults to the current UTC time.
|
||||
"""
|
||||
utcnow.override_time = override_time or datetime.datetime.utcnow()
|
||||
|
||||
|
||||
def advance_time_delta(timedelta):
|
||||
"""Advance overridden time using a datetime.timedelta."""
|
||||
assert utcnow.override_time is not None
|
||||
try:
|
||||
for dt in utcnow.override_time:
|
||||
dt += timedelta
|
||||
except TypeError:
|
||||
utcnow.override_time += timedelta
|
||||
|
||||
|
||||
def advance_time_seconds(seconds):
|
||||
"""Advance overridden time by seconds."""
|
||||
advance_time_delta(datetime.timedelta(0, seconds))
|
||||
|
||||
|
||||
def clear_time_override():
|
||||
"""Remove the overridden time."""
|
||||
utcnow.override_time = None
|
||||
|
||||
|
||||
def marshall_now(now=None):
|
||||
"""Make an rpc-safe datetime with microseconds.
|
||||
|
||||
Note: tzinfo is stripped, but not required for relative times.
|
||||
"""
|
||||
if not now:
|
||||
now = utcnow()
|
||||
return dict(day=now.day, month=now.month, year=now.year, hour=now.hour,
|
||||
minute=now.minute, second=now.second,
|
||||
microsecond=now.microsecond)
|
||||
|
||||
|
||||
def unmarshall_time(tyme):
|
||||
"""Unmarshall a datetime dict."""
|
||||
return datetime.datetime(day=tyme['day'],
|
||||
month=tyme['month'],
|
||||
year=tyme['year'],
|
||||
hour=tyme['hour'],
|
||||
minute=tyme['minute'],
|
||||
second=tyme['second'],
|
||||
microsecond=tyme['microsecond'])
|
||||
|
||||
|
||||
def delta_seconds(before, after):
|
||||
"""Return the difference between two timing objects.
|
||||
|
||||
Compute the difference in seconds between two date, time, or
|
||||
datetime objects (as a float, to microsecond resolution).
|
||||
"""
|
||||
delta = after - before
|
||||
return total_seconds(delta)
|
||||
|
||||
|
||||
def total_seconds(delta):
|
||||
"""Return the total seconds of datetime.timedelta object.
|
||||
|
||||
Compute total seconds of datetime.timedelta, datetime.timedelta
|
||||
doesn't have method total_seconds in Python2.6, calculate it manually.
|
||||
"""
|
||||
try:
|
||||
return delta.total_seconds()
|
||||
except AttributeError:
|
||||
return ((delta.days * 24 * 3600) + delta.seconds +
|
||||
float(delta.microseconds) / (10 ** 6))
|
||||
|
||||
|
||||
def is_soon(dt, window):
|
||||
"""Determines if time is going to happen in the next window seconds.
|
||||
|
||||
:param dt: the time
|
||||
:param window: minimum seconds to remain to consider the time not soon
|
||||
|
||||
:return: True if expiration is within the given duration
|
||||
"""
|
||||
soon = (utcnow() + datetime.timedelta(seconds=window))
|
||||
return normalize_time(dt) <= soon
|
||||
Loading…
Reference in New Issue