From 0542f1f3dac457aa836623cfe1298ab18d0ee339 Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Mon, 27 Jul 2015 15:22:44 +0300 Subject: [PATCH] Update project for the latest libs Change-Id: I036bfc8b25376742ff5a882dab2fd93d333d6dd0 --- .testr.conf | 2 +- driverlog/dashboard/decorators.py | 3 +- driverlog/dashboard/parameters.py | 3 +- driverlog/dashboard/vault.py | 3 +- driverlog/dashboard/web.py | 4 +- driverlog/openstack/common/importutils.py | 68 --- driverlog/openstack/common/jsonutils.py | 169 ------ driverlog/openstack/common/log.py | 557 ------------------ driverlog/openstack/common/timeutils.py | 210 ------- driverlog/processor/__init__.py | 1 - driverlog/processor/config.py | 2 +- driverlog/processor/main.py | 4 +- driverlog/processor/rcs.py | 3 +- driverlog/processor/utils.py | 3 +- driverlog/{openstack => tests}/__init__.py | 0 .../common => tests/unit}/__init__.py | 0 .../tests}/unit/test_config_files.py | 2 +- .../unit/test_data/sample_default_data.json | 0 .../tests}/unit/test_data/sample_review.json | 0 {tests => driverlog/tests}/unit/test_main.py | 8 +- requirements.txt | 17 +- setup.py | 12 +- test-requirements.txt | 15 +- tests/__init__.py | 1 - tests/unit/__init__.py | 1 - 25 files changed, 47 insertions(+), 1041 deletions(-) delete mode 100644 driverlog/openstack/common/importutils.py delete mode 100644 driverlog/openstack/common/jsonutils.py delete mode 100644 driverlog/openstack/common/log.py delete mode 100644 driverlog/openstack/common/timeutils.py rename driverlog/{openstack => tests}/__init__.py (100%) rename driverlog/{openstack/common => tests/unit}/__init__.py (100%) rename {tests => driverlog/tests}/unit/test_config_files.py (98%) rename {tests => driverlog/tests}/unit/test_data/sample_default_data.json (100%) rename {tests => driverlog/tests}/unit/test_data/sample_review.json (100%) rename {tests => driverlog/tests}/unit/test_main.py (96%) delete mode 100644 tests/__init__.py delete mode 100644 tests/unit/__init__.py diff --git a/.testr.conf b/.testr.conf index 2109af6..dccb2db 100644 --- a/.testr.conf +++ b/.testr.conf @@ -1,4 +1,4 @@ [DEFAULT] -test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION +test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ ./driverlog/tests $LISTOPT $IDOPTION test_id_option=--load-list $IDFILE test_list_option=--list diff --git a/driverlog/dashboard/decorators.py b/driverlog/dashboard/decorators.py index 305b891..d74223f 100644 --- a/driverlog/dashboard/decorators.py +++ b/driverlog/dashboard/decorators.py @@ -17,10 +17,9 @@ import functools import json import flask +from oslo_log import log as logging from werkzeug import exceptions -from driverlog.openstack.common import log as logging - LOG = logging.getLogger(__name__) diff --git a/driverlog/dashboard/parameters.py b/driverlog/dashboard/parameters.py index 52dbbe0..05f3dd0 100644 --- a/driverlog/dashboard/parameters.py +++ b/driverlog/dashboard/parameters.py @@ -14,10 +14,9 @@ # limitations under the License. import flask +from oslo_log import log as logging from six.moves.urllib import parse -from driverlog.openstack.common import log as logging - LOG = logging.getLogger(__name__) diff --git a/driverlog/dashboard/vault.py b/driverlog/dashboard/vault.py index 584d075..f4c89b4 100644 --- a/driverlog/dashboard/vault.py +++ b/driverlog/dashboard/vault.py @@ -17,8 +17,7 @@ import re import flask import memcache - -from driverlog.openstack.common import log as logging +from oslo_log import log as logging LOG = logging.getLogger(__name__) diff --git a/driverlog/dashboard/web.py b/driverlog/dashboard/web.py index 8c878c3..7f9c7f5 100644 --- a/driverlog/dashboard/web.py +++ b/driverlog/dashboard/web.py @@ -16,10 +16,10 @@ import os import flask -from oslo.config import cfg +from oslo_config import cfg +from oslo_log import log as logging from driverlog.dashboard import api -from driverlog.openstack.common import log as logging from driverlog.processor import config diff --git a/driverlog/openstack/common/importutils.py b/driverlog/openstack/common/importutils.py deleted file mode 100644 index aea041d..0000000 --- a/driverlog/openstack/common/importutils.py +++ /dev/null @@ -1,68 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Import related utilities and helper functions. -""" - -import sys -import traceback - - -def import_class(import_str): - """Returns a class from a string including module and class.""" - mod_str, _sep, class_str = import_str.rpartition('.') - try: - __import__(mod_str) - return getattr(sys.modules[mod_str], class_str) - except (ValueError, AttributeError): - raise ImportError('Class %s cannot be found (%s)' % - (class_str, - traceback.format_exception(*sys.exc_info()))) - - -def import_object(import_str, *args, **kwargs): - """Import a class and return an instance of it.""" - return import_class(import_str)(*args, **kwargs) - - -def import_object_ns(name_space, import_str, *args, **kwargs): - """Tries to import object from default namespace. - -Imports a class and return an instance of it, first by trying -to find the class in a default namespace, then failing back to -a full path if not found in the default namespace. -""" - import_value = "%s.%s" % (name_space, import_str) - try: - return import_class(import_value)(*args, **kwargs) - except ImportError: - return import_class(import_str)(*args, **kwargs) - - -def import_module(import_str): - """Import a module.""" - __import__(import_str) - return sys.modules[import_str] - - -def try_import(import_str, default=None): - """Try to import a module and if it fails return default.""" - try: - return import_module(import_str) - except ImportError: - return default diff --git a/driverlog/openstack/common/jsonutils.py b/driverlog/openstack/common/jsonutils.py deleted file mode 100644 index 3ebbed2..0000000 --- a/driverlog/openstack/common/jsonutils.py +++ /dev/null @@ -1,169 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2011 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -''' -JSON related utilities. - -This module provides a few things: - - 1) A handy function for getting an object down to something that can be - JSON serialized. See to_primitive(). - - 2) Wrappers around loads() and dumps(). The dumps() wrapper will - automatically use to_primitive() for you if needed. - - 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson - is available. -''' - - -import datetime -import functools -import inspect -import itertools -import json -import xmlrpclib - -import six - -from driverlog.openstack.common import timeutils - - -_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod, - inspect.isfunction, inspect.isgeneratorfunction, - inspect.isgenerator, inspect.istraceback, inspect.isframe, - inspect.iscode, inspect.isbuiltin, inspect.isroutine, - inspect.isabstract] - -_simple_types = (six.string_types + six.integer_types - + (type(None), bool, float)) - - -def to_primitive(value, convert_instances=False, convert_datetime=True, - level=0, max_depth=3): - """Convert a complex object into primitives. - - Handy for JSON serialization. We can optionally handle instances, - but since this is a recursive function, we could have cyclical - data structures. - - To handle cyclical data structures we could track the actual objects - visited in a set, but not all objects are hashable. Instead we just - track the depth of the object inspections and don't go too deep. - - Therefore, convert_instances=True is lossy ... be aware. - - """ - # handle obvious types first - order of basic types determined by running - # full tests on nova project, resulting in the following counts: - # 572754 - # 460353 - # 379632 - # 274610 - # 199918 - # 114200 - # 51817 - # 26164 - # 6491 - # 283 - # 19 - if isinstance(value, _simple_types): - return value - - if isinstance(value, datetime.datetime): - if convert_datetime: - return timeutils.strtime(value) - else: - return value - - # value of itertools.count doesn't get caught by nasty_type_tests - # and results in infinite loop when list(value) is called. - if type(value) == itertools.count: - return six.text_type(value) - - # FIXME(vish): Workaround for LP bug 852095. Without this workaround, - # tests that raise an exception in a mocked method that - # has a @wrap_exception with a notifier will fail. If - # we up the dependency to 0.5.4 (when it is released) we - # can remove this workaround. - if getattr(value, '__module__', None) == 'mox': - return 'mock' - - if level > max_depth: - return '?' - - # The try block may not be necessary after the class check above, - # but just in case ... - try: - recursive = functools.partial(to_primitive, - convert_instances=convert_instances, - convert_datetime=convert_datetime, - level=level, - max_depth=max_depth) - if isinstance(value, dict): - return dict((k, recursive(v)) for k, v in six.iteritems(value)) - elif isinstance(value, (list, tuple)): - return [recursive(lv) for lv in value] - - # It's not clear why xmlrpclib created their own DateTime type, but - # for our purposes, make it a datetime type which is explicitly - # handled - if isinstance(value, xmlrpclib.DateTime): - value = datetime.datetime(*tuple(value.timetuple())[:6]) - - if convert_datetime and isinstance(value, datetime.datetime): - return timeutils.strtime(value) - elif hasattr(value, 'iteritems'): - return recursive(dict(value.iteritems()), level=level + 1) - elif hasattr(value, '__iter__'): - return recursive(list(value)) - elif convert_instances and hasattr(value, '__dict__'): - # Likely an instance of something. Watch for cycles. - # Ignore class member vars. - return recursive(value.__dict__, level=level + 1) - else: - if any(test(value) for test in _nasty_type_tests): - return six.text_type(value) - return value - except TypeError: - # Class objects are tricky since they may define something like - # __iter__ defined but it isn't callable as list(). - return six.text_type(value) - - -def dumps(value, default=to_primitive, **kwargs): - return json.dumps(value, default=default, **kwargs) - - -def loads(s): - return json.loads(s) - - -def load(s): - return json.load(s) - - -try: - import anyjson -except ImportError: - pass -else: - anyjson._modules.append((__name__, 'dumps', TypeError, - 'loads', ValueError, 'load')) - anyjson.force_implementation(__name__) diff --git a/driverlog/openstack/common/log.py b/driverlog/openstack/common/log.py deleted file mode 100644 index f693ed9..0000000 --- a/driverlog/openstack/common/log.py +++ /dev/null @@ -1,557 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack Foundation. -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Openstack logging handler. - -This module adds to logging functionality by adding the option to specify -a context object when calling the various log methods. If the context object -is not specified, default formatting is used. Additionally, an instance uuid -may be passed as part of the log message, which is intended to make it easier -for admins to find messages related to a specific instance. - -It also allows setting of formatting information through conf. - -""" - -import ConfigParser -import cStringIO -import inspect -import itertools -import logging -import logging.config -import logging.handlers -import os -import sys -import traceback - -from oslo.config import cfg - -from driverlog.openstack.common import importutils -from driverlog.openstack.common import jsonutils - - -_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" - -common_cli_opts = [ - cfg.BoolOpt('debug', - short='d', - default=False, - help='Print debugging output (set logging level to ' - 'DEBUG instead of default WARNING level).'), - cfg.BoolOpt('verbose', - short='v', - default=False, - help='Print more verbose output (set logging level to ' - 'INFO instead of default WARNING level).'), -] - -logging_cli_opts = [ - cfg.StrOpt('log-config', - metavar='PATH', - help='If this option is specified, the logging configuration ' - 'file specified is used and overrides any other logging ' - 'options specified. Please see the Python logging module ' - 'documentation for details on logging configuration ' - 'files.'), - cfg.StrOpt('log-format', - default=None, - metavar='FORMAT', - help='A logging.Formatter log message format string which may ' - 'use any of the available logging.LogRecord attributes. ' - 'This option is deprecated. Please use ' - 'logging_context_format_string and ' - 'logging_default_format_string instead.'), - cfg.StrOpt('log-date-format', - default=_DEFAULT_LOG_DATE_FORMAT, - metavar='DATE_FORMAT', - help='Format string for %%(asctime)s in log records. ' - 'Default: %(default)s'), - cfg.StrOpt('log-file', - metavar='PATH', - deprecated_name='logfile', - help='(Optional) Name of log file to output to. ' - 'If no default is set, logging will go to stdout.'), - cfg.StrOpt('log-dir', - deprecated_name='logdir', - help='(Optional) The base directory used for relative ' - '--log-file paths'), - cfg.BoolOpt('use-syslog', - default=False, - help='Use syslog for logging.'), - cfg.StrOpt('syslog-log-facility', - default='LOG_USER', - help='syslog facility to receive log lines') -] - -generic_log_opts = [ - cfg.BoolOpt('use_stderr', - default=True, - help='Log output to standard error') -] - -log_opts = [ - cfg.StrOpt('logging_context_format_string', - default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' - '%(name)s [%(request_id)s %(user)s %(tenant)s] ' - '%(instance)s%(message)s', - help='format string to use for log messages with context'), - cfg.StrOpt('logging_default_format_string', - default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' - '%(name)s [-] %(instance)s%(message)s', - help='format string to use for log messages without context'), - cfg.StrOpt('logging_debug_format_suffix', - default='%(funcName)s %(pathname)s:%(lineno)d', - help='data to append to log format when level is DEBUG'), - cfg.StrOpt('logging_exception_prefix', - default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s ' - '%(instance)s', - help='prefix each line of exception output with this format'), - cfg.ListOpt('default_log_levels', - default=[ - 'amqplib=WARN', - 'sqlalchemy=WARN', - 'boto=WARN', - 'suds=INFO', - 'keystone=INFO', - 'eventlet.wsgi.server=WARN' - ], - help='list of logger=LEVEL pairs'), - cfg.BoolOpt('publish_errors', - default=False, - help='publish error events'), - cfg.BoolOpt('fatal_deprecations', - default=False, - help='make deprecations fatal'), - - # NOTE(mikal): there are two options here because sometimes we are handed - # a full instance (and could include more information), and other times we - # are just handed a UUID for the instance. - cfg.StrOpt('instance_format', - default='[instance: %(uuid)s] ', - help='If an instance is passed with the log message, format ' - 'it like this'), - cfg.StrOpt('instance_uuid_format', - default='[instance: %(uuid)s] ', - help='If an instance UUID is passed with the log message, ' - 'format it like this'), -] - -CONF = cfg.CONF -CONF.register_cli_opts(common_cli_opts) -CONF.register_cli_opts(logging_cli_opts) -CONF.register_opts(generic_log_opts) -CONF.register_opts(log_opts) - -# our new audit level -# NOTE(jkoelker) Since we synthesized an audit level, make the logging -# module aware of it so it acts like other levels. -logging.AUDIT = logging.INFO + 1 -logging.addLevelName(logging.AUDIT, 'AUDIT') - - -try: - NullHandler = logging.NullHandler -except AttributeError: # NOTE(jkoelker) NullHandler added in Python 2.7 - class NullHandler(logging.Handler): - def handle(self, record): - pass - - def emit(self, record): - pass - - def createLock(self): - self.lock = None - - -def _dictify_context(context): - if context is None: - return None - if not isinstance(context, dict) and getattr(context, 'to_dict', None): - context = context.to_dict() - return context - - -def _get_binary_name(): - return os.path.basename(inspect.stack()[-1][1]) - - -def _get_log_file_path(binary=None): - logfile = CONF.log_file - logdir = CONF.log_dir - - if logfile and not logdir: - return logfile - - if logfile and logdir: - return os.path.join(logdir, logfile) - - if logdir: - binary = binary or _get_binary_name() - return '%s.log' % (os.path.join(logdir, binary),) - - -class BaseLoggerAdapter(logging.LoggerAdapter): - - def audit(self, msg, *args, **kwargs): - self.log(logging.AUDIT, msg, *args, **kwargs) - - -class LazyAdapter(BaseLoggerAdapter): - def __init__(self, name='unknown', version='unknown'): - self._logger = None - self.extra = {} - self.name = name - self.version = version - - @property - def logger(self): - if not self._logger: - self._logger = getLogger(self.name, self.version) - return self._logger - - -class ContextAdapter(BaseLoggerAdapter): - warn = logging.LoggerAdapter.warning - - def __init__(self, logger, project_name, version_string): - self.logger = logger - self.project = project_name - self.version = version_string - - @property - def handlers(self): - return self.logger.handlers - - def deprecated(self, msg, *args, **kwargs): - stdmsg = _("Deprecated: %s") % msg - if CONF.fatal_deprecations: - self.critical(stdmsg, *args, **kwargs) - raise DeprecatedConfig(msg=stdmsg) - else: - self.warn(stdmsg, *args, **kwargs) - - def process(self, msg, kwargs): - if 'extra' not in kwargs: - kwargs['extra'] = {} - extra = kwargs['extra'] - - context = kwargs.pop('context', None) - # if not context: - # context = getattr(local.store, 'context', None) - if context: - extra.update(_dictify_context(context)) - - instance = kwargs.pop('instance', None) - instance_extra = '' - if instance: - instance_extra = CONF.instance_format % instance - else: - instance_uuid = kwargs.pop('instance_uuid', None) - if instance_uuid: - instance_extra = (CONF.instance_uuid_format - % {'uuid': instance_uuid}) - extra.update({'instance': instance_extra}) - - extra.update({"project": self.project}) - extra.update({"version": self.version}) - extra['extra'] = extra.copy() - return msg, kwargs - - -class JSONFormatter(logging.Formatter): - def __init__(self, fmt=None, datefmt=None): - # NOTE(jkoelker) we ignore the fmt argument, but its still there - # since logging.config.fileConfig passes it. - self.datefmt = datefmt - - def formatException(self, ei, strip_newlines=True): - lines = traceback.format_exception(*ei) - if strip_newlines: - lines = [itertools.ifilter( - lambda x: x, - line.rstrip().splitlines()) for line in lines] - lines = list(itertools.chain(*lines)) - return lines - - def format(self, record): - message = {'message': record.getMessage(), - 'asctime': self.formatTime(record, self.datefmt), - 'name': record.name, - 'msg': record.msg, - 'args': record.args, - 'levelname': record.levelname, - 'levelno': record.levelno, - 'pathname': record.pathname, - 'filename': record.filename, - 'module': record.module, - 'lineno': record.lineno, - 'funcname': record.funcName, - 'created': record.created, - 'msecs': record.msecs, - 'relative_created': record.relativeCreated, - 'thread': record.thread, - 'thread_name': record.threadName, - 'process_name': record.processName, - 'process': record.process, - 'traceback': None} - - if hasattr(record, 'extra'): - message['extra'] = record.extra - - if record.exc_info: - message['traceback'] = self.formatException(record.exc_info) - - return jsonutils.dumps(message) - - -def _create_logging_excepthook(product_name): - def logging_excepthook(type, value, tb): - extra = {} - if CONF.verbose: - extra['exc_info'] = (type, value, tb) - getLogger(product_name).critical(str(value), **extra) - return logging_excepthook - - -class LogConfigError(Exception): - - message = ('Error loading logging config %(log_config)s: %(err_msg)s') - - def __init__(self, log_config, err_msg): - self.log_config = log_config - self.err_msg = err_msg - - def __str__(self): - return self.message % dict(log_config=self.log_config, - err_msg=self.err_msg) - - -def _load_log_config(log_config): - try: - logging.config.fileConfig(log_config) - except ConfigParser.Error as exc: - raise LogConfigError(log_config, str(exc)) - - -def setup(product_name): - """Setup logging.""" - if CONF.log_config: - _load_log_config(CONF.log_config) - else: - _setup_logging_from_conf() - sys.excepthook = _create_logging_excepthook(product_name) - - -def set_defaults(logging_context_format_string): - cfg.set_defaults(log_opts, - logging_context_format_string= - logging_context_format_string) - - -def _find_facility_from_conf(): - facility_names = logging.handlers.SysLogHandler.facility_names - facility = getattr(logging.handlers.SysLogHandler, - CONF.syslog_log_facility, - None) - - if facility is None and CONF.syslog_log_facility in facility_names: - facility = facility_names.get(CONF.syslog_log_facility) - - if facility is None: - valid_facilities = facility_names.keys() - consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON', - 'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS', - 'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP', - 'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3', - 'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7'] - valid_facilities.extend(consts) - raise TypeError(('syslog facility must be one of: %s') % - ', '.join("'%s'" % fac - for fac in valid_facilities)) - - return facility - - -def _setup_logging_from_conf(): - log_root = getLogger(None).logger - for handler in log_root.handlers: - log_root.removeHandler(handler) - - if CONF.use_syslog: - facility = _find_facility_from_conf() - syslog = logging.handlers.SysLogHandler(address='/dev/log', - facility=facility) - log_root.addHandler(syslog) - - logpath = _get_log_file_path() - if logpath: - filelog = logging.handlers.WatchedFileHandler(logpath) - log_root.addHandler(filelog) - - if CONF.use_stderr: - streamlog = ColorHandler() - log_root.addHandler(streamlog) - - elif not CONF.log_file: - # pass sys.stdout as a positional argument - # python2.6 calls the argument strm, in 2.7 it's stream - streamlog = logging.StreamHandler(sys.stdout) - log_root.addHandler(streamlog) - - if CONF.publish_errors: - handler = importutils.import_object( - "quantum.openstack.common.log_handler.PublishErrorsHandler", - logging.ERROR) - log_root.addHandler(handler) - - datefmt = CONF.log_date_format - for handler in log_root.handlers: - # NOTE(alaski): CONF.log_format overrides everything currently. This - # should be deprecated in favor of context aware formatting. - if CONF.log_format: - handler.setFormatter(logging.Formatter(fmt=CONF.log_format, - datefmt=datefmt)) - log_root.info('Deprecated: log_format is now deprecated and will ' - 'be removed in the next release') - else: - handler.setFormatter(ContextFormatter(datefmt=datefmt)) - - if CONF.debug: - log_root.setLevel(logging.DEBUG) - elif CONF.verbose: - log_root.setLevel(logging.INFO) - else: - log_root.setLevel(logging.WARNING) - - for pair in CONF.default_log_levels: - mod, _sep, level_name = pair.partition('=') - level = logging.getLevelName(level_name) - logger = logging.getLogger(mod) - logger.setLevel(level) - -_loggers = {} - - -def getLogger(name='unknown', version='unknown'): - if name not in _loggers: - _loggers[name] = ContextAdapter(logging.getLogger(name), - name, - version) - return _loggers[name] - - -def getLazyLogger(name='unknown', version='unknown'): - """Returns lazy logger. - - Creates a pass-through logger that does not create the real logger - until it is really needed and delegates all calls to the real logger - once it is created. - """ - return LazyAdapter(name, version) - - -class WritableLogger(object): - """A thin wrapper that responds to `write` and logs.""" - - def __init__(self, logger, level=logging.INFO): - self.logger = logger - self.level = level - - def write(self, msg): - self.logger.log(self.level, msg) - - -class ContextFormatter(logging.Formatter): - """A context.RequestContext aware formatter configured through flags. - - The flags used to set format strings are: logging_context_format_string - and logging_default_format_string. You can also specify - logging_debug_format_suffix to append extra formatting if the log level is - debug. - - For information about what variables are available for the formatter see: - http://docs.python.org/library/logging.html#formatter - - """ - - def format(self, record): - """Uses contextstring if request_id is set, otherwise default.""" - # NOTE(sdague): default the fancier formating params - # to an empty string so we don't throw an exception if - # they get used - for key in ('instance', 'color'): - if key not in record.__dict__: - record.__dict__[key] = '' - - if record.__dict__.get('request_id', None): - self._fmt = CONF.logging_context_format_string - else: - self._fmt = CONF.logging_default_format_string - - if (record.levelno == logging.DEBUG and - CONF.logging_debug_format_suffix): - self._fmt += " " + CONF.logging_debug_format_suffix - - # Cache this on the record, Logger will respect our formated copy - if record.exc_info: - record.exc_text = self.formatException(record.exc_info, record) - return logging.Formatter.format(self, record) - - def formatException(self, exc_info, record=None): - """Format exception output with CONF.logging_exception_prefix.""" - if not record: - return logging.Formatter.formatException(self, exc_info) - - stringbuffer = cStringIO.StringIO() - traceback.print_exception(exc_info[0], exc_info[1], exc_info[2], - None, stringbuffer) - lines = stringbuffer.getvalue().split('\n') - stringbuffer.close() - - if CONF.logging_exception_prefix.find('%(asctime)') != -1: - record.asctime = self.formatTime(record, self.datefmt) - - formatted_lines = [] - for line in lines: - pl = CONF.logging_exception_prefix % record.__dict__ - fl = '%s%s' % (pl, line) - formatted_lines.append(fl) - return '\n'.join(formatted_lines) - - -class ColorHandler(logging.StreamHandler): - LEVEL_COLORS = { - logging.DEBUG: '\033[00;32m', # GREEN - logging.INFO: '\033[00;36m', # CYAN - logging.AUDIT: '\033[01;36m', # BOLD CYAN - logging.WARN: '\033[01;33m', # BOLD YELLOW - logging.ERROR: '\033[01;31m', # BOLD RED - logging.CRITICAL: '\033[01;31m', # BOLD RED - } - - def format(self, record): - record.color = self.LEVEL_COLORS[record.levelno] - return logging.StreamHandler.format(self, record) - - -class DeprecatedConfig(Exception): - message = ("Fatal call to deprecated config: %(msg)s") - - def __init__(self, msg): - super(Exception, self).__init__(self.message % dict(msg=msg)) diff --git a/driverlog/openstack/common/timeutils.py b/driverlog/openstack/common/timeutils.py deleted file mode 100644 index d5ed81d..0000000 --- a/driverlog/openstack/common/timeutils.py +++ /dev/null @@ -1,210 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Time related utilities and helper functions. -""" - -import calendar -import datetime -import time - -import iso8601 -import six - - -# ISO 8601 extended time format with microseconds -_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f' -_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S' -PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND - - -def isotime(at=None, subsecond=False): - """Stringify time in ISO 8601 format.""" - if not at: - at = utcnow() - st = at.strftime(_ISO8601_TIME_FORMAT - if not subsecond - else _ISO8601_TIME_FORMAT_SUBSECOND) - tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' - st += ('Z' if tz == 'UTC' else tz) - return st - - -def parse_isotime(timestr): - """Parse time from ISO 8601 format.""" - try: - return iso8601.parse_date(timestr) - except iso8601.ParseError as e: - raise ValueError(six.text_type(e)) - except TypeError as e: - raise ValueError(six.text_type(e)) - - -def strtime(at=None, fmt=PERFECT_TIME_FORMAT): - """Returns formatted utcnow.""" - if not at: - at = utcnow() - return at.strftime(fmt) - - -def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT): - """Turn a formatted time back into a datetime.""" - return datetime.datetime.strptime(timestr, fmt) - - -def normalize_time(timestamp): - """Normalize time in arbitrary timezone to UTC naive object.""" - offset = timestamp.utcoffset() - if offset is None: - return timestamp - return timestamp.replace(tzinfo=None) - offset - - -def is_older_than(before, seconds): - """Return True if before is older than seconds.""" - if isinstance(before, six.string_types): - before = parse_strtime(before).replace(tzinfo=None) - else: - before = before.replace(tzinfo=None) - - return utcnow() - before > datetime.timedelta(seconds=seconds) - - -def is_newer_than(after, seconds): - """Return True if after is newer than seconds.""" - if isinstance(after, six.string_types): - after = parse_strtime(after).replace(tzinfo=None) - else: - after = after.replace(tzinfo=None) - - return after - utcnow() > datetime.timedelta(seconds=seconds) - - -def utcnow_ts(): - """Timestamp version of our utcnow function.""" - if utcnow.override_time is None: - # NOTE(kgriffs): This is several times faster - # than going through calendar.timegm(...) - return int(time.time()) - - return calendar.timegm(utcnow().timetuple()) - - -def utcnow(): - """Overridable version of utils.utcnow.""" - if utcnow.override_time: - try: - return utcnow.override_time.pop(0) - except AttributeError: - return utcnow.override_time - return datetime.datetime.utcnow() - - -def iso8601_from_timestamp(timestamp): - """Returns a iso8601 formated date from timestamp.""" - return isotime(datetime.datetime.utcfromtimestamp(timestamp)) - - -utcnow.override_time = None - - -def set_time_override(override_time=None): - """Overrides utils.utcnow. - - Make it return a constant time or a list thereof, one at a time. - - :param override_time: datetime instance or list thereof. If not - given, defaults to the current UTC time. - """ - utcnow.override_time = override_time or datetime.datetime.utcnow() - - -def advance_time_delta(timedelta): - """Advance overridden time using a datetime.timedelta.""" - assert(not utcnow.override_time is None) - try: - for dt in utcnow.override_time: - dt += timedelta - except TypeError: - utcnow.override_time += timedelta - - -def advance_time_seconds(seconds): - """Advance overridden time by seconds.""" - advance_time_delta(datetime.timedelta(0, seconds)) - - -def clear_time_override(): - """Remove the overridden time.""" - utcnow.override_time = None - - -def marshall_now(now=None): - """Make an rpc-safe datetime with microseconds. - - Note: tzinfo is stripped, but not required for relative times. - """ - if not now: - now = utcnow() - return dict(day=now.day, month=now.month, year=now.year, hour=now.hour, - minute=now.minute, second=now.second, - microsecond=now.microsecond) - - -def unmarshall_time(tyme): - """Unmarshall a datetime dict.""" - return datetime.datetime(day=tyme['day'], - month=tyme['month'], - year=tyme['year'], - hour=tyme['hour'], - minute=tyme['minute'], - second=tyme['second'], - microsecond=tyme['microsecond']) - - -def delta_seconds(before, after): - """Return the difference between two timing objects. - - Compute the difference in seconds between two date, time, or - datetime objects (as a float, to microsecond resolution). - """ - delta = after - before - return total_seconds(delta) - - -def total_seconds(delta): - """Return the total seconds of datetime.timedelta object. - - Compute total seconds of datetime.timedelta, datetime.timedelta - doesn't have method total_seconds in Python2.6, calculate it manually. - """ - try: - return delta.total_seconds() - except AttributeError: - return ((delta.days * 24 * 3600) + delta.seconds + - float(delta.microseconds) / (10 ** 6)) - - -def is_soon(dt, window): - """Determines if time is going to happen in the next window seconds. - - :param dt: the time - :param window: minimum seconds to remain to consider the time not soon - - :return: True if expiration is within the given duration - """ - soon = (utcnow() + datetime.timedelta(seconds=window)) - return normalize_time(dt) <= soon diff --git a/driverlog/processor/__init__.py b/driverlog/processor/__init__.py index c9d84b5..e69de29 100644 --- a/driverlog/processor/__init__.py +++ b/driverlog/processor/__init__.py @@ -1 +0,0 @@ -__author__ = 'ishakhat' diff --git a/driverlog/processor/config.py b/driverlog/processor/config.py index 648e7d0..6a280aa 100644 --- a/driverlog/processor/config.py +++ b/driverlog/processor/config.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from oslo.config import cfg +from oslo_config import cfg OPTS = [ diff --git a/driverlog/processor/main.py b/driverlog/processor/main.py index 161b977..6ca6bc0 100644 --- a/driverlog/processor/main.py +++ b/driverlog/processor/main.py @@ -17,9 +17,9 @@ import re import time import memcache -from oslo.config import cfg +from oslo_config import cfg +from oslo_log import log as logging -from driverlog.openstack.common import log as logging from driverlog.processor import config from driverlog.processor import rcs from driverlog.processor import utils diff --git a/driverlog/processor/rcs.py b/driverlog/processor/rcs.py index e4ea49b..e5004a0 100644 --- a/driverlog/processor/rcs.py +++ b/driverlog/processor/rcs.py @@ -16,11 +16,10 @@ import json import re +from oslo_log import log as logging import paramiko import six -from driverlog.openstack.common import log as logging - LOG = logging.getLogger(__name__) diff --git a/driverlog/processor/utils.py b/driverlog/processor/utils.py index 03e6647..9330e8b 100644 --- a/driverlog/processor/utils.py +++ b/driverlog/processor/utils.py @@ -21,12 +21,11 @@ import re import time import iso8601 +from oslo_log import log as logging import six from six.moves.urllib import parse from six.moves.urllib import request -from driverlog.openstack.common import log as logging - LOG = logging.getLogger(__name__) diff --git a/driverlog/openstack/__init__.py b/driverlog/tests/__init__.py similarity index 100% rename from driverlog/openstack/__init__.py rename to driverlog/tests/__init__.py diff --git a/driverlog/openstack/common/__init__.py b/driverlog/tests/unit/__init__.py similarity index 100% rename from driverlog/openstack/common/__init__.py rename to driverlog/tests/unit/__init__.py diff --git a/tests/unit/test_config_files.py b/driverlog/tests/unit/test_config_files.py similarity index 98% rename from tests/unit/test_config_files.py rename to driverlog/tests/unit/test_config_files.py index 866668a..bd65317 100644 --- a/tests/unit/test_config_files.py +++ b/driverlog/tests/unit/test_config_files.py @@ -61,7 +61,7 @@ class TestConfigFiles(testtools.TestCase): def test_sample_default_data_schema_conformance(self): self._verify_default_data_by_schema( - 'tests/unit/test_data/sample_default_data.json') + 'driverlog/tests/unit/test_data/sample_default_data.json') def test_projects_in_alphabetical_order(self): projects = self._read_file('etc/default_data.json')['projects'] diff --git a/tests/unit/test_data/sample_default_data.json b/driverlog/tests/unit/test_data/sample_default_data.json similarity index 100% rename from tests/unit/test_data/sample_default_data.json rename to driverlog/tests/unit/test_data/sample_default_data.json diff --git a/tests/unit/test_data/sample_review.json b/driverlog/tests/unit/test_data/sample_review.json similarity index 100% rename from tests/unit/test_data/sample_review.json rename to driverlog/tests/unit/test_data/sample_review.json diff --git a/tests/unit/test_main.py b/driverlog/tests/unit/test_main.py similarity index 96% rename from tests/unit/test_main.py rename to driverlog/tests/unit/test_main.py index efd82b8..2b71a22 100644 --- a/tests/unit/test_main.py +++ b/driverlog/tests/unit/test_main.py @@ -24,12 +24,12 @@ import testtools def _read_sample_review(): - with open('tests/unit/test_data/sample_review.json') as fd: + with open('driverlog/tests/unit/test_data/sample_review.json') as fd: return json.load(fd) def _read_sample_default_data(): - with open('tests/unit/test_data/sample_default_data.json') as fd: + with open('driverlog/tests/unit/test_data/sample_default_data.json') as fd: return json.load(fd) @@ -121,7 +121,7 @@ class TestMain(testtools.TestCase): rcs_getter.side_effect = _get_rcs - @mock.patch('oslo.config.cfg.CONF') + @mock.patch('oslo_config.cfg.CONF') @mock.patch('driverlog.processor.rcs.get_rcs') def test_calculate_update(self, rcs_getter, conf): memcached_inst = self._make_test_memcached() @@ -140,7 +140,7 @@ class TestMain(testtools.TestCase): (update[driver_key]['releases']['juno'] ['review_url'])) - @mock.patch('oslo.config.cfg.CONF') + @mock.patch('oslo_config.cfg.CONF') @mock.patch('driverlog.processor.rcs.get_rcs') def test_calculate_update_existing_version_data(self, rcs_getter, conf): # checks that existing data will be overwritten with update diff --git a/requirements.txt b/requirements.txt index 4709188..196282e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,15 @@ -pbr>=0.6,!=0.7,<1.0 +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +pbr<2.0,>=1.3 -Flask>=0.10,<1.0 +Flask<1.0,>=0.10 iso8601>=0.1.9 -oslo.config>=1.2.1 +oslo.config>=1.11.0 # Apache-2.0 +oslo.i18n>=1.5.0 # Apache-2.0 +oslo.log>=1.6.0 # Apache-2.0 +oslo.serialization>=1.4.0 # Apache-2.0 +oslo.utils>=1.9.0 # Apache-2.0 paramiko>=1.13.0 -python-memcached>=1.48 -six>=1.7.0 +python-memcached>=1.48;python_version=='2.7' or python_version=='2.6' +six>=1.9.0 diff --git a/setup.py b/setup.py index c0a24ea..d8080d0 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,8 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass + setuptools.setup( - setup_requires=['pbr'], + setup_requires=['pbr>=1.3'], pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt index a468217..c1029ec 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,13 +1,16 @@ -hacking>=0.9.2,<0.10 +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +hacking<0.10,>=0.9.2 coverage>=3.6 discover # docutils==0.9.1 -fixtures>=0.3.14 -jsonschema>=2.0.0,<3.0.0 -mock>=1.0 -python-subunit +fixtures>=1.3.1 +jsonschema!=2.5.0,<3.0.0,>=2.0.0 +mock>=1.2 +python-subunit>=0.0.18 # sphinx>=1.1.2,<1.2 # sphinxcontrib-httpdomain testrepository>=0.0.18 -testtools>=0.9.34 +testtools>=1.4.0 diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index 241b5c5..0000000 --- a/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'ilyashakhat' diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py deleted file mode 100644 index c9d84b5..0000000 --- a/tests/unit/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'ishakhat'