Update glance.openstack.common

1. Sync openstack.common modules up to latest version
from oslo-inc. The follow modules got updated: 
 * fileutils
 * install_venv_common
 * lockutils
 * log
 * processutils

2. Add _i18n module as the dependency for new code.

3. Remove openstack.common.test module.
Glance and dependencies now use oslotest instead.

4. Remote unused openstack.common.excutils module.
Glance now use the module from oslo.utils instead.

5. Add missing modules back to openstack-common.conf.
Below ones are still using currently. But they will
be removed when the latest policy module get synced.
 * jsonutils
 * strutils

Change-Id: Ibbc2cda81a4e3c13643fe5c5bf6471c3f544add7
Signed-off-by: Zhi Yan Liu <zhiyanl@cn.ibm.com>

glance/openstack/common/__init__.py

@@ -1,17 +0,0 @@
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import six
-
-
-six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))

glance/openstack/common/_i18n.py

@@ -0,0 +1,40 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""oslo.i18n integration module.
+
+See http://docs.openstack.org/developer/oslo.i18n/usage.html
+
+"""
+
+import oslo.i18n
+
+
+# NOTE(dhellmann): This reference to o-s-l-o will be replaced by the
+# application name when this module is synced into the separate
+# repository. It is OK to have more than one translation function
+# using the same domain, since there will still only be one message
+# catalog.
+_translators = oslo.i18n.TranslatorFactory(domain='glance')
+
+# The primary translation function using the well-known name "_"
+_ = _translators.primary
+
+# Translators for log levels.
+#
+# The abbreviated names are meant to reflect the usual use of a short
+# name like '_'. The "L" is for "log" and the other letter comes from
+# the level.
+_LI = _translators.log_info
+_LW = _translators.log_warning
+_LE = _translators.log_error
+_LC = _translators.log_critical

glance/openstack/common/excutils.py

@@ -1,113 +0,0 @@
-# Copyright 2011 OpenStack Foundation.
-# Copyright 2012, Red Hat, Inc.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-"""
-Exception related utilities.
-"""
-
-import logging
-import sys
-import time
-import traceback
-
-import six
-
-from glance.openstack.common.gettextutils import _LE
-
-
-class save_and_reraise_exception(object):
-    """Save current exception, run some code and then re-raise.
-
-    In some cases the exception context can be cleared, resulting in None
-    being attempted to be re-raised after an exception handler is run. This
-    can happen when eventlet switches greenthreads or when running an
-    exception handler, code raises and catches an exception. In both
-    cases the exception context will be cleared.
-
-    To work around this, we save the exception state, run handler code, and
-    then re-raise the original exception. If another exception occurs, the
-    saved exception is logged and the new exception is re-raised.
-
-    In some cases the caller may not want to re-raise the exception, and
-    for those circumstances this context provides a reraise flag that
-    can be used to suppress the exception.  For example::
-
-      except Exception:
-          with save_and_reraise_exception() as ctxt:
-              decide_if_need_reraise()
-              if not should_be_reraised:
-                  ctxt.reraise = False
-
-    If another exception occurs and reraise flag is False,
-    the saved exception will not be logged.
-
-    If the caller wants to raise new exception during exception handling
-    he/she sets reraise to False initially with an ability to set it back to
-    True if needed::
-
-      except Exception:
-          with save_and_reraise_exception(reraise=False) as ctxt:
-              [if statements to determine whether to raise a new exception]
-              # Not raising a new exception, so reraise
-              ctxt.reraise = True
-    """
-    def __init__(self, reraise=True):
-        self.reraise = reraise
-
-    def __enter__(self):
-        self.type_, self.value, self.tb, = sys.exc_info()
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if exc_type is not None:
-            if self.reraise:
-                logging.error(_LE('Original exception being dropped: %s'),
-                              traceback.format_exception(self.type_,
-                                                         self.value,
-                                                         self.tb))
-            return False
-        if self.reraise:
-            six.reraise(self.type_, self.value, self.tb)
-
-
-def forever_retry_uncaught_exceptions(infunc):
-    def inner_func(*args, **kwargs):
-        last_log_time = 0
-        last_exc_message = None
-        exc_count = 0
-        while True:
-            try:
-                return infunc(*args, **kwargs)
-            except Exception as exc:
-                this_exc_message = six.u(str(exc))
-                if this_exc_message == last_exc_message:
-                    exc_count += 1
-                else:
-                    exc_count = 1
-                # Do not log any more frequently than once a minute unless
-                # the exception message changes
-                cur_time = int(time.time())
-                if (cur_time - last_log_time > 60 or
-                        this_exc_message != last_exc_message):
-                    logging.exception(
-                        _LE('Unexpected exception occurred %d time(s)... '
-                            'retrying.') % exc_count)
-                    last_log_time = cur_time
-                    last_exc_message = this_exc_message
-                    exc_count = 0
-                # This should be a very rare event. In case it isn't, do
-                # a sleep.
-                time.sleep(1)
-    return inner_func

glance/openstack/common/fileutils.py

@@ -18,7 +18,8 @@ import errno
 import os
 import tempfile
 
-from glance.openstack.common import excutils
+from oslo.utils import excutils
+
 from glance.openstack.common import log as logging
 
 LOG = logging.getLogger(__name__)
@@ -50,8 +51,8 @@ def read_cached_file(filename, force_reload=False):
     """
     global _FILE_CACHE
 
-    if force_reload and filename in _FILE_CACHE:
+    if force_reload:
-        del _FILE_CACHE[filename]
+        delete_cached_file(filename)
 
     reloaded = False
     mtime = os.path.getmtime(filename)
@@ -66,6 +67,17 @@ def read_cached_file(filename, force_reload=False):
     return (reloaded, cache_info['data'])
 
 
+def delete_cached_file(filename):
+    """Delete cached file if present.
+
+    :param filename: filename to delete
+    """
+    global _FILE_CACHE
+
+    if filename in _FILE_CACHE:
+        del _FILE_CACHE[filename]
+
+
 def delete_if_exists(path, remove=os.unlink):
     """Delete a file, but ignore file not found error.
 

glance/openstack/common/lockutils.py

@@ -15,8 +15,8 @@
 
 import contextlib
 import errno
-import fcntl
 import functools
+import logging
 import os
 import shutil
 import subprocess
@@ -29,8 +29,7 @@ import weakref
 from oslo.config import cfg
 
 from glance.openstack.common import fileutils
-from glance.openstack.common.gettextutils import _, _LE, _LI
+from glance.openstack.common._i18n import _, _LE, _LI
-from glance.openstack.common import log as logging
 
 
 LOG = logging.getLogger(__name__)
@@ -102,10 +101,8 @@ class _FileLock(object):
                     raise threading.ThreadError(_("Unable to acquire lock on"
                                                   " `%(filename)s` due to"
                                                   " %(exception)s") %
-                                                {
+                                                {'filename': self.fname,
-                                                    'filename': self.fname,
+                                                    'exception': e})
-                                                    'exception': e,
-                                                })
 
     def __enter__(self):
         self.acquire()
@@ -149,57 +146,12 @@ class _FcntlLock(_FileLock):
         fcntl.lockf(self.lockfile, fcntl.LOCK_UN)
 
 
-class _PosixLock(object):
-    def __init__(self, name):
-        # Hash the name because it's not valid to have POSIX semaphore
-        # names with things like / in them. Then use base64 to encode
-        # the digest() instead taking the hexdigest() because the
-        # result is shorter and most systems can't have shm sempahore
-        # names longer than 31 characters.
-        h = hashlib.sha1()
-        h.update(name.encode('ascii'))
-        self.name = str((b'/' + base64.urlsafe_b64encode(
-            h.digest())).decode('ascii'))
-
-    def acquire(self, timeout=None):
-        self.semaphore = posix_ipc.Semaphore(self.name,
-                                             flags=posix_ipc.O_CREAT,
-                                             initial_value=1)
-        self.semaphore.acquire(timeout)
-        return self
-
-    def __enter__(self):
-        self.acquire()
-        return self
-
-    def release(self):
-        self.semaphore.release()
-        self.semaphore.close()
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.release()
-
-    def exists(self):
-        try:
-            semaphore = posix_ipc.Semaphore(self.name)
-        except posix_ipc.ExistentialError:
-            return False
-        else:
-            semaphore.close()
-        return True
-
-
 if os.name == 'nt':
     import msvcrt
     InterProcessLock = _WindowsLock
-    FileLock = _WindowsLock
 else:
-    import base64
+    import fcntl
-    import hashlib
+    InterProcessLock = _FcntlLock
-
-    import posix_ipc
-    InterProcessLock = _PosixLock
-    FileLock = _FcntlLock
 
 _semaphores = weakref.WeakValueDictionary()
 _semaphores_lock = threading.Lock()
@@ -216,11 +168,7 @@ def _get_lock_path(name, lock_file_prefix, lock_path=None):
     local_lock_path = lock_path or CONF.lock_path
 
     if not local_lock_path:
-        # NOTE(bnemec): Create a fake lock path for posix locks so we don't
+        raise cfg.RequiredOptError('lock_path')
-        # unnecessarily raise the RequiredOptError below.
-        if InterProcessLock is not _PosixLock:
-            raise cfg.RequiredOptError('lock_path')
-        local_lock_path = 'posixlock:/'
 
     return os.path.join(local_lock_path, name)
 
@@ -231,11 +179,6 @@ def external_lock(name, lock_file_prefix=None, lock_path=None):
 
     lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path)
 
-    # NOTE(bnemec): If an explicit lock_path was passed to us then it
-    # means the caller is relying on file-based locking behavior, so
-    # we can't use posix locks for those calls.
-    if lock_path:
-        return FileLock(lock_file_path)
     return InterProcessLock(lock_file_path)
 
 
@@ -256,11 +199,12 @@ def internal_lock(name):
     with _semaphores_lock:
         try:
             sem = _semaphores[name]
+            LOG.debug('Using existing semaphore "%s"', name)
         except KeyError:
             sem = threading.Semaphore()
             _semaphores[name] = sem
+            LOG.debug('Created new semaphore "%s"', name)
 
-    LOG.debug('Got semaphore "%(lock)s"', {'lock': name})
     return sem
 
 
@@ -282,13 +226,16 @@ def lock(name, lock_file_prefix=None, external=False, lock_path=None):
     """
     int_lock = internal_lock(name)
     with int_lock:
-        if external and not CONF.disable_process_locking:
+        LOG.debug('Acquired semaphore "%(lock)s"', {'lock': name})
-            ext_lock = external_lock(name, lock_file_prefix, lock_path)
+        try:
-            with ext_lock:
+            if external and not CONF.disable_process_locking:
-                yield ext_lock
+                ext_lock = external_lock(name, lock_file_prefix, lock_path)
-        else:
+                with ext_lock:
-            yield int_lock
+                    yield ext_lock
-    LOG.debug('Released semaphore "%(lock)s"', {'lock': name})
+            else:
+                yield int_lock
+        finally:
+            LOG.debug('Releasing semaphore "%(lock)s"', {'lock': name})
 
 
 def synchronized(name, lock_file_prefix=None, external=False, lock_path=None):

glance/openstack/common/log.py

@@ -33,42 +33,24 @@ import logging
 import logging.config
 import logging.handlers
 import os
-import re
+import socket
 import sys
 import traceback
 
 from oslo.config import cfg
+from oslo.serialization import jsonutils
+from oslo.utils import importutils
 import six
 from six import moves
 
-from glance.openstack.common.gettextutils import _
+_PY26 = sys.version_info[0:2] == (2, 6)
-from glance.openstack.common import importutils
+
-from glance.openstack.common import jsonutils
+from glance.openstack.common._i18n import _
 from glance.openstack.common import local
 
 
 _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
 
-_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
-
-# NOTE(ldbragst): Let's build a list of regex objects using the list of
-# _SANITIZE_KEYS we already have. This way, we only have to add the new key
-# to the list of _SANITIZE_KEYS and we can generate regular expressions
-# for XML and JSON automatically.
-_SANITIZE_PATTERNS = []
-_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
-                    r'(<%(key)s>).*?(</%(key)s>)',
-                    r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
-                    r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
-                    r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
-                    '.*?([\'"])',
-                    r'(%(key)s\s*--?[A-z]+\s*).*?([\s])']
-
-for key in _SANITIZE_KEYS:
-    for pattern in _FORMAT_PATTERNS:
-        reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
-        _SANITIZE_PATTERNS.append(reg_ex)
-
 
 common_cli_opts = [
     cfg.BoolOpt('debug',
@@ -138,6 +120,14 @@ generic_log_opts = [
                 help='Log output to standard error.')
 ]
 
+DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN',
+                      'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO',
+                      'oslo.messaging=INFO', 'iso8601=WARN',
+                      'requests.packages.urllib3.connectionpool=WARN',
+                      'urllib3.connectionpool=WARN', 'websocket=WARN',
+                      "keystonemiddleware=WARN", "routes.middleware=WARN",
+                      "stevedore=WARN"]
+
 log_opts = [
     cfg.StrOpt('logging_context_format_string',
                default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
@@ -156,17 +146,7 @@ log_opts = [
                '%(instance)s',
                help='Prefix each line of exception output with this format.'),
     cfg.ListOpt('default_log_levels',
-                default=[
+                default=DEFAULT_LOG_LEVELS,
-                    'amqp=WARN',
-                    'amqplib=WARN',
-                    'boto=WARN',
-                    'qpid=WARN',
-                    'sqlalchemy=WARN',
-                    'suds=INFO',
-                    'oslo.messaging=INFO',
-                    'iso8601=WARN',
-                    'requests.packages.urllib3.connectionpool=WARN'
-                ],
                 help='List of logger=LEVEL pairs.'),
     cfg.BoolOpt('publish_errors',
                 default=False,
@@ -181,11 +161,11 @@ log_opts = [
     cfg.StrOpt('instance_format',
                default='[instance: %(uuid)s] ',
                help='The format for an instance that is passed with the log '
-                    'message. '),
+                    'message.'),
     cfg.StrOpt('instance_uuid_format',
                default='[instance: %(uuid)s] ',
                help='The format for an instance UUID that is passed with the '
-                    'log message. '),
+                    'log message.'),
 ]
 
 CONF = cfg.CONF
@@ -244,45 +224,20 @@ def _get_log_file_path(binary=None):
     return None
 
 
-def mask_password(message, secret="***"):
-    """Replace password with 'secret' in message.
-
-    :param message: The string which includes security information.
-    :param secret: value with which to replace passwords.
-    :returns: The unicode value of message with the password fields masked.
-
-    For example:
-
-    >>> mask_password("'adminPass' : 'aaaaa'")
-    "'adminPass' : '***'"
-    >>> mask_password("'admin_pass' : 'aaaaa'")
-    "'admin_pass' : '***'"
-    >>> mask_password('"password" : "aaaaa"')
-    '"password" : "***"'
-    >>> mask_password("'original_password' : 'aaaaa'")
-    "'original_password' : '***'"
-    >>> mask_password("u'original_password' :   u'aaaaa'")
-    "u'original_password' :   u'***'"
-    """
-    message = six.text_type(message)
-
-    # NOTE(ldbragst): Check to see if anything in message contains any key
-    # specified in _SANITIZE_KEYS, if not then just return the message since
-    # we don't have to mask any passwords.
-    if not any(key in message for key in _SANITIZE_KEYS):
-        return message
-
-    secret = r'\g<1>' + secret + r'\g<2>'
-    for pattern in _SANITIZE_PATTERNS:
-        message = re.sub(pattern, secret, message)
-    return message
-
-
 class BaseLoggerAdapter(logging.LoggerAdapter):
 
     def audit(self, msg, *args, **kwargs):
         self.log(logging.AUDIT, msg, *args, **kwargs)
 
+    def isEnabledFor(self, level):
+        if _PY26:
+            # This method was added in python 2.7 (and it does the exact
+            # same logic, so we need to do the exact same logic so that
+            # python 2.6 has this capability as well).
+            return self.logger.isEnabledFor(level)
+        else:
+            return super(BaseLoggerAdapter, self).isEnabledFor(level)
+
 
 class LazyAdapter(BaseLoggerAdapter):
     def __init__(self, name='unknown', version='unknown'):
@@ -295,6 +250,11 @@ class LazyAdapter(BaseLoggerAdapter):
     def logger(self):
         if not self._logger:
             self._logger = getLogger(self.name, self.version)
+            if six.PY3:
+                # In Python 3, the code fails because the 'manager' attribute
+                # cannot be found when using a LoggerAdapter as the
+                # underlying logger. Work around this issue.
+                self._logger.manager = self._logger.logger.manager
         return self._logger
 
 
@@ -340,11 +300,10 @@ class ContextAdapter(BaseLoggerAdapter):
         self.warn(stdmsg, *args, **kwargs)
 
     def process(self, msg, kwargs):
-        # NOTE(mrodden): catch any Message/other object and
+        # NOTE(jecarey): If msg is not unicode, coerce it into unicode
-        #                coerce to unicode before they can get
+        #                before it can get to the python logging and
-        #                to the python logging and possibly
+        #                possibly cause string encoding trouble
-        #                cause string encoding trouble
+        if not isinstance(msg, six.text_type):
-        if not isinstance(msg, six.string_types):
             msg = six.text_type(msg)
 
         if 'extra' not in kwargs:
@@ -448,7 +407,7 @@ def _load_log_config(log_config_append):
     try:
         logging.config.fileConfig(log_config_append,
                                   disable_existing_loggers=False)
-    except moves.configparser.Error as exc:
+    except (moves.configparser.Error, KeyError) as exc:
         raise LogConfigError(log_config_append, six.text_type(exc))
 
 
@@ -461,9 +420,20 @@ def setup(product_name, version='unknown'):
     sys.excepthook = _create_logging_excepthook(product_name)
 
 
-def set_defaults(logging_context_format_string):
+def set_defaults(logging_context_format_string=None,
-    cfg.set_defaults(
+                 default_log_levels=None):
-        log_opts, logging_context_format_string=logging_context_format_string)
+    # Just in case the caller is not setting the
+    # default_log_level. This is insurance because
+    # we introduced the default_log_level parameter
+    # later in a backwards in-compatible change
+    if default_log_levels is not None:
+        cfg.set_defaults(
+            log_opts,
+            default_log_levels=default_log_levels)
+    if logging_context_format_string is not None:
+        cfg.set_defaults(
+            log_opts,
+            logging_context_format_string=logging_context_format_string)
 
 
 def _find_facility_from_conf():
@@ -512,18 +482,6 @@ def _setup_logging_from_conf(project, version):
     for handler in log_root.handlers:
         log_root.removeHandler(handler)
 
-    if CONF.use_syslog:
-        facility = _find_facility_from_conf()
-        # TODO(bogdando) use the format provided by RFCSysLogHandler
-        #   after existing syslog format deprecation in J
-        if CONF.use_syslog_rfc_format:
-            syslog = RFCSysLogHandler(address='/dev/log',
-                                      facility=facility)
-        else:
-            syslog = logging.handlers.SysLogHandler(address='/dev/log',
-                                                    facility=facility)
-        log_root.addHandler(syslog)
-
     logpath = _get_log_file_path()
     if logpath:
         filelog = logging.handlers.WatchedFileHandler(logpath)
@@ -582,6 +540,20 @@ def _setup_logging_from_conf(project, version):
         else:
             logger.setLevel(level_name)
 
+    if CONF.use_syslog:
+        try:
+            facility = _find_facility_from_conf()
+            # TODO(bogdando) use the format provided by RFCSysLogHandler
+            #   after existing syslog format deprecation in J
+            if CONF.use_syslog_rfc_format:
+                syslog = RFCSysLogHandler(facility=facility)
+            else:
+                syslog = logging.handlers.SysLogHandler(facility=facility)
+            log_root.addHandler(syslog)
+        except socket.error:
+            log_root.error('Unable to add syslog handler. Verify that syslog '
+                           'is running.')
+
 
 _loggers = {}
 
@@ -651,6 +623,12 @@ class ContextFormatter(logging.Formatter):
     def format(self, record):
         """Uses contextstring if request_id is set, otherwise default."""
 
+        # NOTE(jecarey): If msg is not unicode, coerce it into unicode
+        #                before it can get to the python logging and
+        #                possibly cause string encoding trouble
+        if not isinstance(record.msg, six.text_type):
+            record.msg = six.text_type(record.msg)
+
         # store project info
         record.project = self.project
         record.version = self.version

glance/openstack/common/processutils.py

@@ -18,7 +18,7 @@ System-level utilities and helper functions.
 """
 
 import errno
-import logging as stdlib_logging
+import logging
 import multiprocessing
 import os
 import random
@@ -27,10 +27,10 @@ import signal
 
 from eventlet.green import subprocess
 from eventlet import greenthread
+from oslo.utils import strutils
 import six
 
-from glance.openstack.common.gettextutils import _
+from glance.openstack.common._i18n import _
-from glance.openstack.common import log as logging
 
 
 LOG = logging.getLogger(__name__)
@@ -115,8 +115,7 @@ def execute(*cmd, **kwargs):
                             execute this command. Defaults to false.
     :type shell:            boolean
     :param loglevel:        log level for execute commands.
-    :type loglevel:         int.  (Should be stdlib_logging.DEBUG or
+    :type loglevel:         int.  (Should be logging.DEBUG or logging.INFO)
-                            stdlib_logging.INFO)
     :returns:               (stdout, stderr) from process execution
     :raises:                :class:`UnknownArgumentError` on
                             receiving unknown arguments
@@ -132,7 +131,7 @@ def execute(*cmd, **kwargs):
     run_as_root = kwargs.pop('run_as_root', False)
     root_helper = kwargs.pop('root_helper', '')
     shell = kwargs.pop('shell', False)
-    loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG)
+    loglevel = kwargs.pop('loglevel', logging.DEBUG)
 
     if isinstance(check_exit_code, bool):
         ignore_exit_code = not check_exit_code
@@ -141,8 +140,7 @@ def execute(*cmd, **kwargs):
         check_exit_code = [check_exit_code]
 
     if kwargs:
-        raise UnknownArgumentError(_('Got unknown keyword args '
+        raise UnknownArgumentError(_('Got unknown keyword args: %r') % kwargs)
-                                     'to utils.execute: %r') % kwargs)
 
     if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0:
         if not root_helper:
@@ -152,12 +150,12 @@ def execute(*cmd, **kwargs):
         cmd = shlex.split(root_helper) + list(cmd)
 
     cmd = map(str, cmd)
+    sanitized_cmd = strutils.mask_password(' '.join(cmd))
 
     while attempts > 0:
         attempts -= 1
         try:
-            LOG.log(loglevel, 'Running cmd (subprocess): %s',
+            LOG.log(loglevel, _('Running cmd (subprocess): %s'), sanitized_cmd)
-                    ' '.join(logging.mask_password(cmd)))
             _PIPE = subprocess.PIPE  # pylint: disable=E1101
 
             if os.name == 'nt':
@@ -194,16 +192,18 @@ def execute(*cmd, **kwargs):
             LOG.log(loglevel, 'Result was %s' % _returncode)
             if not ignore_exit_code and _returncode not in check_exit_code:
                 (stdout, stderr) = result
+                sanitized_stdout = strutils.mask_password(stdout)
+                sanitized_stderr = strutils.mask_password(stderr)
                 raise ProcessExecutionError(exit_code=_returncode,
-                                            stdout=stdout,
+                                            stdout=sanitized_stdout,
-                                            stderr=stderr,
+                                            stderr=sanitized_stderr,
-                                            cmd=' '.join(cmd))
+                                            cmd=sanitized_cmd)
             return result
         except ProcessExecutionError:
             if not attempts:
                 raise
             else:
-                LOG.log(loglevel, '%r failed. Retrying.', cmd)
+                LOG.log(loglevel, _('%r failed. Retrying.'), sanitized_cmd)
                 if delay_on_retry:
                     greenthread.sleep(random.randint(20, 200) / 100.0)
         finally:
@@ -242,7 +242,8 @@ def trycmd(*args, **kwargs):
 
 def ssh_execute(ssh, cmd, process_input=None,
                 addl_env=None, check_exit_code=True):
-    LOG.debug('Running cmd (SSH): %s', cmd)
+    sanitized_cmd = strutils.mask_password(cmd)
+    LOG.debug('Running cmd (SSH): %s', sanitized_cmd)
     if addl_env:
         raise InvalidArgumentError(_('Environment not supported over SSH'))
 
@@ -256,7 +257,10 @@ def ssh_execute(ssh, cmd, process_input=None,
     # NOTE(justinsb): This seems suspicious...
     # ...other SSH clients have buffering issues with this approach
     stdout = stdout_stream.read()
+    sanitized_stdout = strutils.mask_password(stdout)
     stderr = stderr_stream.read()
+    sanitized_stderr = strutils.mask_password(stderr)
+
     stdin_stream.close()
 
     exit_status = channel.recv_exit_status()
@@ -266,11 +270,11 @@ def ssh_execute(ssh, cmd, process_input=None,
         LOG.debug('Result was %s' % exit_status)
         if check_exit_code and exit_status != 0:
             raise ProcessExecutionError(exit_code=exit_status,
-                                        stdout=stdout,
+                                        stdout=sanitized_stdout,
-                                        stderr=stderr,
+                                        stderr=sanitized_stderr,
-                                        cmd=cmd)
+                                        cmd=sanitized_cmd)
 
-    return (stdout, stderr)
+    return (sanitized_stdout, sanitized_stderr)
 
 
 def get_worker_count():

openstack-common.conf

@@ -1,15 +1,17 @@
 [DEFAULT]
 
 # The list of modules to copy from oslo-incubator
+module=_i18n
 module=fileutils
 module=gettextutils
 module=install_venv_common
+module=jsonutils
 module=local
 module=lockutils
 module=log
 module=policy
 module=processutils
-module=test
+module=strutils
 
 # The base module to hold the copy of openstack.common
 base=glance

tools/install_venv_common.py

@@ -125,7 +125,7 @@ class InstallVenv(object):
         parser.add_option('-n', '--no-site-packages',
                           action='store_true',
                           help="Do not inherit packages from global Python "
-                               "install")
+                               "install.")
         return parser.parse_args(argv[1:])[0]