openstack
/
trove
Code Issues Proposed changes

Sync latest process and str utils from oslo 

This sync required changes to fix these issues:
* Make execute method clean password in exception
* Make sure mask_password works properly

This is not trivial as these fixes relies on many other changes,
only the necessary code have been imported/adapted.

------------------------------------------------
The sync pulls in the following changes (newest to oldest):

63c99a0f - Mask passwords in exceptions and error messages
66142c34 - Make strutils.mask_password more secure
d6b55fb2 - Remove `processutils` dependency on `log`
cb5a804b - Move `mask_password` to strutils

-----------------------------------------------

Closes-Bug: 1343604
Closes-Bug: 1345233
SecurityImpact

Change-Id: I3b49b1d667f6ade9ae3f6765d735440a3e838917
This commit is contained in:
Tristan Cacqueray 2014-09-14 18:17:40 +00:00
parent 71349ca07a
commit a36c2a7732
6 changed files with 121 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
trove/common/auth.py
View File

@ -20,6 +20,7 @@ import wsgi
from trove.common import exception
from trove.openstack.common import log as logging
from trove.openstack.common.gettextutils import _
from trove.openstack.common import strutils
LOG = logging.getLogger(__name__)
@ -59,7 +60,7 @@ class TenantBasedAuth(object):
match_for_tenant = self.tenant_scoped_url.match(request.path_info)
if (match_for_tenant and
tenant_id == match_for_tenant.group('tenant_id')):
LOG.debug(logging.mask_password(
LOG.debug(strutils.mask_password(
_("Authorized tenant '%(tenant_id)s' request: "
"%(request)s") %
{'tenant_id': tenant_id, 'request': request}))

9
trove/extensions/mysql/service.py
View File

@ -27,6 +27,7 @@ from trove.extensions.mysql import views
from trove.guestagent.db import models as guest_models
from trove.openstack.common import log as logging
from trove.openstack.common.gettextutils import _
from trove.openstack.common import strutils
import trove.common.apischema as apischema
@ -82,8 +83,8 @@ class UserController(wsgi.Controller):
def create(self, req, body, tenant_id, instance_id):
"""Creates a set of users"""
LOG.info(_("Creating users for instance '%s'") % instance_id)
LOG.info(logging.mask_password(_("req : '%s'\n\n") % req))
LOG.info(logging.mask_password(_("body : '%s'\n\n") % body))
LOG.info(strutils.mask_password(_("req : '%s'\n\n") % req))
LOG.info(strutils.mask_password(_("body : '%s'\n\n") % body))
context = req.environ[wsgi.CONTEXT_KEY]
users = body['users']
try:
@ -135,7 +136,7 @@ class UserController(wsgi.Controller):
def update(self, req, body, tenant_id, instance_id, id):
"""Change attributes for one user."""
LOG.info(_("Updating user attributes for instance '%s'") % instance_id)
LOG.info(logging.mask_password(_("req : '%s'\n\n") % req))
LOG.info(strutils.mask_password(_("req : '%s'\n\n") % req))
context = req.environ[wsgi.CONTEXT_KEY]
id = correct_id_with_req(id, req)
username, hostname = unquote_user_host(id)
@ -157,7 +158,7 @@ class UserController(wsgi.Controller):
def update_all(self, req, body, tenant_id, instance_id):
"""Change the password of one or more users."""
LOG.info(_("Updating user passwords for instance '%s'") % instance_id)
LOG.info(logging.mask_password(_("req : '%s'\n\n") % req))
LOG.info(strutils.mask_password(_("req : '%s'\n\n") % req))
context = req.environ[wsgi.CONTEXT_KEY]
users = body['users']
model_users = []

5
trove/instance/service.py
View File

@ -27,6 +27,7 @@ from trove.datastore import models as datastore_models
from trove.backup.models import Backup as backup_model
from trove.backup import views as backup_views
from trove.openstack.common import log as logging
from trove.openstack.common import strutils
from trove.openstack.common.gettextutils import _
import trove.common.apischema as apischema
@ -178,8 +179,8 @@ class InstanceController(wsgi.Controller):
def create(self, req, body, tenant_id):
# TODO(hub-cap): turn this into middleware
LOG.info(_("Creating a database instance for tenant '%s'") % tenant_id)
LOG.info(logging.mask_password(_("req : '%s'\n\n") % req))
LOG.info(logging.mask_password(_("body : '%s'\n\n") % body))
LOG.info(strutils.mask_password(_("req : '%s'\n\n") % req))
LOG.info(strutils.mask_password(_("body : '%s'\n\n") % body))
context = req.environ[wsgi.CONTEXT_KEY]
datastore_args = body['instance'].get('datastore', {})
datastore, datastore_version = (

52
trove/openstack/common/log.py
View File

@ -33,7 +33,6 @@ import logging
import logging.config
import logging.handlers
import os
import re
import sys
import traceback
@ -49,23 +48,6 @@ from trove.openstack.common import local
_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
# NOTE(ldbragst): Let's build a list of regex objects using the list of
# _SANITIZE_KEYS we already have. This way, we only have to add the new key
# to the list of _SANITIZE_KEYS and we can generate regular expressions
# for XML and JSON automatically.
_SANITIZE_PATTERNS = []
_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
r'(<%(key)s>).*?(</%(key)s>)',
r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])']
for key in _SANITIZE_KEYS:
for pattern in _FORMAT_PATTERNS:
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
_SANITIZE_PATTERNS.append(reg_ex)
common_cli_opts = [
cfg.BoolOpt('debug',
@ -231,40 +213,6 @@ def _get_log_file_path(binary=None):
return None
def mask_password(message, secret="***"):
"""Replace password with 'secret' in message.
:param message: The string which includes security information.
:param secret: value with which to replace passwords.
:returns: The unicode value of message with the password fields masked.
For example:
>>> mask_password("'adminPass' : 'aaaaa'")
"'adminPass' : '***'"
>>> mask_password("'admin_pass' : 'aaaaa'")
"'admin_pass' : '***'"
>>> mask_password('"password" : "aaaaa"')
'"password" : "***"'
>>> mask_password("'original_password' : 'aaaaa'")
"'original_password' : '***'"
>>> mask_password("u'original_password' : u'aaaaa'")
"u'original_password' : u'***'"
"""
message = six.text_type(message)
# NOTE(ldbragst): Check to see if anything in message contains any key
# specified in _SANITIZE_KEYS, if not then just return the message since
# we don't have to mask any passwords.
if not any(key in message for key in _SANITIZE_KEYS):
return message
secret = r'\g<1>' + secret + r'\g<2>'
for pattern in _SANITIZE_PATTERNS:
message = re.sub(pattern, secret, message)
return message
class BaseLoggerAdapter(logging.LoggerAdapter):
def audit(self, msg, *args, **kwargs):

28
trove/openstack/common/processutils.py
View File

@ -19,6 +19,7 @@
System-level utilities and helper functions.
"""
import logging
import os
import random
import shlex
@ -28,7 +29,7 @@ from eventlet.green import subprocess
from eventlet import greenthread
from trove.openstack.common.gettextutils import _
from trove.openstack.common import log as logging
from trove.openstack.common import strutils
LOG = logging.getLogger(__name__)
@ -97,6 +98,8 @@ def execute(*cmd, **kwargs):
:param shell: whether or not there should be a shell used to
execute this command. Defaults to false.
:type shell: boolean
:param loglevel: log level for execute commands.
:type loglevel: int. (Should be logging.DEBUG or logging.INFO)
:returns: (stdout, stderr) from process execution
:raises: :class:`UnknownArgumentError` on
receiving unknown arguments
@ -111,6 +114,7 @@ def execute(*cmd, **kwargs):
run_as_root = kwargs.pop('run_as_root', False)
root_helper = kwargs.pop('root_helper', '')
shell = kwargs.pop('shell', False)
loglevel = kwargs.pop('loglevel', logging.DEBUG)
if isinstance(check_exit_code, bool):
ignore_exit_code = not check_exit_code
@ -130,11 +134,12 @@ def execute(*cmd, **kwargs):
cmd = shlex.split(root_helper) + list(cmd)
cmd = map(str, cmd)
sanitized_cmd = strutils.mask_password(' '.join(cmd))
while attempts > 0:
attempts -= 1
try:
LOG.debug(_('Running cmd (subprocess): %s'), ' '.join(cmd))
LOG.log(loglevel, _('Running cmd (subprocess): %s'), sanitized_cmd)
_PIPE = subprocess.PIPE # pylint: disable=E1101
if os.name == 'nt':
@ -158,20 +163,21 @@ def execute(*cmd, **kwargs):
result = obj.communicate()
obj.stdin.close() # pylint: disable=E1101
_returncode = obj.returncode # pylint: disable=E1101
if _returncode:
LOG.debug(_('Result was %s') % _returncode)
if not ignore_exit_code and _returncode not in check_exit_code:
(stdout, stderr) = result
raise ProcessExecutionError(exit_code=_returncode,
stdout=stdout,
stderr=stderr,
cmd=' '.join(cmd))
LOG.log(loglevel, 'Result was %s' % _returncode)
if not ignore_exit_code and _returncode not in check_exit_code:
(stdout, stderr) = result
sanitized_stdout = strutils.mask_password(stdout)
sanitized_stderr = strutils.mask_password(stderr)
raise ProcessExecutionError(exit_code=_returncode,
stdout=sanitized_stdout,
stderr=sanitized_stderr,
cmd=sanitized_cmd)
return result
except ProcessExecutionError:
if not attempts:
raise
else:
LOG.debug(_('%r failed. Retrying.'), cmd)
LOG.log(loglevel, _('%r failed. Retrying.'), sanitized_cmd)
if delay_on_retry:
greenthread.sleep(random.randint(20, 200) / 100.0)
finally:

94
trove/openstack/common/strutils.py Normal file
View File

@ -0,0 +1,94 @@
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
System-level utilities and helper functions.
"""
import re
import six
# NOTE(flaper87): The following globals are used by `mask_password`
_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
# NOTE(ldbragst): Let's build a list of regex objects using the list of
# _SANITIZE_KEYS we already have. This way, we only have to add the new key
# to the list of _SANITIZE_KEYS and we can generate regular expressions
# for XML and JSON automatically.
_SANITIZE_PATTERNS_2 = []
_SANITIZE_PATTERNS_1 = []
# NOTE(amrith): Some regular expressions have only one parameter, some
# have two parameters. Use different lists of patterns here.
_FORMAT_PATTERNS_1 = [r'(%(key)s\s*[=]\s*)[^\s^\'^\"]+']
_FORMAT_PATTERNS_2 = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
r'(%(key)s\s+[\"\']).*?([\"\'])',
r'([-]{2}%(key)s\s+)[^\'^\"^=^\s]+([\s]*)',
r'(<%(key)s>).*?(</%(key)s>)',
r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?'
'[\'"]).*?([\'"])',
r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)']
for key in _SANITIZE_KEYS:
for pattern in _FORMAT_PATTERNS_2:
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
_SANITIZE_PATTERNS_2.append(reg_ex)
for pattern in _FORMAT_PATTERNS_1:
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
_SANITIZE_PATTERNS_1.append(reg_ex)
def mask_password(message, secret="***"):
"""Replace password with 'secret' in message.
:param message: The string which includes security information.
:param secret: value with which to replace passwords.
:returns: The unicode value of message with the password fields masked.
For example:
>>> mask_password("'adminPass' : 'aaaaa'")
"'adminPass' : '***'"
>>> mask_password("'admin_pass' : 'aaaaa'")
"'admin_pass' : '***'"
>>> mask_password('"password" : "aaaaa"')
'"password" : "***"'
>>> mask_password("'original_password' : 'aaaaa'")
"'original_password' : '***'"
>>> mask_password("u'original_password' : u'aaaaa'")
"u'original_password' : u'***'"
"""
message = six.text_type(message)
# NOTE(ldbragst): Check to see if anything in message contains any key
# specified in _SANITIZE_KEYS, if not then just return the message since
# we don't have to mask any passwords.
if not any(key in message for key in _SANITIZE_KEYS):
return message
substitute = r'\g<1>' + secret + r'\g<2>'
for pattern in _SANITIZE_PATTERNS_2:
message = re.sub(pattern, substitute, message)
substitute = r'\g<1>' + secret
for pattern in _SANITIZE_PATTERNS_1:
message = re.sub(pattern, substitute, message)
return message