Add python_exec kwarg to processutils.execute()

This commit adds a new kwarg to the process_utils.execute() function to
specify the python executable to use when launching python to check
prlimits. This is necessary when processutils.execute() is called from
inside an API server running with uwsgi. In this case sys.executable is
uwsgi (because uwsgi links libpython.so and is actually the interpreter)
This doesn't work with the execute() function because it assumes the
cpython interpreter CLI is used for the arguments it uses to call the
prlimits module. To workaround this and enable API servers that may run
under uwsgi to use this those applications can simply pass in an
executable to use.

Longer term it might be better to migrate the prlimits usage to call
multiprocessing instead of subprocessing python. But that would require
a more significant rewrite of both processutils and prlimit to
facilitate that.

Change-Id: I0ae60f0b4cc3700c783f6018e837358f0e053a09
Closes-Bug: #1712463
This commit is contained in:
Matthew Treinish 2018-01-05 15:11:17 -05:00
parent 844198a041
commit 55e06261aa
No known key found for this signature in database
GPG Key ID: FD12A0F214C9E177
3 changed files with 24 additions and 1 deletions

View File

@ -262,6 +262,10 @@ def execute(*cmd, **kwargs):
:param prlimit: Set resource limits on the child process. See :param prlimit: Set resource limits on the child process. See
below for a detailed description. below for a detailed description.
:type prlimit: :class:`ProcessLimits` :type prlimit: :class:`ProcessLimits`
:param python_exec: The python executable to use for enforcing
prlimits. If this is not set it will default to use
sys.executable.
:type python_exec: string
:returns: (stdout, stderr) from process execution :returns: (stdout, stderr) from process execution
:raises: :class:`UnknownArgumentError` on :raises: :class:`UnknownArgumentError` on
receiving unknown arguments receiving unknown arguments
@ -314,6 +318,7 @@ def execute(*cmd, **kwargs):
on_completion = kwargs.pop('on_completion', None) on_completion = kwargs.pop('on_completion', None)
preexec_fn = kwargs.pop('preexec_fn', None) preexec_fn = kwargs.pop('preexec_fn', None)
prlimit = kwargs.pop('prlimit', None) prlimit = kwargs.pop('prlimit', None)
python_exec = kwargs.pop('python_exec', sys.executable)
if isinstance(check_exit_code, bool): if isinstance(check_exit_code, bool):
ignore_exit_code = not check_exit_code ignore_exit_code = not check_exit_code
@ -350,7 +355,7 @@ def execute(*cmd, **kwargs):
_('Process resource limits are ignored as ' _('Process resource limits are ignored as '
'this feature is not supported on Windows.')) 'this feature is not supported on Windows.'))
else: else:
args = [sys.executable, '-m', 'oslo_concurrency.prlimit'] args = [python_exec, '-m', 'oslo_concurrency.prlimit']
args.extend(prlimit.prlimit_args()) args.extend(prlimit.prlimit_args())
args.append('--') args.append('--')
args.extend(cmd) args.extend(cmd)

View File

@ -968,3 +968,16 @@ class PrlimitTestCase(test_base.BaseTestCase):
stderr=mock.ANY, close_fds=mock.ANY, stderr=mock.ANY, close_fds=mock.ANY,
preexec_fn=mock.ANY, shell=mock.ANY, preexec_fn=mock.ANY, shell=mock.ANY,
cwd=mock.ANY, env=mock.ANY) cwd=mock.ANY, env=mock.ANY)
@mock.patch.object(processutils.subprocess, 'Popen')
def test_python_exec(self, sub_mock):
mock_subprocess = mock.MagicMock()
mock_subprocess.communicate.return_value = (b'', b'')
sub_mock.return_value = mock_subprocess
args = ['/a/command']
prlimit = self.limit_address_space()
processutils.execute(*args, prlimit=prlimit, check_exit_code=False,
python_exec='/fake_path')
python_path = sub_mock.mock_calls[0][1][0][0]
self.assertEqual('/fake_path', python_path)

View File

@ -0,0 +1,5 @@
---
features:
- A new kwarg, ``python_exec`` is added to the execute() function in the
processutils module. This option is used to specify the path to the python
executable to use for prlimits enforcement.