openstack
/
ironic-lib
Code Issues Proposed changes

Limit the number of malloc arenas for qemu-img convert 

Under heavy load, the qemu-img convert method may attempt to create
additional malloc arenas to complete its work. By default this number
is 8 * ncpu * which at about 250 MB of memory which can be consumed
in this mannor should the.

Since this is only something which should realistically occur under
heavy cross-thread load where memory locking prevents a thread from
unlocking a range of memory, (such as what can happen with hypervisors
and VMs inside those hypervisors), then a new arena gets created, and
the memory consumption spikes.

More information is avialable at:
https://www.gnu.org/software/libc/manual/html_node/Memory-Allocation-Tunables.html

Change-Id: I8b2b490f2cc0ac5f47c3aaaaa249ce59db6602d6
Story: 2008928
Task: 42529
This commit is contained in:
Julia Kreger 2021-05-26 13:36:59 -07:00
parent 36b8486419
commit 76cbb9623f
2 changed files with 37 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
ironic_lib/disk_utils.py
View File

@ -488,10 +488,23 @@ def convert_image(source, dest, out_format, run_as_root=False, cache=None,
if out_of_order:
cmd.append('-W')
cmd += [source, dest]
# NOTE(TheJulia): Staticly set the MALLOC_ARENA_MAX to prevent leaking
# and the creation of new malloc arenas which will consume the system
# memory. If limited to 1, qemu-img consumes ~250 MB of RAM, but when
# another thread tries to access a locked section of memory in use with
# another thread, then by default a new malloc arena is created,
# which essentially balloons the memory requirement of the machine.
# Default for qemu-img is 8 * nCPU * ~250MB (based on defaults +
# thread/code/process/library overhead. In other words, 64 GB. Limiting
# this to 3 keeps the memory utilization in happy cases below the overall
# threshold which is in place in case a malicious image is attempted to
# be passed through qemu-img.
env_vars = {'MALLOC_ARENA_MAX': '3'}
try:
utils.execute(*cmd, run_as_root=run_as_root,
prlimit=_qemu_img_limits(),
use_standard_locale=True)
use_standard_locale=True,
env_variables=env_vars)
except processutils.ProcessExecutionError as e:
if ('Resource temporarily unavailable' in e.stderr
or 'Cannot allocate memory' in e.stderr):

38
ironic_lib/tests/test_disk_utils.py
View File

@ -1093,22 +1093,26 @@ class OtherFunctionTestCase(base.IronicLibTestCase):
@mock.patch.object(utils, 'execute', autospec=True)
def test_convert_image(self, execute_mock):
disk_utils.convert_image('source', 'dest', 'out_format')
execute_mock.assert_called_once_with('qemu-img', 'convert', '-O',
'out_format', 'source', 'dest',
run_as_root=False,
prlimit=mock.ANY,
use_standard_locale=True)
execute_mock.assert_called_once_with(
'qemu-img', 'convert', '-O',
'out_format', 'source', 'dest',
run_as_root=False,
prlimit=mock.ANY,
use_standard_locale=True,
env_variables={'MALLOC_ARENA_MAX': '3'})
@mock.patch.object(utils, 'execute', autospec=True)
def test_convert_image_flags(self, execute_mock):
disk_utils.convert_image('source', 'dest', 'out_format',
cache='directsync', out_of_order=True)
execute_mock.assert_called_once_with('qemu-img', 'convert', '-O',
'out_format', '-t', 'directsync',
'-W', 'source', 'dest',
run_as_root=False,
prlimit=mock.ANY,
use_standard_locale=True)
execute_mock.assert_called_once_with(
'qemu-img', 'convert', '-O',
'out_format', '-t', 'directsync',
'-W', 'source', 'dest',
run_as_root=False,
prlimit=mock.ANY,
use_standard_locale=True,
env_variables={'MALLOC_ARENA_MAX': '3'})
@mock.patch.object(utils, 'execute', autospec=True)
def test_convert_image_retries(self, execute_mock):
@ -1124,7 +1128,8 @@ class OtherFunctionTestCase(base.IronicLibTestCase):
'out_format', 'source', 'dest',
run_as_root=False,
prlimit=mock.ANY,
use_standard_locale=True)
use_standard_locale=True,
env_variables={'MALLOC_ARENA_MAX': '3'})
execute_mock.assert_has_calls([
convert_call,
mock.call('sync'),
@ -1147,7 +1152,8 @@ class OtherFunctionTestCase(base.IronicLibTestCase):
'out_format', 'source', 'dest',
run_as_root=False,
prlimit=mock.ANY,
use_standard_locale=True)
use_standard_locale=True,
env_variables={'MALLOC_ARENA_MAX': '3'})
execute_mock.assert_has_calls([
convert_call,
mock.call('sync'),
@ -1173,7 +1179,8 @@ class OtherFunctionTestCase(base.IronicLibTestCase):
'out_format', 'source', 'dest',
run_as_root=False,
prlimit=mock.ANY,
use_standard_locale=True)
use_standard_locale=True,
env_variables={'MALLOC_ARENA_MAX': '3'})
execute_mock.assert_has_calls([
convert_call,
mock.call('sync'),
@ -1196,7 +1203,8 @@ class OtherFunctionTestCase(base.IronicLibTestCase):
'out_format', 'source', 'dest',
run_as_root=False,
prlimit=mock.ANY,
use_standard_locale=True)
use_standard_locale=True,
env_variables={'MALLOC_ARENA_MAX': '3'})
execute_mock.assert_has_calls([
convert_call,
])