RBD: Wrap RBD calls in native threads
librbd methods call lower level C code which runs in native thread and isn't aware about the eventlet threads hence hangs the eventlet loop until the native thread is executed. This could cause problems when we are creating multiple images with large size where one call to librados can cause the process to hang and other operations can starve for execution and error out. This patch wraps each RBD call in it's own native thread that won't affect other RBD call from executing. Change-Id: I8efb0460df9fcba050b5ce949eb10caea325c851
This commit is contained in:
parent
1f56f5a423
commit
27ab8a6aeb
|
@ -22,6 +22,7 @@ import logging
|
||||||
import math
|
import math
|
||||||
import urllib
|
import urllib
|
||||||
|
|
||||||
|
from eventlet import tpool
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_utils import encodeutils
|
from oslo_utils import encodeutils
|
||||||
from oslo_utils import units
|
from oslo_utils import units
|
||||||
|
@ -290,6 +291,9 @@ class Store(driver.Store):
|
||||||
def get_schemes(self):
|
def get_schemes(self):
|
||||||
return ('rbd',)
|
return ('rbd',)
|
||||||
|
|
||||||
|
def RBDProxy(self):
|
||||||
|
return tpool.Proxy(rbd.RBD())
|
||||||
|
|
||||||
@contextlib.contextmanager
|
@contextlib.contextmanager
|
||||||
def get_connection(self, conffile, rados_id):
|
def get_connection(self, conffile, rados_id):
|
||||||
client = rados.Rados(conffile=conffile, rados_id=rados_id)
|
client = rados.Rados(conffile=conffile, rados_id=rados_id)
|
||||||
|
@ -431,12 +435,12 @@ class Store(driver.Store):
|
||||||
|
|
||||||
:returns: `glance_store.rbd.StoreLocation` object
|
:returns: `glance_store.rbd.StoreLocation` object
|
||||||
"""
|
"""
|
||||||
librbd = rbd.RBD()
|
|
||||||
features = conn.conf_get('rbd_default_features')
|
features = conn.conf_get('rbd_default_features')
|
||||||
if ((features is None) or (int(features) == 0)):
|
if ((features is None) or (int(features) == 0)):
|
||||||
features = rbd.RBD_FEATURE_LAYERING
|
features = rbd.RBD_FEATURE_LAYERING
|
||||||
librbd.create(ioctx, image_name, size, order, old_format=False,
|
self.RBDProxy().create(ioctx, image_name, size, order,
|
||||||
features=int(features))
|
old_format=False,
|
||||||
|
features=int(features))
|
||||||
return StoreLocation({
|
return StoreLocation({
|
||||||
'fsid': fsid,
|
'fsid': fsid,
|
||||||
'pool': self.pool,
|
'pool': self.pool,
|
||||||
|
@ -496,7 +500,7 @@ class Store(driver.Store):
|
||||||
raise exceptions.InUseByStore()
|
raise exceptions.InUseByStore()
|
||||||
|
|
||||||
# Then delete image.
|
# Then delete image.
|
||||||
rbd.RBD().remove(ioctx, image_name)
|
self.RBDProxy().remove(ioctx, image_name)
|
||||||
except rbd.ImageHasSnapshots:
|
except rbd.ImageHasSnapshots:
|
||||||
log_msg = (_LW("Remove image %(img_name)s failed. "
|
log_msg = (_LW("Remove image %(img_name)s failed. "
|
||||||
"It has snapshot(s) left.") %
|
"It has snapshot(s) left.") %
|
||||||
|
|
|
@ -692,3 +692,53 @@ class TestStore(base.StoreBaseTest,
|
||||||
self.assertEqual(self.called_commands_expected,
|
self.assertEqual(self.called_commands_expected,
|
||||||
self.called_commands_actual)
|
self.called_commands_actual)
|
||||||
super(TestStore, self).tearDown()
|
super(TestStore, self).tearDown()
|
||||||
|
|
||||||
|
def test_create_image_in_native_thread(self):
|
||||||
|
# Tests that we use non-0 features from ceph.conf and cast to int.
|
||||||
|
fsid = 'fake'
|
||||||
|
features = '3'
|
||||||
|
conf_get_mock = mock.Mock(return_value=features)
|
||||||
|
conn = mock.Mock(conf_get=conf_get_mock)
|
||||||
|
ioctxt = mock.sentinel.ioctxt
|
||||||
|
name = '1'
|
||||||
|
size = 1024
|
||||||
|
order = 3
|
||||||
|
fake_proxy = mock.MagicMock()
|
||||||
|
fake_rbd = mock.MagicMock()
|
||||||
|
|
||||||
|
with mock.patch.object(rbd_store.tpool, 'Proxy') as tpool_mock, \
|
||||||
|
mock.patch.object(rbd_store.rbd, 'RBD') as rbd_mock:
|
||||||
|
tpool_mock.return_value = fake_proxy
|
||||||
|
rbd_mock.return_value = fake_rbd
|
||||||
|
location = self.store._create_image(
|
||||||
|
fsid, conn, ioctxt, name, size, order)
|
||||||
|
self.assertEqual(fsid, location.specs['fsid'])
|
||||||
|
self.assertEqual(rbd_store.DEFAULT_POOL, location.specs['pool'])
|
||||||
|
self.assertEqual(name, location.specs['image'])
|
||||||
|
self.assertEqual(rbd_store.DEFAULT_SNAPNAME,
|
||||||
|
location.specs['snapshot'])
|
||||||
|
|
||||||
|
tpool_mock.assert_called_once_with(fake_rbd)
|
||||||
|
fake_proxy.create.assert_called_once_with(ioctxt, name, size, order,
|
||||||
|
old_format=False, features=3)
|
||||||
|
|
||||||
|
def test_delete_image_in_native_thread(self):
|
||||||
|
fake_proxy = mock.MagicMock()
|
||||||
|
fake_rbd = mock.MagicMock()
|
||||||
|
fake_ioctx = mock.MagicMock()
|
||||||
|
|
||||||
|
with mock.patch.object(rbd_store.tpool, 'Proxy') as tpool_mock, \
|
||||||
|
mock.patch.object(rbd_store.rbd, 'RBD') as rbd_mock, \
|
||||||
|
mock.patch.object(self.store, 'get_connection') as mock_conn:
|
||||||
|
|
||||||
|
mock_get_conn = mock_conn.return_value.__enter__.return_value
|
||||||
|
mock_ioctx = mock_get_conn.open_ioctx.return_value.__enter__
|
||||||
|
mock_ioctx.return_value = fake_ioctx
|
||||||
|
tpool_mock.return_value = fake_proxy
|
||||||
|
rbd_mock.return_value = fake_rbd
|
||||||
|
|
||||||
|
self.store._delete_image('fake_pool', self.location.image)
|
||||||
|
|
||||||
|
tpool_mock.assert_called_once_with(fake_rbd)
|
||||||
|
fake_proxy.remove.assert_called_once_with(fake_ioctx,
|
||||||
|
self.location.image)
|
||||||
|
|
Loading…
Reference in New Issue