Fix: API returns 503 if one of the store is mis-configured

If user mis-configures one of the multiple stores defined in glance-api
configuratio file, then service returns 503 for all API requests.

Made provision to exclude faulty store and resume service normally
to function wiht other configured stores.

Closes-Bug: #1875281
Change-Id: I685a0ecbfba7cf7dbce3fd2eb84f97bd8ffbfcf3
(cherry picked from commit 98b9091129)
This commit is contained in:
Abhishek Kekane 2020-04-24 15:39:26 +00:00
parent 9693c70312
commit 17b66ab6a5
3 changed files with 19 additions and 5 deletions

View File

@ -271,10 +271,16 @@ class Store(driver.Store):
try:
client.connect(timeout=self.connect_timeout)
except rados.Error:
msg = _LE("Error connecting to ceph cluster.")
LOG.exception(msg)
raise exceptions.BackendException()
except (rados.Error, rados.ObjectNotFound) as e:
if self.backend_group and len(self.conf.enabled_backends) > 1:
reason = _("Error in store configuration: %s") % e
LOG.debug(reason)
raise exceptions.BadStoreConfiguration(
store_name=self.backend_group, reason=reason)
else:
msg = _LE("Error connecting to ceph cluster.")
LOG.exception(msg)
raise exceptions.BackendException()
try:
yield client
finally:

View File

@ -35,6 +35,9 @@ class MockRados(object):
class Error(Exception):
pass
class ObjectNotFound(Exception):
pass
class ioctx(object):
def __init__(self, *args, **kwargs):
pass
@ -442,11 +445,12 @@ class TestMultiStore(base.MultiStoreBaseTest,
@mock.patch.object(MockRados.Rados, 'connect', side_effect=MockRados.Error)
def test_rados_connect_error(self, _):
rbd_store.rados.Error = MockRados.Error
rbd_store.rados.ObjectNotFound = MockRados.ObjectNotFound
def test():
with self.store.get_connection('conffile', 'rados_id'):
pass
self.assertRaises(exceptions.BackendException, test)
self.assertRaises(exceptions.BadStoreConfiguration, test)
def test_create_image_conf_features(self):
# Tests that we use non-0 features from ceph.conf and cast to int.

View File

@ -34,6 +34,9 @@ class MockRados(object):
class Error(Exception):
pass
class ObjectNotFound(Exception):
pass
class ioctx(object):
def __init__(self, *args, **kwargs):
pass
@ -450,6 +453,7 @@ class TestStore(base.StoreBaseTest,
@mock.patch.object(MockRados.Rados, 'connect', side_effect=MockRados.Error)
def test_rados_connect_error(self, _):
rbd_store.rados.Error = MockRados.Error
rbd_store.rados.ObjectNotFound = MockRados.ObjectNotFound
def test():
with self.store.get_connection('conffile', 'rados_id'):