Retry to fetch paginated volumes if we get 404 for next link
When we get a volume list, it's possible for a volume to disappear causing the pagination to bork. When that happens we retry to get the list from scratch for 5 times. If all the attempts fail we just return what we found. Change-Id: Ia88d1d8a6b3558f7d5d364a9c78bbf834836d3f7 Signed-off-by: Rosario Di Somma <rosario.disomma@dreamhost.com>
This commit is contained in:
Rosario Di Somma
parent
77b2a441cf
commit
6b325282d3
|
|
@ -1700,21 +1700,53 @@ class OpenStackCloud(
|
|||
|
||||
"""
|
||||
def _list(data):
|
||||
volumes.extend(meta.get_and_munchify('volumes', data))
|
||||
volumes.extend(data.get('volumes', []))
|
||||
endpoint = None
|
||||
for l in data.get('volumes_links', []):
|
||||
if 'rel' in l and 'next' == l['rel']:
|
||||
endpoint = l['href']
|
||||
break
|
||||
if endpoint:
|
||||
_list(self._volume_client.get(endpoint))
|
||||
try:
|
||||
_list(self._volume_client.get(endpoint))
|
||||
except OpenStackCloudURINotFound:
|
||||
# Catch and re-raise here because we are making recursive
|
||||
# calls and we just have context for the log here
|
||||
self.log.debug(
|
||||
"While listing volumes, could not find next link"
|
||||
" {link}.".format(link=data))
|
||||
raise
|
||||
|
||||
if not cache:
|
||||
warnings.warn('cache argument to list_volumes is deprecated. Use '
|
||||
'invalidate instead.')
|
||||
volumes = []
|
||||
_list(self._volume_client.get('/volumes/detail'))
|
||||
return self._normalize_volumes(volumes)
|
||||
|
||||
# Fetching paginated volumes can fails for several reasons, if
|
||||
# something goes wrong we'll have to start fetching volumes from
|
||||
# scratch
|
||||
attempts = 5
|
||||
for _ in range(attempts):
|
||||
volumes = []
|
||||
data = self._volume_client.get('/volumes/detail')
|
||||
if 'volumes_links' not in data:
|
||||
# no pagination needed
|
||||
volumes.extend(data.get('volumes', []))
|
||||
break
|
||||
|
||||
try:
|
||||
_list(data)
|
||||
break
|
||||
except OpenStackCloudURINotFound:
|
||||
pass
|
||||
else:
|
||||
self.log.debug(
|
||||
"List volumes failed to retrieve all volumes after"
|
||||
" {attempts} attempts. Returning what we found.".format(
|
||||
attempts=attempts))
|
||||
# list volumes didn't complete succesfully so just return what
|
||||
# we found
|
||||
return self._normalize_volumes(
|
||||
meta.get_and_munchify(key=None, data=volumes))
|
||||
|
||||
@_utils.cache_on_arguments()
|
||||
def list_volume_types(self, get_extra=True):
|
||||
|
|
|
|||
|
|
@ -319,3 +319,96 @@ class TestVolume(base.RequestsMockTestCase):
|
|||
self.cloud._normalize_volume(vol2)],
|
||||
self.cloud.list_volumes())
|
||||
self.assert_calls()
|
||||
|
||||
def test_list_volumes_with_pagination_next_link_fails_once(self):
|
||||
vol1 = meta.obj_to_munch(fakes.FakeVolume('01', 'available', 'vol1'))
|
||||
vol2 = meta.obj_to_munch(fakes.FakeVolume('02', 'available', 'vol2'))
|
||||
self.register_uris([
|
||||
dict(method='GET',
|
||||
uri=self.get_mock_url(
|
||||
'volumev2', 'public',
|
||||
append=['volumes', 'detail']),
|
||||
json={
|
||||
'volumes': [vol1],
|
||||
'volumes_links': [
|
||||
{'href': self.get_mock_url(
|
||||
'volumev2', 'public',
|
||||
append=['volumes', 'detail'],
|
||||
qs_elements=['marker=01']),
|
||||
'rel': 'next'}]}),
|
||||
dict(method='GET',
|
||||
uri=self.get_mock_url(
|
||||
'volumev2', 'public',
|
||||
append=['volumes', 'detail'],
|
||||
qs_elements=['marker=01']),
|
||||
status_code=404),
|
||||
dict(method='GET',
|
||||
uri=self.get_mock_url(
|
||||
'volumev2', 'public',
|
||||
append=['volumes', 'detail']),
|
||||
json={
|
||||
'volumes': [vol1],
|
||||
'volumes_links': [
|
||||
{'href': self.get_mock_url(
|
||||
'volumev2', 'public',
|
||||
append=['volumes', 'detail'],
|
||||
qs_elements=['marker=01']),
|
||||
'rel': 'next'}]}),
|
||||
dict(method='GET',
|
||||
uri=self.get_mock_url(
|
||||
'volumev2', 'public',
|
||||
append=['volumes', 'detail'],
|
||||
qs_elements=['marker=01']),
|
||||
json={
|
||||
'volumes': [vol2],
|
||||
'volumes_links': [
|
||||
{'href': self.get_mock_url(
|
||||
'volumev2', 'public',
|
||||
append=['volumes', 'detail'],
|
||||
qs_elements=['marker=02']),
|
||||
'rel': 'next'}]}),
|
||||
|
||||
dict(method='GET',
|
||||
uri=self.get_mock_url(
|
||||
'volumev2', 'public',
|
||||
append=['volumes', 'detail'],
|
||||
qs_elements=['marker=02']),
|
||||
json={'volumes': []})])
|
||||
self.assertEqual(
|
||||
[self.cloud._normalize_volume(vol1),
|
||||
self.cloud._normalize_volume(vol2)],
|
||||
self.cloud.list_volumes())
|
||||
self.assert_calls()
|
||||
|
||||
def test_list_volumes_with_pagination_next_link_fails_all_attempts(self):
|
||||
vol1 = meta.obj_to_munch(fakes.FakeVolume('01', 'available', 'vol1'))
|
||||
uris = []
|
||||
attempts = 5
|
||||
for i in range(attempts):
|
||||
uris.extend([
|
||||
dict(method='GET',
|
||||
uri=self.get_mock_url(
|
||||
'volumev2', 'public',
|
||||
append=['volumes', 'detail']),
|
||||
json={
|
||||
'volumes': [vol1],
|
||||
'volumes_links': [
|
||||
{'href': self.get_mock_url(
|
||||
'volumev2', 'public',
|
||||
append=['volumes', 'detail'],
|
||||
qs_elements=['marker=01']),
|
||||
'rel': 'next'}]}),
|
||||
dict(method='GET',
|
||||
uri=self.get_mock_url(
|
||||
'volumev2', 'public',
|
||||
append=['volumes', 'detail'],
|
||||
qs_elements=['marker=01']),
|
||||
status_code=404)])
|
||||
self.register_uris(uris)
|
||||
# Check that found volumes are returned even if pagination didn't
|
||||
# complete because call to get next link 404'ed for all the allowed
|
||||
# attempts
|
||||
self.assertEqual(
|
||||
[self.cloud._normalize_volume(vol1)],
|
||||
self.cloud.list_volumes())
|
||||
self.assert_calls()
|
||||
|
|
|
|||
Loading…
Reference in New Issue