HNAS: Fix concurrency error when managing snapshots

During manage snapshot operation, if HNAS is already running a command
'path-to-object-number' it may cause concurrency issues, since HNAS
can run only one 'path-to-object-number' command at a time.

Fixing it by adding a retry when this message is returned.

Change-Id: If0f0b2d6f7e4ba3203d10c549181a3d31113624e
Closes-bug: #1660288
This commit is contained in:
Alyson Rosa 2017-01-30 10:27:53 -02:00
parent 595a2bd73c
commit 2295151784
4 changed files with 34 additions and 0 deletions

View File

@ -812,6 +812,10 @@ class HNASConnException(ManilaException):
message = _("HNAS Connection Exception: %(msg)s")
class HNASSSCIsBusy(ManilaException):
message = _("HNAS SSC is busy and cannot execute the command: %(msg)s")
class HNASItemNotFoundException(StorageResourceNotFound):
message = _("HNAS Item Not Found Exception: %(msg)s")

View File

@ -309,12 +309,18 @@ class HNASSSHBackend(object):
def delete_directory(self, path):
self._locked_selectfs('delete', path)
@mutils.retry(exception=exception.HNASSSCIsBusy, wait_random=True,
retries=5)
def check_snapshot(self, path):
command = ['path-to-object-number', '-f', self.fs_name, path]
try:
self._execute(command)
except processutils.ProcessExecutionError as e:
if 'path-to-object-number is currently running' in e.stdout:
msg = (_("SSC command path-to-object-number for path %s "
"is currently busy.") % path)
raise exception.HNASSSCIsBusy(msg=msg)
if 'Unable to locate component:' in e.stdout:
LOG.debug("Cannot find %(path)s: %(out)s",
{'path': path, 'out': e.stdout})

View File

@ -936,6 +936,26 @@ class HNASSSHTestCase(test.TestCase):
self.assertTrue(out)
self._driver_ssh._execute.assert_called_with(check_snap_args)
def test_check_snapshot_retry(self):
error_msg = ("Unable to run path-to-object-number as "
"path-to-object-number is currently running on volume "
"39.")
path = ("/snapshots/" + self.snapshot['share_id'] + "/" +
self.snapshot['id'])
check_snap_args = ['path-to-object-number', '-f', self.fs_name, path]
self.mock_object(time, "sleep", mock.Mock())
self.mock_object(ssh.HNASSSHBackend, '_execute',
mock.Mock(side_effect=[putils.ProcessExecutionError(
stdout=error_msg), putils.ProcessExecutionError(
stdout=error_msg), 'Object number: 0x45a4']))
out = self._driver_ssh.check_snapshot(path)
self.assertIs(True, out)
self._driver_ssh._execute.assert_called_with(check_snap_args)
def test_check_inexistent_snapshot(self):
path = "/path/snap1/snapshot07-08-2016"

View File

@ -0,0 +1,4 @@
---
fixes:
- Fixed HNAS driver error when managing snapshots caused by
concurrency in backend.