Consistency Group Support for the Generic Driver

The Generic Driver in Manila has no concept of consistency
groups. This patch proposes to implement a loose
consistency group support mechanism for the Generic Driver,
using existing logic to snapshot shares and create shares
from snapshots.

The actions concerning creating and deleting a consistency
group are handled by the API and the Generic Driver does not
need to remember any relevant information persistently.

The actions concerning consistency group snapshots are not
simultaneous, but are rather let to run in succession.

DocImpact
Partially implements bp manila-consistency-groups

Change-Id: Ie5e51205a1154786f9057691aa82bffcafe04de7
This commit is contained in:
Goutham Pacha Ravi 2015-08-27 19:10:15 -04:00 committed by Clinton Knight
parent fa8faad187
commit 079a3d64f5
4 changed files with 461 additions and 4 deletions

View File

@ -88,7 +88,7 @@ fi
set +o errexit
cd $BASE/new/tempest
export MANILA_TEMPEST_CONCURRENCY=${MANILA_TEMPEST_CONCURRENCY:-12}
export MANILA_TEMPEST_CONCURRENCY=${MANILA_TEMPEST_CONCURRENCY:-6}
export MANILA_TESTS=${MANILA_TESTS:-'manila_tempest_tests.tests.api'}
if [[ "$JOB_NAME" =~ "scenario" ]]; then

View File

@ -108,7 +108,7 @@ def ensure_server(f):
raise exception.ManilaException(
_("Share server handling is not available. "
"But 'share_server' was provided. '%s'. "
"Share network should not be used.") % server['id'])
"Share network should not be used.") % server.get('id'))
elif not server:
raise exception.ManilaException(
_("Share server handling is enabled. But 'share_server' "
@ -556,7 +556,9 @@ class GenericShareDriver(driver.ExecuteMixin, driver.ShareDriver):
data = dict(
share_backend_name=self.backend_name,
storage_protocol='NFS_CIFS',
reserved_percentage=(self.configuration.reserved_share_percentage))
reserved_percentage=self.configuration.reserved_share_percentage,
consistency_group_support='pool',
)
super(GenericShareDriver, self)._update_share_stats(data)
@ensure_server
@ -942,6 +944,180 @@ class GenericShareDriver(driver.ExecuteMixin, driver.ShareDriver):
return size
@ensure_server
def create_consistency_group(self, context, cg_dict, share_server=None):
"""Creates a consistency group.
Since we are faking the CG object, apart from verifying if the
share_server is valid, we do nothing else here.
"""
LOG.debug('Created a Consistency Group with ID: %s.', cg_dict['id'])
msg = _LW('The Generic driver has no means to guarantee consistency '
'group snapshots are actually consistent. This '
'implementation is for reference and testing purposes only.')
LOG.warning(msg)
def delete_consistency_group(self, context, cg_dict, share_server=None):
"""Deletes a consistency group.
Since we are faking the CG object we do nothing here.
"""
LOG.debug('Deleted the consistency group with ID %s.', cg_dict['id'])
def _cleanup_cg_share_snapshot(self, context, share_snapshot,
share_server):
"""Deletes the snapshot of a share belonging to a consistency group."""
try:
self.delete_snapshot(context, share_snapshot, share_server)
except exception.ManilaException:
msg = _LE('Could not delete CG Snapshot %(snap)s '
'for share %(share)s.')
LOG.error(msg % {
'snap': share_snapshot['id'],
'share': share_snapshot['share_id'],
})
raise
@ensure_server
def create_cgsnapshot(self, context, snap_dict, share_server=None):
"""Creates a consistency group snapshot one or more shares."""
LOG.debug('Attempting to create a CG snapshot %s.' % snap_dict['id'])
msg = _LW('The Consistency Group Snapshot being created is '
'not expected to be consistent. This implementation is '
'for reference and testing purposes only.')
LOG.warning(msg)
cg_members = snap_dict.get('cgsnapshot_members', [])
if not cg_members:
LOG.warning(_LW('No shares in Consistency Group to Create CG '
'snapshot.'))
else:
share_snapshots = []
for member in cg_members:
share_snapshot = {
'share_id': member['share_id'],
'id': member['id'],
}
try:
self.create_snapshot(context, share_snapshot, share_server)
share_snapshots.append(share_snapshot)
except exception.ManilaException as e:
msg = _LE('Could not create CG Snapshot. Failed '
'to create share snapshot %(snap)s for '
'share %(share)s.')
LOG.exception(msg % {
'snap': share_snapshot['id'],
'share': share_snapshot['share_id']
})
# clean up any share snapshots previously created
LOG.debug('Attempting to clean up snapshots due to '
'failure...')
for share_snapshot in share_snapshots:
self._cleanup_cg_share_snapshot(context,
share_snapshot,
share_server)
raise e
LOG.debug('Successfully created CG snapshot %s.' % snap_dict['id'])
return None, None
@ensure_server
def delete_cgsnapshot(self, context, snap_dict, share_server=None):
"""Deletes a consistency group snapshot."""
cg_members = snap_dict.get('cgsnapshot_members', [])
LOG.debug('Deleting CG snapshot %s.' % snap_dict['id'])
for member in cg_members:
share_snapshot = {
'share_id': member['share_id'],
'id': member['id'],
}
self._cleanup_cg_share_snapshot(context,
share_snapshot,
share_server)
LOG.debug('Deleted CG snapshot %s.' % snap_dict['id'])
return None, None
@ensure_server
def create_consistency_group_from_cgsnapshot(self, context, cg_dict,
cgsnapshot_dict,
share_server=None):
"""Creates a consistency group from an existing CG snapshot."""
# Ensure that the consistency group snapshot has members
if not cgsnapshot_dict['cgsnapshot_members']:
return None, None
clone_list = self._collate_cg_snapshot_info(cg_dict, cgsnapshot_dict)
share_update_list = list()
LOG.debug('Creating consistency group from CG snapshot %s.',
cgsnapshot_dict['id'])
for clone in clone_list:
kwargs = {}
if self.driver_handles_share_servers:
kwargs['share_server'] = share_server
export_location = (
self.create_share_from_snapshot(
context,
clone['share'],
clone['snapshot'],
**kwargs))
share_update_list.append({
'id': clone['share']['id'],
'export_locations': export_location,
})
return None, share_update_list
def _collate_cg_snapshot_info(self, cg_dict, cgsnapshot_dict):
"""Collate the data for a clone of the CG snapshot.
Given two data structures, a CG snapshot (cgsnapshot_dict) and a new
CG to be cloned from the snapshot (cg_dict), match up both
structures into a list of dicts (share & snapshot) suitable for use
by existing method that clones individual share snapshots.
"""
clone_list = list()
for share in cg_dict['shares']:
clone_info = {'share': share}
for cgsnapshot_member in cgsnapshot_dict['cgsnapshot_members']:
if (share['source_cgsnapshot_member_id'] ==
cgsnapshot_member['id']):
clone_info['snapshot'] = {
'id': cgsnapshot_member['id'],
}
break
if len(clone_info) != 2:
msg = _("Invalid data supplied for creating consistency "
"group from CG snapshot %s.") % cgsnapshot_dict['id']
raise exception.InvalidConsistencyGroup(reason=msg)
clone_list.append(clone_info)
return clone_list
class NASHelperBase(object):
"""Interface to work with share."""

View File

@ -1269,7 +1269,7 @@ class ShareManager(manager.SchedulerDependentManager):
share_network_id = group_ref.get('share_network_id', None)
share_server = None
if parent_share_server_id:
if parent_share_server_id and self.driver.driver_handles_share_servers:
share_server = self.db.share_server_get(context,
parent_share_server_id)
share_network_id = share_server['share_network_id']

View File

@ -56,6 +56,96 @@ def get_fake_manage_share():
}
def get_fake_snap_dict():
snap_dict = {
'status': 'available',
'project_id': '13c0be6290934bd98596cfa004650049',
'user_id': 'a0314a441ca842019b0952224aa39192',
'description': None,
'deleted': '0',
'created_at': '2015-08-10 00:05:58',
'updated_at': '2015-08-10 00:05:58',
'consistency_group_id': '4b04fdc3-00b9-4909-ba1a-06e9b3f88b67',
'cgsnapshot_members': [
{
'status': 'available',
'share_type_id': '1a9ed31e-ee70-483d-93ba-89690e028d7f',
'share_id': 'e14b5174-e534-4f35-bc4f-fe81c1575d6f',
'user_id': 'a0314a441ca842019b0952224aa39192',
'deleted': 'False',
'created_at': '2015-08-10 00:05:58',
'share': {
'id': '03e2f06e-14f2-45a5-9631-0949d1937bd8',
'deleted': False,
},
'updated_at': '2015-08-10 00:05:58',
'share_proto': 'NFS',
'project_id': '13c0be6290934bd98596cfa004650049',
'cgsnapshot_id': 'f6aa3b59-57eb-421e-965c-4e182538e36a',
'deleted_at': None,
'id': '03e2f06e-14f2-45a5-9631-0949d1937bd8',
'size': 1,
},
],
'deleted_at': None,
'id': 'f6aa3b59-57eb-421e-965c-4e182538e36a',
'name': None,
}
return snap_dict
def get_fake_cg_dict():
cg_dict = {
'status': 'creating',
'project_id': '13c0be6290934bd98596cfa004650049',
'user_id': 'a0314a441ca842019b0952224aa39192',
'description': None,
'deleted': 'False',
'created_at': '2015-08-10 00:07:58',
'updated_at': None,
'source_cgsnapshot_id': 'f6aa3b59-57eb-421e-965c-4e182538e36a',
'host': 'openstack2@cmodeSSVMNFS',
'deleted_at': None,
'shares': [
{
'id': '02a32f06e-14f2-45a5-9631-7483f1937bd8',
'deleted': False,
'source_cgsnapshot_member_id':
'03e2f06e-14f2-45a5-9631-0949d1937bd8',
},
],
'share_types': [
{
'id': 'f6aa3b56-45a5-9631-02a32f06e1937b',
'deleted': False,
'consistency_group_id': '4b04fdc3-00b9-4909-ba1a-06e9b3f88b67',
'share_type_id': '1a9ed31e-ee70-483d-93ba-89690e028d7f',
},
],
'id': 'eda52174-0442-476d-9694-a58327466c14',
'name': None
}
return cg_dict
def get_fake_collated_cg_snap_info():
fake_collated_cg_snap_info = [
{
'share': {
'id': '02a32f06e-14f2-45a5-9631-7483f1937bd8',
'deleted': False,
'source_cgsnapshot_member_id':
'03e2f06e-14f2-45a5-9631-0949d1937bd8',
},
'snapshot': {
'id': '03e2f06e-14f2-45a5-9631-0949d1937bd8',
},
},
]
return fake_collated_cg_snap_info
@ddt.ddt
class GenericShareDriverTestCase(test.TestCase):
"""Tests GenericShareDriver."""
@ -127,6 +217,10 @@ class GenericShareDriverTestCase(test.TestCase):
self.access = fake_share.fake_access()
self.snapshot = fake_share.fake_snapshot()
self.mock_object(time, 'sleep')
self.mock_debug_log = self.mock_object(generic.LOG, 'debug')
self.mock_warning_log = self.mock_object(generic.LOG, 'warning')
self.mock_error_log = self.mock_object(generic.LOG, 'error')
self.mock_exception_log = self.mock_object(generic.LOG, 'exception')
def test_do_setup(self):
self.mock_object(volume, 'API')
@ -1645,6 +1739,193 @@ class GenericShareDriverTestCase(test.TestCase):
self.assertEqual(result, actual_result)
def test_create_consistency_group(self):
FAKE_SNAP_DICT = get_fake_snap_dict()
result = self._driver.create_consistency_group(
self._context, FAKE_SNAP_DICT, share_server=self.server)
self.assertEqual(1, self.mock_debug_log.call_count)
self.assertEqual(1, self.mock_warning_log.call_count)
self.assertIsNone(result)
def test_delete_consistency_group(self):
FAKE_SNAP_DICT = get_fake_snap_dict()
result = self._driver.delete_consistency_group(
self._context, FAKE_SNAP_DICT, share_server=self.server)
self.assertEqual(1, self.mock_debug_log.call_count)
self.assertIsNone(result)
def test_create_cgsnapshot_no_cg_members(self):
FAKE_SNAP_DICT = dict(get_fake_snap_dict(), cgsnapshot_members=[])
mock_snapshot_creation = self.mock_object(generic.GenericShareDriver,
'create_snapshot')
result = self._driver.create_cgsnapshot(
self._context, FAKE_SNAP_DICT, share_server=self.server)
self.assertEqual(1, self.mock_debug_log.call_count)
self.assertEqual(2, self.mock_warning_log.call_count)
self.assertFalse(mock_snapshot_creation.called)
self.assertEqual((None, None), result)
@ddt.data(
{
'delete_snap_side_effect': None,
'expected_error_log_call_count': 0,
},
{
'delete_snap_side_effect': exception.ManilaException,
'expected_error_log_call_count': 1,
}
)
@ddt.unpack
def test_create_cgsnapshot_manila_exception_on_create_and_delete(
self, delete_snap_side_effect, expected_error_log_call_count):
FAKE_SNAP_DICT = get_fake_snap_dict()
# Append another fake share
FAKE_SHARE = dict(FAKE_SNAP_DICT['cgsnapshot_members'][0])
FAKE_SNAP_DICT['cgsnapshot_members'].append(FAKE_SHARE)
self.mock_object(generic.GenericShareDriver,
'create_snapshot',
mock.Mock(side_effect=[
None,
exception.ManilaException,
]))
self.mock_object(generic.GenericShareDriver,
'delete_snapshot',
mock.Mock(side_effect=delete_snap_side_effect))
self.assertRaises(exception.ManilaException,
self._driver.create_cgsnapshot,
self._context, FAKE_SNAP_DICT,
share_server=self.server)
self.assertEqual(2, self.mock_debug_log.call_count)
self.assertEqual(1, self.mock_warning_log.call_count)
self.assertEqual(1, self.mock_exception_log.call_count)
self.assertEqual(expected_error_log_call_count,
self.mock_error_log.call_count)
def test_create_cgsnapshot(self):
FAKE_SNAP_DICT = get_fake_snap_dict()
FAKE_SHARE_SNAPSHOT = {
'share_id': 'e14b5174-e534-4f35-bc4f-fe81c1575d6f',
'id': '03e2f06e-14f2-45a5-9631-0949d1937bd8',
}
mock_snapshot_creation = self.mock_object(generic.GenericShareDriver,
'create_snapshot')
result = self._driver.create_cgsnapshot(
self._context, FAKE_SNAP_DICT, share_server=self.server)
mock_snapshot_creation.assert_called_once_with(self._context,
FAKE_SHARE_SNAPSHOT,
self.server)
self.assertEqual(2, self.mock_debug_log.call_count)
self.assertEqual(1, self.mock_warning_log.call_count)
self.assertFalse(self.mock_error_log.called)
self.assertEqual((None, None), result)
def test_delete_cgsnapshot_manila_exception(self):
FAKE_SNAP_DICT = get_fake_snap_dict()
self.mock_object(generic.GenericShareDriver,
'delete_snapshot',
mock.Mock(side_effect=exception.ManilaException))
self.assertRaises(exception.ManilaException,
self._driver.delete_cgsnapshot,
self._context, FAKE_SNAP_DICT,
share_server=self.server)
self.assertEqual(1, self.mock_error_log.call_count)
def test_delete_cgsnapshot(self):
FAKE_SNAP_DICT = get_fake_snap_dict()
FAKE_SHARE_SNAPSHOT = {
'share_id': 'e14b5174-e534-4f35-bc4f-fe81c1575d6f',
'id': '03e2f06e-14f2-45a5-9631-0949d1937bd8',
}
mock_snapshot_creation = self.mock_object(generic.GenericShareDriver,
'delete_snapshot')
result = self._driver.delete_cgsnapshot(
self._context, FAKE_SNAP_DICT, share_server=self.server)
mock_snapshot_creation.assert_called_once_with(self._context,
FAKE_SHARE_SNAPSHOT,
self.server)
self.assertEqual(2, self.mock_debug_log.call_count)
self.assertEqual((None, None), result)
def test_create_consistency_group_from_cgsnapshot_no_members(self):
FAKE_CG_DICT = get_fake_cg_dict()
FAKE_CGSNAP_DICT = dict(get_fake_snap_dict(), cgsnapshot_members=[])
mock_share_creation = self.mock_object(generic.GenericShareDriver,
'create_share_from_snapshot')
result = self._driver.create_consistency_group_from_cgsnapshot(
self._context, FAKE_CG_DICT, FAKE_CGSNAP_DICT,
share_server=self.server)
self.assertFalse(self.mock_debug_log.called)
self.assertFalse(mock_share_creation.called)
self.assertEqual((None, None), result)
def test_create_consistency_group_from_cgsnapshot(self):
FAKE_CG_DICT = get_fake_cg_dict()
FAKE_CGSNAP_DICT = get_fake_snap_dict()
FAKE_COLLATED_INFO = get_fake_collated_cg_snap_info()
FAKE_SHARE_UPDATE_LIST = [
{
'id': '02a32f06e-14f2-45a5-9631-7483f1937bd8',
'export_locations': 'xyzzy',
}
]
self.mock_object(generic.GenericShareDriver,
'_collate_cg_snapshot_info',
mock.Mock(return_value=FAKE_COLLATED_INFO))
mock_share_creation = self.mock_object(generic.GenericShareDriver,
'create_share_from_snapshot',
mock.Mock(return_value='xyzzy'))
result = self._driver.create_consistency_group_from_cgsnapshot(
self._context, FAKE_CG_DICT, FAKE_CGSNAP_DICT,
share_server=self.server)
self.assertEqual((None, FAKE_SHARE_UPDATE_LIST), result)
self.assertEqual(1, self.mock_debug_log.call_count)
mock_share_creation.assert_called_once_with(
self._context,
FAKE_COLLATED_INFO[0]['share'],
FAKE_COLLATED_INFO[0]['snapshot'],
share_server=self.server
)
def test_collate_cg_snapshot_info_invalid_cg(self):
FAKE_CG_DICT = get_fake_cg_dict()
FAKE_CGSNAP_DICT = dict(get_fake_snap_dict(), cgsnapshot_members=[])
self.assertRaises(exception.InvalidConsistencyGroup,
self._driver._collate_cg_snapshot_info,
FAKE_CG_DICT,
FAKE_CGSNAP_DICT)
def test_collate_cg_snapshot(self):
FAKE_CG_DICT = get_fake_cg_dict()
FAKE_CGSNAP_DICT = get_fake_snap_dict()
FAKE_COLLATED_INFO = get_fake_collated_cg_snap_info()
result = self._driver._collate_cg_snapshot_info(
FAKE_CG_DICT, FAKE_CGSNAP_DICT)
self.assertEqual(FAKE_COLLATED_INFO, result)
@generic.ensure_server
def fake(driver_instance, context, share_server=None):