Merge "Add Cinder snapshot plugin implementation for karbor"

This commit is contained in:
Jenkins 2017-06-16 01:42:53 +00:00 committed by Gerrit Code Review
commit 539e3ade1e
7 changed files with 620 additions and 0 deletions

View File

@ -0,0 +1,18 @@
[provider]
name = OS Infra Provider with volume snapshot plugin
description = This provider uses OpenStack's own services (swift, cinder) as storage
id = 90d5bfea-a259-41e6-80c6-dcfcfcd9d827
plugin=karbor-volume-snapshot-plugin
bank=karbor-swift-bank-plugin
[swift_client]
swift_auth_url=http://127.0.0.1/identity
swift_user=demo
swift_key=password
swift_tenant_name=demo
[swift_bank_plugin]
lease_expire_window=120
lease_renew_window=100
lease_validity_window=100

View File

@ -0,0 +1,306 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from functools import partial
import six
from cinderclient import exceptions as cinder_exc
from oslo_config import cfg
from oslo_log import log as logging
from karbor.common import constants
from karbor import exception
from karbor.services.protection.client_factory import ClientFactory
from karbor.services.protection import protection_plugin
from karbor.services.protection.protection_plugins import utils
from karbor.services.protection.protection_plugins.volume \
import volume_snapshot_plugin_schemas as volume_schemas
LOG = logging.getLogger(__name__)
volume_snapshot_opts = [
cfg.IntOpt(
'poll_interval', default=15,
help='Poll interval for Cinder volume status.'
)
]
VOLUME_FAILURE_STATUSES = {'error', 'error_deleting', 'deleting',
'not-found'}
VOLUME_IGNORE_STATUSES = {'attaching', 'creating', 'backing-up',
'restoring-backup'}
def get_snapshot_status(cinder_client, snapshot_id):
return get_resource_status(cinder_client.volume_snapshots, snapshot_id,
'snapshot')
def get_volume_status(cinder_client, volume_id):
return get_resource_status(cinder_client.volumes, volume_id, 'volume')
def get_resource_status(resource_manager, resource_id, resource_type):
LOG.debug('Polling %(resource_type)s (id: %(resource_id)s)',
{'resource_type': resource_type, 'resource_id': resource_id})
try:
resource = resource_manager.get(resource_id)
status = resource.status
except cinder_exc.NotFound:
status = 'not-found'
LOG.debug(
'Polled %(resource_type)s (id: %(resource_id)s) status: %(status)s',
{'resource_type': resource_type, 'resource_id': resource_id,
'status': status}
)
return status
class ProtectOperation(protection_plugin.Operation):
def __init__(self, poll_interval):
super(ProtectOperation, self).__init__()
self._interval = poll_interval
def _create_snapshot(self, cinder_client, volume_id, snapshot_name,
description, force):
snapshot = cinder_client.volume_snapshots.create(
volume_id=volume_id,
name=snapshot_name,
description=description,
force=force
)
snapshot_id = snapshot.id
is_success = utils.status_poll(
partial(get_snapshot_status, cinder_client, snapshot_id),
interval=self._interval,
success_statuses={'available'},
failure_statuses={'error', 'error_deleting', 'deleting',
'not-found'},
ignore_statuses={'creating'},
ignore_unexpected=True
)
if not is_success:
try:
snapshot = cinder_client.volume_snapshots.get(snapshot_id)
except Exception:
reason = 'Unable to find volume snapshot.'
else:
reason = 'The status of snapshot is %s' % snapshot.status
raise exception.CreateResourceFailed(
name="Volume Snapshot",
reason=reason, resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE)
return snapshot_id
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
volume_id = resource.id
bank_section = checkpoint.get_resource_bank_section(volume_id)
cinder_client = ClientFactory.create_client('cinder', context)
LOG.info('Creating volume snapshot, volume_id: %s', volume_id)
bank_section.update_object('status',
constants.RESOURCE_STATUS_PROTECTING)
volume_info = cinder_client.volumes.get(volume_id)
is_success = utils.status_poll(
partial(get_volume_status, cinder_client, volume_id),
interval=self._interval,
success_statuses={'available', 'in-use', 'error_extending',
'error_restoring'},
failure_statuses=VOLUME_FAILURE_STATUSES,
ignore_statuses=VOLUME_IGNORE_STATUSES,
)
if not is_success:
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Volume Snapshot",
reason='Volume is in a error status.',
resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE,
)
resource_metadata = {
'volume_id': volume_id,
'size': volume_info.size
}
snapshot_name = parameters.get('snapshot_name', None)
description = parameters.get('description', None)
force = parameters.get('force', False)
try:
snapshot_id = self._create_snapshot(cinder_client, volume_id,
snapshot_name,
description, force)
except Exception as e:
LOG.error('Error creating snapshot (volume_id: %(volume_id)s '
': %(reason)s', {'volume_id': volume_id, 'reason': e})
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.CreateResourceFailed(
name="Volume Snapshot",
reason=e, resource_id=volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE,
)
resource_metadata['snapshot_id'] = snapshot_id
bank_section.update_object('metadata', resource_metadata)
bank_section.update_object('status',
constants.RESOURCE_STATUS_AVAILABLE)
LOG.info('Snapshot volume (volume_id: %(volume_id)s snapshot_id: '
'%(snapshot_id)s ) successfully',
{'volume_id': volume_id, 'snapshot_id': snapshot_id})
class RestoreOperation(protection_plugin.Operation):
def __init__(self, poll_interval):
super(RestoreOperation, self).__init__()
self._interval = poll_interval
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
original_volume_id = resource.id
bank_section = checkpoint.get_resource_bank_section(original_volume_id)
cinder_client = ClientFactory.create_client('cinder', context)
resource_metadata = bank_section.get_object('metadata')
restore_name = parameters.get('restore_name',
'volume-%s@%s' % (checkpoint.id,
original_volume_id))
restore_description = parameters.get('restore_description', None)
snapshot_id = resource_metadata['snapshot_id']
size = resource_metadata['size']
restore = kwargs.get('restore')
LOG.info("Restoring a volume from snapshot, "
"original_volume_id: %s", original_volume_id)
try:
volume = cinder_client.volumes.create(
size, snapshot_id=snapshot_id,
name=restore_name,
description=restore_description)
is_success = utils.status_poll(
partial(get_volume_status, cinder_client, volume.id),
interval=self._interval,
success_statuses={'available', 'in-use', 'error_extending',
'error_restoring'},
failure_statuses=VOLUME_FAILURE_STATUSES,
ignore_statuses=VOLUME_IGNORE_STATUSES,
)
if is_success is not True:
LOG.error('The status of volume is invalid. status:%s',
volume.status)
reason = 'Invalid status: %s' % volume.status
restore.update_resource_status(
constants.VOLUME_RESOURCE_TYPE,
volume.id, volume.status, reason)
restore.save()
raise exception.RestoreResourceFailed(
name="Volume Snapshot",
resource_id=original_volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE)
restore.update_resource_status(constants.VOLUME_RESOURCE_TYPE,
volume.id, volume.status)
restore.save()
except Exception as e:
LOG.error("Restore volume from snapshot failed, volume_id: %s",
original_volume_id)
raise exception.RestoreResourceFailed(
name="Volume Snapshot",
reason=e, resource_id=original_volume_id,
resource_type=constants.VOLUME_RESOURCE_TYPE)
LOG.info("Finish restoring a volume from snapshot, volume_id: %s",
original_volume_id)
class DeleteOperation(protection_plugin.Operation):
def __init__(self, poll_interval):
super(DeleteOperation, self).__init__()
self._interval = poll_interval
def on_main(self, checkpoint, resource, context, parameters, **kwargs):
resource_id = resource.id
bank_section = checkpoint.get_resource_bank_section(resource_id)
snapshot_id = None
try:
bank_section.update_object('status',
constants.RESOURCE_STATUS_DELETING)
resource_metadata = bank_section.get_object('metadata')
snapshot_id = resource_metadata['snapshot_id']
cinder_client = ClientFactory.create_client('cinder', context)
try:
snapshot = cinder_client.volume_snapshots.get(snapshot_id)
cinder_client.volume_snapshots.delete(snapshot)
except cinder_exc.NotFound:
LOG.info('Snapshot id: %s not found. Assuming deleted',
snapshot_id)
is_success = utils.status_poll(
partial(get_snapshot_status, cinder_client, snapshot_id),
interval=self._interval,
success_statuses={'deleted', 'not-found'},
failure_statuses={'error', 'error_deleting'},
ignore_statuses={'deleting'},
ignore_unexpected=True
)
if not is_success:
raise exception.NotFound()
bank_section.delete_object('metadata')
bank_section.update_object('status',
constants.RESOURCE_STATUS_DELETED)
except Exception as e:
LOG.error('Delete volume snapshot failed, snapshot_id: %s',
snapshot_id)
bank_section.update_object('status',
constants.RESOURCE_STATUS_ERROR)
raise exception.DeleteResourceFailed(
name="Volume Snapshot",
reason=six.text_type(e),
resource_id=resource_id,
resource_type=constants.VOLUME_RESOURCE_TYPE
)
class VolumeSnapshotProtectionPlugin(protection_plugin.ProtectionPlugin):
_SUPPORT_RESOURCE_TYPES = [constants.VOLUME_RESOURCE_TYPE]
def __init__(self, config=None):
super(VolumeSnapshotProtectionPlugin, self).__init__(config)
self._config.register_opts(volume_snapshot_opts,
'volume_snapshot_plugin')
self._plugin_config = self._config.volume_snapshot_plugin
self._poll_interval = self._plugin_config.poll_interval
@classmethod
def get_supported_resources_types(cls):
return cls._SUPPORT_RESOURCE_TYPES
@classmethod
def get_options_schema(cls, resources_type):
return volume_schemas.OPTIONS_SCHEMA
@classmethod
def get_restore_schema(cls, resources_type):
return volume_schemas.RESTORE_SCHEMA
@classmethod
def get_saved_info_schema(cls, resources_type):
return volume_schemas.SAVED_INFO_SCHEMA
@classmethod
def get_saved_info(cls, metadata_store, resource):
pass
def get_protect_operation(self, resource):
return ProtectOperation(self._poll_interval)
def get_restore_operation(self, resource):
return RestoreOperation(self._poll_interval)
def get_delete_operation(self, resource):
return DeleteOperation(self._poll_interval)

View File

@ -0,0 +1,64 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
OPTIONS_SCHEMA = {
"title": "Volume Snapshot Options",
"type": "object",
"properties": {
"snapshot_name": {
"type": "string",
"title": "Snapshot Name",
"description": "The name of the snapshot.",
"default": None
},
"description": {
"type": "string",
"title": "Description",
"description": "The description of the volume."
},
"force": {
"type": "boolean",
"title": "Force",
"description": "Allows or disallows snapshot of a volume when the"
"volume is attached to an instance.",
"default": False
}
},
"required": ["snapshot_name", "force"]
}
RESTORE_SCHEMA = {
"title": "Volume snapshot Restore",
"type": "object",
"properties": {
"restore_name": {
"type": "string",
"title": "Restore Volume Name",
"description": "The name of the restore volume",
"default": None
},
"restore_description": {
"type": "string",
"title": "Restore Description",
"description": "The description of the restored volume.",
"default": None
}
},
"required": ["restore_name"]
}
SAVED_INFO_SCHEMA = {
"title": "Volume Protection Saved Info",
"type": "object",
"properties": {},
"required": []
}

View File

@ -155,6 +155,8 @@ class KarborBaseTest(base.BaseTestCase):
self.provider_id_noop = 'b766f37c-d011-4026-8228-28730d734a3f'
self.provider_id_os = 'cf56bd3e-97a7-4078-b6d5-f36246333fd9'
self.provider_id_fs_bank = '6659007d-6f66-4a0f-9cb4-17d6aded0bb9'
self.provider_id_os_volume_snapshot = (
'90d5bfea-a259-41e6-80c6-dcfcfcd9d827')
def store(self, obj, close_func=None):
return self._testcase_store.store(obj, close_func)

View File

@ -212,3 +212,34 @@ class CheckpointsTest(karbor_base.KarborBaseTest):
backups = self.manila_client.share_snapshots.list(
search_opts=search_opts)
self.assertEqual(1, len(backups))
def test_checkpoint_volume_snapshot(self):
volume = self.store(objects.Volume())
volume.create(1)
plan = self.store(objects.Plan())
volume_parameter_key = "OS::Cinder::Volume#{id}".format(id=volume.id)
snapshot_name = "volume-snapshot-{id}".format(id=volume.id)
parameters = {
"OS::Cinder::Volume": {
"force": False
},
volume_parameter_key: {
"snapshot_name": snapshot_name
}
}
plan.create(self.provider_id_os_volume_snapshot, [volume, ],
parameters=parameters)
checkpoint = self.store(objects.Checkpoint())
checkpoint.create(self.provider_id_os_volume_snapshot, plan.id,
timeout=2400)
search_opts = {"volume_id": volume.id}
snapshots = self.cinder_client.volume_snapshots.list(
search_opts=search_opts)
self.assertEqual(1, len(snapshots))
search_opts = {"name": snapshot_name}
snapshots = self.cinder_client.volume_snapshots.list(
search_opts=search_opts)
self.assertEqual(1, len(snapshots))

View File

@ -0,0 +1,198 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
from karbor.common import constants
from karbor.context import RequestContext
from karbor.resource import Resource
from karbor.services.protection.bank_plugin import Bank
from karbor.services.protection.bank_plugin import BankPlugin
from karbor.services.protection.bank_plugin import BankSection
from karbor.services.protection import client_factory
from karbor.services.protection.protection_plugins. \
volume.volume_snapshot_plugin import VolumeSnapshotProtectionPlugin
from karbor.services.protection.protection_plugins.volume \
import volume_snapshot_plugin_schemas
from karbor.tests import base
import mock
from oslo_config import cfg
from oslo_config import fixture
class FakeBankPlugin(BankPlugin):
def update_object(self, key, value):
return
def get_object(self, key):
return
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
return
def delete_object(self, key):
return
def get_owner_id(self):
return
fake_bank = Bank(FakeBankPlugin())
fake_bank_section = BankSection(bank=fake_bank, section="fake")
ResourceNode = collections.namedtuple(
"ResourceNode",
["value",
"child_nodes"]
)
Volume = collections.namedtuple(
"Volume",
["status"]
)
Snapshot = collections.namedtuple(
"Snapshot",
["id", "status"]
)
def call_hooks(operation, checkpoint, resource, context, parameters, **kwargs):
def noop(*args, **kwargs):
pass
hooks = (
'on_prepare_begin',
'on_prepare_finish',
'on_main',
'on_complete',
)
for hook_name in hooks:
hook = getattr(operation, hook_name, noop)
hook(checkpoint, resource, context, parameters, **kwargs)
class CheckpointCollection(object):
def __init__(self):
self.bank_section = fake_bank_section
def get_resource_bank_section(self, resource_id):
return self.bank_section
class CinderSnapshotProtectionPluginTest(base.TestCase):
def setUp(self):
super(CinderSnapshotProtectionPluginTest, self).setUp()
plugin_config = cfg.ConfigOpts()
plugin_config_fixture = self.useFixture(fixture.Config(plugin_config))
plugin_config_fixture.load_raw_values(
group='volume_snapshot_plugin',
poll_interval=0,
)
self.plugin = VolumeSnapshotProtectionPlugin(plugin_config)
cfg.CONF.set_default('cinder_endpoint',
'http://127.0.0.1:8774/v2.1',
'cinder_client')
service_catalog = [
{'type': 'volumev3',
'endpoints': [{'publicURL': 'http://127.0.0.1:8774/v2.1/abcd'}],
},
]
self.cntxt = RequestContext(user_id='demo',
project_id='abcd',
auth_token='efgh',
service_catalog=service_catalog)
self.cinder_client = client_factory.ClientFactory.create_client(
"cinder", self.cntxt)
self.checkpoint = CheckpointCollection()
def test_get_options_schema(self):
options_schema = self.plugin.get_options_schema(
constants.VOLUME_RESOURCE_TYPE)
self.assertEqual(options_schema,
volume_snapshot_plugin_schemas.OPTIONS_SCHEMA)
def test_get_restore_schema(self):
options_schema = self.plugin.get_restore_schema(
constants.VOLUME_RESOURCE_TYPE)
self.assertEqual(options_schema,
volume_snapshot_plugin_schemas.RESTORE_SCHEMA)
def test_get_saved_info_schema(self):
options_schema = self.plugin.get_saved_info_schema(
constants.VOLUME_RESOURCE_TYPE)
self.assertEqual(options_schema,
volume_snapshot_plugin_schemas.SAVED_INFO_SCHEMA)
@mock.patch('karbor.services.protection.protection_plugins.'
'utils.status_poll')
@mock.patch('karbor.services.protection.clients.cinder.create')
def test_create_snapshot(self, mock_cinder_create, mock_status_poll):
resource = Resource(id="123",
type=constants.VOLUME_RESOURCE_TYPE,
name='fake')
fake_bank_section.update_object = mock.MagicMock()
protect_operation = self.plugin.get_protect_operation(resource)
mock_cinder_create.return_value = self.cinder_client
self.cinder_client.volumes.get = mock.MagicMock()
self.cinder_client.volumes.return_value = Volume(
status="available"
)
fake_bank_section.update_object = mock.MagicMock()
self.cinder_client.volume_snapshots.create = mock.MagicMock()
self.cinder_client.volume_snapshots.create.return_value = Snapshot(
id="1234",
status="available"
)
self.cinder_client.volume_snapshots.get = mock.MagicMock()
self.cinder_client.volume_snapshots.get.return_value = Snapshot(
id="1234",
status="available"
)
mock_status_poll.return_value = True
call_hooks(protect_operation, self.checkpoint, resource, self.cntxt,
{})
@mock.patch('karbor.services.protection.protection_plugins.'
'utils.status_poll')
@mock.patch('karbor.services.protection.clients.cinder.create')
def test_delete_snapshot(self, mock_cinder_create, mock_status_poll):
resource = Resource(id="123",
type=constants.VOLUME_RESOURCE_TYPE,
name='fake')
mock_cinder_create.return_value = self.cinder_client
self.cinder_client.volume_snapshots.get = mock.MagicMock()
self.cinder_client.volume_snapshots.get.return_value = Snapshot(
id="1234",
status="available"
)
self.cinder_client.volume_snapshots.delete = mock.MagicMock()
fake_bank_section.get_object = mock.MagicMock()
fake_bank_section.get_object.return_value = {
"snapshot_id": "1234"}
mock_status_poll.return_value = True
delete_operation = self.plugin.get_delete_operation(resource)
call_hooks(delete_operation, self.checkpoint, resource, self.cntxt,
{})
def test_get_supported_resources_types(self):
types = self.plugin.get_supported_resources_types()
self.assertEqual(types,
[constants.VOLUME_RESOURCE_TYPE])

View File

@ -43,6 +43,7 @@ karbor.protections =
karbor-swift-bank-plugin = karbor.services.protection.bank_plugins.swift_bank_plugin:SwiftBankPlugin
karbor-fs-bank-plugin = karbor.services.protection.bank_plugins.file_system_bank_plugin:FileSystemBankPlugin
karbor-volume-protection-plugin = karbor.services.protection.protection_plugins.volume.cinder_protection_plugin:CinderBackupProtectionPlugin
karbor-volume-snapshot-plugin = karbor.services.protection.protection_plugins.volume.volume_snapshot_plugin:VolumeSnapshotProtectionPlugin
karbor-image-protection-plugin = karbor.services.protection.protection_plugins.image.image_protection_plugin:GlanceProtectionPlugin
karbor-server-protection-plugin = karbor.services.protection.protection_plugins.server.nova_protection_plugin:NovaProtectionPlugin
karbor-share-protection-plugin = karbor.services.protection.protection_plugins.share.share_snapshot_plugin:ManilaSnapshotProtectionPlugin