Fix replication failure when Swift isn't available

* add Swift token verification in create instance method in Trove API
    when slave_id is given
  * catch ConnectionError exception in verify_swift_auth_token
  * add Swift token verification to guestagent before backup and restore
  * add new exception representing Swift connection error
  * set missing fault information when replication snapshot fails
  * mock verify auth token method in replication and restore unit tests

Closes-Bug: #1395523

Change-Id: I6a21ba2ba890a82875f9b6dae3c6b93bc9fdb4b0
Signed-off-by: Dariusz Krol <d.krol@samsung.com>
This commit is contained in:
Dariusz Krol 2018-07-27 14:18:28 +02:00
parent 56050fce95
commit 4d358c8f5d
6 changed files with 30 additions and 1 deletions

View File

@ -15,6 +15,7 @@
"""Model classes that form the core of snapshots functionality."""
from oslo_log import log as logging
from requests.exceptions import ConnectionError
from sqlalchemy import desc
from swiftclient.client import ClientException
@ -29,7 +30,6 @@ from trove.db.models import DatabaseModelBase
from trove.quota.quota import run_with_quotas
from trove.taskmanager import api
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
@ -291,6 +291,8 @@ class Backup(object):
raise exception.SwiftAuthError(tenant_id=context.tenant)
except exception.NoServiceEndpoint:
raise exception.SwiftNotFound(tenant_id=context.tenant)
except ConnectionError:
raise exception.SwiftConnectionError()
def persisted_models():

View File

@ -461,6 +461,10 @@ class SwiftNotFound(TroveError):
message = _("Swift is disabled for tenant %(tenant_id)s.")
class SwiftConnectionError(TroveError):
message = _("Cannot connect to Swift.")
class DatabaseForUserNotInDatabaseListError(TroveError):
message = _("The request indicates that user %(user)s should have access "
"to database %(database)s, but database %(database)s is not "

View File

@ -361,6 +361,7 @@ class Controller(object):
webob.exc.HTTPServerError: [
exception.VolumeCreationFailure,
exception.UpdateGuestError,
exception.SwiftConnectionError,
],
webob.exc.HTTPNotImplemented: [
exception.VolumeNotSupported,

View File

@ -923,6 +923,8 @@ class Instance(BuiltInstance):
target_size = flavor.ephemeral # ephemeral_Storage
if backup_id:
Backup.verify_swift_auth_token(context)
call_args['backup_id'] = backup_id
backup_info = Backup.get_by_id(context, backup_id)
if not backup_info.is_done_successfuly:
@ -946,6 +948,8 @@ class Instance(BuiltInstance):
datastore2=datastore.name)
if slave_of_id:
Backup.verify_swift_auth_token(context)
if databases or users:
raise exception.ReplicaCreateWithUsersDatabasesError()
call_args['replica_of'] = slave_of_id

View File

@ -625,6 +625,12 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
'replica': self.id
}
err = inst_models.InstanceTasks.BUILDING_ERROR_REPLICA
e_create_fault = create_log_fmt % create_fmt_content
e_create_stack = traceback.format_exc()
# we persist fault details to source instance
inst_models.save_instance_fault(slave_of_id, e_create_fault,
e_create_stack)
# if the delete of the 'bad' backup fails, it'll mask the
# create exception, so we trap it here
try:

View File

@ -196,6 +196,12 @@ class CreateInstanceTest(trove_testtools.TestCase):
backup_models.DBBackup.check_swift_object_exist = Mock(
return_value=True)
self.locality = 'affinity'
self.swift_verify_patch = patch.object(models.Backup,
'verify_swift_auth_token')
self.addCleanup(self.swift_verify_patch.stop)
self.swift_verify_patch.start()
super(CreateInstanceTest, self).setUp()
@patch.object(task_api.API, 'get_client', Mock(return_value=Mock()))
@ -363,6 +369,12 @@ class TestReplication(trove_testtools.TestCase):
self.safe_nova_client = models.create_nova_client
models.create_nova_client = nova.fake_create_nova_client
self.swift_verify_patch = patch.object(models.Backup,
'verify_swift_auth_token')
self.addCleanup(self.swift_verify_patch.stop)
self.swift_verify_patch.start()
super(TestReplication, self).setUp()
def tearDown(self):