From 4d358c8f5de4f7feda17053fb363be033146c9dc Mon Sep 17 00:00:00 2001 From: Dariusz Krol Date: Fri, 27 Jul 2018 14:18:28 +0200 Subject: [PATCH] Fix replication failure when Swift isn't available * add Swift token verification in create instance method in Trove API when slave_id is given * catch ConnectionError exception in verify_swift_auth_token * add Swift token verification to guestagent before backup and restore * add new exception representing Swift connection error * set missing fault information when replication snapshot fails * mock verify auth token method in replication and restore unit tests Closes-Bug: #1395523 Change-Id: I6a21ba2ba890a82875f9b6dae3c6b93bc9fdb4b0 Signed-off-by: Dariusz Krol --- trove/backup/models.py | 4 +++- trove/common/exception.py | 4 ++++ trove/common/wsgi.py | 1 + trove/instance/models.py | 4 ++++ trove/taskmanager/models.py | 6 ++++++ .../tests/unittests/instance/test_instance_models.py | 12 ++++++++++++ 6 files changed, 30 insertions(+), 1 deletion(-) diff --git a/trove/backup/models.py b/trove/backup/models.py index 06e5087688..2c5c617c1e 100644 --- a/trove/backup/models.py +++ b/trove/backup/models.py @@ -15,6 +15,7 @@ """Model classes that form the core of snapshots functionality.""" from oslo_log import log as logging +from requests.exceptions import ConnectionError from sqlalchemy import desc from swiftclient.client import ClientException @@ -29,7 +30,6 @@ from trove.db.models import DatabaseModelBase from trove.quota.quota import run_with_quotas from trove.taskmanager import api - CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -291,6 +291,8 @@ class Backup(object): raise exception.SwiftAuthError(tenant_id=context.tenant) except exception.NoServiceEndpoint: raise exception.SwiftNotFound(tenant_id=context.tenant) + except ConnectionError: + raise exception.SwiftConnectionError() def persisted_models(): diff --git a/trove/common/exception.py b/trove/common/exception.py index 094fee41f0..663fa4af78 100644 --- a/trove/common/exception.py +++ b/trove/common/exception.py @@ -461,6 +461,10 @@ class SwiftNotFound(TroveError): message = _("Swift is disabled for tenant %(tenant_id)s.") +class SwiftConnectionError(TroveError): + message = _("Cannot connect to Swift.") + + class DatabaseForUserNotInDatabaseListError(TroveError): message = _("The request indicates that user %(user)s should have access " "to database %(database)s, but database %(database)s is not " diff --git a/trove/common/wsgi.py b/trove/common/wsgi.py index 63d966ac05..e294ea182d 100644 --- a/trove/common/wsgi.py +++ b/trove/common/wsgi.py @@ -361,6 +361,7 @@ class Controller(object): webob.exc.HTTPServerError: [ exception.VolumeCreationFailure, exception.UpdateGuestError, + exception.SwiftConnectionError, ], webob.exc.HTTPNotImplemented: [ exception.VolumeNotSupported, diff --git a/trove/instance/models.py b/trove/instance/models.py index ce07e4d44c..902fd7cf4d 100644 --- a/trove/instance/models.py +++ b/trove/instance/models.py @@ -923,6 +923,8 @@ class Instance(BuiltInstance): target_size = flavor.ephemeral # ephemeral_Storage if backup_id: + Backup.verify_swift_auth_token(context) + call_args['backup_id'] = backup_id backup_info = Backup.get_by_id(context, backup_id) if not backup_info.is_done_successfuly: @@ -946,6 +948,8 @@ class Instance(BuiltInstance): datastore2=datastore.name) if slave_of_id: + Backup.verify_swift_auth_token(context) + if databases or users: raise exception.ReplicaCreateWithUsersDatabasesError() call_args['replica_of'] = slave_of_id diff --git a/trove/taskmanager/models.py b/trove/taskmanager/models.py index b5a3c8b88e..7ffebac301 100755 --- a/trove/taskmanager/models.py +++ b/trove/taskmanager/models.py @@ -625,6 +625,12 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): 'replica': self.id } err = inst_models.InstanceTasks.BUILDING_ERROR_REPLICA + e_create_fault = create_log_fmt % create_fmt_content + e_create_stack = traceback.format_exc() + # we persist fault details to source instance + inst_models.save_instance_fault(slave_of_id, e_create_fault, + e_create_stack) + # if the delete of the 'bad' backup fails, it'll mask the # create exception, so we trap it here try: diff --git a/trove/tests/unittests/instance/test_instance_models.py b/trove/tests/unittests/instance/test_instance_models.py index 83b98455e5..e1151fe68a 100644 --- a/trove/tests/unittests/instance/test_instance_models.py +++ b/trove/tests/unittests/instance/test_instance_models.py @@ -196,6 +196,12 @@ class CreateInstanceTest(trove_testtools.TestCase): backup_models.DBBackup.check_swift_object_exist = Mock( return_value=True) self.locality = 'affinity' + + self.swift_verify_patch = patch.object(models.Backup, + 'verify_swift_auth_token') + self.addCleanup(self.swift_verify_patch.stop) + self.swift_verify_patch.start() + super(CreateInstanceTest, self).setUp() @patch.object(task_api.API, 'get_client', Mock(return_value=Mock())) @@ -363,6 +369,12 @@ class TestReplication(trove_testtools.TestCase): self.safe_nova_client = models.create_nova_client models.create_nova_client = nova.fake_create_nova_client + + self.swift_verify_patch = patch.object(models.Backup, + 'verify_swift_auth_token') + self.addCleanup(self.swift_verify_patch.stop) + self.swift_verify_patch.start() + super(TestReplication, self).setUp() def tearDown(self):