From 0fdff12b260550190be01c59080a8be0c109db28 Mon Sep 17 00:00:00 2001 From: Sven Wegener Date: Fri, 14 Dec 2018 12:56:49 +0100 Subject: [PATCH] Fix ceph: only close rbd image after snapshot iteration is finished list_snaps() returns an iterator object that requires resources from the Image object. Closing the Image object deallocates these resources. On Ceph mimic, which contains changes to the functions being invoked, iterating over the snapshots of a closed image results in the following assertion being triggered: cinder-backup[29383]: /build/ceph-13.2.2/src/common/Mutex.cc: In function 'void Mutex::Lock(bool)' thread 7f35a4812700 time 2018-12-14 10:10:49.843120 cinder-backup[29383]: /build/ceph-13.2.2/src/common/Mutex.cc: 110: FAILED assert(r == 0) cinder-backup[29383]: ceph version 13.2.2 (02899bfda814146b021136e9d8e80eba494e1126) mimic (stable) cinder-backup[29383]: 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x102) [0x7f35903440f2] cinder-backup[29383]: 2: (()+0x3162b7) [0x7f35903442b7] cinder-backup[29383]: 3: (Mutex::Lock(bool)+0x1de) [0x7f359031901e] cinder-backup[29383]: 4: (()+0x8f452) [0x7f358dafc452] cinder-backup[29383]: 5: (()+0x11280d) [0x7f358db7f80d] cinder-backup[29383]: 6: (rbd_snap_get_namespace_type()+0x29) [0x7f358dacf549] The lock being tried to acquire and the object that contained it have already been destroyed by closing the image. The assertion terminates the cinder-backup service via SIGABRT. On Ceph luminous we've seen occasional segfaults of the cinder-backup service, which might have the same underlying cause. Closes-Bug: #1838691 Change-Id: If6043d82cc57d9247a290816d90b95cdf719eaac Signed-off-by: Sven Wegener (cherry picked from commit 76569672599eda764f946ad6ee2ce0bdaddad36b) (cherry picked from commit e78b220f448c7eb0de35aafd1d49b00547384d41) (cherry picked from commit 57b127459189e39e993426f8e401e288f6394d4b) (cherry picked from commit 9e5d33ac244e8dce31ff6d2c7119ee4bd53a9908) --- cinder/backup/drivers/ceph.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cinder/backup/drivers/ceph.py b/cinder/backup/drivers/ceph.py index 8769d684630..319dce0728c 100644 --- a/cinder/backup/drivers/ceph.py +++ b/cinder/backup/drivers/ceph.py @@ -627,16 +627,16 @@ class CephBackupDriver(driver.BackupDriver): base_rbd = self.rbd.Image(client.ioctx, base_name, read_only=True) try: snaps = base_rbd.list_snaps() + + if snaps is None: + return False + + for snap in snaps: + if snap['name'] == snap_name: + return True finally: base_rbd.close() - if snaps is None: - return False - - for snap in snaps: - if snap['name'] == snap_name: - return True - return False def _backup_rbd(self, backup, volume_file, volume_name, length):