560 lines
22 KiB
Python
560 lines
22 KiB
Python
# Copyright 2013 Hewlett-Packard Development Company, L.P.
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import hashlib
|
|
import mock
|
|
import os
|
|
|
|
from mock import Mock, MagicMock, patch, ANY, DEFAULT, call
|
|
from oslo_utils import netutils
|
|
from webob.exc import HTTPNotFound
|
|
|
|
from trove.backup.state import BackupState
|
|
from trove.common.context import TroveContext
|
|
from trove.common.strategies.storage.base import Storage
|
|
from trove.common import utils
|
|
from trove.conductor import api as conductor_api
|
|
from trove.guestagent.backup import backupagent
|
|
from trove.guestagent.common import configuration
|
|
from trove.guestagent.common.configuration import ImportOverrideStrategy
|
|
from trove.guestagent.datastore.experimental.redis.service import RedisApp
|
|
from trove.guestagent.strategies.backup.base import BackupRunner
|
|
from trove.guestagent.strategies.backup.base import UnknownBackupType
|
|
from trove.guestagent.strategies.backup.experimental import couchbase_impl
|
|
from trove.guestagent.strategies.backup.experimental import db2_impl
|
|
from trove.guestagent.strategies.backup.experimental import mongo_impl
|
|
from trove.guestagent.strategies.backup.experimental import redis_impl
|
|
from trove.guestagent.strategies.backup import mysql_impl
|
|
from trove.guestagent.strategies.backup.mysql_impl import MySqlApp
|
|
from trove.guestagent.strategies.restore.base import RestoreRunner
|
|
from trove.tests.unittests import trove_testtools
|
|
|
|
|
|
def create_fake_data():
|
|
from random import choice
|
|
from string import ascii_letters
|
|
|
|
return ''.join([choice(ascii_letters) for _ in range(1024)])
|
|
|
|
|
|
class MockBackup(BackupRunner):
|
|
"""Create a large temporary file to 'backup' with subprocess."""
|
|
|
|
backup_type = 'mock_backup'
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
self.data = create_fake_data()
|
|
self.cmd = 'echo %s' % self.data
|
|
super(MockBackup, self).__init__(*args, **kwargs)
|
|
|
|
def cmd(self):
|
|
return self.cmd
|
|
|
|
|
|
class MockCheckProcessBackup(MockBackup):
|
|
"""Backup runner that fails confirming the process."""
|
|
|
|
def check_process(self):
|
|
return False
|
|
|
|
|
|
class MockLossyBackup(MockBackup):
|
|
"""Fake Incomplete writes to swift."""
|
|
|
|
def read(self, *args):
|
|
results = super(MockLossyBackup, self).read(*args)
|
|
if results:
|
|
# strip a few chars from the stream
|
|
return results[20:]
|
|
|
|
|
|
class MockSwift(object):
|
|
"""Store files in String."""
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
self.store = ''
|
|
self.containers = []
|
|
self.container = "database_backups"
|
|
self.url = 'http://mockswift/v1'
|
|
self.etag = hashlib.md5()
|
|
|
|
def put_container(self, container):
|
|
if container not in self.containers:
|
|
self.containers.append(container)
|
|
return None
|
|
|
|
def put_object(self, container, obj, contents, **kwargs):
|
|
if container not in self.containers:
|
|
raise HTTPNotFound
|
|
while True:
|
|
if not hasattr(contents, 'read'):
|
|
break
|
|
content = contents.read(2 ** 16)
|
|
if not content:
|
|
break
|
|
self.store += content
|
|
self.etag.update(self.store)
|
|
return self.etag.hexdigest()
|
|
|
|
def save(self, filename, stream, metadata=None):
|
|
location = '%s/%s/%s' % (self.url, self.container, filename)
|
|
return True, 'w00t', 'fake-checksum', location
|
|
|
|
def load(self, context, storage_url, container, filename, backup_checksum):
|
|
pass
|
|
|
|
def load_metadata(self, location, checksum):
|
|
return {}
|
|
|
|
def save_metadata(self, location, metadata):
|
|
pass
|
|
|
|
|
|
class MockStorage(Storage):
|
|
|
|
def __call__(self, *args, **kwargs):
|
|
return self
|
|
|
|
def load(self, location, backup_checksum):
|
|
pass
|
|
|
|
def save(self, filename, stream, metadata=None):
|
|
pass
|
|
|
|
def load_metadata(self, location, checksum):
|
|
return {}
|
|
|
|
def save_metadata(self, location, metadata={}):
|
|
pass
|
|
|
|
def is_enabled(self):
|
|
return True
|
|
|
|
|
|
class MockRestoreRunner(RestoreRunner):
|
|
|
|
def __init__(self, storage, **kwargs):
|
|
pass
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
pass
|
|
|
|
def restore(self):
|
|
pass
|
|
|
|
def is_zipped(self):
|
|
return False
|
|
|
|
|
|
class MockStats(object):
|
|
f_blocks = 1024 ** 2
|
|
f_bsize = 4096
|
|
f_bfree = 512 * 1024
|
|
|
|
|
|
class BackupAgentTest(trove_testtools.TestCase):
|
|
|
|
def setUp(self):
|
|
super(BackupAgentTest, self).setUp()
|
|
self.patch_ope = patch.multiple('os.path',
|
|
exists=DEFAULT)
|
|
self.mock_ope = self.patch_ope.start()
|
|
self.addCleanup(self.patch_ope.stop)
|
|
self.patch_pc = patch('trove.guestagent.datastore.service.'
|
|
'BaseDbStatus.prepare_completed')
|
|
self.mock_pc = self.patch_pc.start()
|
|
self.mock_pc.__get__ = Mock(return_value=True)
|
|
self.addCleanup(self.patch_pc.stop)
|
|
self.get_auth_pwd_patch = patch.object(
|
|
MySqlApp, 'get_auth_password', MagicMock(return_value='123'))
|
|
self.get_auth_pwd_mock = self.get_auth_pwd_patch.start()
|
|
self.addCleanup(self.get_auth_pwd_patch.stop)
|
|
self.get_ss_patch = patch.object(
|
|
backupagent, 'get_storage_strategy',
|
|
MagicMock(return_value=MockSwift))
|
|
self.get_ss_mock = self.get_ss_patch.start()
|
|
self.addCleanup(self.get_ss_patch.stop)
|
|
self.statvfs_patch = patch.object(
|
|
os, 'statvfs', MagicMock(return_value=MockStats))
|
|
self.statvfs_mock = self.statvfs_patch.start()
|
|
self.addCleanup(self.statvfs_patch.stop)
|
|
self.orig_utils_execute_with_timeout = utils.execute_with_timeout
|
|
self.orig_os_get_ip_address = netutils.get_my_ipv4
|
|
|
|
def tearDown(self):
|
|
super(BackupAgentTest, self).tearDown()
|
|
utils.execute_with_timeout = self.orig_utils_execute_with_timeout
|
|
netutils.get_my_ipv4 = self.orig_os_get_ip_address
|
|
|
|
def test_backup_impl_MySQLDump(self):
|
|
"""This test is for
|
|
guestagent/strategies/backup/mysql_impl
|
|
"""
|
|
mysql_dump = mysql_impl.MySQLDump(
|
|
'abc', extra_opts='')
|
|
self.assertIsNotNone(mysql_dump.cmd)
|
|
str_mysql_dump_cmd = ('mysqldump'
|
|
' --all-databases'
|
|
' %(extra_opts)s'
|
|
' --opt'
|
|
' --password=123'
|
|
' -u os_admin'
|
|
' | gzip |'
|
|
' openssl enc -aes-256-cbc -salt '
|
|
'-pass pass:default_aes_cbc_key')
|
|
self.assertEqual(str_mysql_dump_cmd, mysql_dump.cmd)
|
|
self.assertIsNotNone(mysql_dump.manifest)
|
|
self.assertEqual('abc.gz.enc', mysql_dump.manifest)
|
|
|
|
@mock.patch.object(
|
|
MySqlApp, 'get_data_dir', return_value='/var/lib/mysql/data')
|
|
def test_backup_impl_InnoBackupEx(self, mock_datadir):
|
|
"""This test is for
|
|
guestagent/strategies/backup/mysql_impl
|
|
"""
|
|
inno_backup_ex = mysql_impl.InnoBackupEx('innobackupex', extra_opts='')
|
|
self.assertIsNotNone(inno_backup_ex.cmd)
|
|
str_innobackup_cmd = ('sudo innobackupex'
|
|
' --stream=xbstream'
|
|
' %(extra_opts)s '
|
|
' --user=os_admin --password=123'
|
|
' /var/lib/mysql/data 2>/tmp/innobackupex.log'
|
|
' | gzip |'
|
|
' openssl enc -aes-256-cbc -salt '
|
|
'-pass pass:default_aes_cbc_key')
|
|
self.assertEqual(str_innobackup_cmd, inno_backup_ex.cmd)
|
|
self.assertIsNotNone(inno_backup_ex.manifest)
|
|
str_innobackup_manifest = 'innobackupex.xbstream.gz.enc'
|
|
self.assertEqual(str_innobackup_manifest, inno_backup_ex.manifest)
|
|
|
|
def test_backup_impl_CbBackup(self):
|
|
netutils.get_my_ipv4 = Mock(return_value="1.1.1.1")
|
|
utils.execute_with_timeout = Mock(return_value=None)
|
|
cbbackup = couchbase_impl.CbBackup('cbbackup', extra_opts='')
|
|
self.assertIsNotNone(cbbackup)
|
|
str_cbbackup_cmd = ("tar cpPf - /tmp/backups | "
|
|
"gzip | openssl enc -aes-256-cbc -salt -pass "
|
|
"pass:default_aes_cbc_key")
|
|
self.assertEqual(str_cbbackup_cmd, cbbackup.cmd)
|
|
self.assertIsNotNone(cbbackup.manifest)
|
|
self.assertIn('gz.enc', cbbackup.manifest)
|
|
|
|
@mock.patch.object(db2_impl.DB2Backup, 'list_dbnames',
|
|
return_value=['testdb1', 'testdb2'])
|
|
def test_backup_impl_DB2Backup(self, _):
|
|
netutils.get_my_ipv4 = Mock(return_value="1.1.1.1")
|
|
db2_backup = db2_impl.DB2Backup('db2backup', extra_opts='')
|
|
self.assertIsNotNone(db2_backup)
|
|
str_db2_backup_cmd = ("sudo tar cPf - /home/db2inst1/db2inst1/backup "
|
|
"| gzip | openssl enc -aes-256-cbc -salt -pass "
|
|
"pass:default_aes_cbc_key")
|
|
self.assertEqual(str_db2_backup_cmd, db2_backup.cmd)
|
|
self.assertIsNotNone(db2_backup.manifest)
|
|
self.assertIn('gz.enc', db2_backup.manifest)
|
|
|
|
@mock.patch.object(ImportOverrideStrategy, '_initialize_import_directory')
|
|
def test_backup_impl_MongoDump(self, _):
|
|
netutils.get_my_ipv4 = Mock(return_value="1.1.1.1")
|
|
utils.execute_with_timeout = Mock(return_value=None)
|
|
mongodump = mongo_impl.MongoDump('mongodump', extra_opts='')
|
|
self.assertIsNotNone(mongodump)
|
|
str_mongodump_cmd = ("sudo tar cPf - /var/lib/mongodb/dump | "
|
|
"gzip | openssl enc -aes-256-cbc -salt -pass "
|
|
"pass:default_aes_cbc_key")
|
|
self.assertEqual(str_mongodump_cmd, mongodump.cmd)
|
|
self.assertIsNotNone(mongodump.manifest)
|
|
self.assertIn('gz.enc', mongodump.manifest)
|
|
|
|
@patch.object(utils, 'execute_with_timeout', return_value=('0', ''))
|
|
@patch.object(configuration.ConfigurationManager, 'parse_configuration',
|
|
Mock(return_value={'dir': '/var/lib/redis',
|
|
'dbfilename': 'dump.rdb'}))
|
|
@patch.object(RedisApp, 'get_config_command_name',
|
|
Mock(return_value='fakeconfig'))
|
|
def test_backup_impl_RedisBackup(self, *mocks):
|
|
netutils.get_my_ipv4 = Mock(return_value="1.1.1.1")
|
|
redis_backup = redis_impl.RedisBackup('redisbackup', extra_opts='')
|
|
self.assertIsNotNone(redis_backup)
|
|
str_redis_backup_cmd = ("sudo cat /var/lib/redis/dump.rdb | "
|
|
"gzip | openssl enc -aes-256-cbc -salt -pass "
|
|
"pass:default_aes_cbc_key")
|
|
self.assertEqual(str_redis_backup_cmd, redis_backup.cmd)
|
|
self.assertIsNotNone(redis_backup.manifest)
|
|
self.assertIn('gz.enc', redis_backup.manifest)
|
|
|
|
def test_backup_base(self):
|
|
"""This test is for
|
|
guestagent/strategies/backup/base
|
|
"""
|
|
BackupRunner.cmd = "%s"
|
|
backup_runner = BackupRunner('sample', cmd='echo command')
|
|
if backup_runner.is_zipped:
|
|
self.assertEqual('.gz', backup_runner.zip_manifest)
|
|
self.assertIsNotNone(backup_runner.zip_manifest)
|
|
self.assertIsNotNone(backup_runner.zip_cmd)
|
|
self.assertEqual(' | gzip', backup_runner.zip_cmd)
|
|
else:
|
|
self.assertIsNone(backup_runner.zip_manifest)
|
|
self.assertIsNone(backup_runner.zip_cmd)
|
|
self.assertEqual('BackupRunner', backup_runner.backup_type)
|
|
|
|
@patch('os.killpg')
|
|
def test_backup_runner_exits_with_exception(self, mock_kill_pg):
|
|
"""This test is for
|
|
guestagent/strategies/backup/base,
|
|
ensures that when backup runner exits with an exception,
|
|
all child processes are also killed.
|
|
"""
|
|
BackupRunner.cmd = "%s"
|
|
backup_runner = BackupRunner('sample', cmd='echo command')
|
|
|
|
def test_backup_runner_reraise_exception():
|
|
mock_func = mock.Mock(side_effect=RuntimeError)
|
|
|
|
with backup_runner:
|
|
mock_func()
|
|
|
|
self.assertRaises(RuntimeError,
|
|
test_backup_runner_reraise_exception)
|
|
self.assertTrue(mock_kill_pg.called)
|
|
|
|
@patch.object(conductor_api.API, 'get_client', Mock(return_value=Mock()))
|
|
@patch.object(conductor_api.API, 'update_backup',
|
|
Mock(return_value=Mock()))
|
|
def test_execute_backup(self):
|
|
"""This test should ensure backup agent
|
|
ensures that backup and storage is not running
|
|
resolves backup instance
|
|
starts backup
|
|
starts storage
|
|
reports status
|
|
"""
|
|
agent = backupagent.BackupAgent()
|
|
backup_info = {'id': '123',
|
|
'location': 'fake-location',
|
|
'type': 'InnoBackupEx',
|
|
'checksum': 'fake-checksum',
|
|
'datastore': 'mysql',
|
|
'datastore_version': '5.5'
|
|
}
|
|
agent.execute_backup(context=None, backup_info=backup_info,
|
|
runner=MockBackup)
|
|
|
|
conductor_api.API.update_backup.assert_has_calls([
|
|
call(
|
|
ANY,
|
|
backup_id=backup_info['id'],
|
|
sent=ANY,
|
|
size=ANY,
|
|
state=BackupState.BUILDING
|
|
),
|
|
call(
|
|
ANY,
|
|
backup_id=backup_info['id'],
|
|
checksum='fake-checksum',
|
|
location=ANY,
|
|
note='w00t',
|
|
sent=ANY,
|
|
size=ANY,
|
|
backup_type=MockBackup.backup_type,
|
|
state=BackupState.COMPLETED,
|
|
success=True
|
|
)
|
|
])
|
|
|
|
@patch.object(conductor_api.API, 'get_client', Mock(return_value=Mock()))
|
|
@patch.object(conductor_api.API, 'update_backup',
|
|
Mock(return_value=Mock()))
|
|
@patch('trove.guestagent.backup.backupagent.LOG')
|
|
def test_execute_bad_process_backup(self, mock_logging):
|
|
agent = backupagent.BackupAgent()
|
|
backup_info = {'id': '123',
|
|
'location': 'fake-location',
|
|
'type': 'InnoBackupEx',
|
|
'checksum': 'fake-checksum',
|
|
'datastore': 'mysql',
|
|
'datastore_version': '5.5'
|
|
}
|
|
|
|
self.assertRaises(backupagent.BackupError, agent.execute_backup,
|
|
context=None, backup_info=backup_info,
|
|
runner=MockCheckProcessBackup)
|
|
|
|
conductor_api.API.update_backup.assert_has_calls([
|
|
call(
|
|
ANY,
|
|
backup_id=backup_info['id'],
|
|
sent=ANY,
|
|
size=ANY,
|
|
state=BackupState.BUILDING
|
|
),
|
|
call(
|
|
ANY,
|
|
backup_id=backup_info['id'],
|
|
checksum='fake-checksum',
|
|
location=ANY,
|
|
note='w00t',
|
|
sent=ANY,
|
|
size=ANY,
|
|
backup_type=MockCheckProcessBackup.backup_type,
|
|
state=BackupState.FAILED,
|
|
success=True
|
|
)
|
|
])
|
|
|
|
@patch.object(conductor_api.API, 'get_client', Mock(return_value=Mock()))
|
|
@patch.object(conductor_api.API, 'update_backup',
|
|
Mock(return_value=Mock()))
|
|
@patch('trove.guestagent.backup.backupagent.LOG')
|
|
def test_execute_lossy_backup(self, mock_logging):
|
|
"""This test verifies that incomplete writes to swift will fail."""
|
|
with patch.object(MockSwift, 'save',
|
|
return_value=(False, 'Error', 'y', 'z')):
|
|
|
|
agent = backupagent.BackupAgent()
|
|
|
|
backup_info = {'id': '123',
|
|
'location': 'fake-location',
|
|
'type': 'InnoBackupEx',
|
|
'checksum': 'fake-checksum',
|
|
'datastore': 'mysql',
|
|
'datastore_version': '5.5'
|
|
}
|
|
|
|
self.assertRaises(backupagent.BackupError, agent.execute_backup,
|
|
context=None, backup_info=backup_info,
|
|
runner=MockLossyBackup)
|
|
|
|
conductor_api.API.update_backup.assert_has_calls([
|
|
call(ANY,
|
|
backup_id=backup_info['id'],
|
|
sent=ANY,
|
|
size=ANY,
|
|
state=BackupState.BUILDING
|
|
),
|
|
call(
|
|
ANY,
|
|
backup_id=backup_info['id'],
|
|
checksum='y',
|
|
location='z',
|
|
note='Error',
|
|
sent=ANY,
|
|
size=ANY,
|
|
backup_type=MockLossyBackup.backup_type,
|
|
state=BackupState.FAILED,
|
|
success=False
|
|
)]
|
|
)
|
|
|
|
def test_execute_restore(self):
|
|
"""This test should ensure backup agent
|
|
resolves backup instance
|
|
determines backup/restore type
|
|
transfers/downloads data and invokes the restore module
|
|
reports status
|
|
"""
|
|
with patch.object(backupagent, 'get_storage_strategy',
|
|
return_value=MockStorage):
|
|
|
|
with patch.object(backupagent, 'get_restore_strategy',
|
|
return_value=MockRestoreRunner):
|
|
|
|
agent = backupagent.BackupAgent()
|
|
|
|
bkup_info = {'id': '123',
|
|
'location': 'fake-location',
|
|
'type': 'InnoBackupEx',
|
|
'checksum': 'fake-checksum',
|
|
}
|
|
agent.execute_restore(TroveContext(),
|
|
bkup_info,
|
|
'/var/lib/mysql/data')
|
|
|
|
@patch('trove.guestagent.backup.backupagent.LOG')
|
|
def test_restore_unknown(self, mock_logging):
|
|
with patch.object(backupagent, 'get_restore_strategy',
|
|
side_effect=ImportError):
|
|
|
|
agent = backupagent.BackupAgent()
|
|
|
|
bkup_info = {'id': '123',
|
|
'location': 'fake-location',
|
|
'type': 'foo',
|
|
'checksum': 'fake-checksum',
|
|
}
|
|
|
|
self.assertRaises(UnknownBackupType, agent.execute_restore,
|
|
context=None, backup_info=bkup_info,
|
|
restore_location='/var/lib/mysql/data')
|
|
|
|
@patch.object(MySqlApp, 'get_data_dir', return_value='/var/lib/mysql/data')
|
|
@patch.object(conductor_api.API, 'get_client', Mock(return_value=Mock()))
|
|
@patch.object(MockSwift, 'load_metadata', return_value={'lsn': '54321'})
|
|
@patch.object(MockSwift, 'save')
|
|
@patch.object(backupagent, 'get_storage_strategy', return_value=MockSwift)
|
|
@patch('trove.guestagent.backup.backupagent.LOG')
|
|
def test_backup_incremental_metadata(self, mock_logging,
|
|
get_storage_strategy_mock,
|
|
save_mock,
|
|
load_metadata_mock,
|
|
get_datadir_mock):
|
|
meta = {
|
|
'lsn': '12345',
|
|
'parent_location': 'fake',
|
|
'parent_checksum': 'md5',
|
|
}
|
|
with patch.multiple(mysql_impl.InnoBackupExIncremental,
|
|
metadata=MagicMock(return_value=meta),
|
|
_run=MagicMock(return_value=True),
|
|
__exit__=MagicMock(return_value=True)):
|
|
agent = backupagent.BackupAgent()
|
|
|
|
expected_metadata = {'datastore': 'mysql',
|
|
'datastore_version': 'bo.gus'}
|
|
bkup_info = {'id': '123',
|
|
'location': 'fake-location',
|
|
'type': 'InnoBackupEx',
|
|
'checksum': 'fake-checksum',
|
|
'parent': {'location': 'fake', 'checksum': 'md5'}}
|
|
bkup_info.update(expected_metadata)
|
|
|
|
agent.execute_backup(TroveContext(),
|
|
bkup_info,
|
|
'/var/lib/mysql/data')
|
|
|
|
save_mock.assert_called_once_with(
|
|
ANY, ANY, metadata=expected_metadata)
|
|
|
|
@patch.object(conductor_api.API, 'get_client', Mock(return_value=Mock()))
|
|
@patch('trove.guestagent.backup.backupagent.LOG')
|
|
def test_backup_incremental_bad_metadata(self, mock_logging):
|
|
with patch.object(backupagent, 'get_storage_strategy',
|
|
return_value=MockSwift):
|
|
|
|
agent = backupagent.BackupAgent()
|
|
|
|
bkup_info = {'id': '123',
|
|
'location': 'fake-location',
|
|
'type': 'InnoBackupEx',
|
|
'checksum': 'fake-checksum',
|
|
'parent': {'location': 'fake', 'checksum': 'md5'}
|
|
}
|
|
|
|
self.assertRaises(
|
|
AttributeError,
|
|
agent.execute_backup, TroveContext(), bkup_info, 'location')
|