diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 0b4e9041f9..3d4b28de1e 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -217,6 +217,7 @@ function configure_trove { iniset_conditional $TROVE_CONF DEFAULT resize_time_out $TROVE_RESIZE_TIME_OUT iniset_conditional $TROVE_CONF DEFAULT usage_timeout $TROVE_USAGE_TIMEOUT iniset_conditional $TROVE_CONF DEFAULT state_change_wait_time $TROVE_STATE_CHANGE_WAIT_TIME + iniset_conditional $TROVE_CONF DEFAULT reboot_time_out 300 configure_keystone_authtoken_middleware $TROVE_CONF trove iniset $TROVE_CONF service_credentials username trove @@ -239,7 +240,6 @@ function configure_trove { iniset $TROVE_CONF DEFAULT remote_nova_client trove.common.clients_admin.nova_client_trove_admin iniset $TROVE_CONF DEFAULT remote_cinder_client trove.common.clients_admin.cinder_client_trove_admin iniset $TROVE_CONF DEFAULT remote_neutron_client trove.common.clients_admin.neutron_client_trove_admin - iniset $TROVE_CONF DEFAULT remote_swift_client trove.common.clients_admin.swift_client_trove_admin iniset $TROVE_CONF DEFAULT remote_glance_client trove.common.clients_admin.glance_client_trove_admin iniset $TROVE_CONF cassandra tcp_ports 7000,7001,7199,9042,9160 @@ -278,7 +278,6 @@ function configure_trove { iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_nova_client trove.common.clients_admin.nova_client_trove_admin iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_cinder_client trove.common.clients_admin.cinder_client_trove_admin iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_neutron_client trove.common.clients_admin.neutron_client_trove_admin - iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_swift_client trove.common.clients_admin.swift_client_trove_admin iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_glance_client trove.common.clients_admin.glance_client_trove_admin # To avoid 'Connection timed out' error of sudo command inside the guest agent diff --git a/trove/guestagent/backup/backupagent.py b/trove/guestagent/backup/backupagent.py index 68288eb64d..1fd7342a8e 100644 --- a/trove/guestagent/backup/backupagent.py +++ b/trove/guestagent/backup/backupagent.py @@ -166,6 +166,7 @@ class BackupAgent(object): LOG.info("Restoring instance from backup %(id)s to " "%(restore_location)s", backup_info) + content_size = runner.restore() LOG.info("Restore from backup %(id)s completed successfully " "to %(restore_location)s", backup_info) diff --git a/trove/guestagent/strategies/backup/experimental/mariadb_impl.py b/trove/guestagent/strategies/backup/experimental/mariadb_impl.py index d32c69b406..2ff310caa6 100644 --- a/trove/guestagent/strategies/backup/experimental/mariadb_impl.py +++ b/trove/guestagent/strategies/backup/experimental/mariadb_impl.py @@ -15,6 +15,7 @@ import re from oslo_log import log as logging +from trove.common.i18n import _ from trove.guestagent.datastore.mysql import service as mysql_service from trove.guestagent.datastore.mysql_common import service as common_service from trove.guestagent.strategies.backup import base @@ -29,13 +30,13 @@ class MariaBackup(base.BackupRunner): @property def user_and_pass(self): - return (' --user=%(user)s --password=%(password)s --host=127.0.0.1 ' % + return ('--user=%(user)s --password=%(password)s --host=127.0.0.1' % {'user': common_service.ADMIN_USER_NAME, 'password': mysql_service.MySqlApp.get_auth_password()}) @property def cmd(self): - cmd = ('sudo mariabackup --backup --stream=xbstream' + + cmd = ('sudo mariabackup --backup --stream=xbstream ' + self.user_and_pass + ' 2>' + BACKUP_LOG) return cmd + self.zip_cmd + self.encrypt_cmd @@ -61,10 +62,49 @@ class MariaBackup(base.BackupRunner): return True + def metadata(self): + LOG.debug('Getting metadata for backup %s', self.base_filename) + + meta = {} + lsn = re.compile(r"The latest check point \(for incremental\): " + r"'(\d+)'") + with open(BACKUP_LOG, 'r') as backup_log: + output = backup_log.read() + match = lsn.search(output) + if match: + meta = {'lsn': match.group(1)} + + LOG.info("Metadata for backup %s: %s", self.base_filename, meta) + return meta + @property def filename(self): return '%s.xbstream' % self.base_filename class MariaBackupIncremental(MariaBackup): - pass + def __init__(self, *args, **kwargs): + if not kwargs.get('lsn'): + raise AttributeError(_('lsn attribute missing, bad parent?')) + super(MariaBackupIncremental, self).__init__(*args, **kwargs) + self.parent_location = kwargs.get('parent_location') + self.parent_checksum = kwargs.get('parent_checksum') + + @property + def cmd(self): + cmd = ( + 'sudo mariabackup --backup --stream=xbstream' + ' --incremental-lsn=%(lsn)s ' + + self.user_and_pass + + ' 2>' + + BACKUP_LOG + ) + return cmd + self.zip_cmd + self.encrypt_cmd + + def metadata(self): + meta = super(MariaBackupIncremental, self).metadata() + meta.update({ + 'parent_location': self.parent_location, + 'parent_checksum': self.parent_checksum, + }) + return meta diff --git a/trove/guestagent/strategies/backup/mysql_impl.py b/trove/guestagent/strategies/backup/mysql_impl.py index 1386af3ddd..8d5b844a58 100644 --- a/trove/guestagent/strategies/backup/mysql_impl.py +++ b/trove/guestagent/strategies/backup/mysql_impl.py @@ -100,7 +100,7 @@ class InnoBackupEx(base.BackupRunner): return True def metadata(self): - LOG.debug('Getting metadata from backup.') + LOG.debug('Getting metadata for backup %s', self.base_filename) meta = {} lsn = re.compile(r"The latest check point \(for incremental\): " r"'(\d+)'") @@ -109,7 +109,7 @@ class InnoBackupEx(base.BackupRunner): match = lsn.search(output) if match: meta = {'lsn': match.group(1)} - LOG.info("Metadata for backup: %s.", str(meta)) + LOG.info("Metadata for backup %s: %s", self.base_filename, meta) return meta @property diff --git a/trove/guestagent/strategies/restore/experimental/mariadb_impl.py b/trove/guestagent/strategies/restore/experimental/mariadb_impl.py index 99339cbbaa..4829259546 100644 --- a/trove/guestagent/strategies/restore/experimental/mariadb_impl.py +++ b/trove/guestagent/strategies/restore/experimental/mariadb_impl.py @@ -11,22 +11,32 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import glob +import os from oslo_log import log as logging +from trove.common import cfg +from trove.common import utils from trove.guestagent.common import operating_system from trove.guestagent.datastore.experimental.mariadb import service from trove.guestagent.datastore.mysql_common import service as mysql_service +from trove.guestagent.strategies.restore import base from trove.guestagent.strategies.restore import mysql_impl LOG = logging.getLogger(__name__) +PREPARE_LOG = '/tmp/innoprepare.log' -class MariaBackup(mysql_impl.InnoBackupEx): +class MariaBackup(base.RestoreRunner, mysql_impl.MySQLRestoreMixin): __strategy_name__ = 'mariabackup' base_restore_cmd = ('sudo mbstream -x -C %(restore_location)s ' '2>/tmp/xbstream_extract.log') + def __init__(self, *args, **kwargs): + self._app = None + super(MariaBackup, self).__init__(*args, **kwargs) + @property def app(self): if self._app is None: @@ -35,6 +45,14 @@ class MariaBackup(mysql_impl.InnoBackupEx): ) return self._app + def pre_restore(self): + self.app.stop_db() + LOG.debug("Cleaning out restore location: %s.", self.restore_location) + operating_system.chmod(self.restore_location, + operating_system.FileMode.SET_FULL, + as_root=True) + utils.clean_out(self.restore_location) + def post_restore(self): operating_system.chown(self.restore_location, 'mysql', None, force=True, as_root=True) @@ -45,6 +63,11 @@ class MariaBackup(mysql_impl.InnoBackupEx): self.app.start_mysql() LOG.debug("Finished post restore.") + def _delete_old_binlogs(self): + files = glob.glob(os.path.join(self.restore_location, "ib_logfile*")) + for f in files: + os.unlink(f) + def check_process(self): LOG.debug('Checking return code of mbstream restore process.') return_code = self.process.wait() @@ -56,4 +79,80 @@ class MariaBackup(mysql_impl.InnoBackupEx): class MariaBackupIncremental(MariaBackup): - pass + __strategy_name__ = 'mariabackupincremental' + incremental_prep = ('sudo mariabackup --prepare ' + '--target-dir=%(restore_location)s ' + '%(incremental_args)s ' + '2>/tmp/innoprepare.log') + + def __init__(self, *args, **kwargs): + super(MariaBackupIncremental, self).__init__(*args, **kwargs) + self.content_length = 0 + + def _incremental_restore_cmd(self, incremental_dir): + """Return a command for a restore with a incremental location.""" + args = {'restore_location': incremental_dir} + return (self.decrypt_cmd + + self.unzip_cmd + + (self.base_restore_cmd % args)) + + def _incremental_prepare_cmd(self, incremental_dir): + if incremental_dir is not None: + incremental_arg = '--incremental-dir=%s' % incremental_dir + else: + incremental_arg = '' + + args = { + 'restore_location': self.restore_location, + 'incremental_args': incremental_arg, + } + + return self.incremental_prep % args + + def _incremental_prepare(self, incremental_dir): + prepare_cmd = self._incremental_prepare_cmd(incremental_dir) + + LOG.debug("Running mariabackup prepare: %s.", prepare_cmd) + utils.execute(prepare_cmd, shell=True) + LOG.debug("mariabackup prepare finished successfully.") + + def _incremental_restore(self, location, checksum): + """Recursively apply backups from all parents. + If we are the parent then we restore to the restore_location and + we apply the logs to the restore_location only. + Otherwise if we are an incremental we restore to a subfolder to + prevent stomping on the full restore data. Then we run apply log + with the '--incremental-dir' flag + """ + metadata = self.storage.load_metadata(location, checksum) + incremental_dir = None + if 'parent_location' in metadata: + LOG.info("Restoring parent: %(parent_location)s" + " checksum: %(parent_checksum)s.", metadata) + parent_location = metadata['parent_location'] + parent_checksum = metadata['parent_checksum'] + # Restore parents recursively so backup are applied sequentially + self._incremental_restore(parent_location, parent_checksum) + # for *this* backup set the incremental_dir + # just use the checksum for the incremental path as it is + # sufficiently unique /var/lib/mysql/ + incremental_dir = os.path.join( + cfg.get_configuration_property('mount_point'), checksum) + operating_system.create_directory(incremental_dir, as_root=True) + command = self._incremental_restore_cmd(incremental_dir) + else: + # The parent (full backup) use the same command from InnobackupEx + # super class and do not set an incremental_dir. + command = self.restore_cmd + + self.content_length += self._unpack(location, checksum, command) + self._incremental_prepare(incremental_dir) + + # Delete unpacked incremental backup metadata + if incremental_dir: + operating_system.remove(incremental_dir, force=True, as_root=True) + + def _run_restore(self): + """Run incremental restore.""" + self._incremental_restore(self.location, self.checksum) + return self.content_length diff --git a/trove/guestagent/strategies/restore/mysql_impl.py b/trove/guestagent/strategies/restore/mysql_impl.py index cf4529ab5f..eea8c778d2 100644 --- a/trove/guestagent/strategies/restore/mysql_impl.py +++ b/trove/guestagent/strategies/restore/mysql_impl.py @@ -211,8 +211,7 @@ class InnoBackupEx(base.RestoreRunner, MySQLRestoreMixin): def pre_restore(self): self.app.stop_db() - LOG.info("Cleaning out restore location: %s.", - self.restore_location) + LOG.debug("Cleaning out restore location: %s.", self.restore_location) operating_system.chmod(self.restore_location, FileMode.SET_FULL, as_root=True) utils.clean_out(self.restore_location) @@ -313,7 +312,7 @@ class InnoBackupExIncremental(InnoBackupEx): prepare_cmd = self._incremental_prepare_cmd(incremental_dir) LOG.debug("Running innobackupex prepare: %s.", prepare_cmd) utils.execute(prepare_cmd, shell=True) - LOG.info("Innobackupex prepare finished successfully.") + LOG.debug("Innobackupex prepare finished successfully.") def _incremental_restore(self, location, checksum): """Recursively apply backups from all parents. diff --git a/trove/taskmanager/models.py b/trove/taskmanager/models.py index bf8a1024a1..c1b3d2208f 100755 --- a/trove/taskmanager/models.py +++ b/trove/taskmanager/models.py @@ -1092,7 +1092,8 @@ class BuiltInstanceTasks(BuiltInstance, NotifyMixin, ConfigurationMixin): action.execute() def create_backup(self, backup_info): - LOG.info("Initiating backup for instance %s.", self.id) + LOG.info("Initiating backup for instance %s, backup_info: %s", self.id, + backup_info) self.guest.create_backup(backup_info) def backup_required_for_replication(self): diff --git a/trove/tests/int_tests.py b/trove/tests/int_tests.py index c662028ab8..c386fafd60 100644 --- a/trove/tests/int_tests.py +++ b/trove/tests/int_tests.py @@ -319,7 +319,7 @@ register( ["mariadb_supported"], single=[common_groups, backup_groups, - # backup_incremental_groups, + backup_incremental_groups, configuration_groups, database_actions_groups, root_actions_groups,