Add new option to perform an on-demand backup of MariaDB

blueprint database-backup-recovery

Introduce a new option, mariadb_backup, which takes a backup of all
databases hosted in MariaDB.

Backups are performed using XtraBackup, the output of which is saved to
a dedicated Docker volume on the target host (which defaults to the
first node in the MariaDB cluster).

It supports either full (the default) or incremental backups.

Change-Id: Ied224c0d19b8734aa72092aaddd530155999dbc3
This commit is contained in:
Nick Jones 2017-12-14 18:51:30 +00:00 committed by Mark Goddard
parent 86f29f7c11
commit f704a78029
13 changed files with 311 additions and 1 deletions

View File

@ -545,6 +545,7 @@ enable_trove_singletenant: "no"
enable_vitrage: "no"
enable_vmtp: "no"
enable_watcher: "no"
enable_xtrabackup: "no"
enable_zookeeper: "{{ enable_kafka | bool }}"
enable_zun: "no"

View File

@ -0,0 +1,23 @@
---
- name: Detect openstack_release variable
hosts: mariadb
gather_facts: false
tasks:
- name: Get current kolla-ansible version number
local_action: command python -c "import pbr.version; print(pbr.version.VersionInfo('kolla-ansible'))"
register: kolla_ansible_version
changed_when: false
when: openstack_release == "auto"
- name: Set openstack_release variable
set_fact:
openstack_release: "{{ kolla_ansible_version.stdout }}"
when: openstack_release == "auto"
tags: always
- name: Backup MariaDB
hosts: mariadb
roles:
- { role: mariadb,
tags: mariadb,
when: enable_xtrabackup | bool }

View File

@ -62,3 +62,16 @@ mariadb_dimensions: "{{ default_container_dimensions }}"
# Vars used within recover_cluster.yml
########################################
mariadb_service: "{{ mariadb_services['mariadb'] }}"
####################
# Backups
####################
xtrabackup_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-xtrabackup"
xtrabackup_tag: "{{ openstack_release }}"
xtrabackup_image_full: "{{ xtrabackup_image }}:{{ xtrabackup_tag }}"
mariadb_backup_host: "{{ groups['mariadb'][0] }}"
mariadb_backup_database_schema: "PERCONA_SCHEMA"
mariadb_backup_database_user: "backup"
mariadb_backup_database_address: "{{ database_address }}"
mariadb_backup_type: "full"

View File

@ -0,0 +1,19 @@
---
- name: Taking {{ mariadb_backup_type }} database backup via XtraBackup
kolla_docker:
action: "start_container"
common_options: "{{ docker_common_options }}"
image: "{{ xtrabackup_image_full }}"
name: "xtrabackup"
restart_policy: "never"
remove_on_exit: True
environment:
BACKUP_TYPE: "{{ mariadb_backup_type }}"
volumes:
- "{{ node_config_directory }}xtrabackup:/etc/mysql:ro"
- "/etc/localtime:/etc/localtime:ro"
- "mariadb_backup:/backup"
volumes_from:
- "mariadb"
when:
- inventory_hostname == mariadb_backup_host

View File

@ -12,6 +12,33 @@
- item.value.enabled | bool
with_dict: "{{ mariadb_services }}"
- name: Ensuring database backup config directory exists
file:
path: "{{ node_config_directory }}xtrabackup"
state: "directory"
owner: "{{ config_owner_user }}"
group: "{{ config_owner_group }}"
mode: "0770"
become: true
when:
- enable_xtrabackup | bool
- inventory_hostname == mariadb_backup_host
- name: Copying over my.cnf for xtrabackup
merge_configs:
sources:
- "{{ role_path }}/templates/backup.my.cnf.j2"
- "{{ node_custom_config }}/backup.my.cnf"
- "{{ node_custom_config }}/mariadb/{{ inventory_hostname }}/backup.my.cnf"
dest: "{{ node_config_directory }}xtrabackup/my.cnf"
owner: "{{ config_owner_user }}"
group: "{{ config_owner_group }}"
mode: "0660"
become: true
when:
- enable_xtrabackup | bool
- inventory_hostname == mariadb_backup_host
- name: Copying over config.json files for services
vars:
service_name: "mariadb"

View File

@ -13,6 +13,53 @@
priv: "*.*:USAGE"
run_once: True
- name: Creating the Percona XtraBackup database
kolla_toolbox:
module_name: mysql_db
module_args:
login_host: "{{ database_address }}"
login_port: "{{ database_port }}"
login_user: "{{ database_user }}"
login_password: "{{ database_password }}"
name: "{{ mariadb_backup_database_schema }}"
run_once: True
when:
- enable_xtrabackup | bool
- name: Creating database backup user and setting permissions
kolla_toolbox:
module_name: mysql_user
module_args:
login_host: "{{ database_address }}"
login_port: "{{ database_port }}"
login_user: "{{ database_user }}"
login_password: "{{ database_password }}"
name: "{{ mariadb_backup_database_user }}"
password: "{{ mariadb_backup_database_password }}"
host: "%"
priv: "*.*:CREATE TABLESPACE,RELOAD,PROCESS,SUPER,LOCK TABLES,REPLICATION CLIENT"
append_privs: True
run_once: True
when:
- enable_xtrabackup | bool
- name: Granting permissions on XtraBackup database to backup user
kolla_toolbox:
module_name: mysql_user
module_args:
login_host: "{{ database_address }}"
login_port: "{{ database_port }}"
login_user: "{{ database_user }}"
login_password: "{{ database_password }}"
name: "{{ mariadb_backup_database_user }}"
password: "{{ mariadb_backup_database_password }}"
host: "%"
priv: "{{ mariadb_backup_database_schema }}.*:CREATE,INSERT,SELECT"
append_privs: True
run_once: True
when:
- enable_xtrabackup | bool
- name: Cleaning up facts
set_fact:
delegate_host: "bootstraped"

View File

@ -0,0 +1,6 @@
[client]
default-character-set=utf8
user={{ mariadb_backup_database_user }}
password={{ mariadb_backup_database_password }}
host={{ database_address }}
port={{ database_port }}

View File

@ -6,5 +6,6 @@ Admin Guides
:maxdepth: 2
advanced-configuration
mariadb-backup-and-restore
production-architecture-guide
deployment-philosophy

View File

@ -0,0 +1,140 @@
.. _mariadb-backup-and-restore:
===================================
MariaDB database backup and restore
===================================
Kolla-Ansible can facilitate either full or incremental backups of data
hosted in MariaDB. It achieves this using Percona's Xtrabackup, a tool
designed to allow for 'hot backups' - an approach which means that consistent
backups can be taken without any downtime for your database or your cloud.
.. note::
By default, backups will be performed on the first node in your Galera cluster
or on the MariaDB node itself if you just have the one. Backup files are saved
to a dedicated Docker volume - ``mariadb_backup`` - and it's the contents of
this that you should target for transferring backups elsewhere.
Enabling Backup Functionality
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
For backups to work, some reconfiguration of MariaDB is required - this is to
enable appropriate permissions for the backup client, and also to create an
additional database in order to store backup information.
Firstly, enable backups via ``globals.yml``:
.. code-block:: console
enable_xtrabackup: "yes"
Then, kick off a reconfiguration of MariaDB:
``kolla-ansible -i INVENTORY reconfigure -t mariadb``
Once that has run successfully, you should then be able to take full and
incremental backups as described below.
Backup Procedure
~~~~~~~~~~~~~~~~
To perform a full backup, run the following command:
``kolla-ansible -i INVENTORY mariadb_backup``
Or to perform an incremental backup:
``kolla-ansible -i INVENTORY mariadb_backup --incremental``
Kolla doesn't currently manage the scheduling of these backups, so you'll
need to configure an appropriate scheduler (i.e cron) to run these commands
on your behalf should you require regular snapshots of your data. A suggested
schedule would be:
* Daily full, retained for two weeks
* Hourly incremental, retained for one day
Backups are performed on your behalf on the designated database node using
permissions defined during the configuration step; no password is required to
invoke these commands.
Furthermore, backup actions can be triggered from a node with a minimal
installation of Kolla-Ansible, specifically one which doesn't require a copy of
``passwords.yml``. This is of note if you're looking to implement automated
backups scheduled via a cron job.
Restoring backups
~~~~~~~~~~~~~~~~~
Owing to the way in which XtraBackup performs hot backups, there are some
steps that must be performed in order to prepare your data before it can be
copied into place for use by MariaDB. This process is currently manual, but
the Kolla XtraBackup image includes the tooling necessary to successfully
prepare backups. Two examples are given below.
Full
----
For a full backup, start a new container using the XtraBackup image with the
following options on the master database node:
.. code-block:: console
docker run -it --volumes-from mariadb --name dbrestore \
-v mariadb_backup:/backup kolla/centos-binary-xtrabackup:rocky \
/bin/bash
cd /backup
mkdir -p /restore/full
cat mysqlbackup-04-10-2018.xbc.xbs | xbstream -x -C /restore/full/
innobackupex --decompress /restore/full
find /restore -name *.qp -exec rm {} \;
innobackupex --apply-log /restore/full
Then stop the MariaDB instance, delete the old data files (or move
them elsewhere), and copy the backup into place:
.. code-block:: console
docker stop mariadb
rm -rf /var/lib/mysql/* /var/lib/mysql/.*
innobackupex --copy-back /restore/full
Then you can restart MariaDB with the restored data in place:
.. code-block:: console
docker start mariadb
docker logs mariadb
81004 15:48:27 mysqld_safe WSREP: Running position recovery with --log_error='/var/lib/mysql//wsrep_recovery.BDTAm8' --pid-file='/var/lib/mysql//scratch-recover.pid'
181004 15:48:30 mysqld_safe WSREP: Recovered position 9388319e-c7bd-11e8-b2ce-6e9ec70d9926:58
Incremental
-----------
This starts off similar to the full backup restore procedure above, but we
must apply the logs from the incremental backups first of all before doing
the final preparation required prior to restore. In the example below, I have
a full backup - ``mysqlbackup-06-11-2018-1541505206.qp.xbc.xbs``, and an
incremental backup,
``incremental-11-mysqlbackup-06-11-2018-1541505223.qp.xbc.xbs``.
.. code-block:: console
docker run -it --volumes-from mariadb --name dbrestore \
-v mariadb_backup:/backup kolla/centos-binary-xtrabackup:rocky \
/bin/bash
cd /backup
mkdir -p /restore/full
mkdir -p /restore/inc/11
cat mysqlbackup-06-11-2018-1541505206.qp.xbc.xbs | xbstream -x -C /restore/full/
cat incremental-11-mysqlbackup-06-11-2018-1541505223.qp.xbc.xbs | xbstream -x -C /restore/inc/11
innobackupex --decompress /restore/full
innobackupex --decompress /restore/inc/11
find /restore -name *.qp -exec rm {} \;
innobackupex --apply-log --redo-only /restore/full
innobackupex --apply-log --redo-only --incremental-dir=/restore/inc/11 /restore/full
innobackupex --apply-log /restore/full
At this point the backup is prepared and ready to be copied back into place,
as per the previous example.

View File

@ -284,6 +284,7 @@ kolla_internal_vip_address: "10.10.10.254"
#enable_vitrage: "no"
#enable_vmtp: "no"
#enable_watcher: "no"
#enable_xtrabackup: "no"
#enable_zookeeper: "no"
#enable_zun: "no"

View File

@ -15,6 +15,8 @@ cinder_rbd_secret_uuid:
# Database options
####################
database_password:
# Password for the dedicated backup user account
mariadb_backup_database_password:
####################
# Docker options

View File

@ -0,0 +1,10 @@
---
features:
- |
Supports taking a backup of all MariaDB-hosted databases using Percona XtraBackup.
security:
- |
When the MariaDB backup option is enabled, it will create a new database
which is used to keep track of backup-related metadata, along with a new
backup user with a specific set of permissions limited to backup-related
actions only.

View File

@ -54,6 +54,9 @@ Commands:
prechecks Do pre-deployment checks for hosts
check Do post-deployment smoke tests
mariadb_recovery Recover a completely stopped mariadb cluster
mariadb_backup Take a backup of MariaDB databases
--full (default)
--incremental
bootstrap-servers Bootstrap servers with kolla deploy dependencies
destroy Destroy Kolla containers, volumes and host configuration
--include-images to also destroy Kolla images
@ -91,6 +94,7 @@ cat <<EOF
prechecks
check
mariadb_recovery
mariadb_backup
bootstrap-servers
destroy
deploy
@ -107,7 +111,7 @@ EOF
}
SHORT_OPTS="hi:p:t:k:e:v"
LONG_OPTS="help,inventory:,playbook:,skip-tags:,tags:,key:,extra:,verbose,configdir:,passwords:,limit:,forks:,vault-id:,ask-vault-pass,vault-password-file:,yes-i-really-really-mean-it,include-images,include-dev"
LONG_OPTS="help,inventory:,playbook:,skip-tags:,tags:,key:,extra:,verbose,configdir:,passwords:,limit:,forks:,vault-id:,ask-vault-pass,vault-password-file:,yes-i-really-really-mean-it,include-images,include-dev:,full,incremental"
RAW_ARGS="$*"
ARGS=$(getopt -o "${SHORT_OPTS}" -l "${LONG_OPTS}" --name "$0" -- "$@") || { usage >&2; exit 2; }
@ -125,6 +129,7 @@ PASSWORDS_FILE="${CONFIG_DIR}/passwords.yml"
DANGER_CONFIRM=
INCLUDE_IMAGES=
INCLUDE_DEV=
BACKUP_TYPE="full"
# Serial is not recommended and disabled by default. Users can enable it by
# configuring ANSIBLE_SERIAL variable.
ANSIBLE_SERIAL=${ANSIBLE_SERIAL:-0}
@ -221,6 +226,16 @@ while [ "$#" -gt 0 ]; do
shift 2
;;
(--full)
BACKUP_TYPE="full"
shift 1
;;
(--incremental)
BACKUP_TYPE="incremental"
shift 1
;;
(--help|-h)
usage
shift
@ -254,6 +269,11 @@ case "$1" in
EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=deploy -e common_run=true"
PLAYBOOK="${BASEDIR}/ansible/mariadb_recovery.yml"
;;
(mariadb_backup)
ACTION="Backup MariaDB databases"
EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=backup -e mariadb_backup_type=${BACKUP_TYPE} -e common_run=true"
PLAYBOOK="${BASEDIR}/ansible/mariadb_backup.yml"
;;
(destroy)
ACTION="Destroy Kolla containers, volumes and host configuration"
PLAYBOOK="${BASEDIR}/ansible/destroy.yml"