diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index e2e5c9a5f4..4974e51a21 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -545,6 +545,7 @@ enable_trove_singletenant: "no" enable_vitrage: "no" enable_vmtp: "no" enable_watcher: "no" +enable_xtrabackup: "no" enable_zookeeper: "{{ enable_kafka | bool }}" enable_zun: "no" diff --git a/ansible/mariadb_backup.yml b/ansible/mariadb_backup.yml new file mode 100644 index 0000000000..2bd9f9af90 --- /dev/null +++ b/ansible/mariadb_backup.yml @@ -0,0 +1,23 @@ +--- +- name: Detect openstack_release variable + hosts: mariadb + gather_facts: false + tasks: + - name: Get current kolla-ansible version number + local_action: command python -c "import pbr.version; print(pbr.version.VersionInfo('kolla-ansible'))" + register: kolla_ansible_version + changed_when: false + when: openstack_release == "auto" + + - name: Set openstack_release variable + set_fact: + openstack_release: "{{ kolla_ansible_version.stdout }}" + when: openstack_release == "auto" + tags: always + +- name: Backup MariaDB + hosts: mariadb + roles: + - { role: mariadb, + tags: mariadb, + when: enable_xtrabackup | bool } diff --git a/ansible/roles/mariadb/defaults/main.yml b/ansible/roles/mariadb/defaults/main.yml index 0b07e68035..c1f4546dea 100644 --- a/ansible/roles/mariadb/defaults/main.yml +++ b/ansible/roles/mariadb/defaults/main.yml @@ -62,3 +62,16 @@ mariadb_dimensions: "{{ default_container_dimensions }}" # Vars used within recover_cluster.yml ######################################## mariadb_service: "{{ mariadb_services['mariadb'] }}" + +#################### +# Backups +#################### +xtrabackup_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-xtrabackup" +xtrabackup_tag: "{{ openstack_release }}" +xtrabackup_image_full: "{{ xtrabackup_image }}:{{ xtrabackup_tag }}" + +mariadb_backup_host: "{{ groups['mariadb'][0] }}" +mariadb_backup_database_schema: "PERCONA_SCHEMA" +mariadb_backup_database_user: "backup" +mariadb_backup_database_address: "{{ database_address }}" +mariadb_backup_type: "full" diff --git a/ansible/roles/mariadb/tasks/backup.yml b/ansible/roles/mariadb/tasks/backup.yml new file mode 100644 index 0000000000..601d5b3012 --- /dev/null +++ b/ansible/roles/mariadb/tasks/backup.yml @@ -0,0 +1,19 @@ +--- +- name: Taking {{ mariadb_backup_type }} database backup via XtraBackup + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + image: "{{ xtrabackup_image_full }}" + name: "xtrabackup" + restart_policy: "never" + remove_on_exit: True + environment: + BACKUP_TYPE: "{{ mariadb_backup_type }}" + volumes: + - "{{ node_config_directory }}xtrabackup:/etc/mysql:ro" + - "/etc/localtime:/etc/localtime:ro" + - "mariadb_backup:/backup" + volumes_from: + - "mariadb" + when: + - inventory_hostname == mariadb_backup_host diff --git a/ansible/roles/mariadb/tasks/config.yml b/ansible/roles/mariadb/tasks/config.yml index 99510b4145..83c1a273f9 100644 --- a/ansible/roles/mariadb/tasks/config.yml +++ b/ansible/roles/mariadb/tasks/config.yml @@ -12,6 +12,33 @@ - item.value.enabled | bool with_dict: "{{ mariadb_services }}" +- name: Ensuring database backup config directory exists + file: + path: "{{ node_config_directory }}xtrabackup" + state: "directory" + owner: "{{ config_owner_user }}" + group: "{{ config_owner_group }}" + mode: "0770" + become: true + when: + - enable_xtrabackup | bool + - inventory_hostname == mariadb_backup_host + +- name: Copying over my.cnf for xtrabackup + merge_configs: + sources: + - "{{ role_path }}/templates/backup.my.cnf.j2" + - "{{ node_custom_config }}/backup.my.cnf" + - "{{ node_custom_config }}/mariadb/{{ inventory_hostname }}/backup.my.cnf" + dest: "{{ node_config_directory }}xtrabackup/my.cnf" + owner: "{{ config_owner_user }}" + group: "{{ config_owner_group }}" + mode: "0660" + become: true + when: + - enable_xtrabackup | bool + - inventory_hostname == mariadb_backup_host + - name: Copying over config.json files for services vars: service_name: "mariadb" diff --git a/ansible/roles/mariadb/tasks/register.yml b/ansible/roles/mariadb/tasks/register.yml index 76030f2b46..bdcd12eb2e 100644 --- a/ansible/roles/mariadb/tasks/register.yml +++ b/ansible/roles/mariadb/tasks/register.yml @@ -13,6 +13,53 @@ priv: "*.*:USAGE" run_once: True +- name: Creating the Percona XtraBackup database + kolla_toolbox: + module_name: mysql_db + module_args: + login_host: "{{ database_address }}" + login_port: "{{ database_port }}" + login_user: "{{ database_user }}" + login_password: "{{ database_password }}" + name: "{{ mariadb_backup_database_schema }}" + run_once: True + when: + - enable_xtrabackup | bool + +- name: Creating database backup user and setting permissions + kolla_toolbox: + module_name: mysql_user + module_args: + login_host: "{{ database_address }}" + login_port: "{{ database_port }}" + login_user: "{{ database_user }}" + login_password: "{{ database_password }}" + name: "{{ mariadb_backup_database_user }}" + password: "{{ mariadb_backup_database_password }}" + host: "%" + priv: "*.*:CREATE TABLESPACE,RELOAD,PROCESS,SUPER,LOCK TABLES,REPLICATION CLIENT" + append_privs: True + run_once: True + when: + - enable_xtrabackup | bool + +- name: Granting permissions on XtraBackup database to backup user + kolla_toolbox: + module_name: mysql_user + module_args: + login_host: "{{ database_address }}" + login_port: "{{ database_port }}" + login_user: "{{ database_user }}" + login_password: "{{ database_password }}" + name: "{{ mariadb_backup_database_user }}" + password: "{{ mariadb_backup_database_password }}" + host: "%" + priv: "{{ mariadb_backup_database_schema }}.*:CREATE,INSERT,SELECT" + append_privs: True + run_once: True + when: + - enable_xtrabackup | bool + - name: Cleaning up facts set_fact: delegate_host: "bootstraped" diff --git a/ansible/roles/mariadb/templates/backup.my.cnf.j2 b/ansible/roles/mariadb/templates/backup.my.cnf.j2 new file mode 100644 index 0000000000..0620e046ab --- /dev/null +++ b/ansible/roles/mariadb/templates/backup.my.cnf.j2 @@ -0,0 +1,6 @@ +[client] +default-character-set=utf8 +user={{ mariadb_backup_database_user }} +password={{ mariadb_backup_database_password }} +host={{ database_address }} +port={{ database_port }} diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst index 7233c1695c..8f01b5a3d8 100644 --- a/doc/source/admin/index.rst +++ b/doc/source/admin/index.rst @@ -6,5 +6,6 @@ Admin Guides :maxdepth: 2 advanced-configuration + mariadb-backup-and-restore production-architecture-guide deployment-philosophy diff --git a/doc/source/admin/mariadb-backup-and-restore.rst b/doc/source/admin/mariadb-backup-and-restore.rst new file mode 100644 index 0000000000..dc6b8517c5 --- /dev/null +++ b/doc/source/admin/mariadb-backup-and-restore.rst @@ -0,0 +1,140 @@ +.. _mariadb-backup-and-restore: + +=================================== +MariaDB database backup and restore +=================================== + +Kolla-Ansible can facilitate either full or incremental backups of data +hosted in MariaDB. It achieves this using Percona's Xtrabackup, a tool +designed to allow for 'hot backups' - an approach which means that consistent +backups can be taken without any downtime for your database or your cloud. + +.. note:: + + By default, backups will be performed on the first node in your Galera cluster + or on the MariaDB node itself if you just have the one. Backup files are saved + to a dedicated Docker volume - ``mariadb_backup`` - and it's the contents of + this that you should target for transferring backups elsewhere. + +Enabling Backup Functionality +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For backups to work, some reconfiguration of MariaDB is required - this is to +enable appropriate permissions for the backup client, and also to create an +additional database in order to store backup information. + +Firstly, enable backups via ``globals.yml``: + +.. code-block:: console + + enable_xtrabackup: "yes" + +Then, kick off a reconfiguration of MariaDB: + +``kolla-ansible -i INVENTORY reconfigure -t mariadb`` + +Once that has run successfully, you should then be able to take full and +incremental backups as described below. + +Backup Procedure +~~~~~~~~~~~~~~~~ + +To perform a full backup, run the following command: + +``kolla-ansible -i INVENTORY mariadb_backup`` + +Or to perform an incremental backup: + +``kolla-ansible -i INVENTORY mariadb_backup --incremental`` + +Kolla doesn't currently manage the scheduling of these backups, so you'll +need to configure an appropriate scheduler (i.e cron) to run these commands +on your behalf should you require regular snapshots of your data. A suggested +schedule would be: + +* Daily full, retained for two weeks +* Hourly incremental, retained for one day + +Backups are performed on your behalf on the designated database node using +permissions defined during the configuration step; no password is required to +invoke these commands. + +Furthermore, backup actions can be triggered from a node with a minimal +installation of Kolla-Ansible, specifically one which doesn't require a copy of +``passwords.yml``. This is of note if you're looking to implement automated +backups scheduled via a cron job. + +Restoring backups +~~~~~~~~~~~~~~~~~ + +Owing to the way in which XtraBackup performs hot backups, there are some +steps that must be performed in order to prepare your data before it can be +copied into place for use by MariaDB. This process is currently manual, but +the Kolla XtraBackup image includes the tooling necessary to successfully +prepare backups. Two examples are given below. + +Full +---- + +For a full backup, start a new container using the XtraBackup image with the +following options on the master database node: + +.. code-block:: console + + docker run -it --volumes-from mariadb --name dbrestore \ + -v mariadb_backup:/backup kolla/centos-binary-xtrabackup:rocky \ + /bin/bash + cd /backup + mkdir -p /restore/full + cat mysqlbackup-04-10-2018.xbc.xbs | xbstream -x -C /restore/full/ + innobackupex --decompress /restore/full + find /restore -name *.qp -exec rm {} \; + innobackupex --apply-log /restore/full + +Then stop the MariaDB instance, delete the old data files (or move +them elsewhere), and copy the backup into place: + +.. code-block:: console + + docker stop mariadb + rm -rf /var/lib/mysql/* /var/lib/mysql/.* + innobackupex --copy-back /restore/full + +Then you can restart MariaDB with the restored data in place: + +.. code-block:: console + + docker start mariadb + docker logs mariadb + 81004 15:48:27 mysqld_safe WSREP: Running position recovery with --log_error='/var/lib/mysql//wsrep_recovery.BDTAm8' --pid-file='/var/lib/mysql//scratch-recover.pid' + 181004 15:48:30 mysqld_safe WSREP: Recovered position 9388319e-c7bd-11e8-b2ce-6e9ec70d9926:58 + +Incremental +----------- + +This starts off similar to the full backup restore procedure above, but we +must apply the logs from the incremental backups first of all before doing +the final preparation required prior to restore. In the example below, I have +a full backup - ``mysqlbackup-06-11-2018-1541505206.qp.xbc.xbs``, and an +incremental backup, +``incremental-11-mysqlbackup-06-11-2018-1541505223.qp.xbc.xbs``. + +.. code-block:: console + + docker run -it --volumes-from mariadb --name dbrestore \ + -v mariadb_backup:/backup kolla/centos-binary-xtrabackup:rocky \ + /bin/bash + cd /backup + mkdir -p /restore/full + mkdir -p /restore/inc/11 + cat mysqlbackup-06-11-2018-1541505206.qp.xbc.xbs | xbstream -x -C /restore/full/ + cat incremental-11-mysqlbackup-06-11-2018-1541505223.qp.xbc.xbs | xbstream -x -C /restore/inc/11 + innobackupex --decompress /restore/full + innobackupex --decompress /restore/inc/11 + find /restore -name *.qp -exec rm {} \; + innobackupex --apply-log --redo-only /restore/full + innobackupex --apply-log --redo-only --incremental-dir=/restore/inc/11 /restore/full + innobackupex --apply-log /restore/full + +At this point the backup is prepared and ready to be copied back into place, +as per the previous example. diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index d159b45e07..62da37ddec 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -284,6 +284,7 @@ kolla_internal_vip_address: "10.10.10.254" #enable_vitrage: "no" #enable_vmtp: "no" #enable_watcher: "no" +#enable_xtrabackup: "no" #enable_zookeeper: "no" #enable_zun: "no" diff --git a/etc/kolla/passwords.yml b/etc/kolla/passwords.yml index ebbda636d2..8d60d26180 100644 --- a/etc/kolla/passwords.yml +++ b/etc/kolla/passwords.yml @@ -15,6 +15,8 @@ cinder_rbd_secret_uuid: # Database options #################### database_password: +# Password for the dedicated backup user account +mariadb_backup_database_password: #################### # Docker options diff --git a/releasenotes/notes/mariadb-xtrabackup-d4f48464dd6baaea.yaml b/releasenotes/notes/mariadb-xtrabackup-d4f48464dd6baaea.yaml new file mode 100644 index 0000000000..da38004f20 --- /dev/null +++ b/releasenotes/notes/mariadb-xtrabackup-d4f48464dd6baaea.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Supports taking a backup of all MariaDB-hosted databases using Percona XtraBackup. +security: + - | + When the MariaDB backup option is enabled, it will create a new database + which is used to keep track of backup-related metadata, along with a new + backup user with a specific set of permissions limited to backup-related + actions only. diff --git a/tools/kolla-ansible b/tools/kolla-ansible index 86dac7418c..0706d7a763 100755 --- a/tools/kolla-ansible +++ b/tools/kolla-ansible @@ -54,6 +54,9 @@ Commands: prechecks Do pre-deployment checks for hosts check Do post-deployment smoke tests mariadb_recovery Recover a completely stopped mariadb cluster + mariadb_backup Take a backup of MariaDB databases + --full (default) + --incremental bootstrap-servers Bootstrap servers with kolla deploy dependencies destroy Destroy Kolla containers, volumes and host configuration --include-images to also destroy Kolla images @@ -91,6 +94,7 @@ cat <&2; exit 2; } @@ -125,6 +129,7 @@ PASSWORDS_FILE="${CONFIG_DIR}/passwords.yml" DANGER_CONFIRM= INCLUDE_IMAGES= INCLUDE_DEV= +BACKUP_TYPE="full" # Serial is not recommended and disabled by default. Users can enable it by # configuring ANSIBLE_SERIAL variable. ANSIBLE_SERIAL=${ANSIBLE_SERIAL:-0} @@ -221,6 +226,16 @@ while [ "$#" -gt 0 ]; do shift 2 ;; + (--full) + BACKUP_TYPE="full" + shift 1 + ;; + + (--incremental) + BACKUP_TYPE="incremental" + shift 1 + ;; + (--help|-h) usage shift @@ -254,6 +269,11 @@ case "$1" in EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=deploy -e common_run=true" PLAYBOOK="${BASEDIR}/ansible/mariadb_recovery.yml" ;; +(mariadb_backup) + ACTION="Backup MariaDB databases" + EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=backup -e mariadb_backup_type=${BACKUP_TYPE} -e common_run=true" + PLAYBOOK="${BASEDIR}/ansible/mariadb_backup.yml" + ;; (destroy) ACTION="Destroy Kolla containers, volumes and host configuration" PLAYBOOK="${BASEDIR}/ansible/destroy.yml"