diff --git a/docker/services/database/mysql.yaml b/docker/services/database/mysql.yaml index 64f54959d3..7c5789e4d4 100644 --- a/docker/services/database/mysql.yaml +++ b/docker/services/database/mysql.yaml @@ -191,7 +191,7 @@ outputs: mysql: start_order: 2 image: *mysql_image - restart: always + restart: unless-stopped net: host healthcheck: test: /openstack/healthcheck @@ -240,7 +240,64 @@ outputs: Log files from mysql containers can be found under /var/log/containers/mysql. ignore_errors: true - upgrade_tasks: [] + upgrade_tasks: + # LP 1810136 + # After upgrade, the new mariadb (e.g. 10.3) might not be able + # to replay the redo log of an older one (e.g. 10.1) if mysql + # stopped unexpectedly. Force it to stop and clean the redo log + # here to avoid issue after upgrade. + - name: Stop MySQL server and ensure redo log is cleaned up before upgrade + when: step|int == 2 + block: + - name: Retrieve mysql container configuration from paunch + shell: + str_replace: + # Use a Jinja template below to force str_replace + # to quote the resulting string + # TODO: remove the grep filter once #1811384 is fixed + template: &mysql_template '{{ "paunch" }} list --default-runtime RUNTIME -f json -c container -c image | grep -v "isn''t supported by" | jq -r ''.[] | select(.container=="mysql") + {"cli":"RUNTIME"}''' + params: + RUNTIME: "{{ container_cli }}" + register: mysql_paunch + - name: Retrieve mysql container configuration from paunch (docker) + shell: + str_replace: + template: *mysql_template + params: + RUNTIME: "docker" + register: mysql_paunch_docker + when: '"mysql" not in mysql_paunch.stdout' + - name: Mysql container facts + set_fact: + mysql_json: "{{ mysql_paunch_docker.stdout if mysql_paunch_docker.changed else mysql_paunch.stdout | from_json }}" + - name: Redo log clean-up script + set_fact: + # The purpose of this script is to start mysql so that it + # replays the redo log, and shutdown mysql cleanly + mysql_clean_up_script: + list_join: + - ' ' + - - 'kolla_set_configs;' + - 'mysqld_safe --user=mysql --skip-networking &' + - 'timeout 180 sh -c ''while ! mysqladmin ping --silent; do sleep 1; done'';' + - 'mysqladmin shutdown' + - name: Bind mounts for temporary clean-up container + set_fact: + mysql_clean_up_volumes: *mysql_volumes + - name: Stop the current mysql container + shell: | + {{ mysql_json.cli }} update --restart=unless-stopped mysql + {{ mysql_json.cli }} stop mysql + - name: Clean up redo log by running a transient mysql server + shell: + str_replace: + template: + "{{ mysql_json.cli }} run --rm -u root --net=host ENV VOLUMES \"IMAGE\" /bin/bash -ecx \"SCRIPT\"" + params: + ENV: '-e "KOLLA_CONFIG_STRATEGY=COPY_ALWAYS"' + IMAGE: "{{ mysql_json.image }}" + VOLUMES: "-v {{ mysql_clean_up_volumes | join(' -v ') }}" + SCRIPT: "{{ mysql_clean_up_script }}" post_upgrade_tasks: - when: step|int == 1 import_role: