From d839e2e4f950b08b6fb28169da16f56449797fb8 Mon Sep 17 00:00:00 2001
From: Jesse Pretorius <jesse.pretorius@rackspace.co.uk>
Date: Mon, 11 Jan 2016 16:36:49 +0000
Subject: [PATCH] Resolve MariaDB/Galera cluster startup/logging issues

This patch ensures that MariaDB is given adequate time to start on a
resources constrained system (180s versus the default of 30s),
ensures that the error log is appropriately populated and also
provides a failback restart in the case where there may be a corrupt
sst directory.

In the handler changes:
 - the environment variable "MYSQLD_STARTUP_TIMEOUT" is now being
   passed into the init script because the defaults are not being
   sourced at the init script runtime.
 - the temporary "sst" directory is cleaned up should the handler
   restart fail. This ensurez that a node is in a clean state if a
   leftover sst directory was on the disk which would cause a node
   to fail to join a cluster or bootstrap.

In the task changes a new configuration file, that is part of the
mariadb package, is being removed which has unforeseen options within
it causing no logs to be created.

The default option "galera_innodb_additional_mem_pool_size" was removed
because its no longer valid within MariaDB10 and we'd never caught that
error message until now.

This patch is based on:
 - https://review.openstack.org/256016
 - https://review.openstack.org/266265

Closes-Bug: #1532761
Closes-Bug: #1533126
Change-Id: I16af30c660790656fc2d59f9943c172b88098905
---
 .../roles/galera_server/defaults/main.yml     |  1 -
 .../roles/galera_server/handlers/main.yml     | 29 +++++++++++++++++--
 .../tasks/galera_post_install.yml             |  7 +++++
 .../roles/galera_server/templates/my.cnf.j2   |  2 +-
 4 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/playbooks/roles/galera_server/defaults/main.yml b/playbooks/roles/galera_server/defaults/main.yml
index 13602196ba..b9f6840407 100644
--- a/playbooks/roles/galera_server/defaults/main.yml
+++ b/playbooks/roles/galera_server/defaults/main.yml
@@ -45,7 +45,6 @@ galera_wait_timeout: 28800
 
 ## innodb options
 galera_innodb_buffer_pool_size: 4096M
-galera_innodb_additional_mem_pool_size: 24M
 galera_innodb_log_file_size: 1024M
 galera_innodb_log_buffer_size: 128M
 
diff --git a/playbooks/roles/galera_server/handlers/main.yml b/playbooks/roles/galera_server/handlers/main.yml
index a4f02c97be..7c60131635 100644
--- a/playbooks/roles/galera_server/handlers/main.yml
+++ b/playbooks/roles/galera_server/handlers/main.yml
@@ -18,8 +18,33 @@
     name: mysql
     state: restarted
     args: "{{ (not galera_existing_cluster | bool and inventory_hostname == galera_server_bootstrap_node) | ternary('--wsrep-new-cluster', '') }}"
+  environment:
+    MYSQLD_STARTUP_TIMEOUT: 180
   when: not galera_running_and_bootstrapped | bool
   register: galera_restart
-  until: galera_restart|success
+  # notifies are only fired when status is "changed"
+  changed_when: galera_restart | failed
+  failed_when: false
+  notify:
+    - "remove stale .sst"
+    - "Restart mysql fall back"
+
+- name: remove stale .sst
+  file:
+    path: "/var/lib/mysql/.sst"
+    state: absent
+  when: galera_restart | failed
+
+- name: Restart mysql fall back
+  service:
+    name: mysql
+    state: restarted
+    args: "{{ (not galera_existing_cluster | bool and inventory_hostname == galera_server_bootstrap_node) | ternary('--wsrep-new-cluster', '') }}"
+  environment:
+    MYSQLD_STARTUP_TIMEOUT: 180
+  when: galera_restart | failed
+  register: galera_restart_fall_back
+  until: galera_restart_fall_back | success
   retries: 3
-  delay: 2
+  delay: 5
+
diff --git a/playbooks/roles/galera_server/tasks/galera_post_install.yml b/playbooks/roles/galera_server/tasks/galera_post_install.yml
index 63866d3692..b2a71f77e1 100644
--- a/playbooks/roles/galera_server/tasks/galera_post_install.yml
+++ b/playbooks/roles/galera_server/tasks/galera_post_install.yml
@@ -66,6 +66,13 @@
   tags:
     - galera-config
 
+- name: remove default mysql_safe_syslog
+  file:
+    path: "/etc/mysql/conf.d/mysqld_safe_syslog.cnf"
+    state: absent
+  tags:
+    - galera-config
+
 - name: Remove policy-rc
   file:
     path: "/usr/sbin/policy-rc.d"
diff --git a/playbooks/roles/galera_server/templates/my.cnf.j2 b/playbooks/roles/galera_server/templates/my.cnf.j2
index 12d118da15..0f9a496a2b 100644
--- a/playbooks/roles/galera_server/templates/my.cnf.j2
+++ b/playbooks/roles/galera_server/templates/my.cnf.j2
@@ -15,6 +15,7 @@ socket = /var/run/mysqld/mysqld.sock
 [mysqld_safe]
 socket = /var/run/mysqld/mysqld.sock
 nice = 0
+log_error = /var/log/mysql_logs/galera_server_error.log
 
 
 [mysql]
@@ -66,7 +67,6 @@ table-open-cache = 10240
 
 # INNODB #
 innodb-flush-method = O_DIRECT
-innodb-additional-mem-pool-size = {{ galera_innodb_additional_mem_pool_size }}
 innodb-log-file-size = {{ galera_innodb_log_file_size }}
 innodb-flush-log-at-trx-commit = 1
 innodb-file-per-table = 1