Fix upgrades for multinode galera

During an upgrade, we run the upgrade in serial, and we attempt to
bootstrap the cluster from the "galera_server_bootstrap_node". This
causes issues when there is an existing cluster that is still up.

To avoid this, we can simply attempt to start the cluster normally, if
it fails to start, and join the existing cluster,  we can bootstrap as
usual.

Additionally, we can make the bootstrap slightly more efficient by only
running it against the bootstrap host and splitting out the
"galera_upgrade_post.yml" tasks.

Finally - run the upgrade in serial which mirrors our approach in the
integrated repo.

Change-Id: Ic4d69f0fa75c1eea81d10a76cca9a8d9c3822094
Closes-Bug: #1667103
(cherry picked from commit db64210b12)

tasks/galera_bootstrap.yml

@@ -18,12 +18,26 @@
     num_retries: 2
     wait_delay: 3
 
+# If there is an existing cluster join it by starting mysql normally
+- name: Start mysql node normally
+  service:
+    name: mysql
+    state: started
+  environment:
+    MYSQLD_STARTUP_TIMEOUT: 180
+  until: mysql_started | success
+  retries: 3
+  delay: 5
+  when: mysql_running.rc != 0
+  register: mysql_started
+  ignore_errors: true
+
 - name: Start galera cluster
   command: galera_new_cluster
   when:
     - ansible_service_mgr == 'systemd'
-    - inventory_hostname == galera_server_bootstrap_node
     - mysql_running.rc != 0
+    - mysql_started | failed
   register: start_cluster
   failed_when: false
   tags:
@@ -34,8 +48,8 @@
   command: "/etc/init.d/mysql bootstrap"
   when:
     - ansible_service_mgr == 'systemd'
+    - mysql_started | failed
     - start_cluster | failed
-    - inventory_hostname == galera_server_bootstrap_node
     - mysql_running.rc != 0
   tags:
     - galera-bootstrap
@@ -48,7 +62,6 @@
   retries: 6
   delay: 5
   when:
-    - inventory_hostname == galera_server_bootstrap_node
     - mysql_running.rc != 0
   tags:
     - galera-operational
@@ -58,13 +71,7 @@
   set_fact:
     galera_running_and_bootstrapped: true
   when:
-    - inventory_hostname == galera_server_bootstrap_node
     - mysql_running.rc != 0
   tags:
     - galera-operational
     - galera-bootstrap
-
-- include: galera_upgrade_post.yml
-  static: no
-  when:
-    - galera_upgrade | bool

tasks/main.yml

@@ -52,8 +52,16 @@
   tags: galera_server-config
 
 - include: galera_bootstrap.yml
+  when:
+    - inventory_hostname == galera_server_bootstrap_node
   tags: galera_server-config
 
+- include: galera_upgrade_post.yml
+  static: no
+  tags: galera_server-upgrade
+  when:
+    - galera_upgrade | bool
+
 - name: Flush handlers
   meta: flush_handlers
 

tests/test-install-galera-upgrade.yml

@@ -0,0 +1,25 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Deploy galera
+  hosts: galera_all
+  user: root
+  serial: 1
+  max_fail_percentage: 20
+  gather_facts: true
+  roles:
+    - role: "galera_server"
+  vars_files:
+    - common/test-vars.yml

tests/test.yml

@@ -29,7 +29,7 @@
 - include: test-galera-server-functional.yml
 
 # Install current version Galera server
-- include: common/test-install-galera.yml
+- include: test-install-galera-upgrade.yml
   vars:
     galera_upgrade: true