tripleo-ansible/playbooks/pre-flight_check.yml

# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
- hosts: localhost
  roles:
    - { role: pre_flight_disk_check, fail_percent: 20, exclude_mounts: [] }
- hosts: undercloud
  roles:
    - { role: pre_flight_disk_check, fail_percent: 20, exclude_mounts: [], when: instance_status == "ACTIVE" }
- hosts: controller
  vars:
    num_bootstrap_hosts: "{{ groups['controller-bootstrap'] | length }}"
    expected_bootstrap_hosts: 1
    num_controller_hosts: "{{ groups['controller'] | length }}"
    expected_controller_hosts: 2
  tasks:
    - name: "Verify correct number of controller bootstrap nodes"
      fail: >
        msg="The number of bootstrap nodes is "{{ num_bootstrap_hosts }}" \
             must be exactly "{{ expected_bootstrap_hosts }}"!"        
      when: num_bootstrap_hosts | int != expected_bootstrap_hosts
    - name: "Verify correct number of controller non-bootstrap nodes"
      fail: >
        msg="The number of controller non-bootstrap nodes is \
             "{{ num_controller_hosts }}" must be exactly \
             "{{ expected_controller_hosts }}"!"        
      when: num_controller_hosts | int != expected_controller_hosts
- hosts: controller-bootstrap:controller
  name: "Check Controller Node Status - Ensuring MySQL is running"
  sudo: yes
  gather_facts: yes
  max_fail_percentage: 0
  tasks:
    - name: "Verify controller nodes are ACTIVE"
      fail: >
        msg="One of more controller nodes don't have a status of \
             ACTIVE in nova, cannot proceed with update"        
      when: instance_status != "ACTIVE"
    - name: "Rename MySQL upstart configuration, if necessary, to obtain correct results."
      sudo: yes
      command: mv -f /etc/init/mysql.conf /etc/init/mysql-boot-control.conf removes=/etc/init/mysql.conf
      when: instance_status == "ACTIVE"
    - name: "Ensuring MySQL is running - If this fails, the cluster is likely not in a healthy state, and manual checks/recovery will be required."
      service: name=mysql state=started
      when: instance_status == "ACTIVE"
- hosts: controller-bootstrap
  name: Check RabbitMQ Bootstrap Node
  sudo: yes
  gather_facts: yes
  max_fail_percentage: 0
  tasks:
    - name: "Execute RabbitMQ status Check to verify RabbitMQ is running."
      sudo: yes
      shell: rabbitmqctl -n rabbit@$(hostname) status
      when: instance_status == "ACTIVE"
      register: rabbitmq_status
      ignore_errors: yes
    - name: "Attempting to start RabbitMQ - Bootstrap Node"
      service: name=rabbitmq-server state=started
      when: rabbitmq_status.rc != 0
    - name: "Checking RabbitMQ Cluster Status - Bootstrap Node"
      shell: rabbitmqctl -n rabbit@$(hostname) status
- hosts: controller
  name: "Check RabbitMQ server status"
  sudo: yes
  gather_facts: yes
  max_fail_percentage: 0
  tasks:
    - name: "Execute RabbitMQ status Check to verify RabbitMQ is running."
      shell: rabbitmqctl -n rabbit@$(hostname) status
      when: instance_status == "ACTIVE"
      register: rabbitmq_status
      ignore_errors: yes
    - name: "Attempting to start RabbitMQ"
      service: name=rabbitmq-server state=started
      when: rabbitmq_status.rc != 0
    - name: "Checking RabbitMQ Cluster Status"
      shell: rabbitmqctl -n rabbit@$(hostname) status
- hosts: controller-bootstrap:controller
  name: Check controller MySQL Sync status
  gather_facts: no
  max_fail_percentage: 0
  tasks:
    - name: "Querying MySQL to determine Galera cluster status"
      sudo: yes
      shell: mysql --defaults-file=/mnt/state/root/metadata.my.cnf --socket /var/run/mysqld/mysqld.sock -N -e "SHOW STATUS LIKE 'wsrep_local_state'"|cut -f2
      when: helion is not defined
      register: wsrep_local_state
    - name: "Querying MySQL to determine Galera cluster size"
      sudo: yes
      shell: mysql --defaults-file=/mnt/state/root/metadata.my.cnf --socket /var/run/mysqld/mysqld.sock -N -e "SHOW STATUS LIKE 'wsrep_cluster_size'"|cut -f2
      when: helion is not defined
      register: wsrep_cluster_size
    - fail: msg="Galera wsrep_local_state is not indicating a healthy state, cluster node may be out of sync, manual intervention required."
      when: helion is not defined and wsrep_local_state.stdout != "4"
    - fail: msg="Galera cluster size is being reported as a single node.  The cluster is in an unsafe, possibly split brain state."
      when: helion is not defined and wsrep_cluster_size.stdout == "1"
    # Helion
    - name: "Querying MySQL to determine Galera cluster status - Helion"
      sudo: yes
      shell: mysql --defaults-file=/mnt/state/root/metadata.my.cnf -N -e "SHOW STATUS LIKE 'wsrep_local_state'"|cut -f2
      when: helion is defined
      register: wsrep_local_state
    - name: "Querying MySQL to determine Galera cluster size - Helion"
      sudo: yes
      shell: mysql --defaults-file=/mnt/state/root/metadata.my.cnf -N -e "SHOW STATUS LIKE 'wsrep_cluster_size'"|cut -f2
      when: helion is defined
      register: wsrep_cluster_size
    - fail: msg="Galera wsrep_local_state is not indicating a healthy state, cluster node may be out of sync, manual intervention required."
      when: helion is defined and wsrep_local_state.stdout != "4"
    - fail: msg="Galera cluster size is being reported as a single node.  The cluster is in an unsafe, possibly split brain state."
      when: helion is defined and wsrep_cluster_size.stdout == "1"