Merge "Added checks for failed cinder services"

tasks/cinder_db_sync.yml

@@ -12,22 +12,42 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 - name: Perform a cinder DB sync
   command: "{{ cinder_bin }}/cinder-manage db sync"
   become: yes
   become_user: "{{ cinder_system_user_name }}"
   changed_when: false
 
-- name: Perform online data migrations
-  command: "{{ cinder_bin }}/cinder-manage db online_data_migrations"
+- name: Perform data migrations if necessary and all cinder services are up
+  block:
+
+    - name: Perform online data migrations
+      command: "{{ cinder_bin }}/cinder-manage db online_data_migrations"
+      register: data_migrations
+      changed_when: false
+
+  rescue:
+
+    - name: Check if all cinder services are up
+      shell: "{{ cinder_bin }}/cinder-manage service list | /usr/bin/awk '{ if(NR>1) print $5 }' | /bin/egrep -vq ':-)'"
+      register: failed_cinder_services
+      ignore_errors: True
+
+    - name: Fail if any cinder service is down
+      fail:
+        msg: At least one cinder service is down. You need to fix this before re-running the playbook.
+      when: failed_cinder_services.rc != 1
+
+    - name: All cinder services are up, but migration failed nontheless.
+      fail:
+        msg: An unexpected error occured.
+      when: failed_cinder_services.rc == 1
+
   become: yes
   become_user: "{{ cinder_system_user_name }}"
   when:
     - "(cinder_all_software_updated | default('no')) | bool"
     - "ansible_local['openstack_ansible']['cinder']['need_online_data_migrations'] | bool"
-  changed_when: false
-  register: data_migrations
 
 - name: Disable the online migrations requirement
   ini_file: