Improve docker/distribution reload handlers

Move the wait-for-registry-up loop and systemd daemon reloading
commands behind the user-faced "restart docker(-registry)"
handlers. So users may just refer to it w/o manual daemon reload
tasks and w/o knowing of other advanced sub-steps details. But
those handlers may still be notified directly as well.

Add a fixed delay of a 10s for the docker service reload handler,
which is nice to have for the loaded systems running that command
way too long.

Add wait-for-docker-service-up loop into the docker service
reload handler for the better failures detection.

Simplify future support on CoreOS/atomic distributions
(which require reloading docker and socket in particular order)

Based on
https://github.com/kubernetes-incubator/kubespray/blob/ \
  810596c6d8f1e13afa934ddeef996d4f42b29890/roles/docker/handlers/main.yml

Depends-On: I26cc07aa05912c3e84d59003686eae210e924a16
Change-Id: I28e8c682005e77b87a347f4d2a2dff219fab488c
Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>

handlers/main.yml

@@ -1,13 +1,56 @@
-# handlers file for ansible-role-container-registry
-
+---
 - name: restart docker
-  service:
-    name: docker
-    state: restarted
+  command: /bin/true
+  notify:
+    - Docker | reload systemd
+    - Docker | reload docker
+    - Docker | pause while Docker restarts
+    - Docker | wait for docker
   listen: "restart docker service"
 
 - name: restart docker-distribution
+  command: /bin/true
+  notify:
+    - Docker | reload systemd
+    - Docker | reload docker-distribution
+    - Docker | wait for registry
+  listen: "restart docker-distribution service"
+
+- name: Docker | reload systemd
+  systemd:
+    daemon_reload: yes
+  when: ansible_service_mgr == 'systemd'
+
+- name: Docker | reload docker
+  service:
+    name: docker
+    state: restarted
+
+- name: Docker | pause while Docker restarts
+  pause:
+    seconds: 10
+    prompt: "Waiting for docker restart"
+
+- name: Docker | wait for docker
+  command: /usr/bin/docker images
+  register: docker_ready
+  retries: 10
+  delay: 5
+  until: docker_ready.rc == 0
+
+- name: Docker | reload docker-distribution
   service:
     name: docker-distribution
     state: restarted
-  listen: "restart docker-distribution service"
+
+# NOTE(bogdando): import caveates https://github.com/ansible/ansible/issues/42621
+- name: Docker | wait for registry
+  uri:
+    # Just checking API version should be fine
+    # https://docs.docker.com/registry/spec/api/#api-version-check
+    url: "http://{{ container_registry_host }}:{{ container_registry_port }}/v2/"
+    return_content: yes
+  register: registry_status
+  retries: 10
+  delay: 5
+  until: "registry_status.status|int == 200 and 'OK' in registry_status.msg"

tasks/docker-distribution.yml

@@ -11,14 +11,11 @@
     dest: /etc/docker-distribution/registry/config.yml
   notify: restart docker-distribution service
 
+- name: force systemd to reread configs
+  meta: flush_handlers
+
 - name: enable and start docker-distribution
   systemd:
     enabled: true
     state: started
     name: docker-distribution
-
-- name: validate that registry is up
-  uri:
-    # Just checking API version should be fine
-    # https://docs.docker.com/registry/spec/api/#api-version-check
-    url: "http://{{ container_registry_host }}:{{ container_registry_port }}/v2/"

tasks/docker.yml

@@ -94,9 +94,7 @@
   when: container_registry_deployment_user != ""
 
 - name: force systemd to reread configs
-  systemd:
-    daemon_reload: yes
-  when: ansible_service_mgr == 'systemd'
+  meta: flush_handlers
 
 - name: enable and start docker
   systemd: