Add override for gluster host used for bootstrap operations

This change permits overriding of the host used to bootstrap the
cluster. This is necessary when the cluster already exists and
a new (or upgraded) host needs to join an existing cluster. This
only works when actions are performed from an existing cluster
member.

This patch additionally resolves an issue where the volume
creation step can fail if the bootstrap host's peer names don't
exactly match those being passed to it (such as when they end with
.openstack.local). A restart of the service fixes this by reading
the correct hostnames back from the peer files.

Change-Id: I7127cb86e81abc982290681d24b8a6554a46f58b

roles/glusterfs/defaults/main.yml

@@ -21,4 +21,6 @@ glusterfs_server_backing_dir: "/gluster/bricks/1"
 glusterfs_server_bricks: "{{ glusterfs_server_backing_dir }}"
 glusterfs_server_volume_name: "gfs"
 
-glusterfs_package_state: "{{ package_state | default('latest') }}"
+glusterfs_bootstrap_node: "{{ groups[glusterfs_server_group_name][0] }}"
+
+glusterfs_package_state: "{{ package_state | default('latest') }}"

roles/glusterfs/tasks/main.yml

@@ -100,7 +100,7 @@
 - name: Create gluster peers
   gluster.gluster.gluster_peer:
     nodes: "{{ glusterfs_server_cluster_members }}"
-  when: _glusterfs_is_first_play_host
+  when: _glusterfs_is_bootstrap_host
 
 - name: Ensure glusterfs backing directory exists
   file:
@@ -129,16 +129,40 @@
     - gluster volume reset-brick {{ glusterfs_server_volume_name }} {{ brick }} start
     - gluster volume reset-brick {{ glusterfs_server_volume_name }} {{ brick }} {{ brick }} commit force
 
-- name: Create gluster volume
-  vars:
-    num_cluster_members: "{{ glusterfs_server_cluster_members | length }}"
-    cluster_has_replicas: "{{ (glusterfs_server_cluster_members | length) > 1 }}"
-  gluster.gluster.gluster_volume:
-    state: present
-    name: "{{ glusterfs_server_volume_name }}"
-    bricks: "{{ glusterfs_server_bricks }}"
-    replicas: "{{ cluster_has_replicas | ternary(num_cluster_members, omit) }}"
-    cluster: "{{ glusterfs_server_cluster_members | map('regex_replace', '_', '-') | list }}"
-    force: true
-  when: _glusterfs_is_first_play_host
+- name: Handle volume creation
+  block:
+    # NOTE: When first establishing a peer relationship, a reverse DNS lookup happens which can cause a
+    #       peer's active hostname not to match the one stored on disk. Restarting the service resolves
+    #       this and prevents failures upon calls to create volumes.
+    - name: Find existing peers' runtime hostnames
+      shell: "gluster peer status | grep Hostname: | cut -d ' ' -f 2"
+      changed_when: false
+      register: _existing_peer_hostnames
+      when: (glusterfs_server_cluster_members | length) > 1
 
+    - name: Restart glusterfs server
+      service:
+        name: "{{ glusterfs_server_service }}"
+        state: restarted
+      when:
+        - _existing_peer_hostnames is not skipped
+        - ((_existing_peer_hostnames.stdout_lines | intersect(glusterfs_server_cluster_members)) | length) != (_existing_peer_hostnames.stdout_lines | length)
+
+    # Retry as the service may have just restarted
+    - name: Create gluster volume
+      vars:
+        num_cluster_members: "{{ glusterfs_server_cluster_members | length }}"
+        cluster_has_replicas: "{{ (glusterfs_server_cluster_members | length) > 1 }}"
+      gluster.gluster.gluster_volume:
+        state: present
+        name: "{{ glusterfs_server_volume_name }}"
+        bricks: "{{ glusterfs_server_bricks }}"
+        replicas: "{{ cluster_has_replicas | ternary(num_cluster_members, omit) }}"
+        cluster: "{{ glusterfs_server_cluster_members | map('regex_replace', '_', '-') | list }}"
+        force: true
+      register: _volume_create
+      delay: 1
+      retries: 5
+      until: _volume_create is success
+
+  when: _glusterfs_is_bootstrap_host

roles/glusterfs/vars/main.yml

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-_glusterfs_is_first_play_host: "{{ (inventory_hostname == (ansible_play_hosts | first)) | bool }}"
+_glusterfs_is_bootstrap_host: "{{ ((inventory_hostname | regex_replace('_', '-')) == (glusterfs_bootstrap_node | regex_replace('_', '-'))) | bool }}"