Ensure disable/enable haproxy backends exists for all services

Right now we ensure that services are enabled/disabled while running
playbooks only for core services. At the same time some services still do
not have this mechanism, that might result in unexpected outages.

So we ensure that all service playbooks will behave in the same way and
disable backends in advance before playbook will do any modifications.

With that, setting variable `haproxy_drain: true` will ensure that moving
backend to the MAINT state will be graceful and all current connections
will close normally unless a timeout is reached, which is 2 min by default.

Closes-Bug: #2047017
Change-Id: I8554defec4df54d14be72ae9a1560907ff1aaddf

playbooks/os-aodh-install.yml

@@ -43,6 +43,19 @@
   tags:
     - aodh
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: aodh_api-back
+        haproxy_state: disabled
+      when:
+        - "'aodh_api' in group_names"
+        - "groups['aodh_api'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       when: not is_metal
@@ -54,3 +67,16 @@
 
   roles:
     - role: "os_aodh"
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: aodh_api-back
+        haproxy_state: enabled
+      when:
+        - "'aodh_api' in group_names"
+        - "groups['aodh_api'] | length > 1"

playbooks/os-barbican-install.yml

@@ -43,6 +43,19 @@
   tags:
     - barbican
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: barbican-back
+        haproxy_state: disabled
+      when:
+        - "'barbican_api' in group_names"
+        - "groups['barbican_api'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       when: not is_metal
@@ -53,3 +66,16 @@
         - hostvars['localhost']['resolvconf_enabled'] | bool
   roles:
     - role: "os_barbican"
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: barbican-back
+        haproxy_state: enabled
+      when:
+        - "'barbican_api' in group_names"
+        - "groups['barbican_api'] | length > 1"

playbooks/os-blazar-install.yml

@@ -43,6 +43,19 @@
   tags:
     - blazar
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: blazar_api-back
+        haproxy_state: disabled
+      when:
+        - "'blazar_api' in group_names"
+        - "groups['blazar_api'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       when: not is_metal
@@ -57,3 +70,16 @@
     - role: "openstack.osa.system_crontab_coordination"
       tags:
         - crontab
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: blazar_api-back
+        haproxy_state: enabled
+      when:
+        - "'blazar_api' in group_names"
+        - "groups['blazar_api'] | length > 1"

playbooks/os-designate-install.yml

@@ -46,6 +46,19 @@
   tags:
     - designate
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: designate_api-back
+        haproxy_state: disabled
+      when:
+        - "'designate_api' in group_names"
+        - "groups['designate_api'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       when: not is_metal
@@ -56,3 +69,16 @@
         - hostvars['localhost']['resolvconf_enabled'] | bool
   roles:
     - role: "os_designate"
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: designate_api-back
+        haproxy_state: enabled
+      when:
+        - "'designate_api' in group_names"
+        - "groups['designate_api'] | length > 1"

playbooks/os-gnocchi-install.yml

@@ -43,6 +43,19 @@
   tags:
     - gnocchi
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: gnocchi-back
+        haproxy_state: disabled
+      when:
+        - "'gnocchi_all' in group_names"
+        - "groups['gnocchi_all'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       vars:
@@ -57,3 +70,16 @@
         - hostvars['localhost']['resolvconf_enabled'] | bool
   roles:
     - role: "os_gnocchi"
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: gnocchi-back
+        haproxy_state: enabled
+      when:
+        - "'gnocchi_all' in group_names"
+        - "groups['gnocchi_all'] | length > 1"

playbooks/os-heat-install.yml

@@ -43,6 +43,19 @@
   vars_files:
     - "defaults/{{ install_method }}_install.yml"
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: heat_api-back
+        haproxy_state: disabled
+      when:
+        - "'heat_api' in group_names"
+        - "groups['heat_api'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       when: not is_metal
@@ -56,3 +69,16 @@
     - role: "openstack.osa.system_crontab_coordination"
       tags:
         - crontab
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: heat_api-back
+        haproxy_state: enabled
+      when:
+        - "'heat_api' in group_names"
+        - "groups['heat_api'] | length > 1"

playbooks/os-horizon-install.yml

@@ -43,6 +43,19 @@
   tags:
     - horizon
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: horizon-back
+        haproxy_state: disabled
+      when:
+        - "'horizon_all' in group_names"
+        - "groups['horizon_all'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       when: not is_metal
@@ -55,3 +68,16 @@
     - role: "openstack.osa.system_crontab_coordination"
       tags:
         - crontab
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: horizon-back
+        haproxy_state: enabled
+      when:
+        - "'horizon_all' in group_names"
+        - "groups['horizon_all'] | length > 1"

playbooks/os-ironic-install.yml

@@ -43,6 +43,19 @@
   tags:
     - ironic
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: ironic_api-back
+        haproxy_state: disabled
+      when:
+        - "'ironic_api' in group_names"
+        - "groups['ironic_api'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       when: not is_metal
@@ -56,3 +69,16 @@
     - role: "openstack.osa.system_crontab_coordination"
       tags:
         - crontab
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: ironic_api-back
+        haproxy_state: enabled
+      when:
+        - "'ironic_api' in group_names"
+        - "groups['ironic_api'] | length > 1"

playbooks/os-keystone-install.yml

@@ -81,7 +81,9 @@
       vars:
         haproxy_backend: "keystone_service-back"
         haproxy_state: disabled
-      when: "groups['keystone_all'] | length > 1"
+      when:
+        - "'keystone_all' in group_names"
+        - "groups['keystone_all'] | length > 1"
 
     - name: Configure container
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
@@ -110,7 +112,9 @@
       vars:
         haproxy_backend: "keystone_service-back"
         haproxy_state: enabled
-      when: "groups['keystone_all'] | length > 1"
+      when:
+        - "'keystone_all' in group_names"
+        - "groups['keystone_all'] | length > 1"
 
 # These facts are set against the deployment host to ensure that
 # they are fast to access. This is done in preference to setting

playbooks/os-magnum-install.yml

@@ -46,6 +46,19 @@
   tags:
     - magnum
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: magnum-back
+        haproxy_state: disabled
+      when:
+        - "'magnum_all' in group_names"
+        - "groups['magnum_all'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       when: not is_metal
@@ -57,3 +70,16 @@
 
   roles:
     - role: "os_magnum"
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: magnum-back
+        haproxy_state: enabled
+      when:
+        - "'magnum_all' in group_names"
+        - "groups['magnum_all'] | length > 1"

playbooks/os-mistral-install.yml

@@ -43,6 +43,19 @@
   tags:
     - mistral
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: mistral-back
+        haproxy_state: disabled
+      when:
+        - "'mistral_all' in group_names"
+        - "groups['mistral_all'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       when: not is_metal
@@ -57,3 +70,16 @@
     - role: "openstack.osa.system_crontab_coordination"
       tags:
         - crontab
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: mistral-back
+        haproxy_state: enabled
+      when:
+        - "'mistral_all' in group_names"
+        - "groups['mistral_all'] | length > 1"

playbooks/os-murano-install.yml

@@ -43,6 +43,19 @@
   tags:
     - murano
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: murano-back
+        haproxy_state: disabled
+      when:
+        - "'murano_all' in group_names"
+        - "groups['murano_all'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       when: not is_metal
@@ -57,3 +70,16 @@
     - role: "openstack.osa.system_crontab_coordination"
       tags:
         - crontab
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: murano-back
+        haproxy_state: enabled
+      when:
+        - "'murano_all' in group_names"
+        - "groups['murano_all'] | length > 1"

playbooks/os-octavia-install.yml

@@ -43,6 +43,19 @@
   tags:
     - octavia
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: octavia-back
+        haproxy_state: disabled
+      when:
+        - "'octavia_all' in group_names"
+        - "groups['octavia_all'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       when: not is_metal
@@ -56,3 +69,16 @@
     - role: "openstack.osa.system_crontab_coordination"
       tags:
         - crontab
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: octavia-back
+        haproxy_state: enabled
+      when:
+        - "'octavia_all' in group_names"
+        - "groups['octavia_all'] | length > 1"

playbooks/os-placement-install.yml

@@ -43,6 +43,19 @@
   tags:
     - placement
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: placement-back
+        haproxy_state: disabled
+      when:
+        - "'placement_all' in group_names"
+        - "groups['placement_all'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       when: not is_metal
@@ -56,3 +69,16 @@
     - role: "openstack.osa.system_crontab_coordination"
       tags:
         - crontab
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: placement-back
+        haproxy_state: enabled
+      when:
+        - "'placement_all' in group_names"
+        - "groups['placement_all'] | length > 1"

playbooks/os-sahara-install.yml

@@ -41,6 +41,19 @@
   tags:
     - sahara
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: sahara_api-back
+        haproxy_state: disabled
+      when:
+        - "'sahara_api' in group_names"
+        - "groups['sahara_api'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       when: not is_metal
@@ -51,3 +64,16 @@
         - hostvars['localhost']['resolvconf_enabled'] | bool
   roles:
     - role: "os_sahara"
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: sahara_api-back
+        haproxy_state: enabled
+      when:
+        - "'sahara_api' in group_names"
+        - "groups['sahara_api'] | length > 1"

playbooks/os-tacker-install.yml

@@ -43,8 +43,35 @@
   tags:
     - tacker
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: tacker-back
+        haproxy_state: disabled
+      when:
+        - "'tacker_all' in group_names"
+        - "groups['tacker_all'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       when: not is_metal
+
   roles:
     - role: "os_tacker"
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: tacker-back
+        haproxy_state: enabled
+      when:
+        - "'tacker_all' in group_names"
+        - "groups['tacker_all'] | length > 1"

playbooks/os-trove-install.yml

@@ -46,6 +46,19 @@
   tags:
     - trove
   pre_tasks:
+    # In order to ensure that any container, software or
+    # config file changes which causes a container/service
+    # restart do not cause an unexpected outage, we drain
+    # the load balancer back end for this container.
+    - name: Disabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: trove-back
+        haproxy_state: disabled
+      when:
+        - "'trove_api' in group_names"
+        - "groups['trove_api'] | length > 1"
+
     - name: Including container-setup tasks
       include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       when: not is_metal
@@ -56,3 +69,16 @@
         - hostvars['localhost']['resolvconf_enabled'] | bool
   roles:
     - role: "os_trove"
+
+  post_tasks:
+    # Now that container changes are done, we can set
+    # the load balancer back end for this container
+    # to available again.
+    - name: Enabling haproxy backends
+      include_tasks: common-tasks/haproxy-endpoint-manage.yml
+      vars:
+        haproxy_backend: trove-back
+        haproxy_state: enabled
+      when:
+        - "'trove_api' in group_names"
+        - "groups['trove_api'] | length > 1"