From 9694ae8c2346daec443c5d74f288c146eb05d93f Mon Sep 17 00:00:00 2001 From: Dmitriy Rabotyagov Date: Fri, 29 Dec 2023 17:41:58 +0100 Subject: [PATCH] Ensure disable/enable haproxy backends exists for all services Right now we ensure that services are enabled/disabled while running playbooks only for core services. At the same time some services still do not have this mechanism, that might result in unexpected outages. So we ensure that all service playbooks will behave in the same way and disable backends in advance before playbook will do any modifications. With that, setting variable `haproxy_drain: true` will ensure that moving backend to the MAINT state will be graceful and all current connections will close normally unless a timeout is reached, which is 2 min by default. Closes-Bug: #2047017 Change-Id: I8554defec4df54d14be72ae9a1560907ff1aaddf --- playbooks/os-aodh-install.yml | 26 ++++++++++++++++++++++++++ playbooks/os-barbican-install.yml | 26 ++++++++++++++++++++++++++ playbooks/os-blazar-install.yml | 26 ++++++++++++++++++++++++++ playbooks/os-designate-install.yml | 26 ++++++++++++++++++++++++++ playbooks/os-gnocchi-install.yml | 26 ++++++++++++++++++++++++++ playbooks/os-heat-install.yml | 26 ++++++++++++++++++++++++++ playbooks/os-horizon-install.yml | 26 ++++++++++++++++++++++++++ playbooks/os-ironic-install.yml | 26 ++++++++++++++++++++++++++ playbooks/os-keystone-install.yml | 8 ++++++-- playbooks/os-magnum-install.yml | 26 ++++++++++++++++++++++++++ playbooks/os-mistral-install.yml | 26 ++++++++++++++++++++++++++ playbooks/os-murano-install.yml | 26 ++++++++++++++++++++++++++ playbooks/os-octavia-install.yml | 26 ++++++++++++++++++++++++++ playbooks/os-placement-install.yml | 26 ++++++++++++++++++++++++++ playbooks/os-sahara-install.yml | 26 ++++++++++++++++++++++++++ playbooks/os-tacker-install.yml | 27 +++++++++++++++++++++++++++ playbooks/os-trove-install.yml | 26 ++++++++++++++++++++++++++ 17 files changed, 423 insertions(+), 2 deletions(-) diff --git a/playbooks/os-aodh-install.yml b/playbooks/os-aodh-install.yml index 53189f40f5..11f1530270 100644 --- a/playbooks/os-aodh-install.yml +++ b/playbooks/os-aodh-install.yml @@ -43,6 +43,19 @@ tags: - aodh pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: aodh_api-back + haproxy_state: disabled + when: + - "'aodh_api' in group_names" + - "groups['aodh_api'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" when: not is_metal @@ -54,3 +67,16 @@ roles: - role: "os_aodh" + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: aodh_api-back + haproxy_state: enabled + when: + - "'aodh_api' in group_names" + - "groups['aodh_api'] | length > 1" diff --git a/playbooks/os-barbican-install.yml b/playbooks/os-barbican-install.yml index 6ea39cfa0d..a52d496b40 100644 --- a/playbooks/os-barbican-install.yml +++ b/playbooks/os-barbican-install.yml @@ -43,6 +43,19 @@ tags: - barbican pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: barbican-back + haproxy_state: disabled + when: + - "'barbican_api' in group_names" + - "groups['barbican_api'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" when: not is_metal @@ -53,3 +66,16 @@ - hostvars['localhost']['resolvconf_enabled'] | bool roles: - role: "os_barbican" + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: barbican-back + haproxy_state: enabled + when: + - "'barbican_api' in group_names" + - "groups['barbican_api'] | length > 1" diff --git a/playbooks/os-blazar-install.yml b/playbooks/os-blazar-install.yml index dcf6be4663..7f95b28cbb 100644 --- a/playbooks/os-blazar-install.yml +++ b/playbooks/os-blazar-install.yml @@ -43,6 +43,19 @@ tags: - blazar pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: blazar_api-back + haproxy_state: disabled + when: + - "'blazar_api' in group_names" + - "groups['blazar_api'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" when: not is_metal @@ -57,3 +70,16 @@ - role: "openstack.osa.system_crontab_coordination" tags: - crontab + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: blazar_api-back + haproxy_state: enabled + when: + - "'blazar_api' in group_names" + - "groups['blazar_api'] | length > 1" diff --git a/playbooks/os-designate-install.yml b/playbooks/os-designate-install.yml index d8e175ef79..7dc4f7298e 100644 --- a/playbooks/os-designate-install.yml +++ b/playbooks/os-designate-install.yml @@ -46,6 +46,19 @@ tags: - designate pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: designate_api-back + haproxy_state: disabled + when: + - "'designate_api' in group_names" + - "groups['designate_api'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" when: not is_metal @@ -56,3 +69,16 @@ - hostvars['localhost']['resolvconf_enabled'] | bool roles: - role: "os_designate" + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: designate_api-back + haproxy_state: enabled + when: + - "'designate_api' in group_names" + - "groups['designate_api'] | length > 1" diff --git a/playbooks/os-gnocchi-install.yml b/playbooks/os-gnocchi-install.yml index 0fa02be793..60cd3104db 100644 --- a/playbooks/os-gnocchi-install.yml +++ b/playbooks/os-gnocchi-install.yml @@ -43,6 +43,19 @@ tags: - gnocchi pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: gnocchi-back + haproxy_state: disabled + when: + - "'gnocchi_all' in group_names" + - "groups['gnocchi_all'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" vars: @@ -57,3 +70,16 @@ - hostvars['localhost']['resolvconf_enabled'] | bool roles: - role: "os_gnocchi" + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: gnocchi-back + haproxy_state: enabled + when: + - "'gnocchi_all' in group_names" + - "groups['gnocchi_all'] | length > 1" diff --git a/playbooks/os-heat-install.yml b/playbooks/os-heat-install.yml index b9c2a81f4a..13e4b0dc12 100644 --- a/playbooks/os-heat-install.yml +++ b/playbooks/os-heat-install.yml @@ -43,6 +43,19 @@ vars_files: - "defaults/{{ install_method }}_install.yml" pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: heat_api-back + haproxy_state: disabled + when: + - "'heat_api' in group_names" + - "groups['heat_api'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" when: not is_metal @@ -56,3 +69,16 @@ - role: "openstack.osa.system_crontab_coordination" tags: - crontab + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: heat_api-back + haproxy_state: enabled + when: + - "'heat_api' in group_names" + - "groups['heat_api'] | length > 1" diff --git a/playbooks/os-horizon-install.yml b/playbooks/os-horizon-install.yml index e6f49438a7..d53c9e891b 100644 --- a/playbooks/os-horizon-install.yml +++ b/playbooks/os-horizon-install.yml @@ -43,6 +43,19 @@ tags: - horizon pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: horizon-back + haproxy_state: disabled + when: + - "'horizon_all' in group_names" + - "groups['horizon_all'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" when: not is_metal @@ -55,3 +68,16 @@ - role: "openstack.osa.system_crontab_coordination" tags: - crontab + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: horizon-back + haproxy_state: enabled + when: + - "'horizon_all' in group_names" + - "groups['horizon_all'] | length > 1" diff --git a/playbooks/os-ironic-install.yml b/playbooks/os-ironic-install.yml index a3d93e6974..a42ac2098a 100644 --- a/playbooks/os-ironic-install.yml +++ b/playbooks/os-ironic-install.yml @@ -43,6 +43,19 @@ tags: - ironic pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: ironic_api-back + haproxy_state: disabled + when: + - "'ironic_api' in group_names" + - "groups['ironic_api'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" when: not is_metal @@ -56,3 +69,16 @@ - role: "openstack.osa.system_crontab_coordination" tags: - crontab + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: ironic_api-back + haproxy_state: enabled + when: + - "'ironic_api' in group_names" + - "groups['ironic_api'] | length > 1" diff --git a/playbooks/os-keystone-install.yml b/playbooks/os-keystone-install.yml index 39dcb7a330..cd03c3a8a9 100644 --- a/playbooks/os-keystone-install.yml +++ b/playbooks/os-keystone-install.yml @@ -81,7 +81,9 @@ vars: haproxy_backend: "keystone_service-back" haproxy_state: disabled - when: "groups['keystone_all'] | length > 1" + when: + - "'keystone_all' in group_names" + - "groups['keystone_all'] | length > 1" - name: Configure container include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" @@ -110,7 +112,9 @@ vars: haproxy_backend: "keystone_service-back" haproxy_state: enabled - when: "groups['keystone_all'] | length > 1" + when: + - "'keystone_all' in group_names" + - "groups['keystone_all'] | length > 1" # These facts are set against the deployment host to ensure that # they are fast to access. This is done in preference to setting diff --git a/playbooks/os-magnum-install.yml b/playbooks/os-magnum-install.yml index 3c90adf7ab..d74d22cef3 100644 --- a/playbooks/os-magnum-install.yml +++ b/playbooks/os-magnum-install.yml @@ -46,6 +46,19 @@ tags: - magnum pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: magnum-back + haproxy_state: disabled + when: + - "'magnum_all' in group_names" + - "groups['magnum_all'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" when: not is_metal @@ -57,3 +70,16 @@ roles: - role: "os_magnum" + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: magnum-back + haproxy_state: enabled + when: + - "'magnum_all' in group_names" + - "groups['magnum_all'] | length > 1" diff --git a/playbooks/os-mistral-install.yml b/playbooks/os-mistral-install.yml index 49ed0fc5f6..6cae34ceb5 100644 --- a/playbooks/os-mistral-install.yml +++ b/playbooks/os-mistral-install.yml @@ -43,6 +43,19 @@ tags: - mistral pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: mistral-back + haproxy_state: disabled + when: + - "'mistral_all' in group_names" + - "groups['mistral_all'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" when: not is_metal @@ -57,3 +70,16 @@ - role: "openstack.osa.system_crontab_coordination" tags: - crontab + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: mistral-back + haproxy_state: enabled + when: + - "'mistral_all' in group_names" + - "groups['mistral_all'] | length > 1" diff --git a/playbooks/os-murano-install.yml b/playbooks/os-murano-install.yml index 3a45cadc6a..8adc70fe15 100644 --- a/playbooks/os-murano-install.yml +++ b/playbooks/os-murano-install.yml @@ -43,6 +43,19 @@ tags: - murano pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: murano-back + haproxy_state: disabled + when: + - "'murano_all' in group_names" + - "groups['murano_all'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" when: not is_metal @@ -57,3 +70,16 @@ - role: "openstack.osa.system_crontab_coordination" tags: - crontab + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: murano-back + haproxy_state: enabled + when: + - "'murano_all' in group_names" + - "groups['murano_all'] | length > 1" diff --git a/playbooks/os-octavia-install.yml b/playbooks/os-octavia-install.yml index 3ecb3712f9..25a00af37b 100644 --- a/playbooks/os-octavia-install.yml +++ b/playbooks/os-octavia-install.yml @@ -43,6 +43,19 @@ tags: - octavia pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: octavia-back + haproxy_state: disabled + when: + - "'octavia_all' in group_names" + - "groups['octavia_all'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" when: not is_metal @@ -56,3 +69,16 @@ - role: "openstack.osa.system_crontab_coordination" tags: - crontab + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: octavia-back + haproxy_state: enabled + when: + - "'octavia_all' in group_names" + - "groups['octavia_all'] | length > 1" diff --git a/playbooks/os-placement-install.yml b/playbooks/os-placement-install.yml index 76da5fa971..d44b318446 100644 --- a/playbooks/os-placement-install.yml +++ b/playbooks/os-placement-install.yml @@ -43,6 +43,19 @@ tags: - placement pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: placement-back + haproxy_state: disabled + when: + - "'placement_all' in group_names" + - "groups['placement_all'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" when: not is_metal @@ -56,3 +69,16 @@ - role: "openstack.osa.system_crontab_coordination" tags: - crontab + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: placement-back + haproxy_state: enabled + when: + - "'placement_all' in group_names" + - "groups['placement_all'] | length > 1" diff --git a/playbooks/os-sahara-install.yml b/playbooks/os-sahara-install.yml index 9272f2265e..cfd0823ec6 100644 --- a/playbooks/os-sahara-install.yml +++ b/playbooks/os-sahara-install.yml @@ -41,6 +41,19 @@ tags: - sahara pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: sahara_api-back + haproxy_state: disabled + when: + - "'sahara_api' in group_names" + - "groups['sahara_api'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" when: not is_metal @@ -51,3 +64,16 @@ - hostvars['localhost']['resolvconf_enabled'] | bool roles: - role: "os_sahara" + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: sahara_api-back + haproxy_state: enabled + when: + - "'sahara_api' in group_names" + - "groups['sahara_api'] | length > 1" diff --git a/playbooks/os-tacker-install.yml b/playbooks/os-tacker-install.yml index 9c5704421f..f7fdc6acfb 100644 --- a/playbooks/os-tacker-install.yml +++ b/playbooks/os-tacker-install.yml @@ -43,8 +43,35 @@ tags: - tacker pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: tacker-back + haproxy_state: disabled + when: + - "'tacker_all' in group_names" + - "groups['tacker_all'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" when: not is_metal + roles: - role: "os_tacker" + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: tacker-back + haproxy_state: enabled + when: + - "'tacker_all' in group_names" + - "groups['tacker_all'] | length > 1" diff --git a/playbooks/os-trove-install.yml b/playbooks/os-trove-install.yml index 16d4bea36f..5782d1e200 100644 --- a/playbooks/os-trove-install.yml +++ b/playbooks/os-trove-install.yml @@ -46,6 +46,19 @@ tags: - trove pre_tasks: + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - name: Disabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: trove-back + haproxy_state: disabled + when: + - "'trove_api' in group_names" + - "groups['trove_api'] | length > 1" + - name: Including container-setup tasks include_tasks: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml" when: not is_metal @@ -56,3 +69,16 @@ - hostvars['localhost']['resolvconf_enabled'] | bool roles: - role: "os_trove" + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - name: Enabling haproxy backends + include_tasks: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: trove-back + haproxy_state: enabled + when: + - "'trove_api' in group_names" + - "groups['trove_api'] | length > 1"