Apply Nova rolling upgrade logic

This patchset apply Nova rolling upgrade logic [1]

[1] https://docs.openstack.org/nova/latest/user/upgrade.html

Implements: blueprint apply-service-upgrade-procedure

Co-Authored-By: Ha Manh Dong <donghm@vn.fujitsu.com>
Depends-On: https://review.openstack.org/#/c/558765/
Change-Id: I20531a25dc7f5b05dc70ba771cf4c4222ade33e3
This commit is contained in:
Duong Ha-Quang 2018-04-04 16:52:51 +07:00 committed by Ha Manh Dong
parent b46efe5dc4
commit c79c5d1f36
8 changed files with 202 additions and 107 deletions

View File

@ -290,6 +290,8 @@ openstack_nova_auth: "{{ openstack_auth }}"
openstack_placement_auth: "{{ openstack_auth }}"
nova_compute_host_rp_filter_mode: 0
nova_enable_rolling_upgrade: "yes"
nova_safety_upgrade: "no"
nova_libvirt_port: "16509"
nova_ssh_port: "8022"

View File

@ -1,4 +1,56 @@
---
- name: Restart placement-api container
vars:
service_name: "placement-api"
service: "{{ nova_services[service_name] }}"
config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
placement_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
volumes: "{{ service.volumes|reject('equalto', '')|list }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- inventory_hostname in groups[service.group]
- service.enabled | bool
- config_json.changed | bool
or nova_conf.changed | bool
or policy_overwriting.changed | bool
or placement_api_wsgi_conf | changed
or placement_api_container.changed | bool
- name: Restart nova-conductor container
vars:
service_name: "nova-conductor"
service: "{{ nova_services[service_name] }}"
config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
nova_conductor_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
privileged: "{{ service.privileged | default(False) }}"
volumes: "{{ service.volumes|reject('equalto', '')|list }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- inventory_hostname in groups[service.group]
- service.enabled | bool
- config_json.changed | bool
or nova_conf.changed | bool
or policy_overwriting.changed | bool
or nova_conductor_container.changed | bool
- name: Restart nova-ssh container
vars:
service_name: "nova-ssh"
@ -52,58 +104,6 @@
or nova_libvirt_confs.changed | bool
or nova_libvirt_container.changed | bool
- name: Restart placement-api container
vars:
service_name: "placement-api"
service: "{{ nova_services[service_name] }}"
config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
placement_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
volumes: "{{ service.volumes|reject('equalto', '')|list }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- inventory_hostname in groups[service.group]
- service.enabled | bool
- config_json.changed | bool
or nova_conf.changed | bool
or policy_overwriting.changed | bool
or placement_api_wsgi_conf | changed
or placement_api_container.changed | bool
- name: Restart nova-api container
vars:
service_name: "nova-api"
service: "{{ nova_services[service_name] }}"
config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
nova_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
privileged: "{{ service.privileged | default(False) }}"
volumes: "{{ service.volumes|reject('equalto', '')|list }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- inventory_hostname in groups[service.group]
- service.enabled | bool
- config_json.changed | bool
or nova_conf.changed | bool
or policy_overwriting.changed | bool
or nova_api_container.changed | bool
- name: Restart nova-scheduler container
vars:
service_name: "nova-scheduler"
@ -130,32 +130,6 @@
or policy_overwriting.changed | bool
or nova_scheduler_container.changed | bool
- name: Restart nova-conductor container
vars:
service_name: "nova-conductor"
service: "{{ nova_services[service_name] }}"
config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
nova_conductor_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
privileged: "{{ service.privileged | default(False) }}"
volumes: "{{ service.volumes|reject('equalto', '')|list }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- inventory_hostname in groups[service.group]
- service.enabled | bool
- config_json.changed | bool
or nova_conf.changed | bool
or policy_overwriting.changed | bool
or nova_conductor_container.changed | bool
- name: Restart nova-consoleauth container
vars:
service_name: "nova-consoleauth"
@ -260,6 +234,32 @@
or policy_overwriting.changed | bool
or nova_serialproxy_container.changed | bool
- name: Restart nova-api container
vars:
service_name: "nova-api"
service: "{{ nova_services[service_name] }}"
config_json: "{{ config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
nova_conf: "{{ nova_confs.results|selectattr('item.key', 'equalto', service_name)|first }}"
policy_overwriting: "{{ nova_policy_overwriting.results|selectattr('item.key', 'equalto', service_name)|first }}"
nova_api_container: "{{ check_nova_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
privileged: "{{ service.privileged | default(False) }}"
volumes: "{{ service.volumes|reject('equalto', '')|list }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- inventory_hostname in groups[service.group]
- service.enabled | bool
- config_json.changed | bool
or nova_conf.changed | bool
or policy_overwriting.changed | bool
or nova_api_container.changed | bool
- name: Restart nova-compute container
vars:
service_name: "nova-compute"

View File

@ -2,14 +2,18 @@
- name: Running Nova bootstrap container
vars:
nova_api: "{{ nova_services['nova-api'] }}"
bootstrap_environment:
KOLLA_BOOTSTRAP:
KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
upgrade_environment:
KOLLA_UPGRADE:
KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
become: true
kolla_docker:
action: "start_container"
common_options: "{{ docker_common_options }}"
detach: False
environment:
KOLLA_BOOTSTRAP:
KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
environment: "{{ upgrade_environment if nova_enable_rolling_upgrade|bool else bootstrap_environment }}"
image: "{{ nova_api.image }}"
labels:
BOOTSTRAP:

View File

@ -0,0 +1,30 @@
---
- include_tasks: config.yml
- include_tasks: bootstrap_service.yml
- name: Checking if conductor container needs upgrading
become: true
kolla_docker:
action: "compare_image"
common_options: "{{ docker_common_options }}"
name: "nova_conductor"
image: "{{ nova_conductor_image_full }}"
when: inventory_hostname in groups['nova-conductor']
register: conductor_differs
# Short downtime here, but from user perspective his call will just timeout or execute later
- name: Stopping all nova_conductor containers
become: true
kolla_docker:
action: "stop_container"
common_options: "{{ docker_common_options }}"
name: "nova_conductor"
when:
- inventory_hostname in groups['nova-conductor']
- conductor_differs['result']
- name: Flush handlers
meta: flush_handlers
- include_tasks: reload.yml

View File

@ -0,0 +1,50 @@
---
# Create new set of configs on nodes
- include_tasks: pull.yml
- include_tasks: config.yml
- include_tasks: bootstrap_service.yml
- name: Stopping all nova services except nova-compute
become: true
kolla_docker:
action: "stop_container"
common_options: "{{ docker_common_options }}"
name: "{{ item.value.container_name }}"
with_dict: "{{ nova_services }}"
when:
- "'nova-compute' not in item.key"
- inventory_hostname in groups[item.value.group]
- item.value.enabled | bool
- nova_safety_upgrade | bool
# TODO(donghm): Flush_handlers to restart nova services
# should be run in serial nodes to decrease downtime if
# the previous task did not run. Update when the
# Ansible strategy module for rolling upgrade is finished.
- name: Flush handlers
meta: flush_handlers
- include_tasks: reload.yml
- name: Migrate Nova database
vars:
nova_api: "{{ nova_services['nova-api'] }}"
become: true
kolla_docker:
action: "start_container"
common_options: "{{ docker_common_options }}"
detach: False
environment:
KOLLA_OSM:
KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
image: "{{ nova_api.image }}"
labels:
BOOTSTRAP:
name: "bootstrap_nova"
restart_policy: "never"
volumes: "{{ nova_api.volumes }}"
run_once: True
delegate_to: "{{ groups[nova_api.group][0] }}"

View File

@ -1,31 +1,21 @@
---
# Create new set of configs on nodes
- include_tasks: config.yml
- include_tasks: bootstrap_service.yml
- name: Checking if conductor container needs upgrading
- name: Check nova upgrade status
become: true
kolla_docker:
action: "compare_image"
common_options: "{{ docker_common_options }}"
name: "nova_conductor"
image: "{{ nova_conductor_image_full }}"
when: inventory_hostname in groups['nova-conductor']
register: conductor_differs
command: docker exec -t nova_api nova-status upgrade check
register: nova_upgrade_check_stdout
when: inventory_hostname == groups['nova-api'][0]
# Short downtime here, but from user perspective his call will just timeout or execute later
- name: Stopping all nova_conductor containers
become: true
kolla_docker:
action: "stop_container"
common_options: "{{ docker_common_options }}"
name: "nova_conductor"
when:
- inventory_hostname in groups['nova-conductor']
- conductor_differs['result']
- name: Upgrade status check result
fail:
msg:
- "There was an upgrade status check warning or failure!"
- "See the detail at https://docs.openstack.org/nova/latest/cli/nova-status.html#nova-status-checks"
vars:
first_nova_api_host: "{{ groups['nova-api'][0] }}"
when: hostvars[first_nova_api_host]['nova_upgrade_check_stdout']['rc'] != 0
- name: Flush handlers
meta: flush_handlers
- include_tasks: legacy_upgrade.yml
when: not nova_enable_rolling_upgrade | bool
- include_tasks: reload.yml
- include_tasks: rolling_upgrade.yml
when: nova_enable_rolling_upgrade | bool

View File

@ -401,6 +401,21 @@ glance_enable_rolling_upgrade: "no"
# The number of fake driver per compute node
#num_nova_fake_per_node: 5
# Configure nova upgrade option, due to currently kolla support
# two upgrade ways for nova: legacy_upgrade and rolling_upgrade
# The variable "nova_enable_rolling_upgrade: yes" is meaning
# rolling_upgrade were enabled and opposite
#nova_enable_rolling_upgrade: "yes"
# The flag "nova_safety_upgrade" need to be consider when
# "nova_enable_rolling_upgrade" is enabled. The "nova_safety_upgrade"
# controls whether the nova services are all stopped before rolling
# upgrade to the new version, for the safety and availability.
# If "nova_safety_upgrade" is "yes", that will stop all nova services (except
# nova-compute) for no failed API operations before upgrade to the
# new version. And opposite.
#nova_safety_upgrade: "no"
#################
# Hyper-V options
#################

View File

@ -0,0 +1,4 @@
---
features:
- |
Implement Nova rolling upgrade logic