Apply Glance database migration

Apply Glance database upgrade procedure:

https://docs.openstack.org/glance/latest/admin/zero-downtime-db-upgrade.html

Rolling upgrade in Glance requires that glance-registry is not deployed and run

Depends-On: https://review.openstack.org/532128
Depends-On: https://review.openstack.org/566804

Change-Id: Ia5d0cee2385facefda9d423051b74ab51d075f13
Implements: blueprint apply-service-upgrade-procedure

ansible/group_vars/all.yml

@@ -591,6 +591,7 @@ glance_backend_ceph: "{{ enable_ceph }}"
 glance_backend_vmware: "no"
 glance_backend_swift: "no"
 glance_file_datadir_volume: "glance"
+glance_enable_rolling_upgrade: "no"
 
 #######################
 # Barbican options

ansible/roles/glance/handlers/main.yml

@@ -22,6 +22,7 @@
       or glance_swift_conf.changed | bool
       or policy_overwriting.changed | bool
       or glance_api_container.changed | bool
+      or glance_upgrading | bool
 
 - name: Restart glance-registry container
   vars:

ansible/roles/glance/tasks/legacy_upgrade.yml

@@ -0,0 +1,7 @@
+---
+- include: config.yml
+
+- include: bootstrap_service.yml
+
+- name: Flush handlers
+  meta: flush_handlers

ansible/roles/glance/tasks/rolling_upgrade.yml

@@ -0,0 +1,122 @@
+---
+- set_fact:
+    glance_upgrading: True
+
+# Upgrade first node (NEW NODE in the Glance documentation)
+- include: pull.yml
+  when: inventory_hostname == groups["glance-api"][0]
+
+- include: config.yml
+  when: inventory_hostname == groups["glance-api"][0]
+
+- include: stop_service.yml
+  when: inventory_hostname == groups["glance-api"][0]
+
+- name: Enable log_bin_trust_function_creators function
+  vars:
+    glance_api: "{{ glance_services['glance-api'] }}"
+  kolla_toolbox:
+    module_name: mysql_variables
+    module_args:
+      login_host: "{{ database_address }}"
+      login_port: "{{ database_port }}"
+      login_user: "{{ database_user }}"
+      login_password: "{{ database_password }}"
+      variable: log_bin_trust_function_creators
+      value: 1
+  run_once: True
+  delegate_to: "{{ groups[glance_api.group][0] }}"
+  when:
+    - not use_preconfigured_databases | bool
+
+- name: Running Glance database expand container
+  vars:
+    glance_api: "{{ glance_services['glance-api'] }}"
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    environment:
+      KOLLA_UPGRADE:
+      GLANCE_DB_EXPAND:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    image: "{{ glance_api.image }}"
+    labels:
+      BOOTSTRAP:
+    name: "bootstrap_glance"
+    restart_policy: "never"
+    volumes: "{{ glance_api.volumes }}"
+  run_once: True
+  delegate_to: "{{ groups[glance_api.group][0] }}"
+
+- name: Running Glance database migrate container
+  vars:
+    glance_api: "{{ glance_services['glance-api'] }}"
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    environment:
+      KOLLA_UPGRADE:
+      GLANCE_DB_MIGRATE:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    image: "{{ glance_api.image }}"
+    labels:
+      BOOTSTRAP:
+    name: "bootstrap_glance"
+    restart_policy: "never"
+    volumes: "{{ glance_api.volumes }}"
+  run_once: True
+  delegate_to: "{{ groups[glance_api.group][0] }}"
+  notify:
+    - Restart glance-api container
+
+# Upgrade remaining node
+- include: pull.yml
+  when: inventory_hostname != groups["glance-api"][0]
+
+- include: config.yml
+  when: inventory_hostname != groups["glance-api"][0]
+
+- name: Flush handlers
+  meta: flush_handlers
+
+- name: Running Glance database contract container
+  vars:
+    glance_api: "{{ glance_services['glance-api'] }}"
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    environment:
+      KOLLA_UPGRADE:
+      GLANCE_DB_CONTRACT:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    image: "{{ glance_api.image }}"
+    labels:
+      BOOTSTRAP:
+    name: "bootstrap_glance"
+    restart_policy: "never"
+    volumes: "{{ glance_api.volumes }}"
+  run_once: True
+  delegate_to: "{{ groups[glance_api.group][0] }}"
+
+- name: Disable log_bin_trust_function_creators function
+  vars:
+    glance_api: "{{ glance_services['glance-api'] }}"
+  kolla_toolbox:
+    module_name: mysql_variables
+    module_args:
+      login_host: "{{ database_address }}"
+      login_port: "{{ database_port }}"
+      login_user: "{{ database_user }}"
+      login_password: "{{ database_password }}"
+      variable: log_bin_trust_function_creators
+      value: 0
+  run_once: True
+  delegate_to: "{{ groups[glance_api.group][0] }}"
+  when:
+    - not use_preconfigured_databases | bool
+
+- set_fact:
+    glance_upgrading: False

ansible/roles/glance/tasks/stop_service.yml

@@ -0,0 +1,8 @@
+---
+- name: Stop glance service
+  vars:
+    glance_api: "{{ glance_services['glance-api'] }}"
+  kolla_docker:
+    action: "stop_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ glance_api.container_name }}"

ansible/roles/glance/tasks/upgrade.yml

@@ -1,7 +1,6 @@
 ---
-- include: config.yml
+- include: rolling_upgrade.yml
+  when: glance_enable_rolling_upgrade | bool
 
-- include: bootstrap_service.yml
-
-- name: Flush handlers
-  meta: flush_handlers
+- include: legacy_upgrade.yml
+  when: not glance_enable_rolling_upgrade | bool

etc/kolla/globals.yml

@@ -300,6 +300,9 @@ kolla_internal_vip_address: "10.10.10.254"
 #glance_backend_file: "yes"
 #glance_backend_swift: "no"
 #glance_backend_vmware: "no"
+# Configure glance upgrade option, due to this feature is experimental
+# in glance, so default value should be set to "no".
+glance_enable_rolling_upgrade: "no"
 
 
 ##################

releasenotes/notes/implement-glance-zero-downtime-upgrade-822fea4739beda62.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Implement Glance zero-downtime upgrade logic.

tests/templates/globals-default.j2

@@ -30,6 +30,9 @@ enable_heat: "no"
 openstack_logging_debug: "True"
 openstack_service_workers: "1"
 
+# This is experimental feature, disable if gate fail.
+glance_enable_rolling_upgrade: "yes"
+
 {% if scenario == "ceph" %}
 enable_ceph: "yes"
 enable_cinder: "yes"