Improve Grafana DB bootstrap

This fixes an issue where multiple Grafana instances would race
to bootstrap the Grafana DB. The following changes are made:

- Only start additional Grafana instances after the DB has been
  configured.

- During upgrade, don't allow old instances to run with an
  upgraded DB schema.

Change-Id: I3e0e077ba6a6f43667df042eb593107418a06c39
Closes-Bug: #1888681
(cherry picked from commit 2c730590d7)
This commit is contained in:
Doug Szumski 2020-07-23 10:26:27 +00:00 committed by Mark Goddard
parent 19acf13113
commit f2031fc8a1
5 changed files with 136 additions and 2 deletions

View File

@ -1,5 +1,6 @@
---
- name: Restart grafana container
- name: Restart first grafana container
listen: Restart grafana container
vars:
service_name: "grafana"
service: "{{ grafana_services[service_name] }}"
@ -13,3 +14,39 @@
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- inventory_hostname == groups[service.group]|first
- name: Waiting for grafana to start on first node
listen: Restart grafana container
vars:
service_name: "grafana"
service: "{{ grafana_services[service_name] }}"
become: true
kolla_toolbox:
module_name: uri
module_args:
url: "http://{{ api_interface_address | put_address_in_context('url') }}:{{ grafana_server_port }}/login"
status_code: 200
register: result
until: result.get('status') == 200
retries: 10
delay: 2
when:
- inventory_hostname == groups[service.group]|first
- name: Restart remaining grafana containers
listen: Restart grafana container
vars:
service_name: "grafana"
service: "{{ grafana_services[service_name] }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
volumes: "{{ service.volumes }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- inventory_hostname != groups[service.group]|first

View File

@ -1,5 +1,31 @@
---
- name: Checking if Grafana container needs upgrading
vars:
service_name: "grafana"
service: "{{ grafana_services[service_name] }}"
become: true
kolla_docker:
action: "compare_image"
common_options: "{{ docker_common_options }}"
name: "{{ project_name }}"
image: "{{ grafana_image_full }}"
when: inventory_hostname in groups['grafana']
register: grafana_differs
- include_tasks: config.yml
# NOTE(dszumski): We don't want old Grafana instances running after
# a new instance has updated the DB schema. Since the first instance
# is upgraded first, we stop all the other ones.
- name: Stopping all Grafana instances but the first node
become: true
kolla_docker:
action: "stop_container"
common_options: "{{ docker_common_options }}"
name: "{{ project_name }}"
when:
- inventory_hostname != groups['grafana']|first
- grafana_differs['result']
- name: Flush handlers
meta: flush_handlers

View File

@ -165,7 +165,8 @@
when:
- kolla_action != "config"
- name: Restart monasca-grafana container
- name: Restart first monasca-grafana container
listen: Restart monasca-grafana container
vars:
service_name: "monasca-grafana"
service: "{{ monasca_services[service_name] }}"
@ -179,3 +180,39 @@
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- inventory_hostname == groups[service.group]|first
- name: Waiting for monasca-grafana to start on first node
listen: Restart monasca-grafana container
vars:
service_name: "monasca-grafana"
service: "{{ monasca_services[service_name] }}"
become: true
kolla_toolbox:
module_name: uri
module_args:
url: "http://{{ api_interface_address | put_address_in_context('url') }}:{{ monasca_grafana_server_port }}/login"
status_code: 200
register: result
until: result.get('status') == 200
retries: 10
delay: 2
when:
- inventory_hostname == groups[service.group]|first
- name: Restart remaining monasca-grafana containers
listen: Restart monasca-grafana container
vars:
service_name: "monasca-grafana"
service: "{{ monasca_services[service_name] }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
volumes: "{{ service.volumes }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- inventory_hostname != groups[service.group]|first

View File

@ -1,6 +1,35 @@
---
- name: Checking if Monasca Grafana container needs upgrading
vars:
service_name: "monasca-grafana"
service: "{{ monasca_services[service_name] }}"
become: true
kolla_docker:
action: "compare_image"
common_options: "{{ docker_common_options }}"
name: "{{ project_name }}"
image: "{{ monasca_grafana_image_full }}"
when: inventory_hostname in groups['monasca-grafana']
register: monasca_grafana_differs
- include_tasks: config.yml
# NOTE(dszumski): We don't want old Grafana instances running after
# a new instance has updated the DB schema. Since the first instance
# is upgraded first, we stop all the other ones.
- name: Stopping all Monasca Grafana instances but the first node
vars:
service_name: "monasca-grafana"
service: "{{ monasca_services[service_name] }}"
become: true
kolla_docker:
action: "stop_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
when:
- inventory_hostname != groups['monasca-grafana']|first
- monasca_grafana_differs['result']
- include_tasks: bootstrap_service.yml
- name: Flush handlers

View File

@ -0,0 +1,5 @@
---
fixes:
- |
Fixes an issue where Grafana instances would race to bootstrap the Grafana
DB. See `LP#1888681 <https://bugs.launchpad.net/kolla-ansible/+bug/1888681>`__.