From 04737f5dbd1568fb21bc21588f618716d9899411 Mon Sep 17 00:00:00 2001 From: Jimmy McCrory Date: Tue, 4 Oct 2016 13:26:05 -0700 Subject: [PATCH] Implement zero downtime upgrades This patch implements upgrading keystone with zero downtime as the default installation process. Handlers have been modified to ensure that the first keystone node is stopped, facilitates the database migrations, and that it is started and available before restarting any other keystone nodes. Migrations also now only occur when there is a change within the installed keystone venv. This process is documented at http://docs.openstack.org/developer/keystone/upgrading.html#upgrading-without-downtime A new test scenario has been added for testing basic upgradability between releases. Implements: blueprint upgrade-testing Change-Id: I0d3cfcb80b64d005d60f4c8445f991855f844796 --- handlers/main.yml | 74 +++++++++++++++++-- ...ero-downtime-upgrade-5f19ab84183490b9.yaml | 5 ++ tasks/keystone_apache.yml | 27 ++++--- tasks/keystone_db_setup.yml | 28 ++++++- tasks/keystone_federation_sp_setup.yml | 9 ++- tasks/keystone_idp_metadata.yml | 6 +- tasks/keystone_idp_self_signed_create.yml | 3 +- tasks/keystone_idp_self_signed_distribute.yml | 3 +- tasks/keystone_init_systemd.yml | 5 +- tasks/keystone_init_upstart.yml | 11 ++- tasks/keystone_install.yml | 12 ++- tasks/keystone_ldap_setup.yml | 12 ++- tasks/keystone_nginx.yml | 16 +++- tasks/keystone_post_install.yml | 18 +++-- tasks/keystone_ssl_key_create.yml | 6 +- tasks/keystone_ssl_user_provided.yml | 9 ++- tasks/keystone_uwsgi.yml | 12 ++- tasks/main.yml | 1 + tests/ansible-role-requirements.yml | 4 + tests/test-install-previous-keystone.yml | 35 +++++++++ tests/test.yml | 6 ++ tox.ini | 15 ++++ 22 files changed, 263 insertions(+), 54 deletions(-) create mode 100644 releasenotes/notes/os-keystone-zero-downtime-upgrade-5f19ab84183490b9.yaml create mode 100644 tests/test-install-previous-keystone.yml diff --git a/handlers/main.yml b/handlers/main.yml index 1a354095..04cee31e 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Restart service +- name: Restart service on first node service: name: "{{ keystone_system_service_name }}" state: restarted @@ -22,9 +22,13 @@ until: _restart|success retries: 5 delay: 2 - when: (keystone_apache_enabled | bool) or (keystone_mod_wsgi_enabled | bool) + when: + - inventory_hostname == groups['keystone_all'][0] + - (keystone_apache_enabled | bool) or (keystone_mod_wsgi_enabled | bool) + notify: + - Wait for keystone service port -- name: Restart Nginx +- name: Restart Nginx on first node service: name: nginx state: restarted @@ -32,9 +36,11 @@ until: keystone_restart | success retries: 5 delay: 2 - when: not keystone_apache_enabled | bool + when: + - inventory_hostname == groups['keystone_all'][0] + - not keystone_apache_enabled | bool -- name: Restart Keystone APIs +- name: Restart Keystone APIs on first node service: name: "{{ item }}" state: "restarted" @@ -43,7 +49,58 @@ retries: 5 delay: 2 with_items: "{{ keystone_wsgi_program_names }}" - when: not keystone_mod_wsgi_enabled | bool + when: + - inventory_hostname == groups['keystone_all'][0] + - not keystone_mod_wsgi_enabled | bool + notify: + - Wait for keystone service port + +- name: Wait for keystone service port + wait_for: + port: "{{ keystone_service_port }}" + timeout: 25 + delay: 10 + register: keystone_wait_check + until: keystone_wait_check | success + retries: 5 + +- name: Restart service on other nodes + service: + name: "{{ keystone_system_service_name }}" + state: restarted + pattern: "{{ keystone_system_service_name }}" + register: _restart + until: _restart|success + retries: 5 + delay: 2 + when: + - inventory_hostname != groups['keystone_all'][0] + - (keystone_apache_enabled | bool) or (keystone_mod_wsgi_enabled | bool) + +- name: Restart Nginx on other nodes + service: + name: nginx + state: restarted + register: keystone_restart + until: keystone_restart | success + retries: 5 + delay: 2 + when: + - inventory_hostname != groups['keystone_all'][0] + - not keystone_apache_enabled | bool + +- name: Restart Keystone APIs on other nodes + service: + name: "{{ item }}" + state: "restarted" + register: keystone_restart + until: keystone_restart | success + retries: 5 + delay: 2 + with_items: "{{ keystone_wsgi_program_names }}" + when: + - inventory_hostname != groups['keystone_all'][0] + - not keystone_mod_wsgi_enabled | bool - name: Restart Shibd service: @@ -54,3 +111,8 @@ until: shibd_restart|success retries: 5 delay: 2 + +- name: Perform a Keystone DB sync contract + command: "{{ keystone_bin }}/keystone-manage db_sync --contract" + become: yes + become_user: "{{ keystone_system_user_name }}" diff --git a/releasenotes/notes/os-keystone-zero-downtime-upgrade-5f19ab84183490b9.yaml b/releasenotes/notes/os-keystone-zero-downtime-upgrade-5f19ab84183490b9.yaml new file mode 100644 index 00000000..c4f2411a --- /dev/null +++ b/releasenotes/notes/os-keystone-zero-downtime-upgrade-5f19ab84183490b9.yaml @@ -0,0 +1,5 @@ +--- +features: + - The os_keystone role now performs a rolling upgrade + without downtime during installation. This process + is documented `here `_. diff --git a/tasks/keystone_apache.yml b/tasks/keystone_apache.yml index ab68742c..75de9a31 100644 --- a/tasks/keystone_apache.yml +++ b/tasks/keystone_apache.yml @@ -43,7 +43,8 @@ when: - ansible_pkg_mgr == 'apt' notify: - - Restart service + - Restart service on first node + - Restart service on other nodes ## NOTE(andymccr): ## We need to enable a module for httpd on RedHat/CentOS using LoadModule inside conf files @@ -55,7 +56,8 @@ when: - ansible_pkg_mgr == 'yum' notify: - - Restart service + - Restart service on first node + - Restart service on other nodes - name: Drop apache2 config files template: @@ -65,7 +67,8 @@ group: "root" with_items: "{{ keystone_apache_configs }}" notify: - - Restart service + - Restart service on first node + - Restart service on other nodes - name: Disable default apache site file: @@ -73,7 +76,8 @@ state: "absent" with_items: "{{ keystone_apache_default_sites }}" notify: - - Restart service + - Restart service on first node + - Restart service on other nodes - name: Enabled keystone vhost file: @@ -84,14 +88,16 @@ - keystone_apache_site_available is defined - keystone_apache_site_enabled is defined notify: - - Restart service + - Restart service on first node + - Restart service on other nodes - name: Ensure Apache ServerName lineinfile: dest: "{{ keystone_apache_conf }}" line: "ServerName {{ ansible_hostname }}" notify: - - Restart service + - Restart service on first node + - Restart service on other nodes - name: Ensure Apache ServerTokens lineinfile: @@ -99,7 +105,8 @@ regexp: '^ServerTokens' line: "ServerTokens {{ keystone_apache_servertokens }}" notify: - - Restart service + - Restart service on first node + - Restart service on other nodes - name: Ensure Apache ServerSignature lineinfile: @@ -107,7 +114,8 @@ regexp: '^ServerSignature' line: "ServerSignature {{ keystone_apache_serversignature }}" notify: - - Restart service + - Restart service on first node + - Restart service on other nodes ## NOTE(mgariepy): ## We need to enable httpd on CentOS if not it won't start when the container is restarted. @@ -116,4 +124,5 @@ name: "{{ keystone_system_service_name }}" enabled: "yes" notify: - - Restart service + - Restart service on first node + - Restart service on other nodes diff --git a/tasks/keystone_db_setup.yml b/tasks/keystone_db_setup.yml index e5714990..8d6d07dd 100644 --- a/tasks/keystone_db_setup.yml +++ b/tasks/keystone_db_setup.yml @@ -13,8 +13,30 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Perform a Keystone DB sync - command: "{{ keystone_bin }}/keystone-manage db_sync" - changed_when: false +- name: Ensure keystone service stopped on first node + service: + name: "{{ item }}" + state: stopped + register: keystone_stop + failed_when: + - keystone_stop.msg is defined + - "'no service or tool' not in keystone_stop.msg" + - "'systemd could not find' not in keystone_stop.msg" + - "'Could not find the requested service' not in keystone_stop.msg" + with_items: + - "{{ keystone_wsgi_program_names }}" + - "{{ keystone_system_service_name }}" + +- name: Perform a Keystone DB sync expand + command: "{{ keystone_bin }}/keystone-manage db_sync --expand" + changed_when: true become: yes become_user: "{{ keystone_system_user_name }}" + +- name: Perform a Keystone DB sync migrate + command: "{{ keystone_bin }}/keystone-manage db_sync --migrate" + changed_when: true + become: yes + become_user: "{{ keystone_system_user_name }}" + notify: + - Perform a Keystone DB sync contract diff --git a/tasks/keystone_federation_sp_setup.yml b/tasks/keystone_federation_sp_setup.yml index f0c39070..ae845b58 100644 --- a/tasks/keystone_federation_sp_setup.yml +++ b/tasks/keystone_federation_sp_setup.yml @@ -33,7 +33,8 @@ changed_when: false when: inventory_hostname == groups['keystone_all'][0] notify: - - Restart service + - Restart service on first node + - Restart service on other nodes - Restart Shibd - name: Store Shibboleth SP key-pair @@ -70,7 +71,8 @@ delay: 2 when: inventory_hostname != groups['keystone_all'][0] notify: - - Restart service + - Restart service on first node + - Restart service on other nodes - Restart Shibd - name: Set appropriate file ownership on the Shibboleth SP key-pair @@ -83,5 +85,6 @@ - "/etc/shibboleth/sp-key.pem" when: inventory_hostname != groups['keystone_all'][0] notify: - - Restart service + - Restart service on first node + - Restart service on other nodes - Restart Shibd diff --git a/tasks/keystone_idp_metadata.yml b/tasks/keystone_idp_metadata.yml index 974b180e..593fb2f3 100644 --- a/tasks/keystone_idp_metadata.yml +++ b/tasks/keystone_idp_metadata.yml @@ -20,5 +20,7 @@ become_user: "{{ keystone_system_user_name }}" when: keystone_idp != {} notify: - - Restart Keystone APIs - - Restart service + - Restart Keystone APIs on first node + - Restart Keystone APIs on other nodes + - Restart service on first node + - Restart service on other nodes diff --git a/tasks/keystone_idp_self_signed_create.yml b/tasks/keystone_idp_self_signed_create.yml index 22243a49..2b1d467c 100644 --- a/tasks/keystone_idp_self_signed_create.yml +++ b/tasks/keystone_idp_self_signed_create.yml @@ -33,7 +33,8 @@ when: > inventory_hostname == groups['keystone_all'][0] notify: - - Restart service + - Restart service on first node + - Restart service on other nodes - name: Set appropriate file ownership on the IdP self-signed cert file: diff --git a/tasks/keystone_idp_self_signed_distribute.yml b/tasks/keystone_idp_self_signed_distribute.yml index c2f71226..f9fec911 100644 --- a/tasks/keystone_idp_self_signed_distribute.yml +++ b/tasks/keystone_idp_self_signed_distribute.yml @@ -30,7 +30,8 @@ retries: 5 delay: 2 notify: - - Restart service + - Restart service on first node + - Restart service on other nodes - name: Set appropriate file ownership on the IdP self-signed cert file: diff --git a/tasks/keystone_init_systemd.yml b/tasks/keystone_init_systemd.yml index 373944f6..78914cdc 100644 --- a/tasks/keystone_init_systemd.yml +++ b/tasks/keystone_init_systemd.yml @@ -45,4 +45,7 @@ command: "systemctl daemon-reload" when: systemd_init | changed notify: - - Restart Keystone APIs + - Restart Keystone APIs on first node + - Restart Keystone APIs on other nodes + - Restart service on first node + - Restart service on other nodes diff --git a/tasks/keystone_init_upstart.yml b/tasks/keystone_init_upstart.yml index 69c7d443..0e545279 100644 --- a/tasks/keystone_init_upstart.yml +++ b/tasks/keystone_init_upstart.yml @@ -21,11 +21,18 @@ owner: "root" group: "root" register: upstart_init - notify: Restart Keystone APIs + notify: + - Restart Keystone APIs on first node + - Restart Keystone APIs on other nodes + - Restart service on first node + - Restart service on other nodes - name: Reload init scripts command: initctl reload-configuration changed_when: false when: upstart_init | changed notify: - - Restart Keystone APIs + - Restart Keystone APIs on first node + - Restart Keystone APIs on other nodes + - Restart service on first node + - Restart service on other nodes diff --git a/tasks/keystone_install.yml b/tasks/keystone_install.yml index 81ad2f94..81bdcb0f 100644 --- a/tasks/keystone_install.yml +++ b/tasks/keystone_install.yml @@ -84,8 +84,10 @@ - not keystone_developer_mode | bool - keystone_get_venv | changed or keystone_venv_dir | changed notify: - - Restart Keystone APIs - - Restart service + - Restart Keystone APIs on first node + - Restart Keystone APIs on other nodes + - Restart service on first node + - Restart service on other nodes - name: Install pip packages pip: @@ -103,8 +105,10 @@ delay: 2 when: keystone_developer_mode | bool notify: - - Restart Keystone APIs - - Restart service + - Restart Keystone APIs on first node + - Restart Keystone APIs on other nodes + - Restart service on first node + - Restart service on other nodes - name: Update virtualenv path command: > diff --git a/tasks/keystone_ldap_setup.yml b/tasks/keystone_ldap_setup.yml index e65cde12..b38271e6 100644 --- a/tasks/keystone_ldap_setup.yml +++ b/tasks/keystone_ldap_setup.yml @@ -35,8 +35,10 @@ mode: "0644" with_dict: "{{ keystone_ldap }}" notify: - - Restart Keystone APIs - - Restart service + - Restart Keystone APIs on first node + - Restart Keystone APIs on other nodes + - Restart service on first node + - Restart service on other nodes # Bug 1547542 - Older versions of the keystone role would deploy a blank # keystone.Default.conf and this will cause errors when adding LDAP-backed @@ -47,5 +49,7 @@ state: absent when: keystone_ldap.Default is not defined notify: - - Restart Keystone APIs - - Restart service + - Restart Keystone APIs on first node + - Restart Keystone APIs on other nodes + - Restart service on first node + - Restart service on other nodes diff --git a/tasks/keystone_nginx.yml b/tasks/keystone_nginx.yml index 1efa76af..872e7127 100644 --- a/tasks/keystone_nginx.yml +++ b/tasks/keystone_nginx.yml @@ -24,14 +24,18 @@ file: path: /etc/nginx/sites-enabled/default state: absent - notify: Restart Nginx + notify: + - Restart Nginx on first node + - Restart Nginx on other nodes - name: Configure custom nginx log format lineinfile: insertbefore: access_log dest: "/etc/nginx/nginx.conf" line: "log_format custom '{{ keystone_nginx_access_log_format_combined }} {{ keystone_nginx_access_log_format_extras }}';" - notify: Restart Nginx + notify: + - Restart Nginx on first node + - Restart Nginx on other nodes # Configure app - name: Configure virtual hosts @@ -39,7 +43,9 @@ src: keystone_nginx.conf.j2 dest: "/etc/nginx/{{ keystone_nginx_conf_path }}/{{ item }}.conf" with_items: "{{ keystone_wsgi_program_names }}" - notify: Restart Nginx + notify: + - Restart Nginx on first node + - Restart Nginx on other nodes - name: Link to enable virtual hosts file: @@ -48,4 +54,6 @@ state: link with_items: "{{ keystone_wsgi_program_names }}" when: ansible_os_family == "Debian" - notify: Restart Nginx + notify: + - Restart Nginx on first node + - Restart Nginx on other nodes diff --git a/tasks/keystone_post_install.yml b/tasks/keystone_post_install.yml index 4cb9bbf1..7e4b722b 100644 --- a/tasks/keystone_post_install.yml +++ b/tasks/keystone_post_install.yml @@ -36,8 +36,10 @@ config_overrides: "{{ keystone_policy_overrides }}" config_type: "json" notify: - - Restart Keystone APIs - - Restart service + - Restart Keystone APIs on first node + - Restart Keystone APIs on other nodes + - Restart service on first node + - Restart service on other nodes - name: Copy Keystone Federation SP SSO callback template copy: @@ -49,8 +51,10 @@ when: - keystone_idp != {} notify: - - Restart Keystone APIs - - Restart service + - Restart Keystone APIs on first node + - Restart Keystone APIs on other nodes + - Restart service on first node + - Restart service on other nodes - name: Clean up Keystone Federation SP SSO callback template file: @@ -59,5 +63,7 @@ when: - keystone_idp == {} notify: - - Restart Keystone APIs - - Restart service + - Restart Keystone APIs on first node + - Restart Keystone APIs on other nodes + - Restart service on first node + - Restart service on other nodes diff --git a/tasks/keystone_ssl_key_create.yml b/tasks/keystone_ssl_key_create.yml index d9eec8a8..2f3ddcbd 100644 --- a/tasks/keystone_ssl_key_create.yml +++ b/tasks/keystone_ssl_key_create.yml @@ -29,7 +29,8 @@ -extensions v3_ca creates={{ keystone_ssl_cert }} notify: - - Restart service + - Restart service on first node + - Restart service on other nodes - name: Ensure keystone user owns the self-signed key and certificate file: @@ -41,4 +42,5 @@ - "{{ keystone_ssl_key }}" - "{{ keystone_ssl_cert }}" notify: - - Restart service + - Restart service on first node + - Restart service on other nodes diff --git a/tasks/keystone_ssl_user_provided.yml b/tasks/keystone_ssl_user_provided.yml index 3a197bc4..25976b00 100644 --- a/tasks/keystone_ssl_user_provided.yml +++ b/tasks/keystone_ssl_user_provided.yml @@ -22,7 +22,8 @@ mode: "0644" when: keystone_user_ssl_cert is defined notify: - - Restart service + - Restart service on first node + - Restart service on other nodes - name: Drop user provided ssl key copy: @@ -33,7 +34,8 @@ mode: "0640" when: keystone_user_ssl_key is defined notify: - - Restart service + - Restart service on first node + - Restart service on other nodes - name: Drop user provided ssl CA cert copy: @@ -44,4 +46,5 @@ mode: "0644" when: keystone_user_ssl_ca_cert is defined notify: - - Restart service + - Restart service on first node + - Restart service on other nodes diff --git a/tasks/keystone_uwsgi.yml b/tasks/keystone_uwsgi.yml index 5d8309c8..06a074fc 100644 --- a/tasks/keystone_uwsgi.yml +++ b/tasks/keystone_uwsgi.yml @@ -27,7 +27,9 @@ config_overrides: "{{ keystone_uwsgi_ini_overrides }}" config_type: ini with_items: "{{ keystone_wsgi_program_names }}" - notify: Restart Keystone APIs + notify: + - Restart Keystone APIs on first node + - Restart Keystone APIs on other nodes - include: keystone_init_common.yml vars: @@ -36,7 +38,9 @@ system_user: "{{ keystone_system_user_name }}" system_group: "{{ keystone_system_group_name }}" service_home: "{{ keystone_system_user_home }}" - notify: Restart Keystone APIs + notify: + - Restart Keystone APIs on first node + - Restart Keystone APIs on other nodes - include: keystone_init_common.yml vars: @@ -45,7 +49,9 @@ system_user: "{{ keystone_system_user_name }}" system_group: "{{ keystone_system_group_name }}" service_home: "{{ keystone_system_user_home }}" - notify: Restart Keystone APIs + notify: + - Restart Keystone APIs on first node + - Restart Keystone APIs on other nodes - name: Ensure uwsgi service started service: diff --git a/tasks/main.yml b/tasks/main.yml index e54f1e3d..2ce0181d 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -85,6 +85,7 @@ when: - keystone_database_enabled | bool - inventory_hostname == groups['keystone_all'][0] + - keystone_get_venv | changed or keystone_venv_dir | changed or install_packages | changed tags: - keystone-install diff --git a/tests/ansible-role-requirements.yml b/tests/ansible-role-requirements.yml index 904ca78e..abf81c79 100644 --- a/tests/ansible-role-requirements.yml +++ b/tests/ansible-role-requirements.yml @@ -38,6 +38,10 @@ src: https://git.openstack.org/openstack/openstack-ansible-openstack_openrc scm: git version: master +- name: os_previous_keystone + src: https://git.openstack.org/openstack/openstack-ansible-os_keystone + scm: git + version: stable/newton - name: os_tempest src: https://git.openstack.org/openstack/openstack-ansible-os_tempest scm: git diff --git a/tests/test-install-previous-keystone.yml b/tests/test-install-previous-keystone.yml new file mode 100644 index 00000000..03ca362b --- /dev/null +++ b/tests/test-install-previous-keystone.yml @@ -0,0 +1,35 @@ +--- +# Copyright 2015, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Playbook for deploying previous keystone + hosts: keystone_all + user: root + gather_facts: true + pre_tasks: + - name: Set keystone_messaging fact + set_fact: + keystone_messaging_enabled: "{{ groups['rabbitmq_all'] is defined }}" + - include: common/ensure-rabbitmq.yml + vhost_name: "{{ keystone_rabbitmq_vhost }}" + user_name: "{{ keystone_rabbitmq_userid }}" + user_password: "{{ keystone_rabbitmq_password }}" + when: "{{ groups['rabbitmq_all'] is defined }}" + - include: common/create-grant-db.yml + db_name: "{{ keystone_galera_database }}" + db_password: "{{ keystone_container_mysql_password }}" + roles: + - role: "os_previous_keystone" + vars_files: + - common/previous/test-vars.yml diff --git a/tests/test.yml b/tests/test.yml index 569a7031..cf2ed45d 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -25,6 +25,12 @@ # Install RabbitMQ/MariaDB - include: common/test-install-infra.yml +# Install previous Keystone +- include: test-install-previous-keystone.yml + when: + - keystone_upgrade is defined + - keystone_upgrade | bool + # Install Keystone - include: common/test-install-keystone.yml diff --git a/tox.ini b/tox.ini index 6cd7e810..349de187 100644 --- a/tox.ini +++ b/tox.ini @@ -110,6 +110,21 @@ commands = bash -c "{toxinidir}/tests/common/test-ansible-functional.sh" +[testenv:upgrade] +deps = + {[testenv:ansible]deps} +setenv = + {[testenv]setenv} + ANSIBLE_PARAMETERS=-vvv -e keystone_upgrade=True +commands = + {[testenv:tests_clone]commands} + bash -c "if [ ! -d "{toxinidir}/tests/common/previous" ]; then \ + git clone https://git.openstack.org/openstack/openstack-ansible-tests -b stable/newton \ + {toxinidir}/tests/common/previous; \ + fi" + bash -c "{toxinidir}/tests/common/test-ansible-functional.sh" + + [testenv:uwsgi_apache] deps = {[testenv:ansible]deps}