From b2bc3a4081b6eff947acdb13e32e2d64ceb1823c Mon Sep 17 00:00:00 2001 From: Sofer Athlan-Guyot Date: Thu, 2 Feb 2017 12:04:15 +0100 Subject: [PATCH] Ensure working compute node during the whole migration. Fix subtile error introduced during the attempt to get the vm migration after controller upgrade working. Namely the server.cnf in the my.cnf.d directory was updated with default values, which include listening to 127.0.0.1. This overwrite the galera.cnf configuration and break the link between the haproxy and the database, making the database unavailable for services. This happens only at convergence step where the database is eventually restarted. Beside fixing this bug, this is a rewrite at having the possibility for the operator to manipulate instance during those states: - controllers upgraded/compute not upgraded (vm migration) - controllers upgraded/part of compute upgraded (vm migration) - controllers upgraded/all computes upgraded/no convergence Change-Id: I22a9f1cde99c010d00c653861cc7cf34ab4cc06e Closes-Bug: #1661202 --- .../liberty_to_mitaka_nova_api_upgrade.pp | 67 ++++++++++++++----- ...liberty_to_mitaka_nova_api_upgrade_conf.pp | 32 +++++++++ extraconfig/tasks/major_upgrade_compute.sh | 16 +++-- .../tasks/major_upgrade_pacemaker.yaml | 59 ++++++++++++---- .../tasks/major_upgrade_pacemaker_init.yaml | 14 ---- 5 files changed, 140 insertions(+), 48 deletions(-) create mode 100644 extraconfig/tasks/liberty_to_mitaka_nova_api_upgrade_conf.pp diff --git a/extraconfig/tasks/liberty_to_mitaka_nova_api_upgrade.pp b/extraconfig/tasks/liberty_to_mitaka_nova_api_upgrade.pp index fd090b8eaf..94fc9ff072 100644 --- a/extraconfig/tasks/liberty_to_mitaka_nova_api_upgrade.pp +++ b/extraconfig/tasks/liberty_to_mitaka_nova_api_upgrade.pp @@ -15,23 +15,58 @@ # This puppet manifest is to be used only during a Liberty->Mitaka upgrade # It creates the nova api database during the controller upgrade instead of -# during the converge step. - -class { '::mysql::server': - service_manage => false, -} - +# during the converge step. Executed on the controllers only. include ::tripleo::packages $pacemaker_master = hiera('bootstrap_nodeid') -if downcase($pacemaker_master) == $::hostname { - class {'::nova::db::mysql_api': - user => hiera('nova::db::mysql_api::dbname'), - password => hiera('nova::db::mysql_api::password'), - dbname => hiera('nova::db::mysql_api::dbname'), - host => hiera('nova_api_vip'), - charset => 'utf8', - collate => 'utf8_general_ci', - allowed_hosts => undef, - } +# Will prevent any reboot of any service. +Service <| |> { + hasrestart => true, + restart => '/bin/true', + start => '/bin/true', + stop => '/bin/true', + enable => '/bin/true', + provider => 'base', +} + +# Restart of the nova-* services will be done in another step. +nova_config { + 'api_database/connection': value => hiera('nova::api_database_connection') +} + +if downcase($pacemaker_master) == $::hostname { + # You have to open up the class here to avoid any change to the + # /etc/my.cnf.d/server.cnf. Ultimately, "include ::mysql::server" + # is the one that modify it. + $dbname = 'nova_api' + mysql_database { $dbname: + ensure => present, + charset => 'utf8', + collate => 'utf8_general_ci', + notify => Exec['nova-db-sync-api'], + } + + $allowed_hosts_list = unique(concat(any2array(hiera('nova::db::mysql::allowed_hosts')), [hiera('nova_api_vip')])) + $real_allowed_hosts = prefix($allowed_hosts_list, "${dbname}_") + + openstacklib::db::mysql::host_access { $real_allowed_hosts: + user => hiera('nova::db::mysql_api::dbname'), + password_hash => mysql_password(hiera('nova::db::mysql_api::password')), + database => $dbname, + privileges => 'ALL', + notify => Exec['nova-db-sync-api'], + } + + exec { 'nova-db-sync-api': + command => '/usr/bin/nova-manage api_db sync', + refreshonly => true, + logoutput => on_failure, + notify => Exec['nova-db-online-data-migration'] + } + + exec { 'nova-db-online-data-migration': + command => '/usr/bin/nova-manage db online_data_migration', + refreshonly => true, + logoutput => on_failure, + } } diff --git a/extraconfig/tasks/liberty_to_mitaka_nova_api_upgrade_conf.pp b/extraconfig/tasks/liberty_to_mitaka_nova_api_upgrade_conf.pp new file mode 100644 index 0000000000..94bc8e5a71 --- /dev/null +++ b/extraconfig/tasks/liberty_to_mitaka_nova_api_upgrade_conf.pp @@ -0,0 +1,32 @@ +# Copyright 2017 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This puppet manifest is to be used only during a Liberty->Mitaka +# upgrade It ensures proper configuration of nova.conf on controller +# and compute nodes. +nova_config { + 'neutron/auth_plugin': value => 'v3password'; + 'neutron/username': value => hiera('nova::network::neutron::neutron_username'); + 'neutron/password': value => hiera('nova::network::neutron::neutron_password'); + 'neutron/auth_url': value => hiera('nova::network::neutron::neutron_auth_url'); + 'neutron/project_name': value => hiera('nova::network::neutron::neutron_project_name'); + 'neutron/user_domain_name': value => 'Default'; + 'neutron/project_domain_name': value => 'Default'; + 'DEFAULT/use_neutron': value => 'True'; + 'neutron/auth_strategy': ensure => absent; + 'neutron/admin_tenant_name': ensure => absent; + 'neutron/admin_username': ensure => absent; + 'neutron/admin_auth_url': ensure => absent; +} diff --git a/extraconfig/tasks/major_upgrade_compute.sh b/extraconfig/tasks/major_upgrade_compute.sh index 24bf0920ba..ade6367303 100644 --- a/extraconfig/tasks/major_upgrade_compute.sh +++ b/extraconfig/tasks/major_upgrade_compute.sh @@ -33,10 +33,18 @@ fi yum -y install python-zaqarclient # needed for os-collect-config yum -y update -# Update nova neutron auth parameters, to be able to launch and migrate the -# workloads before running the converge step -puppet apply /root/liberty_to_mitaka_neutron_auth_param.pp -systemctl restart openstack-nova-compute +# Problem creating vif if not restarted. +if systemctl is-enabled openvswitch; then + systemctl restart openvswitch +fi + +# Look like it is required after the installation of the new openvswitch. +if systemctl is-enabled neutron-openvswitch-agent; then + if systemctl is-failed neutron-openvswitch-agent; then + systemctl restart neutron-openvswitch-agent + fi +fi + ENDOFCAT # ensure the permissions are OK diff --git a/extraconfig/tasks/major_upgrade_pacemaker.yaml b/extraconfig/tasks/major_upgrade_pacemaker.yaml index cdf9473add..f12ed76fce 100644 --- a/extraconfig/tasks/major_upgrade_pacemaker.yaml +++ b/extraconfig/tasks/major_upgrade_pacemaker.yaml @@ -100,7 +100,10 @@ resources: properties: group: puppet config: - get_file: liberty_to_mitaka_nova_api_upgrade.pp + list_join: + - '' + - - get_file: liberty_to_mitaka_nova_api_upgrade.pp + - get_file: liberty_to_mitaka_nova_api_upgrade_conf.pp NovaApiDatabaseUpgradeDeployment: type: OS::Heat::SoftwareDeploymentGroup @@ -110,26 +113,54 @@ resources: servers: {get_param: controller_servers} config: {get_resource: NovaApiDatabaseUpgradeConfig} - NovaApiDatabasePostConfig: + NovaApiDatabasePostUgradeRestartConfig: type: OS::Heat::SoftwareConfig properties: group: script config: | #!/bin/bash - set -eux - crudini --set /etc/nova/nova.conf api_database connection "$(hiera nova::api_database_connection)" - crudini --set /etc/nova/nova.conf neutron auth_plugin v3password - crudini --set /etc/nova/nova.conf neutron auth_url "$(hiera nova::api::identity_uri)" - if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then - nova-manage api_db sync - nova-manage db online_data_migrations - fi - systemctl restart openstack-nova-api - NovaApiDatabasePostUpgradeDeployment: + set -eux + # As per http://docs.openstack.org/developer/nova/upgrade.html + # restart conductor first and nova-api list. + + # This is the best way to restart the whole nova stack. It + # 100% of time leads to no transiant errors in pcs resources. + # conductor is the leaf of the pacemaker dependency graph and + # must be restarted first. consoleauth is the root of this + # branch and will restart all nova service in this order: + # openstack-nova-consoleauth openstack-nova-novncproxy openstack-nova-api openstack-nova-scheduler openstack-nova-conductor + if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then + for res in openstack-nova-conductor openstack-nova-consoleauth ; do + pcs resource restart ${res}-clone --wait=600 + done + fi + + NovaApiDatabaseUpgradePostUpgradeRestartDeployment: type: OS::Heat::SoftwareDeploymentGroup depends_on: NovaApiDatabaseUpgradeDeployment properties: - name: NovaApiDatabasePostUpgradeDeployment + name: NovaApiDatabaseUpgradePostUpgradeRestartDeployment servers: {get_param: controller_servers} - config: {get_resource: NovaApiDatabasePostConfig} + config: {get_resource: NovaApiDatabasePostUgradeRestartConfig} + + NovaApiComputePostConfig: + type: OS::Heat::SoftwareConfig + properties: + group: puppet + config: + list_join: + - '' + - - get_file: liberty_to_mitaka_nova_api_upgrade_conf.pp + - | + exec { '/bin/systemctl restart openstack-nova-compute': refreshonly => true } + Nova_config<| |> ~> Exec['/bin/systemctl restart openstack-nova-compute'] + + + NovaApiComputePostUpgradeDeployment: + type: OS::Heat::SoftwareDeploymentGroup + depends_on: NovaApiDatabaseUpgradePostUpgradeRestartDeployment + properties: + name: NovaApiComputePostUpgradeDeployment + servers: {get_param: compute_servers} + config: {get_resource: NovaApiComputePostConfig} diff --git a/extraconfig/tasks/major_upgrade_pacemaker_init.yaml b/extraconfig/tasks/major_upgrade_pacemaker_init.yaml index 3f1f7eabd3..623549a085 100644 --- a/extraconfig/tasks/major_upgrade_pacemaker_init.yaml +++ b/extraconfig/tasks/major_upgrade_pacemaker_init.yaml @@ -97,13 +97,6 @@ resources: UPGRADE_LEVEL_NOVA_COMPUTE: {get_param: UpgradeLevelNovaCompute} - get_file: major_upgrade_compute.sh - NovaNeutronAuthParamUpgradeConfig: - type: OS::Heat::SoftwareConfig - properties: - group: script - config: - get_file: liberty_to_mitaka_neutron_auth_param.sh - ComputeDeliverUpgradeScriptDeployment: type: OS::Heat::SoftwareDeploymentGroup properties: @@ -111,13 +104,6 @@ resources: config: {get_resource: ComputeDeliverUpgradeScriptConfig} input_values: {get_param: input_values} - NovaNeutronAuthParamUpgradeDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - servers: {get_param: compute_servers} - config: {get_resource: NovaNeutronAuthParamUpgradeConfig} - input_values: {get_param: input_values} - ObjectStorageDeliverUpgradeScriptConfig: type: OS::Heat::SoftwareConfig properties: