diff --git a/ansible/overcloud-etc-hosts-fixup.yml b/ansible/overcloud-etc-hosts-fixup.yml new file mode 100644 index 000000000..d51e87741 --- /dev/null +++ b/ansible/overcloud-etc-hosts-fixup.yml @@ -0,0 +1,51 @@ +--- +# For some currently unknown reason, overcloud hosts end up with multiple +# entries in /etc/hosts that map their own hostname to their provisioning +# network IP address, in addition to one that maps their own hostname to their +# internal network IP address. This causes RabbitMQ upgrades to fail, as +# RabbitMQ expects the system's hostname to resolve to the IP address on +# which it is listening. As a workaround, we remove the stale entries from +# /etc/hosts. See https://github.com/stackhpc/kayobe/issues/14. + +- name: Ensure overcloud hosts' /etc/hosts does not contain provisioning network IP + hosts: overcloud + tasks: + - name: Ensure overcloud hosts' /etc/hosts does not contain provisioning network IP + lineinfile: + dest: /etc/hosts + regexp: "^{{ provision_oc_net_name | net_ip }}[ \t]*{{ inventory_hostname }}" + state: absent + when: provision_oc_net_name | net_ip != None + become: True + +- name: Ensure rabbitmq containers' /etc/hosts does not contain provisioning network IP + hosts: controllers + vars: + rabbitmq_containers: + - rabbitmq + - outward_rabbitmq + tasks: + - block: + - name: Check whether rabbitmq container is running + command: docker inspect -f {{ '{{.Id}}' }} {{ item }} + changed_when: False + failed_when: False + with_items: "{{ rabbitmq_containers }}" + register: ps_result + + - name: Ensure rabbitmq containers' /etc/hosts does not contain provisioning network IP + command: > + docker exec -u root {{ item.item }} + bash -c + 'cp /etc/hosts /tmp/hosts && + sed -i -e "/^{{ provision_oc_net_name | net_ip }}[ \t]*{{ inventory_hostname }}/d" /tmp/hosts && + if ! diff -q /tmp/hosts /etc/hosts >/dev/null; then + cp /tmp/hosts /etc/hosts && + echo changed + fi && + rm /tmp/hosts' + changed_when: "'changed' in sed_result.stdout" + with_items: "{{ ps_result.results }}" + when: item.rc == 0 + register: sed_result + when: provision_oc_net_name | net_ip != None diff --git a/doc/source/upgrading.rst b/doc/source/upgrading.rst index 9eeed8ec8..3bea4d57f 100644 --- a/doc/source/upgrading.rst +++ b/doc/source/upgrading.rst @@ -87,6 +87,17 @@ should be obtained either by building them locally or pulling them from an image registry. Second, the overcloud services should be replaced with new containers created from the new container images. +Upgrading Host Services +----------------------- + +Prior to upgrading the OpenStack control plane, the overcloud host services +should be upgraded:: + + (kayobe-venv) $ kayobe overcloud host upgrade + +Note that this will not perform full configuration of the host, and will +instead perform a targeted upgrade of specific services where necessary. + Upgrading the Ironic Deployment Images -------------------------------------- diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index 6157a4638..84b89b494 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -442,7 +442,7 @@ class OvercloudDeprovision(KayobeAnsibleMixin, VaultMixin, Command): class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, Command): - """Configure the overcloud host OS.""" + """Configure the overcloud host services.""" def get_parser(self, prog_name): parser = super(OvercloudHostConfigure, self).get_parser(prog_name) @@ -479,6 +479,21 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, self.run_kayobe_playbooks(parsed_args, playbooks, limit="overcloud") +class OvercloudHostUpgrade(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, + Command): + """Upgrade the overcloud host services. + + Performs the changes necessary to make the host services suitable for the + configured OpenStack release. + """ + + def take_action(self, parsed_args): + self.app.LOG.debug("Upgrading overcloud host services") + playbooks = _build_playbook_list( + "overcloud-etc-hosts-fixup") + self.run_kayobe_playbooks(parsed_args, playbooks) + + class OvercloudServiceDeploy(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, Command): """Deploy the overcloud services.""" diff --git a/setup.py b/setup.py index c21e4d0eb..3ffdda66b 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ setup( 'overcloud_deprovision = kayobe.cli.commands:OvercloudDeprovision', 'overcloud_hardware_inspect = kayobe.cli.commands:OvercloudHardwareInspect', 'overcloud_host_configure = kayobe.cli.commands:OvercloudHostConfigure', + 'overcloud_host_upgrade = kayobe.cli.commands:OvercloudHostUpgrade', 'overcloud_introspection_data_save = kayobe.cli.commands:OvercloudIntrospectionDataSave', 'overcloud_inventory_discover = kayobe.cli.commands:OvercloudInventoryDiscover', 'overcloud_post_configure = kayobe.cli.commands:OvercloudPostConfigure',