Merge "Add automated migration of neutron agents to bare metal" into stable/queens

doc/source/admin/upgrades/major-upgrades-manual-upgrade.rst

@@ -252,6 +252,26 @@ see :ref:`memcached-flush`.
 
     # openstack-ansible "${UPGRADE_PLAYBOOKS}/memcached-flush.yml"
 
+Implement inventory to deploy neutron agents on network_hosts
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In releases prior to Queens, neutron agents were deployed in a container. This
+turned out to be problematic in major upgrades where the LXC container
+configuration may have changed, resulting in the containers restarting and
+therefore all L3 networking going down for some time.
+
+To prevent this happening in the future, the neutron agents are now deployed
+on the network_hosts directly (not in containers). This ensures that whenever
+an upgrade is run, the L3 networks do not go down.
+
+In order to handle this transition, we need to temporarily implement a
+temporary inventory change which adds the network_hosts into each of the
+agent groups so that the os-neutron-install playbook installs agents on them.
+
+.. code-block:: console
+
+    # openstack-ansible "${UPGRADE_PLAYBOOKS}/neutron-tmp-inventory.yml"
+
 Upgrade OpenStack
 ~~~~~~~~~~~~~~~~~
 
@@ -279,6 +299,9 @@ changes to the container/service setup:
   ``ironic_conductor_container`` can be removed.
 # All nova services are consolidated into the ``nova_api_container``
   and the rest of the nova containers can be removed.
+# All neutron agents are moved from containers onto the network_hosts.
+  The previously implemented ``neutron_agents_container`` can therefore
+  be removed.
 # All trove services have been consolidated into the
   ``trove_api_container``. The previously implemented
   ``trove_conductor_container`` and ``trove_taskmanager_container``
@@ -303,3 +326,4 @@ from the haproxy configuration.
     # openstack-ansible "${UPGRADE_PLAYBOOKS}/cleanup-nova.yml" -e force_containers_destroy=yes -e force_containers_data_destroy=yes
     # openstack-ansible "${UPGRADE_PLAYBOOKS}/cleanup-trove.yml" -e force_containers_destroy=yes -e force_containers_data_destroy=yes
     # openstack-ansible --tags haproxy_server-config haproxy-install.yml
+    # openstack-ansible "${UPGRADE_PLAYBOOKS}/cleanup-neutron.yml" -e force_containers_destroy=yes -e force_containers_data_destroy=yes

releasenotes/notes/neutron-agent-auto-migration-a8b160509d7f83d0.yaml

@@ -0,0 +1,9 @@
+---
+upgrade:
+  - |
+    During an upgrade using the run-upgrade script, the neutron agents will
+    now automatically be migrated from the neutron_agents containers on to
+    the network_hosts. The neutron_agents containers will be deleted as they
+    are no longer necessary. Any environments which previously upgraded to
+    Queens can make use of the same playbooks to handle to migration, or
+    inspect the playbooks to determine how to do it by hand if preferred.

scripts/run-upgrade.sh

@@ -196,6 +196,7 @@ function main {
         RUN_TASKS+=("utility-install.yml")
         RUN_TASKS+=("rsyslog-install.yml")
         RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/memcached-flush.yml")
+        RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/neutron-tmp-inventory.yml")
         RUN_TASKS+=("setup-openstack.yml")
         # clean up the containers which are no longer required
         # now that the services are hyperconverged
@@ -204,7 +205,12 @@ function main {
         RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/cleanup-heat.yml -e force_containers_destroy=yes -e force_containers_data_destroy=yes")
         RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/cleanup-ironic.yml -e force_containers_destroy=yes -e force_containers_data_destroy=yes")
         RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/cleanup-trove.yml -e force_containers_destroy=yes -e force_containers_data_destroy=yes")
+        # reconfigure haproxy to ensure that the old back-ends are removed
         RUN_TASKS+=("haproxy-install-rerun.yml --tags=haproxy_server-config")
+        # finalise the migration of the neutron agents to bare
+        # metal and clean up the neutron agent containers which
+        # are no longer required
+        RUN_TASKS+=("${UPGRADE_PLAYBOOKS}/cleanup-neutron.yml -e force_containers_destroy=yes -e force_containers_data_destroy=yes")
         # Run the tasks in order
         for item in ${!RUN_TASKS[@]}; do
           run_lock $item "${RUN_TASKS[$item]}"

scripts/upgrade-utilities/files/neutron-migration-inventory.ini

@@ -0,0 +1,33 @@
+# This is an Ansible inventory to be used for the Pike->Queens
+# transition of neutron agents to the bare metal hosts. Once
+# the agents have been transitioned, this file should be removed.
+
+[neutron_agent:children]
+network_hosts
+
+[neutron_bgp_dragent:children]
+network_hosts
+
+[neutron_dhcp_agent:children]
+network_hosts
+
+[neutron_l3_agent:children]
+network_hosts
+
+[neutron_lbaas_agent:children]
+network_hosts
+
+[neutron_linuxbridge_agent:children]
+network_hosts
+
+[neutron_metadata_agent:children]
+network_hosts
+
+[neutron_metering_agent:children]
+network_hosts
+
+[neutron_openvswitch_agent:children]
+network_hosts
+
+[neutron_sriov_nic_agent:children]
+network_hosts

scripts/upgrade-utilities/playbooks/cleanup-neutron.yml

@@ -0,0 +1,121 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: neutron_agents_container
+  become: yes
+  gather_facts: no
+  tasks:
+    - name: Gather the necessary facts
+      setup:
+        gather_subset: "!facter,!ohai"
+
+- hosts: utility[0]
+  become: yes
+  gather_facts: no
+  tasks:
+    - name: Collect the ID's for the neutron services to delete
+      command: >-
+        openstack --os-cloud default network agent list
+                  --format value --column ID
+                  --host {{ service_host }}
+      register: _neutron_service_to_remove
+      until: _neutron_service_to_remove is succeeded
+      retries: 5
+      delay: 2
+      vars:
+        service_host: "{{ hostvars[item]['ansible_hostname'] }}"
+      with_items: "{{ groups['neutron_agents_container'] }}"
+
+    - name: Disable neutron services in containers which will be deleted
+      command: >-
+        openstack --os-cloud default network agent set --disable {{ item }}
+      with_items: "{{ _neutron_service_to_remove.results | json_query('[].stdout_lines[]') }}"
+      register: _neutron_service_disable
+      until: _neutron_service_disable is succeeded
+      retries: 5
+      delay: 2
+
+- hosts: neutron_agents_container
+  become: yes
+  gather_facts: no
+  tasks:
+    - name: Discover the list of services to shut down in containers which will be deleted
+      shell: "systemctl list-unit-files --state=enabled --type=service | awk '/neutron.* enabled$/ {print $1}'"
+      args:
+        executable: "/bin/bash"
+      register: _enabled_services
+      changed_when: false
+
+    - name: Shut down neutron services in containers which will be deleted
+      service:
+        name: "{{ item }}"
+        enabled: no
+        state: stopped
+      with_items: "{{ _enabled_services.stdout_lines }}"
+
+- include: "{{ playbook_dir }}/../../../playbooks/lxc-containers-destroy.yml"
+  vars:
+    container_group: "neutron_agents_container"
+
+- hosts: utility[0]
+  become: yes
+  gather_facts: no
+  tasks:
+    - name: Collect the ID's for the neutron services to delete
+      command: >-
+        openstack --os-cloud default network agent list
+                  --format value --column ID
+                  --host {{ service_host }}
+      register: _neutron_service_to_remove
+      until: _neutron_service_to_remove is succeeded
+      retries: 5
+      delay: 2
+      vars:
+        service_host: "{{ hostvars[item]['ansible_hostname'] }}"
+      with_items: "{{ groups['neutron_agents_container'] }}"
+
+    - name: Delete the neutron services which were running in the deleted containers
+      command: >-
+        openstack --os-cloud default network agent delete {{ item }}
+      with_items: "{{ _neutron_service_to_remove.results | json_query('[].stdout_lines[]') }}"
+      register: _neutron_service_delete
+      until: _neutron_service_delete is succeeded
+      retries: 5
+      delay: 2
+
+- hosts: localhost
+  connection: local
+  become: yes
+  gather_facts: no
+  tasks:
+    - name: Remove the inventory entries for the deleted containers
+      command: >-
+        {{ playbook_dir }}/../../inventory-manage.py -r {{ item }}
+      with_items: "{{ groups['neutron_agents_container'] }}"
+
+    - name: Remove the transitional user-space inventory file
+      shell: |
+        set -e
+        if [[ -e /etc/openstack_deploy/inventory.ini.org ]]; then
+          echo "Reverting the inventory.ini in /etc/openstack_deploy to the original backup"
+          mv /etc/openstack_deploy/inventory.ini.org /etc/openstack_deploy/inventory.ini
+          exit 2
+        fi
+      args:
+        executable: /bin/bash
+      register: _remove_ini
+      changed_when: _remove_ini.rc == 2
+      failed_when: _remove_ini.rc not in [0,2]
+

scripts/upgrade-utilities/playbooks/neutron-tmp-inventory.yml

@@ -0,0 +1,40 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: localhost
+  connection: local
+  become: yes
+  gather_facts: no
+  tasks:
+    - name: Implement the user-space transitional inventory
+      shell: |
+        set -e
+        if [[ ! -e /etc/openstack_deploy/inventory.ini.org ]]; then
+          if [[ -e /etc/openstack_deploy/inventory.ini ]]; then
+            echo "Backing up the existing inventory.ini in /etc/openstack_deploy"
+            mv /etc/openstack_deploy/inventory.ini /etc/openstack_deploy/inventory.ini.org
+          else
+            echo "Adding a new inventory.ini into /etc/openstack_deploy"
+            echo '[all]' > /etc/openstack_deploy/inventory.ini.org
+          fi
+          echo "Copying the transitional inventory.ini for neutron into /etc/openstack_deploy"
+          cp {{ playbook_dir }}/../files/neutron-migration-inventory.ini /etc/openstack_deploy/inventory.ini
+          exit 2
+        fi
+      args:
+        executable: /bin/bash
+      register: _add_ini
+      changed_when: _add_ini.rc == 2
+      failed_when: _add_ini.rc not in [0,2]