Merge "Migration revert plan"

doc/source/ovn/migration.rst

@@ -147,6 +147,14 @@ Perform the following steps in the undercloud
      during migration to ensure a synchronized MTU switch across the networks.
      Default: 30
 
+   * CREATE_BACKUP - Flag to create a backup of the controllers that can be
+     used as a revert mechanism.
+     Default: True
+
+   * BACKUP_MIGRATION_IP - Only used if CREATE_BACKUP is enabled, IP of the
+     server that will be used as a NFS server to store the backup.
+     Default: 192.168.24.1
+
    .. warning::
 
       Please note that VALIDATE_MIGRATION requires enough quota (2

tools/ovn_migration/tripleo_environment/ovn_migration.sh

@@ -41,6 +41,8 @@ LANG=C
 : ${SERVER_USER_NAME:=cirros}
 : ${VALIDATE_MIGRATION:=False}
 : ${DHCP_RENEWAL_TIME:=30}
+: ${CREATE_BACKUP:=True}
+: ${BACKUP_MIGRATION_IP:=192.168.24.1} # TODO: Document this new var
 
 
 check_for_necessary_files() {
@@ -51,11 +53,14 @@ check_for_necessary_files() {
     fi
 
     # Check if the user has generated overcloud-deploy-ovn.sh file
+    # With correct permissions
     # If it is not generated. Exit
-    if [ ! -e $OVERCLOUD_OVN_DEPLOY_SCRIPT ]; then
+    if [ ! -x $OVERCLOUD_OVN_DEPLOY_SCRIPT ]; then
         echo "overcloud deploy migration script :" \
-             "$OVERCLOUD_OVN_DEPLOY_SCRIPT is not present. Please" \
-             "make sure you create that file before running this script."
+             "$OVERCLOUD_OVN_DEPLOY_SCRIPT is not present" \
+             "or execution permission is missing. Please" \
+             "make sure you create that file with correct" \
+             "permissions before running this script."
         exit 1
     fi
 
@@ -96,6 +101,17 @@ check_for_necessary_files() {
         fi
         exit 1
     fi
+    # Check if backup is enabled
+    if [[ $CREATE_BACKUP = True ]]; then
+        # Check if backup server is reachable
+        ping -c4 $BACKUP_MIGRATION_IP
+        if [[ $? -eq 1 ]]; then
+            echo -e "It is not possible to reach the backup migration server IP" \
+                    "($BACKUP_MIGRATION_IP). Make sure this IP is accessible before" \
+                    "starting the migration." \
+                    "Change this value by doing: export BACKUP_MIGRATION_IP=x.x.x.x"
+        fi
+    fi
 }
 
 get_host_ip() {
@@ -297,14 +313,23 @@ reduce_network_mtu () {
 start_migration() {
     source $STACKRC_FILE
     echo "Starting the Migration"
+    local inventory_file="$OOO_WORKDIR/$STACK_NAME/config-download/$STACK_NAME/tripleo-ansible-inventory.yaml"
+    if ! test -f $inventory_file; then
+        inventory_file=''
+    fi
     ansible-playbook  -vv $OPT_WORKDIR/playbooks/ovn-migration.yml \
     -i hosts_for_migration -e working_dir=$OPT_WORKDIR \
     -e public_network_name=$PUBLIC_NETWORK_NAME \
     -e image_name=$IMAGE_NAME \
     -e flavor_name=$FLAVOR_NAME \
+    -e undercloud_node_user=$UNDERCLOUD_NODE_USER \
     -e overcloud_ovn_deploy_script=$OVERCLOUD_OVN_DEPLOY_SCRIPT \
-    -e server_user_name=$SERVER_USER_NAME        \
-    -e overcloudrc=$OVERCLOUDRC_FILE             \
+    -e server_user_name=$SERVER_USER_NAME \
+    -e overcloudrc=$OVERCLOUDRC_FILE \
+    -e stackrc=$STACKRC_FILE \
+    -e backup_migration_ip=$BACKUP_MIGRATION_IP \
+    -e create_backup=$CREATE_BACKUP \
+    -e ansible_inventory=$inventory_file \
     -e validate_migration=$VALIDATE_MIGRATION $*
 
     rc=$?

tools/ovn_migration/tripleo_environment/playbooks/ovn-migration.yml

@@ -1,11 +1,25 @@
 # This is the playbook used by ovn-migration.sh.
 
+#
+# Backup the controllers to have a backup in case the
+# migration fails leaving the testbed on a broken status.
+#
+
+- name: Backup controllers pre-migration
+  hosts: localhost
+  roles:
+    - recovery-backup
+  tags:
+    - recovery-backup
+
+
 #
 # Pre migration and validation tasks will make sure that the initial cloud
 # is functional, and will create resources which will be checked after
 # migration.
 #
 
+
 - name: Pre migration and validation tasks
   hosts: localhost
   roles:
@@ -48,6 +62,7 @@
     - setup
   become: false
 
+
 #
 # Once everything is migrated prepare everything by syncing the neutron DB
 # into the OVN NB database, and then switching the dataplane to br-int
@@ -65,6 +80,7 @@
   tags:
     - migration
 
+
 #
 # Verify that the initial resources are still reachable, remove them,
 # and afterwards create new resources and repeat the connectivity tests.
@@ -78,6 +94,7 @@
   tags:
     - post-migration
 
+
 #
 # Final validation after tripleo update to br-int
 #

tools/ovn_migration/tripleo_environment/playbooks/revert.yml

@@ -0,0 +1,4 @@
+- name: Clean computes
+  hosts: ovn-controllers
+  roles:
+    - revert

tools/ovn_migration/tripleo_environment/playbooks/roles/recovery-backup/defaults/main.yml

@@ -0,0 +1,12 @@
+---
+
+# Name of the group hosts where the NFS instalation will take place
+# If the NFS server is the undercloud (and there is only one) will
+# not be a problem, but if multiple servers exist on the server_name group
+# it is possible that the nfs will be installed on every server, eventho the
+# storage of the backup will only be done in the backup_ip.
+#
+# This can be solved if a new tripleo-inventory is manually created specifying
+# a [BackupNode] section, with the nfs server info
+revert_preparation_server_name: "Undercloud"
+backup_and_recover_temp_folder: /tmp/backup-recover-temp

tools/ovn_migration/tripleo_environment/playbooks/roles/recovery-backup/tasks/main.yml

@@ -0,0 +1,68 @@
+---
+
+- name: Create controller's backup
+  block:
+    - name: Create temp folder related to backup
+      file:
+        state: directory
+        path: "{{ backup_and_recover_temp_folder }}"
+
+    # Using this task on OSP17
+    - name: Copy tripleo-inventory
+      copy:
+        src: "{{ ansible_inventory }}"
+        dest: "{{ backup_and_recover_temp_folder }}/tripleo-inventory.yaml"
+      when:
+        - create_backup|bool
+        - ansible_inventory is defined
+        - ansible_inventory != ""
+
+    # Using this task in OSP16.x
+    - name: Generate tripleo inventory
+      shell: |
+        source {{ stackrc }} &&
+        tripleo-ansible-inventory \
+        --ansible_ssh_user {{ undercloud_node_user }} \
+        --static-yaml-inventory {{ backup_and_recover_temp_folder }}/tripleo-inventory.yaml
+      when:
+        - create_backup|bool
+        - ansible_inventory is not defined or ansible_inventory == ""
+
+    - name: Setup NFS on the backup node using IP {{ backup_migration_ip }}
+      shell: |
+        source {{ stackrc }} &&
+        openstack overcloud backup \
+        --inventory {{ backup_and_recover_temp_folder }}/tripleo-inventory.yaml \
+        --setup-nfs \
+        --extra-vars '{
+                "tripleo_backup_and_restore_server": {{ backup_migration_ip }},
+                "nfs_server_group_name": {{ revert_preparation_server_name }}
+              }'
+
+    - name: Setup REAR on the controllers
+      shell: |
+        source {{ stackrc }} &&
+        openstack overcloud backup \
+        --inventory {{ backup_and_recover_temp_folder }}/tripleo-inventory.yaml \
+        --setup-rear \
+        --extra-vars '{
+                "tripleo_backup_and_restore_server": {{ backup_migration_ip }}
+              }'
+
+    - name: Backup the controllers
+      shell: |
+        source {{ stackrc }} &&
+        openstack overcloud backup \
+        --inventory {{ backup_and_recover_temp_folder }}/tripleo-inventory.yaml
+
+    # Ensure that after the controller backups the api responds
+    - name: Ensure that the OSP api is working
+      shell: >
+        source {{ overcloudrc }} && openstack flavor list
+      retries: 20
+      register: api_rc
+      delay: 5
+      ignore_errors: yes
+      until: api_rc.rc == "0"
+  when: create_backup|bool
+

tools/ovn_migration/tripleo_environment/playbooks/roles/revert/tasks/main.yml

@@ -0,0 +1,29 @@
+---
+- name: Stop ovn containers
+  become: yes
+  shell: |
+    for agent in $(podman ps -a --format {% raw %}"{{.ID}}"{% endraw %} --filter "name=(ovn_.*|ovnmeta)"); do
+        echo "Cleaning up agent $agent"
+        podman rm -f $agent
+    done
+
+- name: Clean OVN netns
+  become: yes
+  shell: |
+    for netns in $(ip netns ls | grep ovnmeta | cut -d' ' -f1); do
+        echo "delete netns $netns"
+        ip netns del $netns
+    done
+
+- name: Delete OVN ports
+  become: yes
+  shell: |
+    for port in $(ovs-vsctl list interface | grep ^name | grep 'ovn-\|patch-provnet\|patch-br-int-to' | cut -d':' -f2); do
+        echo "Removing port $port"
+        ovs-vsctl del-port $port
+    done
+
+- name: Revert cleanup completed.
+  debug:
+    msg: Revert cleanup done, please run overcloud deploy with the OVS configuration.
+