Move network overlay configuration to Ansible playbooks

Co-Authored-By: Ricardo Carrillo Cruz <ricardo.carrillo.cruz@gmail.com>
Co-Authored-By: David Moreau Simard <dmsimard@redhat.com>

Change-Id: I5d7ee315cac1f5f8d293c1972edb85ac8bd3ebf7

devstack-vm-gate.sh

@@ -98,12 +98,21 @@ function setup_nova_net_networking {
     # We always setup multinode connectivity to work around an
     # issue with nova net configuring br100 to take over eth0
     # by default.
-    # TODO (clarkb): figure out how to make bridge setup sane with ansible.
-    ovs_vxlan_bridge "br_pub" $primary_node "True" 1 \
-                    $FLOATING_HOST_PREFIX $FLOATING_HOST_MASK \
-                    $sub_nodes
-    ovs_vxlan_bridge "br_flat" $primary_node "False" 128 \
-                    $sub_nodes
+    $ANSIBLE_PLAYBOOK -f 5 -i "$WORKSPACE/inventory" "$WORKSPACE/devstack-gate/playbooks/ovs_vxlan_bridge.yaml" \
+        -e "bridge_name=br_pub" \
+        -e "host_ip=$primary_node" \
+        -e "set_ips=True" \
+        -e "ovs_starting_offset=1" \
+        -e "pub_addr_prefix=$FLOATING_HOST_PREFIX" \
+        -e "pub_addr_mask=$FLOATING_HOST_MASK" \
+        -e "peer_ips=$sub_nodes"
+
+    $ANSIBLE_PLAYBOOK -f 5 -i "$WORKSPACE/inventory" "$WORKSPACE/devstack-gate/playbooks/ovs_vxlan_bridge.yaml" \
+        -e "bridge_name=br_flat" \
+        -e "host_ip=$primary_node" \
+        -e "set_ips=False" \
+        -e "ovs_starting_offset=128" \
+        -e "peer_ips=$sub_nodes"
     localrc_set $localrc "FLAT_INTERFACE" "br_flat"
     localrc_set $localrc "PUBLIC_INTERFACE" "br_pub"
 }
@@ -149,16 +158,25 @@ function setup_multinode_connectivity {
         # and on the master
         localrc_set $localconf "MULTI_HOST" "True"
     elif [[ "$DEVSTACK_GATE_NET_OVERLAY" -eq '1' ]]; then
-        ovs_vxlan_bridge "br-ex" $primary_node "True" 1 \
-                        $FLOATING_HOST_PREFIX $FLOATING_HOST_MASK \
-                        $sub_nodes
+        $ANSIBLE_PLAYBOOK -f 5 -i "$WORKSPACE/inventory" "$WORKSPACE/devstack-gate/playbooks/ovs_vxlan_bridge.yaml" \
+            -e "bridge_name=br-ex" \
+            -e "host_ip=$primary_node" \
+            -e "set_ips=True" \
+            -e "ovs_starting_offset=1" \
+            -e "pub_addr_prefix=$FLOATING_HOST_PREFIX" \
+            -e "pub_addr_mask=$FLOATING_HOST_MASK" \
+            -e "peer_ips=$sub_nodes"
     fi
 
     if [[ "$DEVSTACK_GATE_IRONIC" -eq '1' ]]; then
         # NOTE(vsaienko) Ironic VMs will be connected to this bridge
         # in order to have access to VMs on another nodes.
-        ovs_vxlan_bridge "br_ironic_vxlan" $primary_node "False" 128 \
-            $sub_nodes
+        $ANSIBLE_PLAYBOOK -f 5 -i "$WORKSPACE/inventory" "$WORKSPACE/devstack-gate/playbooks/ovs_vxlan_bridge.yaml" \
+            -e "bridge_name=br_ironic_vxlan" \
+            -e "host_ip=$primary_node" \
+            -e "set_ips=False" \
+            -e "ovs_starting_offset=128" \
+            -e "peer_ips=$sub_nodes"
 
         localrc_set "$sub_localconf" "HOST_TOPOLOGY" "multinode"
         localrc_set "$sub_localconf" "HOST_TOPOLOGY_ROLE" "subnode"

functions.sh

@@ -1018,114 +1018,6 @@ function enable_netconsole {
     popd
 }
 
-
-# This function creates an internal gre bridge to connect all external
-# network bridges across the compute and network nodes.
-# bridge_name: Bridge name on each host for logical l2 network
-#              connectivity.
-# host_ip: ip address of the bridge host which is reachable for all peer
-#          the hub for all of our spokes.
-# set_ips: Whether or not to set l3 addresses on our logical l2 network.
-#          This can be helpful for setting up routing tables.
-# offset: starting value for gre tunnel key and the ip addr suffix
-# The next two parameters are only used if set_ips is "True".
-# pub_addr_prefix: The IPv4 address three octet prefix used to give compute
-#                  nodes non conflicting addresses on the pub_if_name'd
-#                  network. Should be provided as X.Y.Z. Offset will be
-#                  applied to this as well as the below mask to get the
-#                  resulting address.
-# pub_addr_mask: the CIDR mask less the '/' for the IPv4 addresses used
-#                above.
-# every additional parameter is considered as a peer host (spokes)
-#
-# For OVS troubleshooting needs:
-#   http://www.yet.org/2014/09/openvswitch-troubleshooting/
-#
-function ovs_vxlan_bridge {
-    if is_suse || is_fedora; then
-        local ovs_package='openvswitch'
-        local ovs_service='openvswitch'
-    elif uses_debs; then
-        local ovs_package='openvswitch-switch'
-        local ovs_service='openvswitch-switch'
-    else
-        echo "Unsupported platform, can't determine openvswitch service"
-        exit 1
-    fi
-    local install_ovs_deps="source $BASE/new/devstack/functions-common; \
-                            install_package ${ovs_package}; \
-                            restart_service ${ovs_service}"
-    local mtu=1450
-    local bridge_name=$1
-    local host_ip=$2
-    local set_ips=$3
-    local offset=$4
-    if [[ "$set_ips" == "True" ]] ; then
-        local pub_addr_prefix=$5
-        local pub_addr_mask=$6
-        shift 6
-    else
-        shift 4
-    fi
-    local peer_ips=$@
-    # neutron uses 1:1000 with default devstack configuration, avoid overlap
-    local additional_vni_offset=1000000
-    eval $install_ovs_deps
-    # create a bridge, just like you would with 'brctl addbr'
-    # if the bridge exists, --may-exist prevents ovs from returning an error
-    sudo ovs-vsctl --may-exist add-br $bridge_name
-    # as for the mtu, look for notes on lp#1301958 in devstack-vm-gate.sh
-    sudo ip link set mtu $mtu dev $bridge_name
-    if [[ "$set_ips" == "True" ]] ; then
-        echo "Set bridge: ${bridge_name}"
-        if ! sudo ip addr show dev ${bridge_name} | grep -q \
-            ${pub_addr_prefix}.${offset}/${pub_addr_mask} ; then
-                sudo ip addr add ${pub_addr_prefix}.${offset}/${pub_addr_mask} \
-                    dev ${bridge_name}
-        fi
-    fi
-    sudo ip link set dev $bridge_name up
-    for node_ip in $peer_ips; do
-        offset=$(( offset+1 ))
-        vni=$(( offset + additional_vni_offset ))
-        # For reference on how to setup a tunnel using OVS see:
-        #   http://openvswitch.org/support/config-cookbooks/port-tunneling/
-        # The command below is equivalent to the sequence of ip/brctl commands
-        # where an interface of vxlan type is created first, and then plugged into
-        # the bridge; options are command specific configuration key-value pairs.
-        #
-        # Create the vxlan tunnel for the Controller/Network Node:
-        #  This establishes a tunnel between remote $node_ip to local $host_ip
-        #  uniquely identified by a key $offset
-        sudo ovs-vsctl --may-exist add-port $bridge_name \
-            ${bridge_name}_${node_ip} \
-            -- set interface ${bridge_name}_${node_ip} type=vxlan \
-            options:remote_ip=${node_ip} \
-            options:key=${vni} \
-            options:local_ip=${host_ip}
-        # Now complete the vxlan tunnel setup for the Compute Node:
-        #  Similarly this establishes the tunnel in the reverse direction
-        remote_command $node_ip "$install_ovs_deps"
-        remote_command $node_ip sudo ovs-vsctl --may-exist add-br $bridge_name
-        remote_command $node_ip sudo ip link set mtu $mtu dev $bridge_name
-        remote_command $node_ip sudo ovs-vsctl --may-exist add-port $bridge_name \
-            ${bridge_name}_${host_ip} \
-            -- set interface ${bridge_name}_${host_ip} type=vxlan \
-            options:remote_ip=${host_ip} \
-            options:key=${vni} \
-            options:local_ip=${node_ip}
-        if [[ "$set_ips" == "True" ]] ; then
-            if ! remote_command $node_ip sudo ip addr show dev ${bridge_name} | \
-                grep -q ${pub_addr_prefix}.${offset}/${pub_addr_mask} ; then
-                    remote_command $node_ip sudo ip addr add \
-                        ${pub_addr_prefix}.${offset}/${pub_addr_mask} \
-                        dev ${bridge_name}
-            fi
-        fi
-        remote_command $node_ip sudo ip link set dev $bridge_name up
-    done
-}
-
 # Timeout hook calls implemented as bash functions. Note this
 # forks and execs a new bash in order to use the timeout utility
 # which cannot operate on bash functions directly.

playbooks/ovs_vxlan_bridge.yaml

@@ -0,0 +1,25 @@
+---
+- name: Set up openvswitch requirements
+  hosts: all
+  gather_facts: yes
+  become: yes
+  roles:
+    - ovs_vxlan_bridge_prereqs
+
+- name: Set up bridge on the primary node
+  hosts: primary
+  gather_facts: no
+  become: yes
+  vars_files:
+    - devstack_gate_vars.yaml
+  roles:
+    - ovs_vxlan_bridge_primary
+
+- name: Set up the bridge on the sub nodes
+  hosts: subnodes
+  gather_facts: no
+  become: yes
+  vars_files:
+    - devstack_gate_vars.yaml
+  roles:
+    - ovs_vxlan_bridge_peers

playbooks/roles/ovs_vxlan_bridge_peers/defaults/main.yaml

@@ -0,0 +1,3 @@
+ovs_starting_offset: 1
+ovs_vni_offset: 1000000
+ovs_bridge_mtu: 1450

playbooks/roles/ovs_vxlan_bridge_peers/tasks/main.yaml

@@ -0,0 +1,54 @@
+---
+# This dynamically configures a unique offset for this peer
+- name: Set offset
+  set_fact:
+    offset: "{{ ovs_starting_offset | int + 1 + groups['all'].index(inventory_hostname) }}"
+
+- name: Add additional vni offset
+  set_fact:
+    vni: "{{ offset | int + ovs_vni_offset | int }}"
+
+# To make things more readable in the following tasks
+- name: Alias the primary node private IP
+  set_fact:
+    primary_private_ip: "{{ hostvars[groups['primary'][0]]['nodepool']['private_ipv4'] }}"
+
+- name: Add port to bridge on primary
+  shell: >-
+    ovs-vsctl --may-exist add-port {{ bridge_name }}
+    {{ bridge_name }}_{{ nodepool['private_ipv4'] }}
+    -- set interface {{ bridge_name}}_{{ nodepool['private_ipv4'] }}
+    type=vxlan options:remote_ip={{ nodepool['private_ipv4'] }} options:key={{ vni }}
+    options:local_ip={{ primary_private_ip }}
+  delegate_to: "{{ groups['primary'][0] }}"
+
+- name: Create bridge on subnode
+  openvswitch_bridge:
+    bridge: "{{ bridge_name }}"
+
+- name: Set MTU on subnode bridge
+  command: ip link set mtu {{ ovs_bridge_mtu }} dev {{ bridge_name }}
+
+- name: Add port to bridge on subnode
+  shell: >-
+    ovs-vsctl --may-exist add-port {{ bridge_name }}
+    {{ bridge_name }}_{{ primary_private_ip }}
+    -- set interface {{ bridge_name}}_{{ primary_private_ip }}
+    type=vxlan options:remote_ip={{ primary_private_ip }} options:key={{ vni }}
+    options:local_ip={{ nodepool['private_ipv4'] }}
+
+- when: set_ips
+  block:
+    - name: Verify if the bridge address is set
+      shell: ip addr show dev {{ bridge_name }} | grep -q {{ pub_addr_prefix }}.{{ offset }}/{{ pub_addr_mask }}
+      register: ip_addr_var
+      failed_when: False
+      changed_when: False
+
+    - name: Set the bridge address
+      command: ip addr add {{ pub_addr_prefix }}.{{ offset }}/{{ pub_addr_mask }} dev {{ bridge_name }}
+      become: yes
+      when: ip_addr_var.rc == 1
+
+- name: Bring subnode bridge interface up
+  command: ip link set dev {{ bridge_name }} up

playbooks/roles/ovs_vxlan_bridge_prereqs/tasks/main.yaml

@@ -0,0 +1,13 @@
+---
+- name: Gather OS specific package and service names
+  include_vars: "{{ ansible_os_family }}.yaml"
+
+- name: Install openvswitch package
+  package:
+    name: "{{ ovs_package }}"
+    state: installed
+
+- name: Start openvswitch service
+  service:
+    name: "{{ ovs_service }}"
+    state: started

playbooks/roles/ovs_vxlan_bridge_prereqs/vars/Debian.yaml

@@ -0,0 +1,3 @@
+---
+ovs_package: "openvswitch-switch"
+ovs_service: "openvswitch-switch"

playbooks/roles/ovs_vxlan_bridge_prereqs/vars/RedHat.yaml

@@ -0,0 +1,3 @@
+---
+ovs_package: "openvswitch"
+ovs_service: "openvswitch"

playbooks/roles/ovs_vxlan_bridge_prereqs/vars/Suse.yaml

@@ -0,0 +1,3 @@
+---
+ovs_package: "openvswitch"
+ovs_service: "openvswitch"

playbooks/roles/ovs_vxlan_bridge_primary/defaults/main.yaml

@@ -0,0 +1,2 @@
+ovs_starting_offset: 1
+ovs_bridge_mtu: 1450

playbooks/roles/ovs_vxlan_bridge_primary/tasks/main.yaml

@@ -0,0 +1,22 @@
+- name: Create bridge
+  openvswitch_bridge:
+    bridge: "{{ bridge_name }}"
+
+- name: Set bridge MTU
+  command: ip link set mtu {{ ovs_bridge_mtu }} dev {{ bridge_name }}
+
+- when: set_ips
+  block:
+    - name: Verify if the bridge address is set
+      shell: ip addr show dev {{ bridge_name }} | grep -q {{ pub_addr_prefix }}.{{ ovs_starting_offset }}/{{ pub_addr_mask }}
+      register: ip_addr_var
+      failed_when: False
+      changed_when: False
+
+    - name: Set the bridge address
+      command: ip addr add {{ pub_addr_prefix }}.{{ ovs_starting_offset }}/{{ pub_addr_mask }} dev {{ bridge_name }}
+      become: yes
+      when: ip_addr_var.rc == 1
+
+- name: Bring bridge interface up
+  command: ip link set dev {{ bridge_name }} up