Enable VxLAN with XenServer

1. Remove the restriction of vxlan on yaml file
2. Configure iptables in Dom0 to enable vxlan port 4789
3. Move br-mesh from compute node to Dom0
4. Add mos-vxlan.service to take use of systemctl and make
   sure when XenServer host reboot, the br-mesh and related
   configurations is still there
5. Persist iptables rules
6. Add cleanup of br-mesh if it exist in in Dom0 when the
   neutron network type is not vxlan

Change-Id: I1ab8ca56714167a2e20513b0f9b0ed4a82d9648e

5 files changed, 260 insertions, 13 deletions

diff --git a/plugin_source/components.yaml b/plugin_source/components.yaml
index a82ed4a..3e98a48 100644
--- a/plugin_source/components.yaml
+++ b/plugin_source/components.yaml
@@ -6,8 +6,6 @@
       description: ''
     - name: 'hypervisor:qemu'
       description: ''
-    - name: 'network:neutron:ml2:tun'
-      description: ''
     - name: 'additional_service:sahara'
       description: ''
     - name: 'additional_service:murano'
diff --git a/plugin_source/deployment_scripts/compute_post_deployment.py b/plugin_source/deployment_scripts/compute_post_deployment.py
index 2c0f0d5..7ba11f4 100755
--- a/plugin_source/deployment_scripts/compute_post_deployment.py
+++ b/plugin_source/deployment_scripts/compute_post_deployment.py
@@ -2,6 +2,7 @@
 
 import ConfigParser
 from distutils.version import LooseVersion
+import ipaddress
 import netifaces
 import os
 import re
@@ -14,6 +15,10 @@ from utils import HIMN_IP
 
 
 INT_BRIDGE = 'br-int'
+MESH_BRIDGE = 'br-mesh'
+AUTO_START_SERVICE = 'mos-vxlan.service'
+AUTO_START_SERVICE_TEMPLATE = 'mos-vxlan-template.service'
+AUTO_SCRIPT = 'fuel-xs-vxlan.sh'
 XS_PLUGIN_ISO = 'xenapi-plugins-mitaka.iso'
 CONNTRACK_CONF_SAMPLE =\
     '/usr/share/doc/conntrack-tools-1.4.2/doc/stats/conntrackd.conf'
@@ -264,7 +269,7 @@ def modify_neutron_rootwrap_conf(himn, username, password):
     LOG.info('Modify file %s successfully', filename)
 
 
-def modify_neutron_ovs_agent_conf(int_br, br_mappings):
+def modify_neutron_ovs_agent_conf(int_br, br_mappings=None, local_ip=None):
     filename = '/etc/neutron/plugins/ml2/openvswitch_agent.ini'
     cf = ConfigParser.ConfigParser()
     try:
@@ -274,7 +279,10 @@ def modify_neutron_ovs_agent_conf(int_br, br_mappings):
         cf.set('agent', 'root_helper_daemon', '')
         cf.set('agent', 'minimize_polling', False)
         cf.set('ovs', 'integration_bridge', int_br)
-        cf.set('ovs', 'bridge_mappings', br_mappings)
+        if br_mappings:
+            cf.set('ovs', 'bridge_mappings', br_mappings)
+        if local_ip:
+            cf.set('ovs', 'local_ip', local_ip)
         with open(filename, 'w') as configfile:
             cf.write(configfile)
     except Exception:
@@ -282,26 +290,28 @@ def modify_neutron_ovs_agent_conf(int_br, br_mappings):
     LOG.info('Modify %s successfully', filename)
 
 
-def get_private_network_ethX():
+def get_network_ethX(bridge_name):
     # find out ethX in DomU which connect to private network
     # br-aux is the auxiliary bridge and in normal case there will be a patch
     # between br-prv and br-aux
     values = astute['network_scheme']['transformations']
     for item in values:
-        if item['action'] == 'add-port' and item['bridge'] == 'br-aux':
+        if item['action'] == 'add-port' and item['bridge'] == bridge_name:
             return item['name']
-    # If cannot find br-aux, the network topo should be public and private
-    # connect to the same network and "Assign public network to all nodes"
-    # is checked, we need to use br-ex to find ethX in domU
+    # If cannot find given bridge, the network topo should be public and
+    # private connecting to the same network and the checkbox
+    # "Assign public network to all nodes" is checked, we need to use br-ex
+    # to find ethX in domU
     for item in values:
         if item['action'] == 'add-port' and item['bridge'] == 'br-ex':
             return item['name']
 
 
-def find_bridge_mappings(astute, himn, username):
-    ethX = get_private_network_ethX()
+def find_dom0_bridge(himn, username, bridge_name):
+    ethX = get_network_ethX(bridge_name)
     if not ethX:
         utils.reportError("Cannot find eth used for private network")
+    ethX = ethX.split('.')[0]
 
     # find the ethX mac in /sys/class/net/ethX/address
     with open('/sys/class/net/%s/address' % ethX, 'r') as fo:
@@ -312,6 +322,12 @@ def find_bridge_mappings(astute, himn, username):
     bridge = utils.ssh(himn, username,
                        ('xe network-param-get param-name=bridge '
                         'uuid=%s') % network_uuid)
+    return bridge
+
+
+def find_physical_network_mappings(astute, himn, username):
+    # find corresponding bridge in Dom0
+    bridge = find_dom0_bridge(himn, username, 'br-aux')
 
     # find physical network name
     phynet_setting = astute['quantum_settings']['L2']['phys_nets']
@@ -458,6 +474,142 @@ def enable_conntrack_service(himn, username):
     utils.ssh(himn, username, 'service', 'conntrackd', 'restart')
 
 
+def configure_dom0_iptables(himn, username):
+    xs_chain = 'XenServer-Neutron-INPUT'
+
+    # Check XenServer specific chain, create if not exist
+    commands = ('iptables -t filter -L %s' % xs_chain,
+                'iptables -t filter --new %s' % xs_chain,
+                'iptables -t filter -I INPUT -j %s' % xs_chain)
+    execute_iptables_commands(himn, username, commands)
+
+    # Check XenServer rule for ovs native mode, create if not exist
+    commands = ('iptables -t filter -C %s -p tcp -m tcp --dport 6640 -j ACCEPT'
+                % xs_chain,
+                'iptables -t filter -I %s -p tcp --dport 6640 -j ACCEPT'
+                % xs_chain)
+    execute_iptables_commands(himn, username, commands)
+
+    # Check XenServer rule for vxlan, create if not exist
+    commands = ('iptables -t filter -C %s -p udp -m multiport --dports 4789 '
+                '-j ACCEPT' % xs_chain,
+                'iptables -t filter -I %s -p udp -m multiport --dport 4789 -j '
+                'ACCEPT' % xs_chain)
+    execute_iptables_commands(himn, username, commands)
+
+    # Persist iptables rules
+    utils.ssh(himn, username, 'service', 'iptables', 'save')
+
+
+def execute_iptables_commands(himn, username, command_list):
+    # Execute first command and continue based on first command result
+    exitcode, _, _ = utils.ssh_detailed(
+        himn, username, command_list[0], allowed_return_codes=[0, 1])
+    if exitcode == 1:
+        for command in command_list[1:]:
+            LOG.info('Execute iptables command %s', command)
+            utils.ssh(himn, username, command)
+
+
+def create_dom0_mesh_bridge(himn, username, dom0_bridge, mesh_info):
+    # Create br-mesh and veth pair if not exist in Dom0
+    exitcode, out, _ = utils.ssh_detailed(himn, username,
+                                          'ip', 'addr', 'show', MESH_BRIDGE,
+                                          allowed_return_codes=[0, 1])
+    if exitcode == 1:
+        # create mesh bridge if it not exist in Dom0
+        create_mesh_bridge = True
+    else:
+        # if mesh bridge exist in Dom0, check its ip, re-configure ip if
+        # it's not the same as what we want to set
+        bridge_info = out.split()
+        try:
+            index_inet = bridge_info.index('inet')
+            # get inet info like '192.168.2.2/24'
+            ipaddr = bridge_info[index_inet + 1].split('/')[0]
+            current_ip = ipaddress.ip_address(unicode(ipaddr))
+            configured_ip = ipaddress.ip_address(unicode(mesh_info['ipaddr']))
+            if current_ip == configured_ip:
+                LOG.info('Bridge %s already exist in Dom0' % MESH_BRIDGE)
+                return
+            else:
+                create_mesh_bridge = True
+                remove_old_mesh_bridge(himn, username, MESH_BRIDGE)
+        except ValueError:
+            create_mesh_bridge = True
+            remove_old_mesh_bridge(himn, username, MESH_BRIDGE)
+
+    if create_mesh_bridge:
+        LOG.debug("Create mesh bridge in Dom0")
+        utils.scp(himn, username, '/etc/sysconfig/network-scripts/',
+                  AUTO_SCRIPT)
+        utils.ssh(himn, username, 'chmod', '+x',
+                  '/etc/sysconfig/network-scripts/%s' % AUTO_SCRIPT)
+
+        start_param = '%(bridge)s %(ip)s %(netmask)s %(broadcast)s %(tag)s' \
+                      % {'bridge': dom0_bridge,
+                         'ip': mesh_info['ipaddr'],
+                         'netmask': mesh_info['netmask'],
+                         'broadcast': mesh_info['broadcast'],
+                         'tag': mesh_info['tag']}
+        with open(AUTO_START_SERVICE_TEMPLATE) as f:
+            contents = f.read()
+        contents = contents.replace('@MESH_INFO@', start_param)
+        with open(AUTO_START_SERVICE, 'w') as f:
+            f.write(contents)
+        utils.scp(himn, username, '/etc/systemd/system', AUTO_START_SERVICE)
+        utils.ssh(himn, username, 'systemctl', 'daemon-reload')
+        utils.ssh(himn, username, 'systemctl', 'enable', AUTO_START_SERVICE)
+        utils.ssh(himn, username, 'systemctl', 'start', AUTO_START_SERVICE)
+
+
+def disable_local_mesh_bridge(bridge):
+    iface_list = netifaces.interfaces()
+    if bridge in iface_list:
+        utils.execute('ifconfig', bridge, '0.0.0.0')
+        utils.execute('ip', 'link', 'set', bridge, 'down')
+
+    filename = '/etc/network/interfaces.d/ifcfg-%s' % bridge
+    if os.path.isfile(filename):
+        utils.execute('rm', '-f', filename)
+
+
+def get_mesh_info(astute, bridge):
+    mesh_nets = astute['network_scheme']['endpoints'][bridge]['IP'][0]
+    mesh_ip = mesh_nets.split('/')[0]
+    ipv4_net = ipaddress.ip_network(unicode(mesh_nets), strict=False)
+    mesh_broadcast = str(ipv4_net.broadcast_address)
+    network_netmask = str(ipv4_net.with_netmask).split('/')
+    mesh_netmask = network_netmask[1]
+    mesh_network = network_netmask[0]
+    mesh_eth = get_network_ethX(bridge)
+    mesh_tag = "''"
+    index = mesh_eth.index('.')
+    if index > 0:
+        mesh_tag = mesh_eth[index+1:]
+    mesh_info = {'ipaddr': mesh_ip, 'network': mesh_network,
+                 'netmask': mesh_netmask, 'broadcast': mesh_broadcast,
+                 'tag': mesh_tag}
+    return mesh_info
+
+
+def remove_old_mesh_bridge(himn, username, bridge):
+    exitcode, _, _ = utils.ssh_detailed(himn, username, 'ip', 'link', 'show',
+                                        bridge, allowed_return_codes=[0, 1])
+    if exitcode == 0:
+        # Allow return code 5 to make sure it won't fail when
+        # mos-vxlan.service isn't exist
+        utils.ssh_detailed(himn, username, 'systemctl', 'stop',
+                           AUTO_START_SERVICE, allowed_return_codes=[0, 5])
+        utils.ssh_detailed(himn, username, 'systemctl', 'disable',
+                           AUTO_START_SERVICE, allowed_return_codes=[0, 1])
+        utils.ssh(himn, username, 'rm', '-f',
+                  '/etc/systemd/system/%s' % AUTO_START_SERVICE)
+        utils.ssh(himn, username, 'rm', '-f',
+                  '/etc/sysconfig/network-scripts/%s' % AUTO_SCRIPT)
+        utils.ssh(himn, username, 'systemctl', 'daemon-reload')
+
+
 if __name__ == '__main__':
     install_xenapi_sdk()
     astute = utils.get_astute()
@@ -493,10 +645,29 @@ if __name__ == '__main__':
             # enable conntrackd service in Dom0
             enable_conntrack_service(HIMN_IP, username)
 
+            # configure iptables in Dom0 to support ovs native mode and VxLAN
+            configure_dom0_iptables(HIMN_IP, username)
+
             # neutron-l2-agent in compute node
             modify_neutron_rootwrap_conf(HIMN_IP, username, password)
-            br_mappings = find_bridge_mappings(astute, HIMN_IP, username)
-            modify_neutron_ovs_agent_conf(INT_BRIDGE, br_mappings)
+            l2_net_type = astute['quantum_settings']['predefined_networks'][
+                'admin_internal_net']['L2']['network_type']
+            br_mappings = None
+            if l2_net_type == 'vlan':
+                br_mappings = find_physical_network_mappings(astute, HIMN_IP,
+                                                             username)
+                remove_old_mesh_bridge(HIMN_IP, username, MESH_BRIDGE)
+            ip = None
+            if l2_net_type == 'tun':
+                dom0_priv_bridge = find_dom0_bridge(HIMN_IP, username,
+                                                    MESH_BRIDGE)
+                mesh_info = get_mesh_info(astute, MESH_BRIDGE)
+                ip = mesh_info['ipaddr']
+                disable_local_mesh_bridge(MESH_BRIDGE)
+                create_dom0_mesh_bridge(HIMN_IP, username, dom0_priv_bridge,
+                                        mesh_info)
+            modify_neutron_ovs_agent_conf(INT_BRIDGE, br_mappings=br_mappings,
+                                          local_ip=ip)
             patch_neutron_ovs_agent()
             restart_services('neutron-openvswitch-agent')
 
diff --git a/plugin_source/deployment_scripts/fuel-xs-vxlan.sh b/plugin_source/deployment_scripts/fuel-xs-vxlan.sh
new file mode 100644
index 0000000..3e66d7e
--- /dev/null
+++ b/plugin_source/deployment_scripts/fuel-xs-vxlan.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+OP=$1
+COUNT=$#
+
+function create_mesh_bridge {
+    local dom0_bridge=$1
+    local mesh_ip=$2
+    local mesh_netmask=$3
+    local mesh_broadcast=$4
+    local tag=$5
+
+    ip link show br-mesh
+    exitcode=$?
+    if [ "$exitcode" == "1" ]; then
+        brctl addbr br-mesh
+        brctl setfd br-mesh 0
+        brctl stp br-mesh off
+        ip link set br-mesh up
+        ip link delete mesh_ovs
+        ip link add mesh_ovs type veth peer name mesh_linux
+        ip link set mesh_ovs up
+        ip link set mesh_ovs promisc on
+        ip link set mesh_linux up
+        ip link set mesh_linux promisc on
+        brctl addif br-mesh mesh_linux
+        ovs-vsctl -- --if-exists del-port mesh_ovs -- add-port $dom0_bridge mesh_ovs
+        ip addr add $mesh_ip/$mesh_netmask broadcast $mesh_broadcast dev br-mesh
+        if [ -n "$tag" ]; then
+            ovs-vsctl -- set Port mesh_ovs tag=$tag
+        fi
+    fi
+}
+
+function delete_mesh_bridge {
+    ip link show br-mesh
+    exitcode=$?
+    if [ "$exitcode" == "0" ]; then
+        ip link set br-mesh down
+        ip link set mesh_ovs down
+        ip link delete mesh_ovs
+        ovs-vsctl -- --if-exist del-port mesh_ovs
+        brctl delbr br-mesh
+    fi
+}
+
+
+if [ "$OP" == "start" ]; then
+    if [ $COUNT -lt 6 ]; then
+        echo "usage: fuel-xs-vlan.sh start BRIDGE IP NETMASK BROADCAST [VLAN_TAG]"
+        echo "Exit due to lack of parameters!"
+        exit 0
+    fi
+
+    dom0_bridge=$2
+    mesh_ip=$3
+    mesh_netmask=$4
+    mesh_broadcast=$5
+    tag=$6
+    create_mesh_bridge $dom0_bridge $mesh_ip $mesh_netmask $mesh_broadcast $tag
+elif [ "$OP" == "stop" ]; then
+    delete_mesh_bridge
+fi
\ No newline at end of file
diff --git a/plugin_source/deployment_scripts/mos-vxlan-template.service b/plugin_source/deployment_scripts/mos-vxlan-template.service
new file mode 100644
index 0000000..dd13a94
--- /dev/null
+++ b/plugin_source/deployment_scripts/mos-vxlan-template.service
@@ -0,0 +1,14 @@
+[Unit]
+Description=Configure Mirantis OpenStack mesh bridge
+Requires=xcp-networkd.service openvswitch-xapi-sync.service
+After=xcp-networkd.service openvswitch-xapi-sync.service
+AssertPathExists=/etc/sysconfig/network-scripts/
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/bash /etc/sysconfig/network-scripts/fuel-xs-vxlan.sh start @MESH_INFO@
+ExecStop=/bin/bash /etc/sysconfig/network-scripts/fuel-xs-vxlan.sh stop
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
diff --git a/plugin_source/deployment_scripts/utils.py b/plugin_source/deployment_scripts/utils.py
index c90dee2..dee5ca8 100644
--- a/plugin_source/deployment_scripts/utils.py
+++ b/plugin_source/deployment_scripts/utils.py
@@ -67,6 +67,7 @@ def detailed_execute(*cmd, **kwargs):
             LOG.info('Swallowed acceptable return code of %d',
                      proc.returncode)
         else:
+            LOG.warn('proc.returncode: %s', proc.returncode)
             raise ExecutionError(err)
 
     return proc.returncode, out, err