Enable VxLAN with XenServer

1. Remove the restriction of vxlan on yaml file
2. Configure iptables in Dom0 to enable vxlan port 4789
3. Move br-mesh from compute node to Dom0
4. Add mos-vxlan.service to take use of systemctl and make
   sure when XenServer host reboot, the br-mesh and related
   configurations is still there
5. Persist iptables rules
6. Add cleanup of br-mesh if it exist in in Dom0 when the
   neutron network type is not vxlan

Change-Id: I1ab8ca56714167a2e20513b0f9b0ed4a82d9648e
This commit is contained in:
Huan Xie 2017-01-12 19:12:47 -08:00
parent 9a2beb2950
commit 6689a9f9c9
5 changed files with 260 additions and 13 deletions

View File

@ -6,8 +6,6 @@
description: ''
- name: 'hypervisor:qemu'
description: ''
- name: 'network:neutron:ml2:tun'
description: ''
- name: 'additional_service:sahara'
description: ''
- name: 'additional_service:murano'

View File

@ -2,6 +2,7 @@
import ConfigParser
from distutils.version import LooseVersion
import ipaddress
import netifaces
import os
import re
@ -14,6 +15,10 @@ from utils import HIMN_IP
INT_BRIDGE = 'br-int'
MESH_BRIDGE = 'br-mesh'
AUTO_START_SERVICE = 'mos-vxlan.service'
AUTO_START_SERVICE_TEMPLATE = 'mos-vxlan-template.service'
AUTO_SCRIPT = 'fuel-xs-vxlan.sh'
XS_PLUGIN_ISO = 'xenapi-plugins-mitaka.iso'
CONNTRACK_CONF_SAMPLE =\
'/usr/share/doc/conntrack-tools-1.4.2/doc/stats/conntrackd.conf'
@ -264,7 +269,7 @@ def modify_neutron_rootwrap_conf(himn, username, password):
LOG.info('Modify file %s successfully', filename)
def modify_neutron_ovs_agent_conf(int_br, br_mappings):
def modify_neutron_ovs_agent_conf(int_br, br_mappings=None, local_ip=None):
filename = '/etc/neutron/plugins/ml2/openvswitch_agent.ini'
cf = ConfigParser.ConfigParser()
try:
@ -274,7 +279,10 @@ def modify_neutron_ovs_agent_conf(int_br, br_mappings):
cf.set('agent', 'root_helper_daemon', '')
cf.set('agent', 'minimize_polling', False)
cf.set('ovs', 'integration_bridge', int_br)
cf.set('ovs', 'bridge_mappings', br_mappings)
if br_mappings:
cf.set('ovs', 'bridge_mappings', br_mappings)
if local_ip:
cf.set('ovs', 'local_ip', local_ip)
with open(filename, 'w') as configfile:
cf.write(configfile)
except Exception:
@ -282,26 +290,28 @@ def modify_neutron_ovs_agent_conf(int_br, br_mappings):
LOG.info('Modify %s successfully', filename)
def get_private_network_ethX():
def get_network_ethX(bridge_name):
# find out ethX in DomU which connect to private network
# br-aux is the auxiliary bridge and in normal case there will be a patch
# between br-prv and br-aux
values = astute['network_scheme']['transformations']
for item in values:
if item['action'] == 'add-port' and item['bridge'] == 'br-aux':
if item['action'] == 'add-port' and item['bridge'] == bridge_name:
return item['name']
# If cannot find br-aux, the network topo should be public and private
# connect to the same network and "Assign public network to all nodes"
# is checked, we need to use br-ex to find ethX in domU
# If cannot find given bridge, the network topo should be public and
# private connecting to the same network and the checkbox
# "Assign public network to all nodes" is checked, we need to use br-ex
# to find ethX in domU
for item in values:
if item['action'] == 'add-port' and item['bridge'] == 'br-ex':
return item['name']
def find_bridge_mappings(astute, himn, username):
ethX = get_private_network_ethX()
def find_dom0_bridge(himn, username, bridge_name):
ethX = get_network_ethX(bridge_name)
if not ethX:
utils.reportError("Cannot find eth used for private network")
ethX = ethX.split('.')[0]
# find the ethX mac in /sys/class/net/ethX/address
with open('/sys/class/net/%s/address' % ethX, 'r') as fo:
@ -312,6 +322,12 @@ def find_bridge_mappings(astute, himn, username):
bridge = utils.ssh(himn, username,
('xe network-param-get param-name=bridge '
'uuid=%s') % network_uuid)
return bridge
def find_physical_network_mappings(astute, himn, username):
# find corresponding bridge in Dom0
bridge = find_dom0_bridge(himn, username, 'br-aux')
# find physical network name
phynet_setting = astute['quantum_settings']['L2']['phys_nets']
@ -458,6 +474,142 @@ def enable_conntrack_service(himn, username):
utils.ssh(himn, username, 'service', 'conntrackd', 'restart')
def configure_dom0_iptables(himn, username):
xs_chain = 'XenServer-Neutron-INPUT'
# Check XenServer specific chain, create if not exist
commands = ('iptables -t filter -L %s' % xs_chain,
'iptables -t filter --new %s' % xs_chain,
'iptables -t filter -I INPUT -j %s' % xs_chain)
execute_iptables_commands(himn, username, commands)
# Check XenServer rule for ovs native mode, create if not exist
commands = ('iptables -t filter -C %s -p tcp -m tcp --dport 6640 -j ACCEPT'
% xs_chain,
'iptables -t filter -I %s -p tcp --dport 6640 -j ACCEPT'
% xs_chain)
execute_iptables_commands(himn, username, commands)
# Check XenServer rule for vxlan, create if not exist
commands = ('iptables -t filter -C %s -p udp -m multiport --dports 4789 '
'-j ACCEPT' % xs_chain,
'iptables -t filter -I %s -p udp -m multiport --dport 4789 -j '
'ACCEPT' % xs_chain)
execute_iptables_commands(himn, username, commands)
# Persist iptables rules
utils.ssh(himn, username, 'service', 'iptables', 'save')
def execute_iptables_commands(himn, username, command_list):
# Execute first command and continue based on first command result
exitcode, _, _ = utils.ssh_detailed(
himn, username, command_list[0], allowed_return_codes=[0, 1])
if exitcode == 1:
for command in command_list[1:]:
LOG.info('Execute iptables command %s', command)
utils.ssh(himn, username, command)
def create_dom0_mesh_bridge(himn, username, dom0_bridge, mesh_info):
# Create br-mesh and veth pair if not exist in Dom0
exitcode, out, _ = utils.ssh_detailed(himn, username,
'ip', 'addr', 'show', MESH_BRIDGE,
allowed_return_codes=[0, 1])
if exitcode == 1:
# create mesh bridge if it not exist in Dom0
create_mesh_bridge = True
else:
# if mesh bridge exist in Dom0, check its ip, re-configure ip if
# it's not the same as what we want to set
bridge_info = out.split()
try:
index_inet = bridge_info.index('inet')
# get inet info like '192.168.2.2/24'
ipaddr = bridge_info[index_inet + 1].split('/')[0]
current_ip = ipaddress.ip_address(unicode(ipaddr))
configured_ip = ipaddress.ip_address(unicode(mesh_info['ipaddr']))
if current_ip == configured_ip:
LOG.info('Bridge %s already exist in Dom0' % MESH_BRIDGE)
return
else:
create_mesh_bridge = True
remove_old_mesh_bridge(himn, username, MESH_BRIDGE)
except ValueError:
create_mesh_bridge = True
remove_old_mesh_bridge(himn, username, MESH_BRIDGE)
if create_mesh_bridge:
LOG.debug("Create mesh bridge in Dom0")
utils.scp(himn, username, '/etc/sysconfig/network-scripts/',
AUTO_SCRIPT)
utils.ssh(himn, username, 'chmod', '+x',
'/etc/sysconfig/network-scripts/%s' % AUTO_SCRIPT)
start_param = '%(bridge)s %(ip)s %(netmask)s %(broadcast)s %(tag)s' \
% {'bridge': dom0_bridge,
'ip': mesh_info['ipaddr'],
'netmask': mesh_info['netmask'],
'broadcast': mesh_info['broadcast'],
'tag': mesh_info['tag']}
with open(AUTO_START_SERVICE_TEMPLATE) as f:
contents = f.read()
contents = contents.replace('@MESH_INFO@', start_param)
with open(AUTO_START_SERVICE, 'w') as f:
f.write(contents)
utils.scp(himn, username, '/etc/systemd/system', AUTO_START_SERVICE)
utils.ssh(himn, username, 'systemctl', 'daemon-reload')
utils.ssh(himn, username, 'systemctl', 'enable', AUTO_START_SERVICE)
utils.ssh(himn, username, 'systemctl', 'start', AUTO_START_SERVICE)
def disable_local_mesh_bridge(bridge):
iface_list = netifaces.interfaces()
if bridge in iface_list:
utils.execute('ifconfig', bridge, '0.0.0.0')
utils.execute('ip', 'link', 'set', bridge, 'down')
filename = '/etc/network/interfaces.d/ifcfg-%s' % bridge
if os.path.isfile(filename):
utils.execute('rm', '-f', filename)
def get_mesh_info(astute, bridge):
mesh_nets = astute['network_scheme']['endpoints'][bridge]['IP'][0]
mesh_ip = mesh_nets.split('/')[0]
ipv4_net = ipaddress.ip_network(unicode(mesh_nets), strict=False)
mesh_broadcast = str(ipv4_net.broadcast_address)
network_netmask = str(ipv4_net.with_netmask).split('/')
mesh_netmask = network_netmask[1]
mesh_network = network_netmask[0]
mesh_eth = get_network_ethX(bridge)
mesh_tag = "''"
index = mesh_eth.index('.')
if index > 0:
mesh_tag = mesh_eth[index+1:]
mesh_info = {'ipaddr': mesh_ip, 'network': mesh_network,
'netmask': mesh_netmask, 'broadcast': mesh_broadcast,
'tag': mesh_tag}
return mesh_info
def remove_old_mesh_bridge(himn, username, bridge):
exitcode, _, _ = utils.ssh_detailed(himn, username, 'ip', 'link', 'show',
bridge, allowed_return_codes=[0, 1])
if exitcode == 0:
# Allow return code 5 to make sure it won't fail when
# mos-vxlan.service isn't exist
utils.ssh_detailed(himn, username, 'systemctl', 'stop',
AUTO_START_SERVICE, allowed_return_codes=[0, 5])
utils.ssh_detailed(himn, username, 'systemctl', 'disable',
AUTO_START_SERVICE, allowed_return_codes=[0, 1])
utils.ssh(himn, username, 'rm', '-f',
'/etc/systemd/system/%s' % AUTO_START_SERVICE)
utils.ssh(himn, username, 'rm', '-f',
'/etc/sysconfig/network-scripts/%s' % AUTO_SCRIPT)
utils.ssh(himn, username, 'systemctl', 'daemon-reload')
if __name__ == '__main__':
install_xenapi_sdk()
astute = utils.get_astute()
@ -493,10 +645,29 @@ if __name__ == '__main__':
# enable conntrackd service in Dom0
enable_conntrack_service(HIMN_IP, username)
# configure iptables in Dom0 to support ovs native mode and VxLAN
configure_dom0_iptables(HIMN_IP, username)
# neutron-l2-agent in compute node
modify_neutron_rootwrap_conf(HIMN_IP, username, password)
br_mappings = find_bridge_mappings(astute, HIMN_IP, username)
modify_neutron_ovs_agent_conf(INT_BRIDGE, br_mappings)
l2_net_type = astute['quantum_settings']['predefined_networks'][
'admin_internal_net']['L2']['network_type']
br_mappings = None
if l2_net_type == 'vlan':
br_mappings = find_physical_network_mappings(astute, HIMN_IP,
username)
remove_old_mesh_bridge(HIMN_IP, username, MESH_BRIDGE)
ip = None
if l2_net_type == 'tun':
dom0_priv_bridge = find_dom0_bridge(HIMN_IP, username,
MESH_BRIDGE)
mesh_info = get_mesh_info(astute, MESH_BRIDGE)
ip = mesh_info['ipaddr']
disable_local_mesh_bridge(MESH_BRIDGE)
create_dom0_mesh_bridge(HIMN_IP, username, dom0_priv_bridge,
mesh_info)
modify_neutron_ovs_agent_conf(INT_BRIDGE, br_mappings=br_mappings,
local_ip=ip)
patch_neutron_ovs_agent()
restart_services('neutron-openvswitch-agent')

View File

@ -0,0 +1,63 @@
#!/bin/bash
OP=$1
COUNT=$#
function create_mesh_bridge {
local dom0_bridge=$1
local mesh_ip=$2
local mesh_netmask=$3
local mesh_broadcast=$4
local tag=$5
ip link show br-mesh
exitcode=$?
if [ "$exitcode" == "1" ]; then
brctl addbr br-mesh
brctl setfd br-mesh 0
brctl stp br-mesh off
ip link set br-mesh up
ip link delete mesh_ovs
ip link add mesh_ovs type veth peer name mesh_linux
ip link set mesh_ovs up
ip link set mesh_ovs promisc on
ip link set mesh_linux up
ip link set mesh_linux promisc on
brctl addif br-mesh mesh_linux
ovs-vsctl -- --if-exists del-port mesh_ovs -- add-port $dom0_bridge mesh_ovs
ip addr add $mesh_ip/$mesh_netmask broadcast $mesh_broadcast dev br-mesh
if [ -n "$tag" ]; then
ovs-vsctl -- set Port mesh_ovs tag=$tag
fi
fi
}
function delete_mesh_bridge {
ip link show br-mesh
exitcode=$?
if [ "$exitcode" == "0" ]; then
ip link set br-mesh down
ip link set mesh_ovs down
ip link delete mesh_ovs
ovs-vsctl -- --if-exist del-port mesh_ovs
brctl delbr br-mesh
fi
}
if [ "$OP" == "start" ]; then
if [ $COUNT -lt 6 ]; then
echo "usage: fuel-xs-vlan.sh start BRIDGE IP NETMASK BROADCAST [VLAN_TAG]"
echo "Exit due to lack of parameters!"
exit 0
fi
dom0_bridge=$2
mesh_ip=$3
mesh_netmask=$4
mesh_broadcast=$5
tag=$6
create_mesh_bridge $dom0_bridge $mesh_ip $mesh_netmask $mesh_broadcast $tag
elif [ "$OP" == "stop" ]; then
delete_mesh_bridge
fi

View File

@ -0,0 +1,14 @@
[Unit]
Description=Configure Mirantis OpenStack mesh bridge
Requires=xcp-networkd.service openvswitch-xapi-sync.service
After=xcp-networkd.service openvswitch-xapi-sync.service
AssertPathExists=/etc/sysconfig/network-scripts/
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/bin/bash /etc/sysconfig/network-scripts/fuel-xs-vxlan.sh start @MESH_INFO@
ExecStop=/bin/bash /etc/sysconfig/network-scripts/fuel-xs-vxlan.sh stop
[Install]
WantedBy=multi-user.target

View File

@ -67,6 +67,7 @@ def detailed_execute(*cmd, **kwargs):
LOG.info('Swallowed acceptable return code of %d',
proc.returncode)
else:
LOG.warn('proc.returncode: %s', proc.returncode)
raise ExecutionError(err)
return proc.returncode, out, err