Add support for sriovnicswitch and VXLAN offload

New configurations for fuel mellanox plugin to use sriovnicswitch
mechanism driver for ethernet networks, it also gives the user the
option to enable vxlan offloading to imporove performance and reduce
CPU overhead.

Change-Id: I480d4efb31fde126a51594bdbf48307a4e63ea74
This commit is contained in:
Rawan Herzallah 2015-12-14 10:01:01 +00:00
parent 28d514eb56
commit c03921affa
14 changed files with 299 additions and 54 deletions

View File

@ -80,3 +80,4 @@ readonly USER_NUM_OF_VFS=`get_mlnx_param num_of_vfs`
readonly ISER=`get_mlnx_param iser`
readonly MAX_VFS=64
readonly MIN_VFS=1
readonly VXLAN_OFFLOADING=`get_mlnx_param vxlan_offloading`

View File

@ -0,0 +1,64 @@
#!/usr/bin/python
# Copyright 2015 Mellanox Technologies, Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import subprocess
def get_pci_list():
command_lspci = "lspci -nn | grep Mellanox | grep -i virtual | awk '{print $1}'"
p_lspci = subprocess.Popen(command_lspci, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
pci_res = list()
for line in p_lspci.stdout.readlines():
pci_res.append(line.rstrip())
retval = p_lspci.wait()
if retval != 0:
print 'Failed to execute command lspci.'
sys.exit()
else:
return pci_res
def exclude_vfs(exclude_vf_numbers, pci_res):
if len( exclude_vf_numbers ) > 0:
# parse the exclude_vf_numbers
exclude_vf_list = [exclude_vf_list.strip() for exclude_vf_list in \
exclude_vf_numbers.split(',')]
for vf_number in exclude_vf_list:
# remove iSER vf in exclude list
command_get_iser_vf = "readlink /sys/class/net/eth*/device/virtfn{0}\
| cut -d':' -f2-".format(vf_number)
p = subprocess.Popen(command_get_iser_vf, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
# to avoid bus IDs duplication when using more than a single port nic
bus = list(set(p.stdout.readlines())).pop().strip()
retval = p.wait()
if retval != 0:
print 'Failed to execute command lspci.'
sys.exit()
if bus in pci_res:
pci_res.remove(bus)
return pci_res
def main(exclude_vf_numbers, physnet):
pci_res = get_pci_list()
pci_res = exclude_vfs(exclude_vf_numbers, pci_res)
pci_res_str = ",".join('{\"address\":\"' + item + '\", \"physical_network\":\"{0}\"}}'.format(physnet) for item in pci_res)
pci_res_str2 = "{0}{1}{2}".format('[',pci_res_str,']')
sys.stdout.write(pci_res_str2)
return pci_res_str2
if __name__=="__main__":
main(sys.argv[1], sys.argv[2])

View File

@ -198,7 +198,7 @@ class MellanoxSettings(object):
mlnx_interfaces = cls.mlnx_interfaces_section
drivers = list()
for network_type, ifc_dict in mlnx_interfaces.iteritems():
if ( ifc_dict['driver'] in MLNX_DRIVERS_LIST):
if ifc_dict['driver'] in MLNX_DRIVERS_LIST:
drivers.append(ifc_dict['driver'])
return list(set(drivers))
@ -210,6 +210,10 @@ class MellanoxSettings(object):
def is_iser_enabled(cls):
return cls.get_mlnx_section()['iser']
@classmethod
def is_vxlan_offloading_enabled(cls):
return cls.get_mlnx_section()['vxlan_offloading']
@classmethod
def update_role_settings(cls):
# realize the driver in use (eth/ib)
@ -295,7 +299,7 @@ class MellanoxSettings(object):
network_type = 'admin'
elif transformation['bridge'] == 'br-ex':
network_type = 'public'
elif transformation['bridge'] == 'br-aux':
elif transformation['bridge'] == 'br-aux' or transformation['bridge'] == 'br-mesh':
network_type = 'private'
elif transformation['bridge'] == 'br-mgmt':
network_type = 'management'

View File

@ -1,10 +1,15 @@
$network_scheme = hiera('network_scheme')
$quantum_settings = hiera('quantum_settings')
$mlnx = hiera('mellanox-plugin')
$firewall_driver = 'neutron.agent.firewall.NoopFirewallDriver'
$exclude_vf = '0'
if ($mlnx['sriov']) {
class { 'mellanox_openstack::compute_sriov' :
physnet => $quantum_settings['predefined_networks']['net04']['L2']['physnet'],
physifc => $mlnx['physical_port'],
physnet => $quantum_settings['predefined_networks']['net04']['L2']['physnet'],
physifc => $mlnx['physical_port'],
mlnx_driver => $mlnx['driver'],
firewall_driver => $firewall_driver,
exclude_vf => $exclude_vf,
}
}

View File

@ -4,18 +4,22 @@ $eswitch_apply_profile_patch = 'True'
$mechanism_drivers = 'openvswitch'
if ($mlnx['sriov']) {
$pci_vendor_devices = generate ("/bin/bash", "-c", 'lspci -nn | grep -i Mellanox | grep -i virtual | awk \'{print $NF}\' | sort -u | tr -d \']\' | tr -d \'[\'')
$agent_required = 'True'
class { 'mellanox_openstack::controller_sriov' :
eswitch_vnic_type => $eswitch_vnic_type,
eswitch_apply_profile_patch => $eswitch_apply_profile_patch,
mechanism_drivers => $mechanism_drivers,
mlnx_driver => $mlnx['driver'],
mlnx_sriov => $mlnx['sriov']
mlnx_sriov => $mlnx['sriov'],
pci_vendor_devices => $pci_vendor_devices,
agent_required => $agent_required,
}
}
# Configure broadcast dnsmasq for IB PV
elsif ($mlnx['driver'] == 'eth_ipoib') {
class { 'mellanox_openstack::controller_ib_pv' :
mlnx_driver => $mlnx['driver'],
mlnx_sriov => $mlnx['sriov']
mlnx_sriov => $mlnx['sriov'],
}
}

View File

@ -1,35 +1,70 @@
class mellanox_openstack::compute_sriov (
$physnet,
$physifc,
$mlnx_driver,
$firewall_driver,
$exclude_vf,
) {
include nova::params
include mellanox_openstack::params
$sriov_agent_service = $::mellanox_openstack::params::sriov_agent_service_name
$sriov_agent_package = $::mellanox_openstack::params::sriov_agent_package_name
$libvirt_service_name = 'libvirtd'
$libvirt_package_name = $nova::params::libvirt_package_name
class { 'mellanox_openstack::eswitchd' :
physnet => $physnet,
physifc => $physifc,
$path_to_generate_pci_script = generate ("/bin/bash", "-c", 'echo /etc/fuel/plugins/mellanox-plugin-*.0/generate_pci_passthrough_whitelist.py | tr -d \'\n \' ')
$pci_passthrough_addresses = generate ("/usr/bin/python", $path_to_generate_pci_script, $exclude_vf, $physnet)
if ( $mlnx_driver == 'mlx4_en' ){
# configure pci_passthrough_whitelist nova compute
nova_config { 'DEFAULT/pci_passthrough_whitelist':
value => check_array_of_hash("${pci_passthrough_addresses}"),
}
# update [securitygroup] section in neutron
neutron_plugin_ml2 { 'securitygroup/firewall_driver':
value => $firewall_driver,
}
package { $sriov_agent_package:
ensure => installed,
}
service { $sriov_agent_service:
ensure => running,
enable => true,
hasstatus => true,
hasrestart => true,
}
Package[$sriov_agent_package] ->
Service[$sriov_agent_service]
} elsif ( $mlnx_driver == 'eth_ipoib' ){
class { 'mellanox_openstack::eswitchd' :
physnet => $physnet,
physifc => $physifc,
}
class { 'mellanox_openstack::agent' :
physnet => $physnet,
physifc => $physifc,
}
package { $libvirt_package_name :
ensure => installed
}
service { $libvirt_service_name :
ensure => running
}
Package[$libvirt_package_name] ->
Service[$libvirt_service_name] ->
Class['mellanox_openstack::eswitchd'] ~>
Class['mellanox_openstack::agent']
}
class { 'mellanox_openstack::agent' :
physnet => $physnet,
physifc => $physifc,
}
class { 'mellanox_openstack::snapshot_patch' : }
package { $libvirt_package_name :
ensure => installed
}
service { $libvirt_service_name :
ensure => running
}
Package[$libvirt_package_name] ->
Service[$libvirt_service_name] ->
Class['mellanox_openstack::eswitchd'] ~>
Class['mellanox_openstack::agent']
}

View File

@ -3,7 +3,9 @@ class mellanox_openstack::controller_sriov (
$eswitch_apply_profile_patch,
$mechanism_drivers,
$mlnx_driver,
$mlnx_sriov
$mlnx_sriov,
$pci_vendor_devices,
$agent_required,
) {
include neutron::params
@ -11,16 +13,30 @@ class mellanox_openstack::controller_sriov (
$dhcp_agent = $neutron::params::dhcp_agent_service
include mellanox_openstack::params
$package = $::mellanox_openstack::params::neutron_mlnx_packages_controller
$package = $::mellanox_openstack::params::neutron_mlnx_packages_controller
package { $package :
ensure => installed,
ensure => installed,
}
neutron_plugin_ml2 {
'eswitch/vnic_type': value => $eswitch_vnic_type;
'eswitch/apply_profile_patch': value => $eswitch_apply_profile_patch;
'ml2/mechanism_drivers': value => "mlnx,${mechanism_drivers}";
if ( $mlnx_driver == 'mlx4_en' ){
nova_config { 'DEFAULT/scheduler_default_filters':
value => 'RetryFilter, AvailabilityZoneFilter, RamFilter, ComputeFilter, ComputeCapabilitiesFilter, ImagePropertiesFilter, PciPassthroughFilter'
}
$ml2_extra_mechanism_driver = 'sriovnicswitch'
neutron_plugin_ml2 {
'ml2/mechanism_drivers': value => "${ml2_extra_mechanism_driver},${mechanism_drivers}";
'ml2_sriov/supported_pci_vendor_devs': value => $pci_vendor_devices;
'ml2_sriov/agent_required': value => $agent_required;
}
}
else {
$ml2_extra_mechanism_driver = 'mlnx'
neutron_plugin_ml2 {
'eswitch/vnic_type': value => $eswitch_vnic_type;
'eswitch/apply_profile_patch': value => $eswitch_apply_profile_patch;
'ml2/mechanism_drivers': value => "${ml2_extra_mechanism_driver},${mechanism_drivers}";
}
}
service { $server_service :
@ -32,6 +48,7 @@ class mellanox_openstack::controller_sriov (
Service[$server_service]
if ( $mlnx_driver == 'eth_ipoib' and $mlnx_sriov == true ){
# To remove in a separated PR for Liberty changes
package { 'mlnx-dnsmasq' :
ensure => installed,
subscribe => Service[$server_service]
@ -49,5 +66,4 @@ class mellanox_openstack::controller_sriov (
subscribe => Neutron_dhcp_agent_config['DEFAULT/dhcp_driver'],
}
}
}

View File

@ -16,10 +16,12 @@ class mellanox_openstack::params {
'Debian': {
$neutron_mlnx_packages_compute = ['neutron-plugin-mlnx','neutron-plugin-mlnx-agent', 'python-networking-mlnx']
$neutron_mlnx_packages_controller = ['python-networking-mlnx']
$agent_service = ['neutron-plugin-mlnx-agent']
$compute_service_name = 'nova-compute'
$openvswitch_mgmt_service = 'openvswitch-switch'
$libvirt_driver_file = '/usr/lib/python2.7/dist-packages/nova/virt/libvirt/driver.py'
$agent_service = ['neutron-plugin-mlnx-agent']
$sriov_agent_service_name = 'neutron-plugin-sriov-agent'
$sriov_agent_package_name = 'neutron-plugin-sriov-agent'
$compute_service_name = 'nova-compute'
$openvswitch_mgmt_service = 'openvswitch-switch'
$libvirt_driver_file = '/usr/lib/python2.7/dist-packages/nova/virt/libvirt/driver.py'
}
}

View File

@ -35,7 +35,9 @@ function get_port_type() {
}
function get_num_probe_vfs () {
if [ $ISER == true ] && [ $DRIVER == 'mlx4_en' ]; then
if [ $SRIOV == true ] && [ $DRIVER == 'mlx4_en' ]; then
probe_vfs=`calculate_total_vfs`
elif [ $ISER == true ] && [ $DRIVER == 'mlx4_en' ]; then
probe_vfs=1
else
probe_vfs=0
@ -79,22 +81,32 @@ function reduce_mac_caching_timeout () {
sysctl_conf set 'net.ipv4.route.gc_timeout' "$timeout"
}
function is_vxlan_offloading_required () {
[ $VXLAN_OFFLOADING == true ]
return $?
}
function set_modprobe_file () {
PROBE_VFS=`get_num_probe_vfs`
MLX4_CORE_FILE="/etc/modprobe.d/mlx4_core.conf"
PORT_TYPE=`get_port_type`
MLX4_CORE_STR="options mlx4_core
enable_64b_cqe_eqe=0
log_num_mgm_entry_size=-1
port_type_array=${PORT_TYPE},${PORT_TYPE}"
debug_level=1"
TOTAL_VFS=$1
if is_vxlan_offloading_required; then
MLX4_CORE_STR="${MLX4_CORE_STR} log_num_mgm_entry_size=-1"
fi
MLX4_CORE_STR="${MLX4_CORE_STR} port_type_array=${PORT_TYPE},${PORT_TYPE}"
if [[ $TOTAL_VFS -gt 0 ]]; then
MLX4_CORE_STR="${MLX4_CORE_STR} num_vfs=${TOTAL_VFS}"
if [[ $PROBE_VFS -gt 0 ]]; then
MLX4_CORE_STR="${MLX4_CORE_STR} probe_vf=${PROBE_VFS}"
MLX4_CORE_STR="${MLX4_CORE_STR} probe_vf=${TOTAL_VFS}"
fi
fi
echo ${MLX4_CORE_STR} > ${MLX4_CORE_FILE}
}
function set_kernel_params () {
@ -164,6 +176,9 @@ function configure_sriov () {
fi
logger_print info "Configuring ${total_vfs} virtual functions
(only even number is currently supported)"
probe_vfs=`get_num_probe_vfs`
port_type=`get_port_type`
set_modprobe_file $total_vfs &&
set_kernel_params &&
burn_vfs_in_fw $total_vfs
@ -203,6 +218,8 @@ function validate_sriov () {
# fallback only if kernel param exists and amount of vfs is not as expcted
logger_print error "Failed , trying to fallback to ${FALLBACK_NUM_VFS}"
probe_vfs=`get_num_probe_vfs`
port_type=`get_port_type`
set_modprobe_file $FALLBACK_NUM_VFS
service openibd restart &> /dev/null
current_num_vfs=`lspci | grep -i mellanox | grep -i virtual | wc -l`

View File

@ -0,0 +1,81 @@
#!/bin/bash -x
# Copyright 2015 Mellanox Technologies, Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
readonly SCRIPT_DIR=$(dirname "$0")
source $SCRIPT_DIR/common
readonly SCRIPT_MODE=$1
function get_port_type() {
if [ $DRIVER == 'mlx4_en' ]; then
port_type=2
elif [ $DRIVER == 'eth_ipoib' ]; then
port_type=1
fi
echo $port_type
}
function is_vxlan_offloading_required () {
[ $VXLAN_OFFLOADING == true ] && [ $DRIVER == 'mlx4_en' ]
return $?
}
function configure_vxlan_offloading() {
if is_vxlan_offloading_required; then
logger_print info "Configuring VXLAN OFFLOADING."
set_modprobe_file
return $?
else
# not required, no need to configure it
logger_print info "Skipping VXLAN OFFLOADING configuration."
return 0
fi
}
function validate_vxlan_offloading() {
# to be filled later
logger_print info "Validating VXLAN OFFLOADING."
return 0
}
function set_modprobe_file () {
MLX4_CORE_FILE="/etc/modprobe.d/mlx4_core.conf"
PORT_TYPE=`get_port_type`
MLX4_CORE_STR="options mlx4_core
enable_64b_cqe_eqe=0
log_num_mgm_entry_size=-1
debug_level=1
port_type_array=${PORT_TYPE},${PORT_TYPE}"
echo ${MLX4_CORE_STR} > ${MLX4_CORE_FILE}
}
#################
configure_vxlan_offloading
case $SCRIPT_MODE in
'configure')
configure_vxlan_offloading
;;
'validate')
# to be added later.
validate_vxlan_offloading
;;
*)
logger_print error "Unsupported execution mode ${SCRIPT_MODE}"
exit 1
;;
esac
exit $?

View File

@ -16,7 +16,7 @@ attributes:
- After the interfaces discovery, perform the following:
* Set the relevant networks to work over Mellanox interfaces, in the interfaces configuration in the Nodes tab.
* Run network verification, after configuring the desired network subnets and VLAN segmentations.
- For more information: https://community.mellanox.com/docs/DOC-2165
- For more information: https://community.mellanox.com/docs/DOC-2374
iser:
value: false
label: "iSER protocol for volumes (Cinder)"
@ -47,3 +47,12 @@ attributes:
restrictions:
- condition: "settings:mellanox-plugin.sriov.value == false"
message: "In order to change the number of virtual NICs, Neutron SR-IOV plugin should be checked."
vxlan_offloading:
value: false
label: "VXLAN Offloading"
description: "By using Mellanox hardware, VXLAN offloading is achieved with better performance and significant CPU overhead reduction. "
weight: 11
type: "checkbox"
restrictions:
- condition: "not (cluster:net_provider == 'neutron' and networking_parameters:segmentation_type == 'tun')"
message: "For VXLAN offloading, network should be neutron with tunnling segmentation."

View File

@ -5,13 +5,13 @@ name: mellanox-plugin
title: Mellanox Openstack features
# Plugin version
version: 2.0.0
version: 2.0.10
# Description
description: Enable features over Mellanox hardware
# Required fuel version
fuel_version: ['6.1', '7.0']
fuel_version: ['6.1', '7.0', '8.0']
# Specify license of your plugin
licenses: ['Apache License Version 2.0']
@ -42,11 +42,11 @@ releases:
mode: ['ha']
deployment_scripts_path: deployment_scripts/
repository_path: repositories/ubuntu
- os: centos
version: 2015.1.0-7.0
- os: ubuntu
version: 2015.2.0-8.0
mode: ['ha']
deployment_scripts_path: deployment_scripts/
repository_path: repositories/centos
repository_path: repositories/ubuntu
# Version of plugin package
package_version: '2.0.0'

View File

@ -72,7 +72,7 @@ deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb
cirros-testvm-mellanox-ib_0.3.2-7_amd64.deb
eswitchd_1.14-3_amd64.deb
mlnx-dnsmasq_2015.1.0-1_all.deb
mlnx-ofed-fuel_2.3-2.0.8_amd64.deb
mlnx-ofed-fuel_3.1-1.5.7_amd64.deb
python-networking-mlnx_2015.1.2-1_amd64.deb"
get_packages "deb" "$old_debs" "$deb_files"

View File

@ -38,6 +38,13 @@
parameters:
cmd: ./install_ofed.sh
timeout: 3600
# add VXLAN offloading parameters
- role: '*'
stage: pre_deployment
type: shell
parameters:
cmd: ./vxlan_offloading.sh configure
timeout: 60
# Configure number of VFs according to the user decision:
# change modprobe file + IOMMU in grub file + change VFs num in FW
- role: '*'
@ -46,7 +53,7 @@
parameters:
cmd: ./sriov.sh configure
timeout: 60
# Rename iSER interface for Eth mode
# if VXLAN is not required. Rename iSER interface for Eth mode
- role: '*'
stage: pre_deployment
type: puppet