diff --git a/lib/puppet/provider/sriov_vf_config/numvfs.rb b/lib/puppet/provider/sriov_vf_config/numvfs.rb index 963311b11..1e9abfb8c 100644 --- a/lib/puppet/provider/sriov_vf_config/numvfs.rb +++ b/lib/puppet/provider/sriov_vf_config/numvfs.rb @@ -10,8 +10,11 @@ Puppet::Type.type(:sriov_vf_config).provide(:numvfs) do def create if File.file?(sriov_numvfs_path) - _set_numvfs - _apply_hw_offload + if ovs_mode == "switchdev" + _apply_hw_offload + else + _set_numvfs + end else warning("#{sriov_numvfs_path} doesn't exist. Check if #{sriov_get_interface} is a valid network interface supporting SR-IOV") end @@ -45,25 +48,53 @@ Puppet::Type.type(:sriov_vf_config).provide(:numvfs) do def _apply_hw_offload # Changing the mode of virtual functions to hw-offload - if ovs_mode == "switchdev" - cur_value = File.read(vendor_path).strip - if cur_value == "0x15b3" - vfs_pcis = get_vfs_pcis - # Unbinding virtual functions - vfs_pcis.each do|vfs_pci| - File.write("/sys/bus/pci/drivers/mlx5_core/unbind",vfs_pci) + + bond_enabled = get_bond_enabled + vendor_id = File.read(vendor_path).strip + + # Adding the VF LAG + if vendor_id == "0x15b3" and bond_enabled + bond_masters = get_bond_masters + bond_master_hash = {} + bond_masters.each do |bond| + bond_interfaces = get_bond_interfaces(bond) + bond_master_hash[bond] = bond_interfaces + end + # Removing the slaves from the bond interfaces + bond_master_hash.each do |bond, bond_interfaces| + bond_interfaces.each do |bond_interface| + %x{echo "-#{bond_interface}" > /sys/class/net/#{bond}/bonding/slaves} end end - # Changing the mode of sriov interface to switchdev mode - %x{/usr/sbin/devlink dev eswitch set pci/#{get_interface_pci} mode switchdev} - if get_interface_device == "0x1013" || get_interface_device == "0x1015" - %x{/usr/sbin/devlink dev eswitch set pci/#{get_interface_pci} inline-mode transport} + end + # Setting the number of vfs + _set_numvfs + + # Applying the hardware offloading + if vendor_id == "0x15b3" + vfs_pcis = get_vfs_pcis + # Unbinding virtual functions + vfs_pcis.each do|vfs_pci| + File.write("/sys/bus/pci/drivers/mlx5_core/unbind",vfs_pci) end - %x{/usr/sbin/ethtool -K #{sriov_get_interface} hw-tc-offload on} - if cur_value == "0x15b3" - # Binding virtual functions - vfs_pcis.each do|vfs_pci| - File.write("/sys/bus/pci/drivers/mlx5_core/bind",vfs_pci) + end + # Changing the mode of sriov interface to switchdev mode + %x{/usr/sbin/devlink dev eswitch set pci/#{get_interface_pci} mode switchdev} + if get_interface_device == "0x1013" || get_interface_device == "0x1015" + %x{/usr/sbin/devlink dev eswitch set pci/#{get_interface_pci} inline-mode transport} + end + %x{/usr/sbin/ethtool -K #{sriov_get_interface} hw-tc-offload on} + if vendor_id == "0x15b3" + # Binding virtual functions + vfs_pcis.each do|vfs_pci| + File.write("/sys/bus/pci/drivers/mlx5_core/bind",vfs_pci) + end + end + if vendor_id and bond_enabled + # Adding the slaves back to the bond interfaces + bond_master_hash.each do |bond, bond_interfaces| + bond_interfaces.each do |bond_interface| + %x{echo "+#{bond_interface}" > /sys/class/net/#{bond}/bonding/slaves} end end end @@ -104,4 +135,33 @@ Puppet::Type.type(:sriov_vf_config).provide(:numvfs) do def get_interface_device %x{cat /sys/class/net/#{sriov_get_interface}/device/device}.strip end + + def get_bond_enabled + if %x{lsmod | grep bonding}.strip.length > 0 + true + else + false + end + end + + def bond_masters_path + "/sys/class/net/bonding_masters" + end + + def get_bond_masters + if File.file?(bond_masters_path) + File.read(bond_masters_path).split() + end + end + + def get_bond_interfaces_path(bond) + "/sys/class/net/#{bond}/bonding/slaves" + end + + def get_bond_interfaces(bond) + if File.file?(get_bond_interfaces_path(bond)) + File.read(get_bond_interfaces_path(bond)).split() + end + end + end diff --git a/releasenotes/notes/vf-lag-sriov-ec194ecd4b447a46.yaml b/releasenotes/notes/vf-lag-sriov-ec194ecd4b447a46.yaml new file mode 100644 index 000000000..0a314183f --- /dev/null +++ b/releasenotes/notes/vf-lag-sriov-ec194ecd4b447a46.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Allows to configure bond over two virtual functions in mellanox interfaces. diff --git a/spec/defines/tripleo_host_sriov_numvfs_persistence_spec.rb b/spec/defines/tripleo_host_sriov_numvfs_persistence_spec.rb index 455e09639..0f5f654a6 100644 --- a/spec/defines/tripleo_host_sriov_numvfs_persistence_spec.rb +++ b/spec/defines/tripleo_host_sriov_numvfs_persistence_spec.rb @@ -68,19 +68,32 @@ describe 'tripleo::host::sriov::numvfs_persistence' do it 'configures persistence' do is_expected.to contain_file('/etc/sysconfig/allocate_vfs').with( :ensure => 'file', - :content => "Hashbang\nset -eux + :content => "Hashbang\nset -ex set -o pipefail if [ \"eth0\" == \"$1\" ] then exec 1> >(logger -s -t $(basename $0)) 2>&1 + vendor_id=\"$(cat /sys/class/net/eth0/device/vendor)\" + if [[ $vendor_id == \"0x15b3\" && $(/sbin/lsmod | grep bonding) ]] + then + declare -A bonds + for bond in `cat /sys/class/net/bonding_masters` + do + bonds[$bond]=`cat \"/sys/class/net/$bond/bonding/slaves\"` + for bond_interface in ${bonds[$bond]} + do + echo \"-$bond_interface\" > \"/sys/class/net/$bond/bonding/slaves\" + done + done + fi if [ \"$(cat /sys/class/net/eth0/device/sriov_numvfs)\" == \"0\" ] then echo 10 > /sys/class/net/eth0/device/sriov_numvfs else exit 0 fi - if [ \"$(cat /sys/class/net/eth0/device/vendor)\" == \"0x15b3\" ] + if [ $vendor_id == \"0x15b3\" ] then vfs_pci_list=$(grep PCI_SLOT_NAME /sys/class/net/eth0/device/virtfn*/uevent | cut -d'=' -f2) for pci in $vfs_pci_list @@ -103,13 +116,23 @@ then /usr/sbin/devlink dev eswitch set pci/\"$interface_pci\" inline-mode transport fi /usr/sbin/ethtool -K eth0 hw-tc-offload on - if [ \"$(cat /sys/class/net/eth0/device/vendor)\" == \"0x15b3\" ] + if [ $vendor_id == \"0x15b3\" ] then for pci in $vfs_pci_list do echo \"$pci\" > /sys/bus/pci/drivers/mlx5_core/bind done fi + if [[ $vendor_id == \"0x15b3\" && $(/sbin/lsmod | grep bonding) ]] + then + for bond in \"${!bonds[@]}\" + do + for bond_interface in ${bonds[$bond]} + do + echo \"+$bond_interface\" > \"/sys/class/net/$bond/bonding/slaves\" + done + done + fi fi\n[ \"eth1\" == \"\$1\" ] && echo 8 > /sys/class/net/eth1/device/sriov_numvfs\n", :group => 'root', :mode => '0755', diff --git a/templates/switchdev/switchdev.epp b/templates/switchdev/switchdev.epp index 5eb1b7689..9981ccb98 100644 --- a/templates/switchdev/switchdev.epp +++ b/templates/switchdev/switchdev.epp @@ -1,17 +1,30 @@ <%- | String $content_string = '', String $interface = '', String $count = '' | -%> -<%=$content_string%>set -eux +<%=$content_string%>set -ex set -o pipefail if [ "<%=$interface%>" == "$1" ] then exec 1> >(logger -s -t $(basename $0)) 2>&1 + vendor_id="$(cat /sys/class/net/<%=$interface%>/device/vendor)" + if [[ $vendor_id == "0x15b3" && $(/sbin/lsmod | grep bonding) ]] + then + declare -A bonds + for bond in `cat /sys/class/net/bonding_masters` + do + bonds[$bond]=`cat "/sys/class/net/$bond/bonding/slaves"` + for bond_interface in ${bonds[$bond]} + do + echo "-$bond_interface" > "/sys/class/net/$bond/bonding/slaves" + done + done + fi if [ "$(cat /sys/class/net/<%=$interface%>/device/sriov_numvfs)" == "0" ] then echo <%=$count%> > /sys/class/net/<%=$interface%>/device/sriov_numvfs else exit 0 fi - if [ "$(cat /sys/class/net/<%=$interface%>/device/vendor)" == "0x15b3" ] + if [ $vendor_id == "0x15b3" ] then vfs_pci_list=$(grep PCI_SLOT_NAME /sys/class/net/<%=$interface%>/device/virtfn*/uevent | cut -d'=' -f2) for pci in $vfs_pci_list @@ -34,11 +47,21 @@ then /usr/sbin/devlink dev eswitch set pci/"$interface_pci" inline-mode transport fi /usr/sbin/ethtool -K <%=$interface%> hw-tc-offload on - if [ "$(cat /sys/class/net/<%=$interface%>/device/vendor)" == "0x15b3" ] + if [ $vendor_id == "0x15b3" ] then for pci in $vfs_pci_list do echo "$pci" > /sys/bus/pci/drivers/mlx5_core/bind done fi + if [[ $vendor_id == "0x15b3" && $(/sbin/lsmod | grep bonding) ]] + then + for bond in "${!bonds[@]}" + do + for bond_interface in ${bonds[$bond]} + do + echo "+$bond_interface" > "/sys/class/net/$bond/bonding/slaves" + done + done + fi fi