Set VFs MAC addresses
Added mac address assignment for created mellanox vfs Change-Id: Ia4045b0799eacbfc01ceebf8d0d23c8db36f8197
This commit is contained in:
parent
4faa9ef57e
commit
8b30f4cb75
|
@ -0,0 +1,204 @@
|
|||
#!/usr/bin/env python
|
||||
# Copyright 2016 Mellanox Technologies, Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import logging
|
||||
import traceback
|
||||
import glob
|
||||
|
||||
LOG_FILE = '/var/log/mellanox-plugin.log'
|
||||
|
||||
class MellanoxVfsSettingsException(Exception):
|
||||
pass
|
||||
|
||||
class MellanoxVfsSettings(object):
|
||||
|
||||
mellanox_vfs = list();
|
||||
|
||||
@classmethod
|
||||
def get_pci_list(cls):
|
||||
cmd_vfs = "lspci -D | grep -i mellanox | grep -i virtual | awk '{print $1}'"
|
||||
p_lspci = subprocess.Popen(cmd_vfs, shell=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
retval = p_lspci.wait()
|
||||
if retval != 0:
|
||||
print 'Failed to execute command lspci.'
|
||||
logging.error("Failed to find mellanox VFs.")
|
||||
sys.exit(1)
|
||||
pci_res = list()
|
||||
for line in p_lspci.stdout.readlines():
|
||||
pci_res.append(line.rstrip())
|
||||
return pci_res
|
||||
|
||||
@classmethod
|
||||
def build_vfs_dict(cls):
|
||||
pci_res = cls.get_pci_list()
|
||||
cmd_vfs_data = "ibdev2netdev -v"
|
||||
p_vfs_data = subprocess.Popen(cmd_vfs_data, shell=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
retval = p_vfs_data.wait()
|
||||
if retval != 0:
|
||||
print 'Failed to execute command ibdev2netdev -v.'
|
||||
logging.error("Failed to execute ibdev2netdev -v")
|
||||
sys.exit(1)
|
||||
|
||||
ibdev = list()
|
||||
for vf_data in p_vfs_data.stdout.readlines():
|
||||
ibdev.append(vf_data.rstrip())
|
||||
|
||||
cmd_vfs_macs = "ip link show | grep vf | awk '{print $4}' | tr -d ','"
|
||||
p_vfs_macs = subprocess.Popen(cmd_vfs_macs, shell=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
retval = p_vfs_macs.wait()
|
||||
if retval != 0:
|
||||
print 'Failed to execute command ip link show | grep vf '
|
||||
logging.error("Failed to find VFs in ip link show command")
|
||||
sys.exit(1)
|
||||
|
||||
mac_list = list()
|
||||
for mac in p_vfs_macs.stdout.readlines():
|
||||
mac_list.append(mac.rstrip())
|
||||
|
||||
count = 0
|
||||
for pci_address in pci_res:
|
||||
vf_dict = dict();
|
||||
vf_info = next((vf_info for vf_info in ibdev if pci_address in vf_info), None)
|
||||
vf_info = vf_info.split();
|
||||
vf_dict['vf_num'] = count;
|
||||
vf_dict['pci_address'] = vf_info[0]
|
||||
vf_dict['port_module'] = vf_info[1]
|
||||
vf_dict['port_num'] = vf_info[12]
|
||||
vf_dict['mac'] = mac_list[count]
|
||||
cls.mellanox_vfs.append(vf_dict)
|
||||
count += 1;
|
||||
|
||||
@classmethod
|
||||
def unbind(cls):
|
||||
for vf in cls.mellanox_vfs:
|
||||
cmd_vfs_pci = "echo {0} ".format(vf['pci_address']) + \
|
||||
">> /sys/bus/pci/drivers/mlx5_core/unbind"
|
||||
p_unbind_pci = subprocess.Popen(cmd_vfs_pci, shell=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
retval = p_unbind_pci.wait()
|
||||
if retval != 0:
|
||||
logging.error("Failed to unbind pci address {0}".format(vf['pci_address']))
|
||||
sys.exit(1)
|
||||
logging.info("Managed to unbind VFs.")
|
||||
|
||||
@classmethod
|
||||
def bind(cls):
|
||||
for vf in cls.mellanox_vfs:
|
||||
cmd_vfs_pci = "echo {0} ".format(vf['pci_address']) + \
|
||||
">> /sys/bus/pci/drivers/mlx5_core/bind"
|
||||
p_bind_pci = subprocess.Popen(cmd_vfs_pci, shell=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
retval = p_bind_pci.wait()
|
||||
if retval != 0:
|
||||
print 'Bind: Error is:', p_bind_pci.stdout.readlines()
|
||||
logging.error("Failed to bind pci address {0}".format(vf['pci_address']) )
|
||||
sys.exit(1)
|
||||
else:
|
||||
print 'Managed to bind pci address ', vf['pci_address']
|
||||
logging.debug("Managed to bind pci address {0}".format(vf['pci_address']) )
|
||||
logging.info("Managed to bind VFs.")
|
||||
|
||||
@classmethod
|
||||
def assign_mac_per_vf(cls):
|
||||
for vf in cls.mellanox_vfs:
|
||||
if "00:00:00:00:00:00" in vf['mac']:
|
||||
cmd_generate_mac = "ibstat {0} {1} |".format(vf['port_module'], vf['port_num']) + \
|
||||
" grep GUID | cut -d' ' -f3 | cut -d'x' -f2"
|
||||
p_cmd_generate_mac = subprocess.Popen(cmd_generate_mac, shell=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
retval = p_cmd_generate_mac.wait()
|
||||
if retval != 0:
|
||||
print 'Failed to get ibstat port guid'
|
||||
logging.error("Failed to find PORT GUID to set it as a MAC address for the VF")
|
||||
sys.exit(1)
|
||||
port_guid = p_cmd_generate_mac.stdout.readlines()[0].rstrip();
|
||||
port_guid_to_mac = port_guid[0:12]
|
||||
port_guid_to_mac = ':'.join(port_guid_to_mac[i:i+2] for i in range \
|
||||
(0, len(port_guid_to_mac), 2))
|
||||
vf['mac'] = port_guid_to_mac
|
||||
|
||||
@classmethod
|
||||
def set_mac_per_vf(cls):
|
||||
cmd_physical_port = "hiera mellanox-plugin | grep physical_port | cut -d'>' -f2 "
|
||||
p = subprocess.Popen(cmd_physical_port ,shell=True,stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
physical_port = p.stdout.readlines()[0].rstrip()
|
||||
physical_port = physical_port.strip(',"')
|
||||
for vf in cls.mellanox_vfs:
|
||||
cmd_set_mac_per_vf = "ip link set " + \
|
||||
"{0} vf {1} mac {2}".format(physical_port,vf['vf_num'], vf['mac'])
|
||||
p_cmd_set_mac_per_vf = subprocess.Popen(cmd_set_mac_per_vf,shell=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
retval = p_cmd_set_mac_per_vf.wait()
|
||||
if retval != 0:
|
||||
print 'Failed to set vf mac', cmd_set_mac_per_vf
|
||||
logging.error("Failed to set MAC address to VF {0}".format(vf['vf_num']))
|
||||
sys.exit(1)
|
||||
|
||||
@classmethod
|
||||
def wait_for_mlx_modules_and_vfs_loaded(cls, total_vfs):
|
||||
retval = 1
|
||||
while (retval != 0):
|
||||
cmd_lsmod = "lsmod | grep mlx_compat"
|
||||
p_lsmod = subprocess.Popen(cmd_lsmod, shell=True,stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
retval = p_lsmod.wait()
|
||||
logging.info("mlx_compat module is loaded!")
|
||||
|
||||
number_of_vfs = 0
|
||||
while ( number_of_vfs != total_vfs):
|
||||
cmd_number_vfs = "lspci | grep -i mellanox | grep -i virtual | wc -l"
|
||||
p_number_vfs = subprocess.Popen(cmd_number_vfs, shell=True,stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
retval = p_number_vfs.wait()
|
||||
if retval != 0:
|
||||
logging.error("Failed to run lspci")
|
||||
sys.exit(1)
|
||||
number_of_vfs = int(p_number_vfs.stdout.readlines()[0].rstrip());
|
||||
logging.info("all VFs are loaded.")
|
||||
|
||||
def main(total_vfs):
|
||||
logging.basicConfig(format='%(asctime)s %(message)s',
|
||||
level=logging.DEBUG, filename=LOG_FILE)
|
||||
try:
|
||||
vfs_configurations = MellanoxVfsSettings()
|
||||
vfs_configurations.wait_for_mlx_modules_and_vfs_loaded(int(total_vfs))
|
||||
vfs_configurations.build_vfs_dict()
|
||||
vfs_configurations.assign_mac_per_vf()
|
||||
vfs_configurations.unbind()
|
||||
vfs_configurations.set_mac_per_vf()
|
||||
vfs_configurations.bind()
|
||||
|
||||
except MellanoxVfsSettingsException, exc:
|
||||
error_msg = "Failed configuring Mellanox vfs: {0}\n".format(exc)
|
||||
sys.stderr.write(error_msg)
|
||||
logging.error(exc)
|
||||
sys.exit(1)
|
||||
success_msg = "Done configuring Mellanox vfs\n"
|
||||
sys.stdout.write(success_msg)
|
||||
logging.info(success_msg)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__=="__main__":
|
||||
main(sys.argv[1])
|
|
@ -119,6 +119,10 @@ function enable_eipoib (){
|
|||
fi
|
||||
}
|
||||
|
||||
function install_gawk (){
|
||||
apt-get -y install gawk
|
||||
}
|
||||
|
||||
if ! is_ofed_installed; then
|
||||
# Install mlnx-ofed-fuel rpm/deb package which extracts OFED installation dir
|
||||
install_mlnx_ofed_src
|
||||
|
@ -132,6 +136,10 @@ if ! is_ofed_installed; then
|
|||
|
||||
# Enable Ethernet IP Over Infiniband in case of eth_ipoib driver
|
||||
enable_eipoib
|
||||
|
||||
# Install gawk package for running ibdev2netdev -v
|
||||
install_gawk
|
||||
|
||||
fi
|
||||
|
||||
# Decrease loglevels for prevent flooding kernel messages to console
|
||||
|
|
|
@ -99,9 +99,6 @@ if [ $PROBED_PORT_NAME != $ISER_NAME ]; then
|
|||
udevadm trigger
|
||||
modprobe mlx4_en
|
||||
modprobe mlx5_ib && modprobe mlx5_core
|
||||
echo "#!/bin/bash" > /etc/network/if-up.d/iser_ifc
|
||||
echo "ifconfig $ISER_NAME up " >> /etc/network/if-up.d/iser_ifc
|
||||
ifup --all
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
logger_print error "Mellanox drivers restart failed."
|
||||
|
|
|
@ -278,16 +278,27 @@ function set_sriov () {
|
|||
logger_print error "Failed to find mlx5 up ports in ibdev2netdev."
|
||||
exit 1
|
||||
else
|
||||
if [ "$(lspci | grep -i mellanox | grep -i virtual | wc -l)" -ne "$TOTAL_VFS" ]; then
|
||||
res=`echo 0 > /sys/class/net/${device_up}/device/mlx5_num_vfs`
|
||||
res=`echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs`
|
||||
if [ ! $? -eq 0 ]; then
|
||||
logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs"
|
||||
exit 1
|
||||
fi
|
||||
echo "#!/bin/bash" > /etc/network/if-up.d/sriov_vfs
|
||||
echo "echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs" >> /etc/network/if-up.d/sriov_vfs
|
||||
chmod +x /etc/network/if-up.d/sriov_vfs
|
||||
ifup --all
|
||||
|
||||
# Give MACs to created VFs
|
||||
python ./configure_mellanox_vfs.py ${TOTAL_VFS}
|
||||
|
||||
# Make number of VFs and their MACs persistent
|
||||
persistent_ifup_script=/etc/network/if-up.d/persistent_mlnx_params
|
||||
echo "#!/bin/bash" > $persistent_ifup_script
|
||||
chmod +x $persistent_ifup_script
|
||||
echo "if ! lspci | grep -i mellanox | grep -i virtual; then" >> $persistent_ifup_script
|
||||
echo "echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs" >> $persistent_ifup_script
|
||||
echo "python /etc/fuel/plugins/mellanox-plugin-*/configure_mellanox_vfs.py ${TOTAL_VFS}" >> $persistent_ifup_script
|
||||
echo "fi" >> $persistent_ifup_script
|
||||
echo "if [ -f /etc/init.d/tgt ]; then /etc/init.d/tgt force-reload; else exit 0; fi" >> $persistent_ifup_script
|
||||
|
||||
if [ ! $? -eq 0 ]; then
|
||||
logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs"
|
||||
exit 1
|
||||
|
@ -295,6 +306,7 @@ function set_sriov () {
|
|||
logger_print debug "Configured total vfs ${TOTAL_VFS} on ${device_up}"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
}
|
||||
#################
|
||||
|
||||
|
|
|
@ -5,7 +5,7 @@ name: mellanox-plugin
|
|||
title: Mellanox ConnectX-4 Openstack Features
|
||||
|
||||
# Plugin version
|
||||
version: 3.1.5
|
||||
version: 3.1.8
|
||||
|
||||
# Description
|
||||
description: Enable features over Mellanox ConnectX-4 Adapters
|
||||
|
|
|
@ -62,7 +62,7 @@ old_debs="${PLUGIN_DIR}/repositories/ubuntu/*.deb"
|
|||
deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb
|
||||
cirros-testvm-mellanox-ib_0.3.2-9_amd64.deb
|
||||
eswitchd_1.0.0-18_amd64.deb
|
||||
mlnx-ofed-fuel_3.2-2.0.0.0_amd64.deb
|
||||
mlnx-ofed-fuel_3.3-0.1.0.0_amd64.deb
|
||||
lldpd_0.9.1-0_amd64.deb
|
||||
python-networking-mlnx_7.0.0-1_all.deb"
|
||||
get_packages "deb" "$old_debs" "$deb_files"
|
||||
|
|
Loading…
Reference in New Issue