Upgrading to Mellanox OFED 3.4-1 GA

Adding some reqiured packages for ofed installation
and adding a workaround for setting interfaces up after
ofed installtion and adding vfs

Change-Id: I482d02eb51f18df71e8aa5e0cba0c14218bf53af
This commit is contained in:
Aviram Bar-Haim 2017-02-15 16:24:11 +02:00 committed by waleedm
parent c864477372
commit 3d7d5fb73c
8 changed files with 104 additions and 75 deletions

View File

@ -16,6 +16,11 @@
OFED_DEBS_DIR=/opt/ofed/MLNX_OFED/DEBS
# Updating linux-headers
kernel_prefix=initrd.img
sudo apt-get install -y $(ls /boot/ | grep $kernel_prefix | \
sed "s/$kernel_prefix/linux-headers/")
# Set mlnx scripts to run on boot
sed -i '1a\$(init_mlnx.sh > \/dev\/null 2>\&1) \&\n' /etc/rc.local
if [ ! -z $1 ]; then

View File

@ -92,72 +92,76 @@ class MellanoxVfsSettings(object):
count += 1;
@classmethod
def unbind(cls):
def unbind(cls, vf_index):
for vf in cls.mellanox_vfs:
cmd_vfs_pci = "echo {0} ".format(vf['pci_address']) + \
">> /sys/bus/pci/drivers/mlx5_core/unbind"
p_unbind_pci = subprocess.Popen(cmd_vfs_pci, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
retval = p_unbind_pci.wait()
if retval != 0:
logging.error("Failed to unbind pci address {0}".format(vf['pci_address']))
sys.exit(1)
logging.info("Managed to unbind VFs.")
@classmethod
def bind(cls):
for vf in cls.mellanox_vfs:
cmd_vfs_pci = "echo {0} ".format(vf['pci_address']) + \
">> /sys/bus/pci/drivers/mlx5_core/bind"
p_bind_pci = subprocess.Popen(cmd_vfs_pci, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
retval = p_bind_pci.wait()
if retval != 0:
print 'Bind: Error is:', p_bind_pci.stdout.readlines()
logging.error("Failed to bind pci address {0}".format(vf['pci_address']) )
sys.exit(1)
else:
print 'Managed to bind pci address ', vf['pci_address']
logging.debug("Managed to bind pci address {0}".format(vf['pci_address']) )
logging.info("Managed to bind VFs.")
@classmethod
def assign_mac_per_vf(cls):
for vf in cls.mellanox_vfs:
if "00:00:00:00:00:00" in vf['mac']:
cmd_generate_mac = "ibstat {0} {1} |".format(vf['port_module'], vf['port_num']) + \
" grep GUID | cut -d' ' -f3 | cut -d'x' -f2"
p_cmd_generate_mac = subprocess.Popen(cmd_generate_mac, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
retval = p_cmd_generate_mac.wait()
if int(vf['vf_num']) == int(vf_index):
cmd_vfs_pci = "echo {0} ".format(vf['pci_address']) + \
">> /sys/bus/pci/drivers/mlx5_core/unbind"
p_unbind_pci = subprocess.Popen(cmd_vfs_pci, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
retval = p_unbind_pci.wait()
if retval != 0:
print 'Failed to get ibstat port guid'
logging.error("Failed to find PORT GUID to set it as a MAC address for the VF")
sys.exit(1)
port_guid = p_cmd_generate_mac.stdout.readlines()[0].rstrip();
port_guid_to_mac = port_guid[0:12]
port_guid_to_mac = ':'.join(port_guid_to_mac[i:i+2] for i in range \
(0, len(port_guid_to_mac), 2))
vf['mac'] = port_guid_to_mac
logging.error("Failed to unbind pci address {0}".format(vf['pci_address']))
sys.exit(1)
logging.info("Managed to unbind VFs.")
@classmethod
def set_mac_per_vf(cls):
def bind(cls, vf_index):
for vf in cls.mellanox_vfs:
if int(vf['vf_num']) == int(vf_index):
cmd_vfs_pci = "echo {0} ".format(vf['pci_address']) + \
">> /sys/bus/pci/drivers/mlx5_core/bind"
p_bind_pci = subprocess.Popen(cmd_vfs_pci, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
retval = p_bind_pci.wait()
if retval != 0:
print 'Bind: Error is:', p_bind_pci.stdout.readlines()
logging.error("Failed to bind pci address {0}".format(vf['pci_address']) )
sys.exit(1)
else:
print 'Managed to bind pci address ', vf['pci_address']
logging.debug("Managed to bind pci address {0}".format(vf['pci_address']) )
logging.info("Managed to bind VFs.")
@classmethod
def assign_mac_per_vf(cls, vf_index):
for vf in cls.mellanox_vfs:
if int(vf['vf_num']) == int(vf_index):
if "00:00:00:00:00:00" in vf['mac']:
cmd_generate_mac = "ibstat {0} {1} |".format(vf['port_module'], vf['port_num']) + \
" grep GUID | cut -d' ' -f3 | cut -d'x' -f2"
p_cmd_generate_mac = subprocess.Popen(cmd_generate_mac, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
retval = p_cmd_generate_mac.wait()
if retval != 0:
print 'Failed to get ibstat port guid'
logging.error("Failed to find PORT GUID to set it as a MAC address for the VF")
sys.exit(1)
port_guid = p_cmd_generate_mac.stdout.readlines()[0].rstrip();
port_guid_to_mac = port_guid[0:12]
port_guid_to_mac = ':'.join(port_guid_to_mac[i:i+2] for i in range \
(0, len(port_guid_to_mac), 2))
vf['mac'] = port_guid_to_mac
@classmethod
def set_mac_per_vf(cls, vf_index):
cmd_physical_port = "hiera mellanox-plugin | grep physical_port | cut -d'>' -f2 "
p = subprocess.Popen(cmd_physical_port ,shell=True,stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
physical_port = p.stdout.readlines()[0].rstrip()
physical_port = physical_port.strip(',"')
for vf in cls.mellanox_vfs:
cmd_set_mac_per_vf = "ip link set " + \
"{0} vf {1} mac {2}".format(physical_port,vf['vf_num'], vf['mac'])
p_cmd_set_mac_per_vf = subprocess.Popen(cmd_set_mac_per_vf,shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
retval = p_cmd_set_mac_per_vf.wait()
if retval != 0:
print 'Failed to set vf mac', cmd_set_mac_per_vf
logging.error("Failed to set MAC address to VF {0}".format(vf['vf_num']))
sys.exit(1)
if int(vf['vf_num']) == int(vf_index):
cmd_set_mac_per_vf = "ip link set " + \
"{0} vf {1} mac {2}".format(physical_port,vf['vf_num'], vf['mac'])
p_cmd_set_mac_per_vf = subprocess.Popen(cmd_set_mac_per_vf,shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
retval = p_cmd_set_mac_per_vf.wait()
if retval != 0:
print 'Failed to set vf mac', cmd_set_mac_per_vf
logging.error("Failed to set MAC address to VF {0}".format(vf['vf_num']))
sys.exit(1)
@classmethod
def wait_for_vfs_loaded(cls, total_vfs):
@ -190,10 +194,10 @@ def main(total_vfs):
vfs_configurations = MellanoxVfsSettings()
vfs_configurations.wait_for_vfs_loaded(int(total_vfs))
vfs_configurations.build_vfs_dict()
vfs_configurations.assign_mac_per_vf()
vfs_configurations.unbind()
vfs_configurations.set_mac_per_vf()
vfs_configurations.bind()
vfs_configurations.assign_mac_per_vf(0)
vfs_configurations.unbind(0)
vfs_configurations.set_mac_per_vf(0)
vfs_configurations.bind(0)
except MellanoxVfsSettingsException, exc:
error_msg = "Failed configuring Mellanox vfs: {0}\n".format(exc)

View File

@ -41,7 +41,7 @@ function install_mlnx_ofed_src () {
if [ "$DISTRO" == "redhat" ]; then
yum install ${OFED_PACKAGE_NAME} -y
elif [ "$DISTRO" == "ubuntu" ]; then
apt-get install ${OFED_PACKAGE_NAME} -y
apt-get install ${OFED_PACKAGE_NAME} -y --force-yes
fi
if [ $? -ne 0 ]; then
logger_print error "Failed installing ${OFED_PACKAGE_NAME} package"
@ -78,6 +78,13 @@ function add_kernel_support () {
fi
}
function install_missing_packages_for_ofed() {
apt-get install -y --force-yes dpatch autoconf libgfortran3 \
chrpath graphviz flex debhelper swig quilt m4 libltdl-dev \
gfortran tcl bison autotools-dev tk python-libxml2 tcl8.4 \
pkg-config automake tk8.4
}
function install_ofed_without_fw_update () {
OFED_INSTALL_SCRIPT="${OFED_DIR}/mlnxofedinstall"
if [ ! -f $OFED_INSTALL_SCRIPT ]; then
@ -85,6 +92,9 @@ function install_ofed_without_fw_update () {
exit 1
fi
logger_print info "Installing missing packges for OFED"
install_missing_packages_for_ofed
logger_print info "Installing OFED drivers"
OFED_INSTALL_SCRIPT_CMD="/usr/bin/perl ${OFED_INSTALL_SCRIPT}"
${OFED_INSTALL_SCRIPT_CMD} --force --without-fw-update
@ -145,4 +155,8 @@ sysctl_conf set 'kernel.printk' '4 4 1 7'
service openibd stop
service openibd start
# Setting interfaces up
for interface in `ifconfig -a | sed 's/[ \t].*//;/^\(lo\|\)$/d' | \
sed 's/://'`;do ifconfig $interface up; done
exit 0

View File

@ -305,13 +305,19 @@ function set_sriov () {
python ./configure_mellanox_vfs.py ${TOTAL_VFS}
fi
# Make number of VFs and their MACs persistent
# Make number of VFs persistent
persistent_ifup_script=/etc/network/if-up.d/persistent_mlnx_params
echo "#!/bin/bash" > $persistent_ifup_script
chmod +x $persistent_ifup_script
echo "# VFs Persistency" >> $persistent_ifup_script
echo "if ! lspci | grep -i mellanox | grep -i virtual; then" >> $persistent_ifup_script
echo "echo 0 > /sys/class/net/${device_up}/device/mlx5_num_vfs" >> $persistent_ifup_script
echo "echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs" >> $persistent_ifup_script
echo "fi" >> $persistent_ifup_script
# Make the MAC for iser vf persistent
echo "# MACs Persistency" >> $persistent_ifup_script
echo 'if ip link show | grep "vf 0"| grep "MAC 00:00:00:00:00:00"; then' >> $persistent_ifup_script
echo "python /etc/fuel/plugins/mellanox-plugin-*/configure_mellanox_vfs.py ${TOTAL_VFS}" >> $persistent_ifup_script
echo "fi" >> $persistent_ifup_script
echo "if [ -f /etc/init.d/tgt ]; then /etc/init.d/tgt force-reload; else exit 0; fi" >> $persistent_ifup_script
@ -342,4 +348,7 @@ case $SCRIPT_MODE in
;;
esac
# Setting interfaces up
for interface in `ifconfig -a | sed 's/[ \t].*//;/^\(lo\|\)$/d' | \
sed 's/://'`;do ifconfig $interface up; done
exit $?

View File

@ -5,13 +5,13 @@ name: mellanox-plugin
title: Mellanox ConnectX-4 Openstack Features
# Plugin version
version: 3.3.0
version: 3.3.1
# Description
description: Enable features over Mellanox ConnectX-4 Adapters
# Required fuel version
fuel_version: ['8.0','9.0']
fuel_version: ['8.0','9.0','9.1','9.2']
# Specify license of your plugin
licenses: ['Apache License Version 2.0']

View File

@ -62,7 +62,7 @@ old_debs="${PLUGIN_DIR}/repositories/ubuntu/*.deb"
deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb
cirros-testvm-mellanox-ib_0.3.2-9_amd64.deb
eswitchd_1.0.0-18_amd64.deb
mlnx-ofed-fuel_3.3-1.5.0.0_amd64.deb
mlnx-ofed-fuel_3.4-1.0.0.0_amd64.deb
lldpd_0.9.1-0_amd64.deb
python-networking-mlnx_7.0.0-1_all.deb"
get_packages "deb" "$old_debs" "$deb_files"

View File

@ -76,7 +76,12 @@ with open("/etc/fuel-bootstrap-cli/fuel_bootstrap_cli.yaml", 'r') as stream:
with open("/etc/fuel-bootstrap-cli/fuel_bootstrap_cli.yaml", "w") as f:
yaml.dump(fuel_bootstrap_config, f)
extra_packages = ['mlnx-ofed-fuel', 'python-libxml2', 'lsof']
extra_packages = ['mlnx-ofed-fuel', 'python-libxml2', 'lsof', 'curl',
'libgfortran3', 'tk', 'swig', 'm4', 'autoconf',
'tcl8.4', 'flex', 'dpatch', 'gfortran', 'libltdl-dev',
'autotools-dev', 'quilt', 'automake', 'pkg-config',
'chrpath', 'debhelper', 'tk8.4', 'graphviz', 'bison',
'tcl']
packages = ''.join(['--package %s ' % p for p in extra_packages])
extra_dir = "/var/www/nailgun/plugins/%s/bootstrap/sync" % plugin
cmd = "fuel-bootstrap build --debug {0} --extra-dir {1} \

View File

@ -1,18 +1,10 @@
EXTRA_SCRIPTS="/sbin/reboot_bootstrap_node
/sbin/create_mellanox_bootstrap"
source /var/www/nailgun/plugins/mellanox-plugin*/scripts/common
# Verify run is over Fuel Master and we are not During upgrade
if [ -d $FUEL_BOOTSTRAP_DIR ] && [ $1 -eq 0 ]; then
source /var/www/nailgun/plugins/mellanox-plugin*/scripts/common
# Return orig active bootstrap
if [ -f $ORIG_BOOTSTRAP_VERSION_FILE ]; then
orig_uid=`cat $ORIG_BOOTSTRAP_VERSION_FILE`
fuel-bootstrap activate $orig_uid
\rm $ORIG_BOOTSTRAP_VERSION_FILE
fi
# Return orig yaml
if [ -f $ORIG_BOOTSTRAP_CLI_YAML ]; then
mv $ORIG_BOOTSTRAP_CLI_YAML $BOOTSTRAP_CLI_YAML