diff --git a/deployment_scripts/configure_mellanox_vfs.py b/deployment_scripts/configure_mellanox_vfs.py new file mode 100755 index 0000000..b6d31b1 --- /dev/null +++ b/deployment_scripts/configure_mellanox_vfs.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python +# Copyright 2016 Mellanox Technologies, Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import subprocess +import logging +import traceback +import glob + +LOG_FILE = '/var/log/mellanox-plugin.log' + +class MellanoxVfsSettingsException(Exception): + pass + +class MellanoxVfsSettings(object): + + mellanox_vfs = list(); + + @classmethod + def get_pci_list(cls): + cmd_vfs = "lspci -D | grep -i mellanox | grep -i virtual | awk '{print $1}'" + p_lspci = subprocess.Popen(cmd_vfs, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + retval = p_lspci.wait() + if retval != 0: + print 'Failed to execute command lspci.' + logging.error("Failed to find mellanox VFs.") + sys.exit(1) + pci_res = list() + for line in p_lspci.stdout.readlines(): + pci_res.append(line.rstrip()) + return pci_res + + @classmethod + def build_vfs_dict(cls): + pci_res = cls.get_pci_list() + cmd_vfs_data = "ibdev2netdev -v" + p_vfs_data = subprocess.Popen(cmd_vfs_data, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + retval = p_vfs_data.wait() + if retval != 0: + print 'Failed to execute command ibdev2netdev -v.' + logging.error("Failed to execute ibdev2netdev -v") + sys.exit(1) + + ibdev = list() + for vf_data in p_vfs_data.stdout.readlines(): + ibdev.append(vf_data.rstrip()) + + cmd_vfs_macs = "ip link show | grep vf | awk '{print $4}' | tr -d ','" + p_vfs_macs = subprocess.Popen(cmd_vfs_macs, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + retval = p_vfs_macs.wait() + if retval != 0: + print 'Failed to execute command ip link show | grep vf ' + logging.error("Failed to find VFs in ip link show command") + sys.exit(1) + + mac_list = list() + for mac in p_vfs_macs.stdout.readlines(): + mac_list.append(mac.rstrip()) + + count = 0 + for pci_address in pci_res: + vf_dict = dict(); + vf_info = next((vf_info for vf_info in ibdev if pci_address in vf_info), None) + vf_info = vf_info.split(); + vf_dict['vf_num'] = count; + vf_dict['pci_address'] = vf_info[0] + vf_dict['port_module'] = vf_info[1] + vf_dict['port_num'] = vf_info[12] + vf_dict['mac'] = mac_list[count] + cls.mellanox_vfs.append(vf_dict) + count += 1; + + @classmethod + def unbind(cls): + for vf in cls.mellanox_vfs: + cmd_vfs_pci = "echo {0} ".format(vf['pci_address']) + \ + ">> /sys/bus/pci/drivers/mlx5_core/unbind" + p_unbind_pci = subprocess.Popen(cmd_vfs_pci, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + retval = p_unbind_pci.wait() + if retval != 0: + logging.error("Failed to unbind pci address {0}".format(vf['pci_address'])) + sys.exit(1) + logging.info("Managed to unbind VFs.") + + @classmethod + def bind(cls): + for vf in cls.mellanox_vfs: + cmd_vfs_pci = "echo {0} ".format(vf['pci_address']) + \ + ">> /sys/bus/pci/drivers/mlx5_core/bind" + p_bind_pci = subprocess.Popen(cmd_vfs_pci, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + retval = p_bind_pci.wait() + if retval != 0: + print 'Bind: Error is:', p_bind_pci.stdout.readlines() + logging.error("Failed to bind pci address {0}".format(vf['pci_address']) ) + sys.exit(1) + else: + print 'Managed to bind pci address ', vf['pci_address'] + logging.debug("Managed to bind pci address {0}".format(vf['pci_address']) ) + logging.info("Managed to bind VFs.") + + @classmethod + def assign_mac_per_vf(cls): + for vf in cls.mellanox_vfs: + if "00:00:00:00:00:00" in vf['mac']: + cmd_generate_mac = "ibstat {0} {1} |".format(vf['port_module'], vf['port_num']) + \ + " grep GUID | cut -d' ' -f3 | cut -d'x' -f2" + p_cmd_generate_mac = subprocess.Popen(cmd_generate_mac, shell=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + retval = p_cmd_generate_mac.wait() + if retval != 0: + print 'Failed to get ibstat port guid' + logging.error("Failed to find PORT GUID to set it as a MAC address for the VF") + sys.exit(1) + port_guid = p_cmd_generate_mac.stdout.readlines()[0].rstrip(); + port_guid_to_mac = port_guid[0:12] + port_guid_to_mac = ':'.join(port_guid_to_mac[i:i+2] for i in range \ + (0, len(port_guid_to_mac), 2)) + vf['mac'] = port_guid_to_mac + + @classmethod + def set_mac_per_vf(cls): + cmd_physical_port = "hiera mellanox-plugin | grep physical_port | cut -d'>' -f2 " + p = subprocess.Popen(cmd_physical_port ,shell=True,stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + physical_port = p.stdout.readlines()[0].rstrip() + physical_port = physical_port.strip(',"') + for vf in cls.mellanox_vfs: + cmd_set_mac_per_vf = "ip link set " + \ + "{0} vf {1} mac {2}".format(physical_port,vf['vf_num'], vf['mac']) + p_cmd_set_mac_per_vf = subprocess.Popen(cmd_set_mac_per_vf,shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + retval = p_cmd_set_mac_per_vf.wait() + if retval != 0: + print 'Failed to set vf mac', cmd_set_mac_per_vf + logging.error("Failed to set MAC address to VF {0}".format(vf['vf_num'])) + sys.exit(1) + + @classmethod + def wait_for_mlx_modules_and_vfs_loaded(cls, total_vfs): + retval = 1 + while (retval != 0): + cmd_lsmod = "lsmod | grep mlx_compat" + p_lsmod = subprocess.Popen(cmd_lsmod, shell=True,stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + retval = p_lsmod.wait() + logging.info("mlx_compat module is loaded!") + + number_of_vfs = 0 + while ( number_of_vfs != total_vfs): + cmd_number_vfs = "lspci | grep -i mellanox | grep -i virtual | wc -l" + p_number_vfs = subprocess.Popen(cmd_number_vfs, shell=True,stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + retval = p_number_vfs.wait() + if retval != 0: + logging.error("Failed to run lspci") + sys.exit(1) + number_of_vfs = int(p_number_vfs.stdout.readlines()[0].rstrip()); + logging.info("all VFs are loaded.") + +def main(total_vfs): + logging.basicConfig(format='%(asctime)s %(message)s', + level=logging.DEBUG, filename=LOG_FILE) + try: + vfs_configurations = MellanoxVfsSettings() + vfs_configurations.wait_for_mlx_modules_and_vfs_loaded(int(total_vfs)) + vfs_configurations.build_vfs_dict() + vfs_configurations.assign_mac_per_vf() + vfs_configurations.unbind() + vfs_configurations.set_mac_per_vf() + vfs_configurations.bind() + + except MellanoxVfsSettingsException, exc: + error_msg = "Failed configuring Mellanox vfs: {0}\n".format(exc) + sys.stderr.write(error_msg) + logging.error(exc) + sys.exit(1) + success_msg = "Done configuring Mellanox vfs\n" + sys.stdout.write(success_msg) + logging.info(success_msg) + sys.exit(0) + + +if __name__=="__main__": + main(sys.argv[1]) diff --git a/deployment_scripts/install_ofed.sh b/deployment_scripts/install_ofed.sh index df7da01..7f05932 100755 --- a/deployment_scripts/install_ofed.sh +++ b/deployment_scripts/install_ofed.sh @@ -119,6 +119,10 @@ function enable_eipoib (){ fi } +function install_gawk (){ + apt-get -y install gawk +} + if ! is_ofed_installed; then # Install mlnx-ofed-fuel rpm/deb package which extracts OFED installation dir install_mlnx_ofed_src @@ -132,6 +136,10 @@ if ! is_ofed_installed; then # Enable Ethernet IP Over Infiniband in case of eth_ipoib driver enable_eipoib + + # Install gawk package for running ibdev2netdev -v + install_gawk + fi # Decrease loglevels for prevent flooding kernel messages to console diff --git a/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb b/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb index c957ea2..d4bc2db 100644 --- a/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb +++ b/deployment_scripts/puppet/modules/mellanox_openstack/templates/iser_rename.erb @@ -99,9 +99,6 @@ if [ $PROBED_PORT_NAME != $ISER_NAME ]; then udevadm trigger modprobe mlx4_en modprobe mlx5_ib && modprobe mlx5_core - echo "#!/bin/bash" > /etc/network/if-up.d/iser_ifc - echo "ifconfig $ISER_NAME up " >> /etc/network/if-up.d/iser_ifc - ifup --all if [ $? -ne 0 ]; then logger_print error "Mellanox drivers restart failed." diff --git a/deployment_scripts/sriov.sh b/deployment_scripts/sriov.sh index d5787f5..a79c64b 100755 --- a/deployment_scripts/sriov.sh +++ b/deployment_scripts/sriov.sh @@ -278,21 +278,33 @@ function set_sriov () { logger_print error "Failed to find mlx5 up ports in ibdev2netdev." exit 1 else - res=`echo 0 > /sys/class/net/${device_up}/device/mlx5_num_vfs` - res=`echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs` - if [ ! $? -eq 0 ]; then - logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs" - exit 1 - fi - echo "#!/bin/bash" > /etc/network/if-up.d/sriov_vfs - echo "echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs" >> /etc/network/if-up.d/sriov_vfs - chmod +x /etc/network/if-up.d/sriov_vfs - ifup --all - if [ ! $? -eq 0 ]; then - logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs" - exit 1 - else - logger_print debug "Configured total vfs ${TOTAL_VFS} on ${device_up}" + if [ "$(lspci | grep -i mellanox | grep -i virtual | wc -l)" -ne "$TOTAL_VFS" ]; then + res=`echo 0 > /sys/class/net/${device_up}/device/mlx5_num_vfs` + res=`echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs` + if [ ! $? -eq 0 ]; then + logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs" + exit 1 + fi + + # Give MACs to created VFs + python ./configure_mellanox_vfs.py ${TOTAL_VFS} + + # Make number of VFs and their MACs persistent + persistent_ifup_script=/etc/network/if-up.d/persistent_mlnx_params + echo "#!/bin/bash" > $persistent_ifup_script + chmod +x $persistent_ifup_script + echo "if ! lspci | grep -i mellanox | grep -i virtual; then" >> $persistent_ifup_script + echo "echo ${TOTAL_VFS} > /sys/class/net/${device_up}/device/mlx5_num_vfs" >> $persistent_ifup_script + echo "python /etc/fuel/plugins/mellanox-plugin-*/configure_mellanox_vfs.py ${TOTAL_VFS}" >> $persistent_ifup_script + echo "fi" >> $persistent_ifup_script + echo "if [ -f /etc/init.d/tgt ]; then /etc/init.d/tgt force-reload; else exit 0; fi" >> $persistent_ifup_script + + if [ ! $? -eq 0 ]; then + logger_print error "Failed to write $TOTAL_VFS > /sys/class/net/${device_up}/device/mlx5_num_vfs" + exit 1 + else + logger_print debug "Configured total vfs ${TOTAL_VFS} on ${device_up}" + fi fi fi } @@ -300,7 +312,7 @@ function set_sriov () { case $SCRIPT_MODE in 'configure') - configure_sriov + configure_sriov ;; 'validate') validate_sriov diff --git a/metadata.yaml b/metadata.yaml index 1dd8dae..b8ba0ca 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -5,7 +5,7 @@ name: mellanox-plugin title: Mellanox ConnectX-4 Openstack Features # Plugin version -version: 3.1.5 +version: 3.1.8 # Description description: Enable features over Mellanox ConnectX-4 Adapters diff --git a/pre_build_hook b/pre_build_hook index c08b515..848b306 100755 --- a/pre_build_hook +++ b/pre_build_hook @@ -62,7 +62,7 @@ old_debs="${PLUGIN_DIR}/repositories/ubuntu/*.deb" deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb cirros-testvm-mellanox-ib_0.3.2-9_amd64.deb eswitchd_1.0.0-18_amd64.deb - mlnx-ofed-fuel_3.2-2.0.0.0_amd64.deb + mlnx-ofed-fuel_3.3-0.1.0.0_amd64.deb lldpd_0.9.1-0_amd64.deb python-networking-mlnx_7.0.0-1_all.deb" get_packages "deb" "$old_debs" "$deb_files"