From c0ed8e85ea9f355d534dc42f7abe4939c06f93d4 Mon Sep 17 00:00:00 2001 From: Hamdy Khader Date: Wed, 28 Jun 2017 01:07:58 +0300 Subject: [PATCH] Add support for ConnectX-5 adapters - upgrade mlnx-ofed-fuel to version 4.0-2.0.0.1 - add maximum number of CX5 VFs to 96 Change-Id: Ie3612d5b9a488ade49dbfc65389be9e9f9442bb0 --- deployment_scripts/common | 1 + deployment_scripts/mellanox_settings.py | 24 ++++++++++++------- .../puppet/manifests/controller.pp | 2 +- deployment_scripts/sriov.sh | 18 ++++++++------ pre_build_hook | 2 +- 5 files changed, 30 insertions(+), 17 deletions(-) diff --git a/deployment_scripts/common b/deployment_scripts/common index 994dbb9..d02f265 100755 --- a/deployment_scripts/common +++ b/deployment_scripts/common @@ -85,6 +85,7 @@ readonly SRIOV=`get_mlnx_param sriov` readonly USER_NUM_OF_VFS=`get_mlnx_param num_of_vfs` readonly ISER=`get_mlnx_param iser` readonly MAX_VFS=62 +readonly MAX_VFS_CX5=96 readonly MIN_VFS=1 readonly CX=`get_mlnx_param cx_card` readonly NETWORK_TYPE=`get_mlnx_param network_type` diff --git a/deployment_scripts/mellanox_settings.py b/deployment_scripts/mellanox_settings.py index 795e171..dd5b3ab 100755 --- a/deployment_scripts/mellanox_settings.py +++ b/deployment_scripts/mellanox_settings.py @@ -27,7 +27,8 @@ MLNX_SECTION = 'mellanox-plugin' SETTINGS_FILE = '/etc/astute.yaml' PLUGIN_OVERRIDE_FILE = '/etc/hiera/override/plugins.yaml' MLNX_DRIVERS_LIST = { 'ConnectX-3': {'eth_driver' : 'mlx4_en', 'ib_driver' : 'eth_ipoib'}, - 'ConnectX-4': {'eth_driver' : 'mlx5_core', 'ib_driver' : 'eth_ipoib'}} + 'ConnectX-4': {'eth_driver' : 'mlx5_core', 'ib_driver' : 'eth_ipoib'}, + 'ConnectX-5': {'eth_driver' : 'mlx5_core', 'ib_driver' : 'eth_ipoib'}} MLNX_DRIVERS = set([MLNX_DRIVERS_LIST[card][net] for card in MLNX_DRIVERS_LIST for net in MLNX_DRIVERS_LIST[card]]) @@ -82,6 +83,17 @@ class MellanoxSettings(object): .index(driver)] return card + @classmethod + def get_card_type_by_interface_name(cls, ifc_name): + card_type = os.popen('mst status -v | grep {0} | grep -o ConnectX[0-9]*' + .format(ifc_name)).readlines() + if len(card_type) == 1: + card_type = card_type[0].replace('ConnectX', 'ConnectX-').strip() + return card_type + else: + logging.error('No driver found for interface {0}'.format(ifc_name)) + exit(1) + @classmethod def add_cx_card(cls): mlnx_interfaces = cls.mlnx_interfaces_section @@ -90,7 +102,7 @@ class MellanoxSettings(object): mlnx = cls.get_mlnx_section() for network_type, ifc_dict in mlnx_interfaces.iteritems(): if 'driver' in ifc_dict and network_type in ['private','management','storage']: - + # The bond interfaces extend the original list, # otherwise, the interface is appended to the list. if(type(ifc_dict['driver']) is list): @@ -115,7 +127,7 @@ class MellanoxSettings(object): mellanox_interface = interfaces_set[0] if current_driver in ETH_DRIVERS: mlnx['network_type'] = 'ethernet' - mlnx['cx_card'] = cls.get_card_type(current_driver) + mlnx['cx_card'] = cls.get_card_type_by_interface_name(mellanox_interface) elif current_driver in IB_DRIVERS: mlnx['network_type'] = 'infiniband' ibdev = os.popen('ibdev2netdev').readlines() @@ -125,11 +137,7 @@ class MellanoxSettings(object): return 0 if ('bonds' in cls.data and mellanox_interface.startswith('bond')): mellanox_interface = cls.data['bonds'][mellanox_interface]['interfaces'][0] - interface_line = [l for l in ibdev if mellanox_interface in l] - if interface_line and 'mlx5' in interface_line.pop(): - mlnx['cx_card'] = 'ConnectX-4' - else: - mlnx['cx_card'] = 'ConnectX-3' + mlnx['cx_card'] = cls.get_card_type_by_interface_name(mellanox_interface) network_info_msg = 'Detected Network Type is: {0} '.format(mlnx['network_type']) card_info_msg = 'Detected Card Type is: {0} '.format(mlnx['cx_card']) diff --git a/deployment_scripts/puppet/manifests/controller.pp b/deployment_scripts/puppet/manifests/controller.pp index 18e8d4f..d2ea68c 100644 --- a/deployment_scripts/puppet/manifests/controller.pp +++ b/deployment_scripts/puppet/manifests/controller.pp @@ -13,7 +13,7 @@ if ( $mlnx['driver'] == 'mlx4_en' and $mlnx['mlnx_qos'] ) { } if ($mlnx['sriov']) { - $pci_vendor_devices = '15b3:1014,15b3:1016' + $pci_vendor_devices = '15b3:1014,15b3:1016,15b3:1018' $agent_required = 'True' class { 'mellanox_openstack::controller_sriov' : eswitch_vnic_type => $eswitch_vnic_type, diff --git a/deployment_scripts/sriov.sh b/deployment_scripts/sriov.sh index 8a62fdd..6c8c866 100755 --- a/deployment_scripts/sriov.sh +++ b/deployment_scripts/sriov.sh @@ -44,12 +44,16 @@ function get_num_probe_vfs () { } function calculate_total_vfs () { - # validate num of vfs is an integer, 0 <= num <= 64 + max_card_vfs=$MAX_VFS + if [ $CX == "ConnectX-5" ]; then + max_card_vfs=$MAX_VFS_CX5 + fi + # validate num of vfs is an integer, 0 <= num <= 96 if [ "${USER_NUM_OF_VFS}" -ne "${USER_NUM_OF_VFS}" ] 2>/dev/null || - [ "${USER_NUM_OF_VFS}" -gt ${MAX_VFS} ] || + [ "${USER_NUM_OF_VFS}" -gt ${max_card_vfs} ] || [ "${USER_NUM_OF_VFS}" -lt ${MIN_VFS} ]; then logger_print error "Illegal number of VFs ${USER_NUM_OF_VFS}, value - should be an integer between ${MIN_VFS},${MAX_VFS}" + should be an integer between ${MIN_VFS},${max_card_vfs}" return 1 fi num_of_vfs=0 @@ -177,7 +181,7 @@ function burn_vfs_in_fw () { done service mst stop &>/dev/null fi - if [ $CX == 'ConnectX-4' ]; then + if [ $CX == 'ConnectX-4' ] || [ $CX == 'ConnectX-5' ]; then # required for mlxconfig to discover mlnx devices service openibd start &>/dev/null service mst start &>/dev/null @@ -218,9 +222,9 @@ function configure_sriov () { logger_print info "Detected: ConnectX-3 card" fi - if [ $CX == 'ConnectX-4' ]; then + if [ $CX == 'ConnectX-4' ] || [ $CX == 'ConnectX-5' ]; then set_sriov $total_vfs && - logger_print info "Detected: ConnectX-4 card" + logger_print info "Detected: $CX card" fi return $? @@ -266,7 +270,7 @@ function validate_sriov () { set_modprobe_file $FALLBACK_NUM_VFS service openibd restart &> /dev/null fi - if [ $CX == 'ConnectX-4' ]; then + if [ $CX == 'ConnectX-4' ] || [ $CX == 'ConnectX-5' ]; then set_sriov $FALLBACK_NUM_VFS fi diff --git a/pre_build_hook b/pre_build_hook index c5eca12..3a8795f 100755 --- a/pre_build_hook +++ b/pre_build_hook @@ -62,7 +62,7 @@ old_debs="${PLUGIN_DIR}/repositories/ubuntu/*.deb" deb_files="cirros-testvm-mellanox_0.3.2-ubuntu3_amd64.deb cirros-testvm-mellanox-ib_0.3.2-9_amd64.deb eswitchd_1.0.0-18_amd64.deb - mlnx-ofed-fuel_3.4-1.0.0.0_amd64.deb + mlnx-ofed-fuel_4.0-2.0.0.1_amd64.deb lldpd_0.9.1-0_amd64.deb python-networking-mlnx_7.0.0-1_all.deb" get_packages "deb" "$old_debs" "$deb_files"